From f829a12add380c78e50ecc86357d4a1b513f21f0 Mon Sep 17 00:00:00 2001 From: boyealkim Date: Thu, 25 Jul 2019 20:42:13 +0900 Subject: fix stream_merge front algo --- src/stream_manager.cc | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/src/stream_manager.cc b/src/stream_manager.cc index 6cd62a2..6e02dcd 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -194,14 +194,17 @@ bool stream_operation::do_operation( gpgpu_sim *gpu ) m_stream->record_next_done(); } break; - case stream_wait_event: { + case stream_wait_event: //only allows next op to go if event is done //otherwise stays in the stream queue printf("stream wait event processing...\n"); - if(m_event->done()) + if(m_event->done()){ printf("stream wait event done\n"); m_stream->record_next_done(); } + else{ + return false; + } break; default: abort(); @@ -232,6 +235,7 @@ stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) m_service_stream_zero = false; m_cuda_launch_blocking = cuda_launch_blocking; pthread_mutex_init(&m_lock,NULL); + m_last_stream = m_streams.begin(); } bool stream_manager::operation( bool * sim) @@ -330,22 +334,31 @@ stream_operation stream_manager::front() m_service_stream_zero = false; } } - if(!m_service_stream_zero) { - std::list::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++) { - CUstream_st *stream = *s; - if( !stream->busy() && !stream->empty() ) { - result = stream->next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = stream; - } - break; - } - } - } + std::list::iterator s = m_last_stream; + if(m_last_stream == m_streams.end()){ + s = m_streams.begin(); + } + else{ + s++; + } + for(size_t ii = 0 ; ii < m_streams.size(); ii++, s++) { + if(s == m_streams.end()){ + s = m_streams.begin(); + } + m_last_stream = s; + CUstream_st *stream = *s; + if( !stream->busy() && !stream->empty() ) { + result = stream->next(); + if( result.is_kernel() ) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = stream; + } + break; + } + } + } return result; } @@ -370,7 +383,8 @@ void stream_manager::destroy_stream( CUstream_st *stream ) break; } } - delete stream; + delete stream; + m_last_stream = m_streams.begin(); pthread_mutex_unlock(&m_lock); } @@ -384,7 +398,6 @@ bool stream_manager::concurrent_streams_empty() for( s=m_streams.begin(); s!=m_streams.end();++s ) { struct CUstream_st *stream = *s; if( !stream->empty() ) { - //stream->print(stdout); result = false; break; } @@ -437,7 +450,6 @@ void stream_manager::print_impl( FILE *fp) void stream_manager::push( stream_operation op ) { struct CUstream_st *stream = op.get_stream(); - // block if stream 0 (or concurrency disabled) and pending concurrent operations exist bool block= !stream || m_cuda_launch_blocking; while(block) { -- cgit v1.3 From b13557baa4ea6d1084bdcc9b28b9eb660c394427 Mon Sep 17 00:00:00 2001 From: boyealkim Date: Fri, 26 Jul 2019 11:55:45 +0900 Subject: fix indentation --- libcuda/cuda_runtime_api.cc | 10 ++++---- src/stream_manager.cc | 58 ++++++++++++++++++++++----------------------- src/stream_manager.h | 17 ++++++------- 3 files changed, 40 insertions(+), 45 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 7005eef..6a7826e 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1709,13 +1709,13 @@ __host__ cudaError_t CUDARTAPI cudaStreamWaitEvent(cudaStream_t stream, cudaEven //reference: https://www.cs.cmu.edu/afs/cs/academic/class/15668-s11/www/cuda-doc/html/group__CUDART__STREAM_gfe68d207dc965685d92d3f03d77b0876.html CUevent_st *e = get_event(event); if( !e ){ - printf("GPGPU-Sim API: Error at cudaStreamWaitEvent. Event is not created .\n"); - return g_last_cudaError = cudaErrorInvalidResourceHandle; + printf("GPGPU-Sim API: Error at cudaStreamWaitEvent. Event is not created .\n"); + return g_last_cudaError = cudaErrorInvalidResourceHandle; } - else if(e->num_issued() == 0){ - printf("GPGPU-Sim API: Warning: cudaEventRecord has not been called on event before calling cudaStreamWaitEvent.\nNothing to be done.\n"); + else if(e->num_issued() == 0){ + printf("GPGPU-Sim API: Warning: cudaEventRecord has not been called on event before calling cudaStreamWaitEvent.\nNothing to be done.\n"); return g_last_cudaError = cudaSuccess; - } + } if (!stream){ g_stream_manager->pushCudaStreamWaitEventToAllStreams(e, flags); } else { diff --git a/src/stream_manager.cc b/src/stream_manager.cc index 6e02dcd..d0eecdd 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -198,13 +198,13 @@ bool stream_operation::do_operation( gpgpu_sim *gpu ) //only allows next op to go if event is done //otherwise stays in the stream queue printf("stream wait event processing...\n"); - if(m_event->done()){ + if(m_event->done()){ printf("stream wait event done\n"); m_stream->record_next_done(); } - else{ - return false; - } + else{ + return false; + } break; default: abort(); @@ -235,7 +235,7 @@ stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) m_service_stream_zero = false; m_cuda_launch_blocking = cuda_launch_blocking; pthread_mutex_init(&m_lock,NULL); - m_last_stream = m_streams.begin(); + m_last_stream = m_streams.begin(); } bool stream_manager::operation( bool * sim) @@ -336,29 +336,25 @@ stream_operation stream_manager::front() } if(!m_service_stream_zero) { - std::list::iterator s = m_last_stream; - if(m_last_stream == m_streams.end()){ - s = m_streams.begin(); - } - else{ - s++; - } - for(size_t ii = 0 ; ii < m_streams.size(); ii++, s++) { - if(s == m_streams.end()){ - s = m_streams.begin(); - } - m_last_stream = s; - CUstream_st *stream = *s; - if( !stream->busy() && !stream->empty() ) { - result = stream->next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = stream; - } - break; - } - } - } + std::list::iterator s = m_last_stream; + if(m_last_stream == m_streams.end()){ s = m_streams.begin(); } + else{ s++; } + for(size_t ii = 0 ; ii < m_streams.size(); ii++, s++) { + if(s == m_streams.end()){ + s = m_streams.begin(); + } + m_last_stream = s; + CUstream_st *stream = *s; + if( !stream->busy() && !stream->empty() ) { + result = stream->next(); + if( result.is_kernel() ) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = stream; + } + break; + } + } + } return result; } @@ -383,8 +379,8 @@ void stream_manager::destroy_stream( CUstream_st *stream ) break; } } - delete stream; - m_last_stream = m_streams.begin(); + delete stream; + m_last_stream = m_streams.begin(); pthread_mutex_unlock(&m_lock); } @@ -398,6 +394,7 @@ bool stream_manager::concurrent_streams_empty() for( s=m_streams.begin(); s!=m_streams.end();++s ) { struct CUstream_st *stream = *s; if( !stream->empty() ) { + //stream->print(stdout); result = false; break; } @@ -450,6 +447,7 @@ void stream_manager::print_impl( FILE *fp) void stream_manager::push( stream_operation op ) { struct CUstream_st *stream = op.get_stream(); + // block if stream 0 (or concurrency disabled) and pending concurrent operations exist bool block= !stream || m_cuda_launch_blocking; while(block) { diff --git a/src/stream_manager.h b/src/stream_manager.h index ae1af9f..eb4c6ff 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -163,6 +163,7 @@ public: void print( FILE *fp ) const; struct CUstream_st *get_stream() { return m_stream; } void set_stream( CUstream_st *stream ) { m_stream = stream; } + private: struct CUstream_st *m_stream; @@ -178,9 +179,9 @@ private: const char *m_symbol; size_t m_offset; - struct CUevent_st *m_event; bool m_sim_mode; kernel_info_t *m_kernel; + struct CUevent_st *m_event; }; struct CUevent_st { @@ -192,7 +193,7 @@ public: m_updates = 0; m_wallclock = 0; m_gpu_tot_sim_cycle = 0; - m_issued = 0; + m_issued = 0; m_done = false; } void update( double cycle, time_t clk ) @@ -207,18 +208,14 @@ public: unsigned num_updates() const { return m_updates; } bool done() const { return m_done; } time_t clock() const { return m_wallclock; } - void issue(){ - m_issued++; - } - unsigned int num_issued() const{ - return m_issued; - } + void issue(){ m_issued++; } + unsigned int num_issued() const{ return m_issued; } private: int m_uid; bool m_blocking; bool m_done; int m_updates; - unsigned int m_issued; + unsigned int m_issued; time_t m_wallclock; double m_gpu_tot_sim_cycle; @@ -275,7 +272,7 @@ private: CUstream_st m_stream_zero; bool m_service_stream_zero; pthread_mutex_t m_lock; - std::list::iterator m_last_stream; + std::list::iterator m_last_stream; }; #endif -- cgit v1.3 From f3ec23390a0798eab1426adf962487680ea89e93 Mon Sep 17 00:00:00 2001 From: boyealkim Date: Sat, 27 Jul 2019 03:54:44 +0900 Subject: fix event done impl --- src/stream_manager.cc | 2 +- src/stream_manager.h | 80 +++++++++++++++++++++++++-------------------------- 2 files changed, 41 insertions(+), 41 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/src/stream_manager.cc b/src/stream_manager.cc index d0eecdd..0d67e10 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -198,7 +198,7 @@ bool stream_operation::do_operation( gpgpu_sim *gpu ) //only allows next op to go if event is done //otherwise stays in the stream queue printf("stream wait event processing...\n"); - if(m_event->done()){ + if(m_event->num_updates()>=m_cnt){ printf("stream wait event done\n"); m_stream->record_next_done(); } diff --git a/src/stream_manager.h b/src/stream_manager.h index eb4c6ff..e73d71a 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -43,6 +43,45 @@ // unsigned m_pending_streams; //}; +struct CUevent_st { +public: + CUevent_st( bool blocking ) + { + m_uid = ++m_next_event_uid; + m_blocking = blocking; + m_updates = 0; + m_wallclock = 0; + m_gpu_tot_sim_cycle = 0; + m_issued = 0; + m_done = false; + } + void update( double cycle, time_t clk ) + { + m_updates++; + m_wallclock=clk; + m_gpu_tot_sim_cycle=cycle; + m_done = true; + } + //void set_done() { assert(!m_done); m_done=true; } + int get_uid() const { return m_uid; } + unsigned num_updates() const { return m_updates; } + bool done() const { return m_updates==m_issued; } + time_t clock() const { return m_wallclock; } + void issue(){ m_issued++; } + unsigned int num_issued() const{ return m_issued; } +private: + int m_uid; + bool m_blocking; + bool m_done; + int m_updates; + unsigned int m_issued; + time_t m_wallclock; + double m_gpu_tot_sim_cycle; + + static int m_next_event_uid; +}; + + enum stream_operation_type { stream_no_op, stream_memcpy_host_to_device, @@ -107,6 +146,7 @@ public: m_kernel=NULL; m_type=stream_wait_event; m_event=e; + m_cnt = m_event->num_issued(); m_stream=stream; m_done=false; } @@ -163,7 +203,6 @@ public: void print( FILE *fp ) const; struct CUstream_st *get_stream() { return m_stream; } void set_stream( CUstream_st *stream ) { m_stream = stream; } - private: struct CUstream_st *m_stream; @@ -183,45 +222,6 @@ private: kernel_info_t *m_kernel; struct CUevent_st *m_event; }; - -struct CUevent_st { -public: - CUevent_st( bool blocking ) - { - m_uid = ++m_next_event_uid; - m_blocking = blocking; - m_updates = 0; - m_wallclock = 0; - m_gpu_tot_sim_cycle = 0; - m_issued = 0; - m_done = false; - } - void update( double cycle, time_t clk ) - { - m_updates++; - m_wallclock=clk; - m_gpu_tot_sim_cycle=cycle; - m_done = true; - } - //void set_done() { assert(!m_done); m_done=true; } - int get_uid() const { return m_uid; } - unsigned num_updates() const { return m_updates; } - bool done() const { return m_done; } - time_t clock() const { return m_wallclock; } - void issue(){ m_issued++; } - unsigned int num_issued() const{ return m_issued; } -private: - int m_uid; - bool m_blocking; - bool m_done; - int m_updates; - unsigned int m_issued; - time_t m_wallclock; - double m_gpu_tot_sim_cycle; - - static int m_next_event_uid; -}; - struct CUstream_st { public: CUstream_st(); -- cgit v1.3 From 26ca8de4a6ec9bfe422a14cbe325a5f257df453b Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 05:29:40 -0400 Subject: Add src/ director formatting --- src/abstract_hardware_model.cc | 2033 +++++++++++++++++------------------ src/abstract_hardware_model.h | 2276 ++++++++++++++++++++-------------------- src/debug.cc | 350 +++--- src/debug.h | 91 +- src/gpgpusim_entrypoint.cc | 439 ++++---- src/gpgpusim_entrypoint.h | 70 +- src/option_parser.cc | 894 ++++++++-------- src/option_parser.h | 50 +- src/statwrapper.cc | 59 +- src/statwrapper.h | 14 +- src/stream_manager.cc | 740 +++++++------ src/stream_manager.h | 423 ++++---- src/tr1_hash_map.h | 43 +- src/trace.cc | 36 +- src/trace.h | 73 +- 15 files changed, 3825 insertions(+), 3766 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 07232ee..33a7080 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,1185 +27,1192 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - #include "abstract_hardware_model.h" +#include +#include +#include +#include +#include "../libcuda/gpgpu_context.h" +#include "cuda-sim/cuda-sim.h" #include "cuda-sim/memory.h" -#include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx-stats.h" -#include "cuda-sim/cuda-sim.h" +#include "cuda-sim/ptx_ir.h" #include "gpgpu-sim/gpu-sim.h" -#include "option_parser.h" #include "gpgpusim_entrypoint.h" -#include -#include -#include -#include -#include "../libcuda/gpgpu_context.h" +#include "option_parser.h" -void mem_access_t::init(gpgpu_context* ctx) -{ - gpgpu_ctx = ctx; - m_uid=++(gpgpu_ctx->sm_next_access_uid); - m_addr=0; - m_req_size=0; +void mem_access_t::init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = ++(gpgpu_ctx->sm_next_access_uid); + m_addr = 0; + m_req_size = 0; } -void warp_inst_t::issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ) -{ - m_warp_active_mask = mask; - m_warp_issued_mask = mask; - m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); - m_warp_id = warp_id; - m_dynamic_warp_id = dynamic_warp_id; - issue_cycle = cycle; - cycles = initiation_interval; - m_cache_hit=false; - m_empty=false; - m_scheduler_id=sch_id; +void warp_inst_t::issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, + int sch_id) { + m_warp_active_mask = mask; + m_warp_issued_mask = mask; + m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); + m_warp_id = warp_id; + m_dynamic_warp_id = dynamic_warp_id; + issue_cycle = cycle; + cycles = initiation_interval; + m_cache_hit = false; + m_empty = false; + m_scheduler_id = sch_id; } -checkpoint::checkpoint() -{ - - struct stat st = {0}; - - if (stat("checkpoint_files", &st) == -1) { - mkdir("checkpoint_files", 0777); - } +checkpoint::checkpoint() { + struct stat st = {0}; + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); + } } -void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) -{ - - FILE * fp2 = fopen(f1name, "r"); - assert(fp2!=NULL); - char line [ 128 ]; /* or other suitable maximum line size */ - unsigned int offset ; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - unsigned int index; - char * pch; - pch = strtok (line," "); - if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') - { - - pch = strtok (NULL, " "); - - std::stringstream ss; - ss << std::hex << pch; - ss >> index; - - offset=0; - } - else { - unsigned int data; - std::stringstream ss; - ss << std::hex << pch; - ss >> data; - temp_mem->write_only(offset,index, 4,&data); - offset= offset+4; - } - //fputs ( line, stdout ); /* write the line */ - } - fclose ( fp2 ); +void checkpoint::load_global_mem(class memory_space *temp_mem, char *f1name) { + FILE *fp2 = fopen(f1name, "r"); + assert(fp2 != NULL); + char line[128]; /* or other suitable maximum line size */ + unsigned int offset; + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + unsigned int index; + char *pch; + pch = strtok(line, " "); + if (pch[0] == 'g' || pch[0] == 's' || pch[0] == 'l') { + pch = strtok(NULL, " "); + + std::stringstream ss; + ss << std::hex << pch; + ss >> index; + + offset = 0; + } else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset, index, 4, &data); + offset = offset + 4; + } + // fputs ( line, stdout ); /* write the line */ + } + fclose(fp2); } -void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) -{ - - FILE * fp3 = fopen(fname, "w"); - assert(fp3!=NULL); - mem->print(format,fp3); - fclose(fp3); +void checkpoint::store_global_mem(class memory_space *mem, char *fname, + char *format) { + FILE *fp3 = fopen(fname, "w"); + assert(fp3 != NULL); + mem->print(format, fp3); + fclose(fp3); } -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) -{ - assert( dst->empty() ); - warp_inst_t* temp = dst; - dst = src; - src = temp; - src->clear(); +void move_warp(warp_inst_t *&dst, warp_inst_t *&src) { + assert(dst->empty()); + warp_inst_t *temp = dst; + dst = src; + src = temp; + src->clear(); } - -void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) -{ - option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, - &m_ptx_use_cuobjdump, - "Use cuobjdump to extract ptx and sass from binaries", +void gpgpu_functional_sim_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, + &m_ptx_use_cuobjdump, + "Use cuobjdump to extract ptx and sass from binaries", #if (CUDART_VERSION >= 4000) - "1" + "1" #else - "0" + "0" #endif - ); - option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, - &m_experimental_lib_support, - "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", - "0"); - option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, - " checkpointing flag (0 = no checkpoint)", - "0"); - option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, - " checkpointing during execution of which kernel (1- 1st kernel)", - "1"); - option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, - " checkpointing after # of CTA (< less than total CTA)", - "0"); - option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, - " resume flag (0 = no resume)", - "0"); - option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, - " Resume from which kernel (1= 1st kernel)", - "0"); - option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, - " resume from which CTA ", - "0"); - - option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, - &m_ptx_convert_to_ptxplus, - "Convert SASS (native ISA) to ptxplus and run ptxplus", - "0"); - option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, - &m_ptx_force_max_capability, - "Force maximum compute capability", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, - &g_ptx_inst_debug_to_file, - "Dump executed instructions' debug information to file", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, - "Executed instructions' debug output file", - "inst_debug.txt"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, &g_ptx_inst_debug_thread_uid, - "Thread UID for executed instructions' debug output", - "1"); + ); + option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, + &m_experimental_lib_support, + "Try to extract code from cuda libraries [Broken " + "because of unknown cudaGetExportTable]", + "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, + &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", "0"); + option_parser_register( + opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", "1"); + option_parser_register( + opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, + &checkpoint_insn_Y, " resume from which CTA ", "0"); + + option_parser_register( + opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, + "Convert SASS (native ISA) to ptxplus and run ptxplus", "0"); + option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, + &m_ptx_force_max_capability, + "Force maximum compute capability", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, &g_ptx_inst_debug_to_file, + "Dump executed instructions' debug information to file", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, + "Executed instructions' debug output file", "inst_debug.txt"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, + &g_ptx_inst_debug_thread_uid, + "Thread UID for executed instructions' debug output", + "1"); } -void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize(unsigned linesize) -{ - m_texcache_linesize = linesize; +void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize( + unsigned linesize) { + m_texcache_linesize = linesize; } -gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ) - : m_function_model_config(config) -{ - gpgpu_ctx = ctx; - m_global_mem = new memory_space_impl<8192>("global",64*1024); - - m_tex_mem = new memory_space_impl<8192>("tex",64*1024); - m_surf_mem = new memory_space_impl<8192>("surf",64*1024); - - m_dev_malloc=GLOBAL_HEAP_START; - checkpoint_option = m_function_model_config.get_checkpoint_option(); - checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); - checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); - resume_option = m_function_model_config.get_resume_option(); - resume_kernel = m_function_model_config.get_resume_kernel(); - resume_CTA = m_function_model_config.get_resume_CTA(); - checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); - checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); - - // initialize texture mappings to empty - m_NameToTextureInfo.clear(); - m_NameToCudaArray.clear(); - m_TextureRefToName.clear(); - m_NameToAttribute.clear(); - - if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) - ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); - - gpu_sim_cycle=0; - gpu_tot_sim_cycle=0; +gpgpu_t::gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx) + : m_function_model_config(config) { + gpgpu_ctx = ctx; + m_global_mem = new memory_space_impl<8192>("global", 64 * 1024); + + m_tex_mem = new memory_space_impl<8192>("tex", 64 * 1024); + m_surf_mem = new memory_space_impl<8192>("surf", 64 * 1024); + + m_dev_malloc = GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + + // initialize texture mappings to empty + m_NameToTextureInfo.clear(); + m_NameToCudaArray.clear(); + m_TextureRefToName.clear(); + m_NameToAttribute.clear(); + + if (m_function_model_config.get_ptx_inst_debug_to_file() != 0) + ptx_inst_debug_file = + fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); + + gpu_sim_cycle = 0; + gpu_tot_sim_cycle = 0; } -address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_size) -{ - //gives the tag for an address based on a given line size - return address & ~(line_size-1); +address_type line_size_based_tag_func(new_addr_type address, + new_addr_type line_size) { + // gives the tag for an address based on a given line size + return address & ~(line_size - 1); } -const char * mem_access_type_str(enum mem_access_type access_type) -{ - #define MA_TUP_BEGIN(X) static const char* access_type_str[] = { - #define MA_TUP(X) #X - #define MA_TUP_END(X) }; - MEM_ACCESS_TYPE_TUP_DEF - #undef MA_TUP_BEGIN - #undef MA_TUP - #undef MA_TUP_END +const char *mem_access_type_str(enum mem_access_type access_type) { +#define MA_TUP_BEGIN(X) static const char *access_type_str[] = { +#define MA_TUP(X) #X +#define MA_TUP_END(X) \ + } \ + ; + MEM_ACCESS_TYPE_TUP_DEF +#undef MA_TUP_BEGIN +#undef MA_TUP +#undef MA_TUP_END - assert(access_type < NUM_MEM_ACCESS_TYPE); + assert(access_type < NUM_MEM_ACCESS_TYPE); - return access_type_str[access_type]; + return access_type_str[access_type]; } - -void warp_inst_t::clear_active( const active_mask_t &inactive ) { - active_mask_t test = m_warp_active_mask; - test &= inactive; - assert( test == inactive ); // verify threads being disabled were active - m_warp_active_mask &= ~inactive; +void warp_inst_t::clear_active(const active_mask_t &inactive) { + active_mask_t test = m_warp_active_mask; + test &= inactive; + assert(test == inactive); // verify threads being disabled were active + m_warp_active_mask &= ~inactive; } -void warp_inst_t::set_not_active( unsigned lane_id ) { - m_warp_active_mask.reset(lane_id); +void warp_inst_t::set_not_active(unsigned lane_id) { + m_warp_active_mask.reset(lane_id); } -void warp_inst_t::set_active( const active_mask_t &active ) { - m_warp_active_mask = active; - if( m_isatomic ) { - for( unsigned i=0; i < m_config->warp_size; i++ ) { - if( !m_warp_active_mask.test(i) ) { - m_per_scalar_thread[i].callback.function = NULL; - m_per_scalar_thread[i].callback.instruction = NULL; - m_per_scalar_thread[i].callback.thread = NULL; - } +void warp_inst_t::set_active(const active_mask_t &active) { + m_warp_active_mask = active; + if (m_isatomic) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (!m_warp_active_mask.test(i)) { + m_per_scalar_thread[i].callback.function = NULL; + m_per_scalar_thread[i].callback.instruction = NULL; + m_per_scalar_thread[i].callback.thread = NULL; } - } + } + } } void warp_inst_t::do_atomic(bool forceDo) { - do_atomic( m_warp_active_mask,forceDo ); + do_atomic(m_warp_active_mask, forceDo); } - -void warp_inst_t::do_atomic( const active_mask_t& access_mask,bool forceDo ) { - assert( m_isatomic && (!m_empty||forceDo) ); - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ) - cb.function(cb.instruction, cb.thread); - } +void warp_inst_t::do_atomic(const active_mask_t &access_mask, bool forceDo) { + assert(m_isatomic && (!m_empty || forceDo)); + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) cb.function(cb.instruction, cb.thread); } + } } -void warp_inst_t::broadcast_barrier_reduction(const active_mask_t& access_mask) -{ - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ){ - cb.function(cb.instruction, cb.thread); - } - } +void warp_inst_t::broadcast_barrier_reduction( + const active_mask_t &access_mask) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) { + cb.function(cb.instruction, cb.thread); + } } + } } -void warp_inst_t::generate_mem_accesses() -{ - if( empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created ) - return; - if (!((op == LOAD_OP) || (op==TENSOR_CORE_LOAD_OP) || (op == STORE_OP)||(op==TENSOR_CORE_STORE_OP))) - return; - if( m_warp_active_mask.count() == 0 ) - return; // predicated off +void warp_inst_t::generate_mem_accesses() { + if (empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created) return; + if (!((op == LOAD_OP) || (op == TENSOR_CORE_LOAD_OP) || (op == STORE_OP) || + (op == TENSOR_CORE_STORE_OP))) + return; + if (m_warp_active_mask.count() == 0) return; // predicated off - const size_t starting_queue_size = m_accessq.size(); + const size_t starting_queue_size = m_accessq.size(); - assert( is_load() || is_store() ); - assert( m_per_scalar_thread_valid ); // need address information per thread + assert(is_load() || is_store()); + assert(m_per_scalar_thread_valid); // need address information per thread - bool is_write = is_store(); + bool is_write = is_store(); - mem_access_type access_type; - switch (space.get_type()) { + mem_access_type access_type; + switch (space.get_type()) { case const_space: - case param_space_kernel: - access_type = CONST_ACC_R; - break; - case tex_space: - access_type = TEXTURE_ACC_R; - break; - case global_space: - access_type = is_write? GLOBAL_ACC_W: GLOBAL_ACC_R; - break; + case param_space_kernel: + access_type = CONST_ACC_R; + break; + case tex_space: + access_type = TEXTURE_ACC_R; + break; + case global_space: + access_type = is_write ? GLOBAL_ACC_W : GLOBAL_ACC_R; + break; case local_space: - case param_space_local: - access_type = is_write? LOCAL_ACC_W: LOCAL_ACC_R; - break; - case shared_space: break; - case sstarr_space: break; - default: assert(0); break; - } + case param_space_local: + access_type = is_write ? LOCAL_ACC_W : LOCAL_ACC_R; + break; + case shared_space: + break; + case sstarr_space: + break; + default: + assert(0); + break; + } - // Calculate memory accesses generated by this warp - new_addr_type cache_block_size = 0; // in bytes + // Calculate memory accesses generated by this warp + new_addr_type cache_block_size = 0; // in bytes - switch( space.get_type() ) { + switch (space.get_type()) { case shared_space: case sstarr_space: { - unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; - unsigned total_accesses=0; - for( unsigned subwarp=0; subwarp < m_config->mem_warp_parts; subwarp++ ) { - - // data structures used per part warp - std::map > bank_accs; // bank -> word address -> access count - - // step 1: compute accesses to words in banks - for( unsigned thread=subwarp*subwarp_size; thread < (subwarp+1)*subwarp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - //FIXME: deferred allocation of shared memory should not accumulate across kernel launches - //assert( addr < m_config->gpgpu_shmem_size ); - unsigned bank = m_config->shmem_bank_func(addr); - new_addr_type word = line_size_based_tag_func(addr,m_config->WORD_SIZE); - bank_accs[bank][word]++; - } - - if (m_config->shmem_limited_broadcast) { - // step 2: look for and select a broadcast bank/word if one occurs - bool broadcast_detected = false; - new_addr_type broadcast_word=(new_addr_type)-1; - unsigned broadcast_bank=(unsigned)-1; - std::map >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank = b->first; - std::map &access_set = b->second; - std::map::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->second > 1 ) { - // found a broadcast - broadcast_detected=true; - broadcast_bank=bank; - broadcast_word=w->first; - break; - } - } - if( broadcast_detected ) - break; - } - - // step 3: figure out max bank accesses performed, taking account of broadcast case - unsigned max_bank_accesses=0; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank_accesses=0; - std::map &access_set = b->second; - std::map::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) - bank_accesses += w->second; - if( broadcast_detected && broadcast_bank == b->first ) { - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->first == broadcast_word ) { - unsigned n = w->second; - assert(n > 1); // or this wasn't a broadcast - assert(bank_accesses >= (n-1)); - bank_accesses -= (n-1); - break; - } - } - } - if( bank_accesses > max_bank_accesses ) - max_bank_accesses = bank_accesses; - } + unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; + unsigned total_accesses = 0; + for (unsigned subwarp = 0; subwarp < m_config->mem_warp_parts; + subwarp++) { + // data structures used per part warp + std::map > + bank_accs; // bank -> word address -> access count + + // step 1: compute accesses to words in banks + for (unsigned thread = subwarp * subwarp_size; + thread < (subwarp + 1) * subwarp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + // FIXME: deferred allocation of shared memory should not accumulate + // across kernel launches + // assert( addr < m_config->gpgpu_shmem_size ); + unsigned bank = m_config->shmem_bank_func(addr); + new_addr_type word = + line_size_based_tag_func(addr, m_config->WORD_SIZE); + bank_accs[bank][word]++; + } - // step 4: accumulate - total_accesses+= max_bank_accesses; - } else { - // step 2: look for the bank with the maximum number of access to different words - unsigned max_bank_accesses=0; - std::map >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - max_bank_accesses = std::max(max_bank_accesses, (unsigned)b->second.size()); + if (m_config->shmem_limited_broadcast) { + // step 2: look for and select a broadcast bank/word if one occurs + bool broadcast_detected = false; + new_addr_type broadcast_word = (new_addr_type)-1; + unsigned broadcast_bank = (unsigned)-1; + std::map >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank = b->first; + std::map &access_set = b->second; + std::map::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->second > 1) { + // found a broadcast + broadcast_detected = true; + broadcast_bank = bank; + broadcast_word = w->first; + break; + } + } + if (broadcast_detected) break; + } + + // step 3: figure out max bank accesses performed, taking account of + // broadcast case + unsigned max_bank_accesses = 0; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank_accesses = 0; + std::map &access_set = b->second; + std::map::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) + bank_accesses += w->second; + if (broadcast_detected && broadcast_bank == b->first) { + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->first == broadcast_word) { + unsigned n = w->second; + assert(n > 1); // or this wasn't a broadcast + assert(bank_accesses >= (n - 1)); + bank_accesses -= (n - 1); + break; } - - // step 3: accumulate - total_accesses+= max_bank_accesses; + } } + if (bank_accesses > max_bank_accesses) + max_bank_accesses = bank_accesses; + } + + // step 4: accumulate + total_accesses += max_bank_accesses; + } else { + // step 2: look for the bank with the maximum number of access to + // different words + unsigned max_bank_accesses = 0; + std::map >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + max_bank_accesses = + std::max(max_bank_accesses, (unsigned)b->second.size()); + } + + // step 3: accumulate + total_accesses += max_bank_accesses; } - assert( total_accesses > 0 && total_accesses <= m_config->warp_size ); - cycles = total_accesses; // shared memory conflicts modeled as larger initiation interval - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); - break; + } + assert(total_accesses > 0 && total_accesses <= m_config->warp_size); + cycles = total_accesses; // shared memory conflicts modeled as larger + // initiation interval + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( + pc, total_accesses); + break; } - case tex_space: - cache_block_size = m_config->gpgpu_cache_texl1_linesize; - break; - case const_space: case param_space_kernel: - cache_block_size = m_config->gpgpu_cache_constl1_linesize; - break; + case tex_space: + cache_block_size = m_config->gpgpu_cache_texl1_linesize; + break; + case const_space: + case param_space_kernel: + cache_block_size = m_config->gpgpu_cache_constl1_linesize; + break; - case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch >= 13) { - if(isatomic()) - memory_coalescing_arch_atomic(is_write, access_type); - else - memory_coalescing_arch(is_write, access_type); - } else abort(); + case global_space: + case local_space: + case param_space_local: + if (m_config->gpgpu_coalesce_arch >= 13) { + if (isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else + abort(); - break; + break; default: - abort(); + abort(); + } + + if (cache_block_size) { + assert(m_accessq.empty()); + mem_access_byte_mask_t byte_mask; + std::map + accesses; // block address -> set of thread offsets in warp + std::map::iterator a; + for (unsigned thread = 0; thread < m_config->warp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, cache_block_size); + accesses[block_address].set(thread); + unsigned idx = addr - block_address; + for (unsigned i = 0; i < data_size; i++) byte_mask.set(idx + i); } + for (a = accesses.begin(); a != accesses.end(); ++a) + m_accessq.push_back(mem_access_t( + access_type, a->first, cache_block_size, is_write, a->second, + byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); + } + + if (space.get_type() == global_space) { + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( + pc, m_accessq.size() - starting_queue_size); + } + m_mem_accesses_created = true; +} - if( cache_block_size ) { - assert( m_accessq.empty() ); - mem_access_byte_mask_t byte_mask; - std::map accesses; // block address -> set of thread offsets in warp - std::map::iterator a; - for( unsigned thread=0; thread < m_config->warp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr,cache_block_size); - accesses[block_address].set(thread); - unsigned idx = addr-block_address; - for( unsigned i=0; i < data_size; i++ ) - byte_mask.set(idx+i); +void warp_inst_t::memory_coalescing_arch(bool is_write, + mem_access_type access_type) { + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } + + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map subwarp_transactions; + + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; + + unsigned data_size_coales = data_size; + unsigned num_accesses = 1; + + if (space.get_type() == local_space || + space.get_type() == param_space_local) { + // Local memory accesses >4B were split into 4B chunks + if (data_size >= 4) { + data_size_coales = 4; + num_accesses = data_size / 4; } - for( a=accesses.begin(); a != accesses.end(); ++a ) - m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); - } + // Otherwise keep the same data_size for sub-4B access to local memory + } - if ( space.get_type() == global_space ) { - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); + assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); + + // for(unsigned access=0; accessmem_warp_parts; - bool sector_segment_size = false; - - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } + // step 2: reduce each transaction size, if possible + std::map::iterator t; + for (t = subwarp_transactions.begin(); t != subwarp_transactions.end(); + t++) { + new_addr_type addr = t->first; + const transaction_info &info = t->second; - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, + segment_size); } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map subwarp_transactions; - - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread4B were split into 4B chunks - if(data_size >= 4) { - data_size_coales = 4; - num_accesses = data_size/4; - } - // Otherwise keep the same data_size for sub-4B access to local memory - } - - - assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); - -// for(unsigned access=0; access::iterator t; - for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { - new_addr_type addr = t->first; - const transaction_info &info = t->second; - - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); + } +} +void warp_inst_t::memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type) { + assert(space.get_type() == + global_space); // Atomics allowed only for global memory + + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } + + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map > + subwarp_transactions; // each block addr maps to a list of transactions + + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; + + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, segment_size); + unsigned chunk = (addr & 127) / 32; // which 32-byte chunk within in a + // 128-byte chunk does this thread + // access? + + // can only write to one segment + assert(block_address == + line_size_based_tag_func(addr + data_size - 1, segment_size)); + + // Find a transaction that does not conflict with this thread's accesses + bool new_transaction = true; + std::list::iterator it; + transaction_info *info; + for (it = subwarp_transactions[block_address].begin(); + it != subwarp_transactions[block_address].end(); it++) { + unsigned idx = (addr & 127); + if (not it->test_bytes(idx, idx + data_size - 1)) { + new_transaction = false; + info = &(*it); + break; } + } + if (new_transaction) { + // Need a new transaction + subwarp_transactions[block_address].push_back(transaction_info()); + info = &subwarp_transactions[block_address].back(); + } + assert(info); + + info->chunks.set(chunk); + info->active.set(thread); + unsigned idx = (addr & 127); + for (unsigned i = 0; i < data_size; i++) { + assert(!info->bytes.test(idx + i)); + info->bytes.set(idx + i); + } } -} -void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) -{ - - assert(space.get_type() == global_space); // Atomics allowed only for global memory - - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; - - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } - - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map > subwarp_transactions; // each block addr maps to a list of transactions - - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread::iterator it; - transaction_info* info; - for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) { - unsigned idx = (addr&127); - if(not it->test_bytes(idx,idx+data_size-1)) { - new_transaction = false; - info = &(*it); - break; - } - } - if(new_transaction) { - // Need a new transaction - subwarp_transactions[block_address].push_back(transaction_info()); - info = &subwarp_transactions[block_address].back(); - } - assert(info); - - info->chunks.set(chunk); - info->active.set(thread); - unsigned idx = (addr&127); - for( unsigned i=0; i < data_size; i++ ) { - assert(!info->bytes.test(idx+i)); - info->bytes.set(idx+i); - } - } - - // step 2: reduce each transaction size, if possible - std::map< new_addr_type, std::list >::iterator t_list; - for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) { - // For each block addr - new_addr_type addr = t_list->first; - const std::list& transaction_list = t_list->second; - - std::list::const_iterator t; - for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { - // For each transaction - const transaction_info &info = *t; - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); - } - } - } + // step 2: reduce each transaction size, if possible + std::map >::iterator t_list; + for (t_list = subwarp_transactions.begin(); + t_list != subwarp_transactions.end(); t_list++) { + // For each block addr + new_addr_type addr = t_list->first; + const std::list &transaction_list = t_list->second; + + std::list::const_iterator t; + for (t = transaction_list.begin(); t != transaction_list.end(); t++) { + // For each transaction + const transaction_info &info = *t; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, + addr, segment_size); + } + } + } } -void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) -{ - assert( (addr & (segment_size-1)) == 0 ); - - const std::bitset<4> &q = info.chunks; - assert( q.count() >= 1 ); - std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) - - unsigned size=segment_size; - if( segment_size == 128 ) { - bool lower_half_used = q[0] || q[1]; - bool upper_half_used = q[2] || q[3]; - if( lower_half_used && !upper_half_used ) { - // only lower 64 bytes used - size = 64; - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else if ( (!lower_half_used) && upper_half_used ) { - // only upper 64 bytes used - addr = addr+64; - size = 64; - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } else { - assert(lower_half_used && upper_half_used); - } - } else if( segment_size == 64 ) { - // need to set halves - if( (addr % 128) == 0 ) { - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else { - assert( (addr % 128) == 64 ); - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } - } - if( size == 64 ) { - bool lower_half_used = h[0]; - bool upper_half_used = h[1]; - if( lower_half_used && !upper_half_used ) { - size = 32; - } else if ( (!lower_half_used) && upper_half_used ) { - addr = addr+32; - size = 32; - } else { - assert(lower_half_used && upper_half_used); - } - } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks,m_config->gpgpu_ctx) ); +void warp_inst_t::memory_coalescing_arch_reduce_and_send( + bool is_write, mem_access_type access_type, const transaction_info &info, + new_addr_type addr, unsigned segment_size) { + assert((addr & (segment_size - 1)) == 0); + + const std::bitset<4> &q = info.chunks; + assert(q.count() >= 1); + std::bitset<2> h; // halves (used to check if 64 byte segment can be + // compressed into a single 32 byte segment) + + unsigned size = segment_size; + if (segment_size == 128) { + bool lower_half_used = q[0] || q[1]; + bool upper_half_used = q[2] || q[3]; + if (lower_half_used && !upper_half_used) { + // only lower 64 bytes used + size = 64; + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else if ((!lower_half_used) && upper_half_used) { + // only upper 64 bytes used + addr = addr + 64; + size = 64; + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } else { + assert(lower_half_used && upper_half_used); + } + } else if (segment_size == 64) { + // need to set halves + if ((addr % 128) == 0) { + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else { + assert((addr % 128) == 64); + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } + } + if (size == 64) { + bool lower_half_used = h[0]; + bool upper_half_used = h[1]; + if (lower_half_used && !upper_half_used) { + size = 32; + } else if ((!lower_half_used) && upper_half_used) { + addr = addr + 32; + size = 32; + } else { + assert(lower_half_used && upper_half_used); + } + } + m_accessq.push_back(mem_access_t(access_type, addr, size, is_write, + info.active, info.bytes, info.chunks, + m_config->gpgpu_ctx)); } -void warp_inst_t::completed( unsigned long long cycle ) const -{ - unsigned long long latency = cycle - issue_cycle; - assert(latency <= cycle); // underflow detection - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency(pc, latency * active_count()); +void warp_inst_t::completed(unsigned long long cycle) const { + unsigned long long latency = cycle - issue_cycle; + assert(latency <= cycle); // underflow detection + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency( + pc, latency * active_count()); } - -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); - - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set=false; +kernel_info_t::kernel_info_t(dim3 gridDim, dim3 blockDim, + class function_info *entry) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); + + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set = false; } -/*A snapshot of the texture mappings needs to be stored in the kernel's info as +/*A snapshot of the texture mappings needs to be stored in the kernel's info as kernels should use the texture bindings seen at the time of launch and textures can be bound/unbound asynchronously with respect to streams. */ -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); - - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set=false; - m_NameToCudaArray = nameToCudaArray; - m_NameToTextureInfo = nameToTextureInfo; +kernel_info_t::kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map nameToCudaArray, + std::map nameToTextureInfo) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); + + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set = false; + m_NameToCudaArray = nameToCudaArray; + m_NameToTextureInfo = nameToTextureInfo; } -kernel_info_t::~kernel_info_t() -{ - assert( m_active_threads.empty() ); - destroy_cta_streams(); - delete m_param_mem; +kernel_info_t::~kernel_info_t() { + assert(m_active_threads.empty()); + destroy_cta_streams(); + delete m_param_mem; } -std::string kernel_info_t::name() const -{ - return m_kernel_entry->get_name(); -} +std::string kernel_info_t::name() const { return m_kernel_entry->get_name(); } -//Jin: parent and child kernel management for CDP -void kernel_info_t::set_parent(kernel_info_t * parent, - dim3 parent_ctaid, dim3 parent_tid) { - m_parent_kernel = parent; - m_parent_ctaid = parent_ctaid; - m_parent_tid = parent_tid; - parent->set_child(this); +// Jin: parent and child kernel management for CDP +void kernel_info_t::set_parent(kernel_info_t *parent, dim3 parent_ctaid, + dim3 parent_tid) { + m_parent_kernel = parent; + m_parent_ctaid = parent_ctaid; + m_parent_tid = parent_tid; + parent->set_child(this); } -void kernel_info_t::set_child(kernel_info_t * child) { - m_child_kernels.push_back(child); +void kernel_info_t::set_child(kernel_info_t *child) { + m_child_kernels.push_back(child); } -void kernel_info_t::remove_child(kernel_info_t * child) { - assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) - != m_child_kernels.end()); - m_child_kernels.remove(child); +void kernel_info_t::remove_child(kernel_info_t *child) { + assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) != + m_child_kernels.end()); + m_child_kernels.remove(child); } bool kernel_info_t::is_finished() { - if(done() && children_all_finished()) - return true; + if (done() && children_all_finished()) + return true; else - return false; + return false; } bool kernel_info_t::children_all_finished() { - if(!m_child_kernels.empty()) - return false; - - return true; + if (!m_child_kernels.empty()) return false; + + return true; } void kernel_info_t::notify_parent_finished() { - if(m_parent_kernel) { - m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= ((m_kernel_entry->get_args_aligned_size() + 255)/256*256); - m_parent_kernel->remove_child(this); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(m_parent_kernel->get_uid()); - } + if (m_parent_kernel) { + m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= + ((m_kernel_entry->get_args_aligned_size() + 255) / 256 * 256); + m_parent_kernel->remove_child(this); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager + ->register_finished_kernel(m_parent_kernel->get_uid()); + } } -CUstream_st * kernel_info_t::create_stream_cta(dim3 ctaid) { - assert(get_default_stream_cta(ctaid)); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); - assert(m_cta_streams[ctaid].size() >= 1); //must have default stream - m_cta_streams[ctaid].push_back(stream); +CUstream_st *kernel_info_t::create_stream_cta(dim3 ctaid) { + assert(get_default_stream_cta(ctaid)); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); + assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); + assert(m_cta_streams[ctaid].size() >= 1); // must have default stream + m_cta_streams[ctaid].push_back(stream); - return stream; + return stream; } -CUstream_st * kernel_info_t::get_default_stream_cta(dim3 ctaid) { - if(m_cta_streams.find(ctaid) != m_cta_streams.end()) { - assert(m_cta_streams[ctaid].size() >= 1); //already created, must have default stream - return *(m_cta_streams[ctaid].begin()); - } - else { - m_cta_streams[ctaid] = std::list(); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - m_cta_streams[ctaid].push_back(stream); - return stream; - } +CUstream_st *kernel_info_t::get_default_stream_cta(dim3 ctaid) { + if (m_cta_streams.find(ctaid) != m_cta_streams.end()) { + assert(m_cta_streams[ctaid].size() >= + 1); // already created, must have default stream + return *(m_cta_streams[ctaid].begin()); + } else { + m_cta_streams[ctaid] = std::list(); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream( + stream); + m_cta_streams[ctaid].push_back(stream); + return stream; + } } -bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st* stream) { - if(m_cta_streams.find(ctaid) == m_cta_streams.end()) - return false; +bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st *stream) { + if (m_cta_streams.find(ctaid) == m_cta_streams.end()) return false; - std::list &stream_list = m_cta_streams[ctaid]; - if(std::find(stream_list.begin(), stream_list.end(), stream) - == stream_list.end()) - return false; - else - return true; + std::list &stream_list = m_cta_streams[ctaid]; + if (std::find(stream_list.begin(), stream_list.end(), stream) == + stream_list.end()) + return false; + else + return true; } void kernel_info_t::print_parent_info() { - if(m_parent_kernel) { - printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", - m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), - m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, - m_parent_tid.x, m_parent_tid.y, m_parent_tid.z); - } + if (m_parent_kernel) { + printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", + m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), + m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, m_parent_tid.x, + m_parent_tid.y, m_parent_tid.z); + } } void kernel_info_t::destroy_cta_streams() { - printf("Destroy streams for kernel %d: ", get_uid()); size_t stream_size = 0; - for(auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { - stream_size += s->second.size(); - for(auto ss = s->second.begin(); ss != s->second.end(); ss++) - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream(*ss); - s->second.clear(); - } - printf("size %lu\n", stream_size); - m_cta_streams.clear(); + printf("Destroy streams for kernel %d: ", get_uid()); + size_t stream_size = 0; + for (auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { + stream_size += s->second.size(); + for (auto ss = s->second.begin(); ss != s->second.end(); ss++) + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream( + *ss); + s->second.clear(); + } + printf("size %lu\n", stream_size); + m_cta_streams.clear(); } -simt_stack::simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu) -{ - m_warp_id=wid; - m_warp_size = warpSize; - m_gpu=gpu; - reset(); +simt_stack::simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu) { + m_warp_id = wid; + m_warp_size = warpSize; + m_gpu = gpu; + reset(); } -void simt_stack::reset() -{ - m_stack.clear(); -} +void simt_stack::reset() { m_stack.clear(); } -void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) -{ - reset(); - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = start_pc; - new_stack_entry.m_calldepth = 1; - new_stack_entry.m_active_mask = active_mask; - new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; - m_stack.push_back(new_stack_entry); +void simt_stack::launch(address_type start_pc, const simt_mask_t &active_mask) { + reset(); + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = start_pc; + new_stack_entry.m_calldepth = 1; + new_stack_entry.m_active_mask = active_mask; + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + m_stack.push_back(new_stack_entry); } -void simt_stack::resume( char * fname ) -{ - reset(); - - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - - char line [ 200 ]; /* or other suitable maximum line size */ - - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - simt_stack_entry new_stack_entry; - char * pch; - pch = strtok (line," "); - for (unsigned j=0; j 0); - return m_stack.back().m_active_mask; -} + char line[200]; /* or other suitable maximum line size */ -void simt_stack::get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const -{ - assert(m_stack.size() > 0); - *pc = m_stack.back().m_pc; - *rpc = m_stack.back().m_recvg_pc; + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + simt_stack_entry new_stack_entry; + char *pch; + pch = strtok(line, " "); + for (unsigned j = 0; j < m_warp_size; j++) { + if (pch[0] == '1') + new_stack_entry.m_active_mask.set(j); + else + new_stack_entry.m_active_mask.reset(j); + pch = strtok(NULL, " "); + } + + new_stack_entry.m_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_calldepth = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_recvg_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_branch_div_cycle = atoi(pch); + pch = strtok(NULL, " "); + if (pch[0] == '0') + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + else + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + } + fclose(fp2); } -unsigned simt_stack::get_rp() const -{ - assert(m_stack.size() > 0); - return m_stack.back().m_recvg_pc; +const simt_mask_t &simt_stack::get_active_mask() const { + assert(m_stack.size() > 0); + return m_stack.back().m_active_mask; } -void simt_stack::print (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - if ( k==0 ) { - fprintf(fout, "w%02d %1u ", m_warp_id, k ); - } else { - fprintf(fout, " %1u ", k ); - } - for (unsigned j=0; jgpgpu_ctx->func_sim->ptx_print_insn( stack_entry.m_pc, fout ); - fprintf(fout,"\n"); - } +void simt_stack::get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const { + assert(m_stack.size() > 0); + *pc = m_stack.back().m_pc; + *rpc = m_stack.back().m_recvg_pc; +} +unsigned simt_stack::get_rp() const { + assert(m_stack.size() > 0); + return m_stack.back().m_recvg_pc; } -void simt_stack::print_checkpoint (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - - for (unsigned j=0; jgpgpu_ctx->func_sim->ptx_print_insn(stack_entry.m_pc, fout); + fprintf(fout, "\n"); + } } -void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ) -{ - assert(m_stack.size() > 0); - - assert( next_pc.size() == m_warp_size ); - - simt_mask_t top_active_mask = m_stack.back().m_active_mask; - address_type top_recvg_pc = m_stack.back().m_recvg_pc; - address_type top_pc = m_stack.back().m_pc; // the pc of the instruction just executed - stack_entry_type top_type = m_stack.back().m_type; - assert(top_pc==next_inst_pc); - assert(top_active_mask.any()); - - const address_type null_pc = -1; - bool warp_diverged = false; - address_type new_recvg_pc = null_pc; - unsigned num_divergent_paths=0; - - std::map divergent_paths; - while (top_active_mask.any()) { - - // extract a group of threads with the same next PC among the active threads in the warp - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - for (int i = m_warp_size - 1; i >= 0; i--) { - if ( top_active_mask.test(i) ) { // is this thread active? - if (thread_done.test(i)) { - top_active_mask.reset(i); // remove completed thread from active mask - } else if (tmp_next_pc == null_pc) { - tmp_next_pc = next_pc[i]; - tmp_active_mask.set(i); - top_active_mask.reset(i); - } else if (tmp_next_pc == next_pc[i]) { - tmp_active_mask.set(i); - top_active_mask.reset(i); - } - } - } +void simt_stack::print_checkpoint(FILE *fout) const { + for (unsigned k = 0; k < m_stack.size(); k++) { + simt_stack_entry stack_entry = m_stack[k]; + + for (unsigned j = 0; j < m_warp_size; j++) + fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j) ? '1' : '0')); + fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc, + stack_entry.m_calldepth, stack_entry.m_recvg_pc, + stack_entry.m_branch_div_cycle, stack_entry.m_type); + fprintf(fout, "%d %d\n", m_warp_id, m_warp_size); + } +} - if(tmp_next_pc == null_pc) { - assert(!top_active_mask.any()); // all threads done - continue; +void simt_stack::update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc) { + assert(m_stack.size() > 0); + + assert(next_pc.size() == m_warp_size); + + simt_mask_t top_active_mask = m_stack.back().m_active_mask; + address_type top_recvg_pc = m_stack.back().m_recvg_pc; + address_type top_pc = + m_stack.back().m_pc; // the pc of the instruction just executed + stack_entry_type top_type = m_stack.back().m_type; + assert(top_pc == next_inst_pc); + assert(top_active_mask.any()); + + const address_type null_pc = -1; + bool warp_diverged = false; + address_type new_recvg_pc = null_pc; + unsigned num_divergent_paths = 0; + + std::map divergent_paths; + while (top_active_mask.any()) { + // extract a group of threads with the same next PC among the active threads + // in the warp + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + for (int i = m_warp_size - 1; i >= 0; i--) { + if (top_active_mask.test(i)) { // is this thread active? + if (thread_done.test(i)) { + top_active_mask.reset(i); // remove completed thread from active mask + } else if (tmp_next_pc == null_pc) { + tmp_next_pc = next_pc[i]; + tmp_active_mask.set(i); + top_active_mask.reset(i); + } else if (tmp_next_pc == next_pc[i]) { + tmp_active_mask.set(i); + top_active_mask.reset(i); } - - divergent_paths[tmp_next_pc]=tmp_active_mask; - num_divergent_paths++; + } } + if (tmp_next_pc == null_pc) { + assert(!top_active_mask.any()); // all threads done + continue; + } - address_type not_taken_pc = next_inst_pc+next_inst_size; - assert(num_divergent_paths<=2); - for(unsigned i=0; i:: iterator it=divergent_paths.begin(); - tmp_next_pc=it->first; - tmp_active_mask=divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); - } - - // HANDLE THE SPECIAL CASES FIRST - if (next_inst_op== CALL_OPS){ - // Since call is not a divergent instruction, all threads should have executed a call instruction - assert(num_divergent_paths == 1); - - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = tmp_next_pc; - new_stack_entry.m_active_mask = tmp_active_mask; - new_stack_entry.m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; - new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; - m_stack.push_back(new_stack_entry); - return; - }else if(next_inst_op == RET_OPS && top_type==STACK_ENTRY_TYPE_CALL){ - // pop the CALL Entry - assert(num_divergent_paths == 1); - m_stack.pop_back(); - - assert(m_stack.size() > 0); - m_stack.back().m_pc=tmp_next_pc;// set the PC of the stack top entry to return PC from the call stack; - // Check if the New top of the stack is reconverging - if (tmp_next_pc == m_stack.back().m_recvg_pc && m_stack.back().m_type!=STACK_ENTRY_TYPE_CALL){ - assert(m_stack.back().m_type==STACK_ENTRY_TYPE_NORMAL); - m_stack.pop_back(); - } - return; - } - - // discard the new entry if its PC matches with reconvergence PC - // that automatically reconverges the entry - // If the top stack entry is CALL, dont reconverge. - if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) continue; - - // this new entry is not converging - // if this entry does not include thread from the warp, divergence occurs - if ((num_divergent_paths>1) && !warp_diverged ) { - warp_diverged = true; - // modify the existing top entry into a reconvergence entry in the pdom stack - new_recvg_pc = recvg_pc; - if (new_recvg_pc != top_recvg_pc) { - m_stack.back().m_pc = new_recvg_pc; - m_stack.back().m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; - - m_stack.push_back(simt_stack_entry()); - } - } + divergent_paths[tmp_next_pc] = tmp_active_mask; + num_divergent_paths++; + } + + address_type not_taken_pc = next_inst_pc + next_inst_size; + assert(num_divergent_paths <= 2); + for (unsigned i = 0; i < num_divergent_paths; i++) { + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + tmp_active_mask.reset(); + if (divergent_paths.find(not_taken_pc) != divergent_paths.end()) { + assert(i == 0); + tmp_next_pc = not_taken_pc; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } else { + std::map::iterator it = + divergent_paths.begin(); + tmp_next_pc = it->first; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } - // discard the new entry if its PC matches with reconvergence PC - if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // HANDLE THE SPECIAL CASES FIRST + if (next_inst_op == CALL_OPS) { + // Since call is not a divergent instruction, all threads should have + // executed a call instruction + assert(num_divergent_paths == 1); + + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = tmp_next_pc; + new_stack_entry.m_active_mask = tmp_active_mask; + new_stack_entry.m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + return; + } else if (next_inst_op == RET_OPS && top_type == STACK_ENTRY_TYPE_CALL) { + // pop the CALL Entry + assert(num_divergent_paths == 1); + m_stack.pop_back(); + + assert(m_stack.size() > 0); + m_stack.back().m_pc = tmp_next_pc; // set the PC of the stack top entry + // to return PC from the call stack; + // Check if the New top of the stack is reconverging + if (tmp_next_pc == m_stack.back().m_recvg_pc && + m_stack.back().m_type != STACK_ENTRY_TYPE_CALL) { + assert(m_stack.back().m_type == STACK_ENTRY_TYPE_NORMAL); + m_stack.pop_back(); + } + return; + } - // update the current top of pdom stack - m_stack.back().m_pc = tmp_next_pc; - m_stack.back().m_active_mask = tmp_active_mask; - if (warp_diverged) { - m_stack.back().m_calldepth = 0; - m_stack.back().m_recvg_pc = new_recvg_pc; - } else { - m_stack.back().m_recvg_pc = top_recvg_pc; - } + // discard the new entry if its PC matches with reconvergence PC + // that automatically reconverges the entry + // If the top stack entry is CALL, dont reconverge. + if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) + continue; + + // this new entry is not converging + // if this entry does not include thread from the warp, divergence occurs + if ((num_divergent_paths > 1) && !warp_diverged) { + warp_diverged = true; + // modify the existing top entry into a reconvergence entry in the pdom + // stack + new_recvg_pc = recvg_pc; + if (new_recvg_pc != top_recvg_pc) { + m_stack.back().m_pc = new_recvg_pc; + m_stack.back().m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; m_stack.push_back(simt_stack_entry()); + } } - assert(m_stack.size() > 0); - m_stack.pop_back(); + // discard the new entry if its PC matches with reconvergence PC + if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // update the current top of pdom stack + m_stack.back().m_pc = tmp_next_pc; + m_stack.back().m_active_mask = tmp_active_mask; if (warp_diverged) { - m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + m_stack.back().m_calldepth = 0; + m_stack.back().m_recvg_pc = new_recvg_pc; + } else { + m_stack.back().m_recvg_pc = top_recvg_pc; } + + m_stack.push_back(simt_stack_entry()); + } + assert(m_stack.size() > 0); + m_stack.pop_back(); + + if (warp_diverged) { + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + } } -void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) -{ - for ( unsigned t=0; t < m_warp_size; t++ ) { - if( inst.active(t) ) { - if(warpId==(unsigned (-1))) - warpId = inst.warp_id(); - unsigned tid=m_warp_size*warpId+t; - m_thread[tid]->ptx_exec_inst(inst,t); - - //virtual function - checkExecutionStatusAndUpdate(inst,t,tid); - } - } +void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) { + for (unsigned t = 0; t < m_warp_size; t++) { + if (inst.active(t)) { + if (warpId == (unsigned(-1))) warpId = inst.warp_id(); + unsigned tid = m_warp_size * warpId + t; + m_thread[tid]->ptx_exec_inst(inst, t); + + // virtual function + checkExecutionStatusAndUpdate(inst, t, tid); + } + } } - -bool core_t::ptx_thread_done( unsigned hw_thread_id ) const -{ - return ((m_thread[ hw_thread_id ]==NULL) || m_thread[ hw_thread_id ]->is_done()); + +bool core_t::ptx_thread_done(unsigned hw_thread_id) const { + return ((m_thread[hw_thread_id] == NULL) || + m_thread[hw_thread_id]->is_done()); } - -void core_t::updateSIMTStack(unsigned warpId, warp_inst_t * inst) -{ - simt_mask_t thread_done; - addr_vector_t next_pc; - unsigned wtid = warpId * m_warp_size; - for (unsigned i = 0; i < m_warp_size; i++) { - if( ptx_thread_done(wtid+i) ) { - thread_done.set(i); - next_pc.push_back( (address_type)-1 ); - } else { - if( inst->reconvergence_pc == RECONVERGE_RETURN_PC ) - inst->reconvergence_pc = get_return_pc(m_thread[wtid+i]); - next_pc.push_back( m_thread[wtid+i]->get_pc() ); - } + +void core_t::updateSIMTStack(unsigned warpId, warp_inst_t *inst) { + simt_mask_t thread_done; + addr_vector_t next_pc; + unsigned wtid = warpId * m_warp_size; + for (unsigned i = 0; i < m_warp_size; i++) { + if (ptx_thread_done(wtid + i)) { + thread_done.set(i); + next_pc.push_back((address_type)-1); + } else { + if (inst->reconvergence_pc == RECONVERGE_RETURN_PC) + inst->reconvergence_pc = get_return_pc(m_thread[wtid + i]); + next_pc.push_back(m_thread[wtid + i]->get_pc()); } - m_simt_stack[warpId]->update(thread_done,next_pc,inst->reconvergence_pc, inst->op,inst->isize,inst->pc); + } + m_simt_stack[warpId]->update(thread_done, next_pc, inst->reconvergence_pc, + inst->op, inst->isize, inst->pc); } //! Get the warp to be executed using the data taken form the SIMT stack -warp_inst_t core_t::getExecuteWarp(unsigned warpId) -{ - unsigned pc,rpc; - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - warp_inst_t wi= *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); - wi.set_active(m_simt_stack[warpId]->get_active_mask()); - return wi; +warp_inst_t core_t::getExecuteWarp(unsigned warpId) { + unsigned pc, rpc; + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + warp_inst_t wi = *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); + wi.set_active(m_simt_stack[warpId]->get_active_mask()); + return wi; } -void core_t::deleteSIMTStack() -{ - if ( m_simt_stack ) { - for (unsigned i = 0; i < m_warp_count; ++i) - delete m_simt_stack[i]; - delete[] m_simt_stack; - m_simt_stack = NULL; - } +void core_t::deleteSIMTStack() { + if (m_simt_stack) { + for (unsigned i = 0; i < m_warp_count; ++i) delete m_simt_stack[i]; + delete[] m_simt_stack; + m_simt_stack = NULL; + } } -void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) -{ - m_simt_stack = new simt_stack*[warp_count]; - for (unsigned i = 0; i < warp_count; ++i) - m_simt_stack[i] = new simt_stack(i,warp_size,m_gpu); - m_warp_size = warp_size; - m_warp_count = warp_count; +void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) { + m_simt_stack = new simt_stack *[warp_count]; + for (unsigned i = 0; i < warp_count; ++i) + m_simt_stack[i] = new simt_stack(i, warp_size, m_gpu); + m_warp_size = warp_size; + m_warp_count = warp_count; } -void core_t::get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const -{ - m_simt_stack[warpId]->get_pdom_stack_top_info(pc,rpc); +void core_t::get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const { + m_simt_stack[warpId]->get_pdom_stack_top_info(pc, rpc); } diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 29e4a30..206ae06 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,45 +35,41 @@ class gpgpu_sim; class kernel_info_t; class gpgpu_context; - -//Set a hard limit of 32 CTAs per shader [cuda only has 8] +// Set a hard limit of 32 CTAs per shader [cuda only has 8] #define MAX_CTA_PER_SHADER 32 #define MAX_BARRIERS_PER_CTA 16 -//After expanding the vector input and output operands +// After expanding the vector input and output operands #define MAX_INPUT_VALUES 24 #define MAX_OUTPUT_VALUES 8 enum _memory_space_t { - undefined_space=0, - reg_space, - local_space, - shared_space, - sstarr_space, - param_space_unclassified, - param_space_kernel, /* global to all threads in a kernel : read-only */ - param_space_local, /* local to a thread : read-writable */ - const_space, - tex_space, - surf_space, - global_space, - generic_space, - instruction_space + undefined_space = 0, + reg_space, + local_space, + shared_space, + sstarr_space, + param_space_unclassified, + param_space_kernel, /* global to all threads in a kernel : read-only */ + param_space_local, /* local to a thread : read-writable */ + const_space, + tex_space, + surf_space, + global_space, + generic_space, + instruction_space }; - -enum FuncCache -{ +enum FuncCache { FuncCachePreferNone = 0, FuncCachePreferShared = 1, FuncCachePreferL1 = 2 }; - #ifdef __cplusplus -#include #include +#include #include typedef unsigned long long new_addr_type; @@ -79,371 +77,357 @@ typedef unsigned long long cudaTextureObject_t; typedef unsigned address_type; typedef unsigned addr_t; -// the following are operations the timing model can see +// the following are operations the timing model can see enum uarch_op_t { - NO_OP=-1, - ALU_OP=1, - SFU_OP, - TENSOR_CORE_OP, - DP_OP, - SP_OP, - INTP_OP, - ALU_SFU_OP, - LOAD_OP, - TENSOR_CORE_LOAD_OP, - TENSOR_CORE_STORE_OP, - STORE_OP, - BRANCH_OP, - BARRIER_OP, - MEMORY_BARRIER_OP, - CALL_OPS, - RET_OPS + NO_OP = -1, + ALU_OP = 1, + SFU_OP, + TENSOR_CORE_OP, + DP_OP, + SP_OP, + INTP_OP, + ALU_SFU_OP, + LOAD_OP, + TENSOR_CORE_LOAD_OP, + TENSOR_CORE_STORE_OP, + STORE_OP, + BRANCH_OP, + BARRIER_OP, + MEMORY_BARRIER_OP, + CALL_OPS, + RET_OPS }; typedef enum uarch_op_t op_type; - -enum uarch_bar_t { - NOT_BAR=-1, - SYNC=1, - ARRIVE, - RED -}; +enum uarch_bar_t { NOT_BAR = -1, SYNC = 1, ARRIVE, RED }; typedef enum uarch_bar_t barrier_type; -enum uarch_red_t { - NOT_RED=-1, - POPC_RED=1, - AND_RED, - OR_RED -}; +enum uarch_red_t { NOT_RED = -1, POPC_RED = 1, AND_RED, OR_RED }; typedef enum uarch_red_t reduction_type; - -enum uarch_operand_type_t { - UN_OP=-1, - INT_OP, - FP_OP -}; +enum uarch_operand_type_t { UN_OP = -1, INT_OP, FP_OP }; typedef enum uarch_operand_type_t types_of_operands; enum special_operations_t { - OTHER_OP, - INT__OP, - INT_MUL24_OP, - INT_MUL32_OP, - INT_MUL_OP, - INT_DIV_OP, - FP_MUL_OP, - FP_DIV_OP, - FP__OP, - FP_SQRT_OP, - FP_LG_OP, - FP_SIN_OP, - FP_EXP_OP + OTHER_OP, + INT__OP, + INT_MUL24_OP, + INT_MUL32_OP, + INT_MUL_OP, + INT_DIV_OP, + FP_MUL_OP, + FP_DIV_OP, + FP__OP, + FP_SQRT_OP, + FP_LG_OP, + FP_SIN_OP, + FP_EXP_OP }; -typedef enum special_operations_t special_ops; // Required to identify for the power model +typedef enum special_operations_t + special_ops; // Required to identify for the power model enum operation_pipeline_t { - UNKOWN_OP, - SP__OP, - DP__OP, - INTP__OP, - SFU__OP, - TENSOR_CORE__OP, - MEM__OP + UNKOWN_OP, + SP__OP, + DP__OP, + INTP__OP, + SFU__OP, + TENSOR_CORE__OP, + MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; -enum mem_operation_t { - NOT_TEX, - TEX -}; +enum mem_operation_t { NOT_TEX, TEX }; typedef enum mem_operation_t mem_operation; -enum _memory_op_t { - no_memory_op = 0, - memory_load, - memory_store -}; +enum _memory_op_t { no_memory_op = 0, memory_load, memory_store }; -#include -#include -#include #include #include -#include -#include #include +#include +#include +#include +#include +#include #if !defined(__VECTOR_TYPES_H__) #include "vector_types.h" #endif struct dim3comp { - bool operator() (const dim3 & a, const dim3 & b) const - { - if(a.z < b.z) - return true; - else if(a.y < b.y) - return true; - else if (a.x < b.x) - return true; - else - return false; - } + bool operator()(const dim3 &a, const dim3 &b) const { + if (a.z < b.z) + return true; + else if (a.y < b.y) + return true; + else if (a.x < b.x) + return true; + else + return false; + } }; -void increment_x_then_y_then_z( dim3 &i, const dim3 &bound); +void increment_x_then_y_then_z(dim3 &i, const dim3 &bound); -//Jin: child kernel information for CDP +// Jin: child kernel information for CDP #include "stream_manager.h" class stream_manager; struct CUstream_st; -//extern stream_manager * g_stream_manager; -//support for pinned memories added -extern std::map pinned_memory; +// extern stream_manager * g_stream_manager; +// support for pinned memories added +extern std::map pinned_memory; extern std::map pinned_memory_size; class kernel_info_t { -public: -// kernel_info_t() -// { -// m_valid=false; -// m_kernel_entry=NULL; -// m_uid=0; -// m_num_cores_running=0; -// m_param_mem=NULL; -// } - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry); - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo); - ~kernel_info_t(); - - void inc_running() { m_num_cores_running++; } - void dec_running() - { - assert( m_num_cores_running > 0 ); - m_num_cores_running--; - } - bool running() const { return m_num_cores_running>0; } - bool done() const - { - return no_more_ctas_to_run() && !running(); - } - class function_info *entry() { return m_kernel_entry; } - const class function_info *entry() const { return m_kernel_entry; } - - size_t num_blocks() const - { - return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; - } - - size_t threads_per_cta() const - { - return m_block_dim.x * m_block_dim.y * m_block_dim.z; - } - - dim3 get_grid_dim() const { return m_grid_dim; } - dim3 get_cta_dim() const { return m_block_dim; } - - void increment_cta_id() - { - increment_x_then_y_then_z(m_next_cta,m_grid_dim); - m_next_tid.x=0; - m_next_tid.y=0; - m_next_tid.z=0; - } - dim3 get_next_cta_id() const { return m_next_cta; } - unsigned get_next_cta_id_single() const - { - return m_next_cta.x + m_grid_dim.x*m_next_cta.y + m_grid_dim.x*m_grid_dim.y*m_next_cta.z; - } - bool no_more_ctas_to_run() const - { - return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || m_next_cta.z >= m_grid_dim.z ); - } - - void increment_thread_id() { increment_x_then_y_then_z(m_next_tid,m_block_dim); } - dim3 get_next_thread_id_3d() const { return m_next_tid; } - unsigned get_next_thread_id() const - { - return m_next_tid.x + m_block_dim.x*m_next_tid.y + m_block_dim.x*m_block_dim.y*m_next_tid.z; - } - bool more_threads_in_cta() const - { - return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && m_next_tid.x < m_block_dim.x; - } - unsigned get_uid() const { return m_uid; } - std::string name() const; - - std::list &active_threads() { return m_active_threads; } - class memory_space *get_param_memory() { return m_param_mem; } - - - //The following functions access texture bindings present at the kernel's launch - - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - -private: - kernel_info_t( const kernel_info_t & ); // disable copy constructor - void operator=( const kernel_info_t & ); // disable copy operator - - class function_info *m_kernel_entry; - - unsigned m_uid; - - //These maps contain the snapshot of the texture mappings at kernel launch - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - - dim3 m_grid_dim; - dim3 m_block_dim; - dim3 m_next_cta; - dim3 m_next_tid; - - unsigned m_num_cores_running; - - std::list m_active_threads; - class memory_space *m_param_mem; - -public: - //Jin: parent and child kernel management for CDP - void set_parent(kernel_info_t * parent, dim3 parent_ctaid, dim3 parent_tid); - void set_child(kernel_info_t * child); - void remove_child(kernel_info_t * child); - bool is_finished(); - bool children_all_finished(); - void notify_parent_finished(); - CUstream_st * create_stream_cta(dim3 ctaid); - CUstream_st * get_default_stream_cta(dim3 ctaid); - bool cta_has_stream(dim3 ctaid, CUstream_st* stream); - void destroy_cta_streams(); - void print_parent_info(); - kernel_info_t * get_parent() { return m_parent_kernel; } - -private: - kernel_info_t * m_parent_kernel; - dim3 m_parent_ctaid; - dim3 m_parent_tid; - std::list m_child_kernels; //child kernel launched - std::map< dim3, std::list, dim3comp > m_cta_streams; //streams created in each CTA - -//Jin: kernel timing -public: - unsigned long long launch_cycle; - unsigned long long start_cycle; - unsigned long long end_cycle; - unsigned m_launch_latency; - - mutable bool volta_cache_config_set; + public: + // kernel_info_t() + // { + // m_valid=false; + // m_kernel_entry=NULL; + // m_uid=0; + // m_num_cores_running=0; + // m_param_mem=NULL; + // } + kernel_info_t(dim3 gridDim, dim3 blockDim, class function_info *entry); + kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map nameToCudaArray, + std::map nameToTextureInfo); + ~kernel_info_t(); + + void inc_running() { m_num_cores_running++; } + void dec_running() { + assert(m_num_cores_running > 0); + m_num_cores_running--; + } + bool running() const { return m_num_cores_running > 0; } + bool done() const { return no_more_ctas_to_run() && !running(); } + class function_info *entry() { + return m_kernel_entry; + } + const class function_info *entry() const { return m_kernel_entry; } + + size_t num_blocks() const { + return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; + } + + size_t threads_per_cta() const { + return m_block_dim.x * m_block_dim.y * m_block_dim.z; + } + + dim3 get_grid_dim() const { return m_grid_dim; } + dim3 get_cta_dim() const { return m_block_dim; } + + void increment_cta_id() { + increment_x_then_y_then_z(m_next_cta, m_grid_dim); + m_next_tid.x = 0; + m_next_tid.y = 0; + m_next_tid.z = 0; + } + dim3 get_next_cta_id() const { return m_next_cta; } + unsigned get_next_cta_id_single() const { + return m_next_cta.x + m_grid_dim.x * m_next_cta.y + + m_grid_dim.x * m_grid_dim.y * m_next_cta.z; + } + bool no_more_ctas_to_run() const { + return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || + m_next_cta.z >= m_grid_dim.z); + } + + void increment_thread_id() { + increment_x_then_y_then_z(m_next_tid, m_block_dim); + } + dim3 get_next_thread_id_3d() const { return m_next_tid; } + unsigned get_next_thread_id() const { + return m_next_tid.x + m_block_dim.x * m_next_tid.y + + m_block_dim.x * m_block_dim.y * m_next_tid.z; + } + bool more_threads_in_cta() const { + return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && + m_next_tid.x < m_block_dim.x; + } + unsigned get_uid() const { return m_uid; } + std::string name() const; + + std::list &active_threads() { + return m_active_threads; + } + class memory_space *get_param_memory() { + return m_param_mem; + } + + // The following functions access texture bindings present at the kernel's + // launch + + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + + private: + kernel_info_t(const kernel_info_t &); // disable copy constructor + void operator=(const kernel_info_t &); // disable copy operator + + class function_info *m_kernel_entry; + + unsigned m_uid; + + // These maps contain the snapshot of the texture mappings at kernel launch + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + + dim3 m_grid_dim; + dim3 m_block_dim; + dim3 m_next_cta; + dim3 m_next_tid; + + unsigned m_num_cores_running; + + std::list m_active_threads; + class memory_space *m_param_mem; + + public: + // Jin: parent and child kernel management for CDP + void set_parent(kernel_info_t *parent, dim3 parent_ctaid, dim3 parent_tid); + void set_child(kernel_info_t *child); + void remove_child(kernel_info_t *child); + bool is_finished(); + bool children_all_finished(); + void notify_parent_finished(); + CUstream_st *create_stream_cta(dim3 ctaid); + CUstream_st *get_default_stream_cta(dim3 ctaid); + bool cta_has_stream(dim3 ctaid, CUstream_st *stream); + void destroy_cta_streams(); + void print_parent_info(); + kernel_info_t *get_parent() { return m_parent_kernel; } + + private: + kernel_info_t *m_parent_kernel; + dim3 m_parent_ctaid; + dim3 m_parent_tid; + std::list m_child_kernels; // child kernel launched + std::map, dim3comp> + m_cta_streams; // streams created in each CTA + + // Jin: kernel timing + public: + unsigned long long launch_cycle; + unsigned long long start_cycle; + unsigned long long end_cycle; + unsigned m_launch_latency; + + mutable bool volta_cache_config_set; }; class core_config { - public: - core_config(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_valid = false; - num_shmem_bank=16; - shmem_limited_broadcast = false; - gpgpu_shmem_sizeDefault=(unsigned)-1; - gpgpu_shmem_sizePrefL1=(unsigned)-1; - gpgpu_shmem_sizePrefShared=(unsigned)-1; - } - virtual void init() = 0; - - bool m_valid; - unsigned warp_size; - // backward pointer - class gpgpu_context* gpgpu_ctx; - - // off-chip memory request architecture parameters - int gpgpu_coalesce_arch; - - // shared memory bank conflict checking parameters - bool shmem_limited_broadcast; - static const address_type WORD_SIZE=4; - unsigned num_shmem_bank; - unsigned shmem_bank_func(address_type addr) const - { - return ((addr/WORD_SIZE) % num_shmem_bank); - } - unsigned mem_warp_parts; - mutable unsigned gpgpu_shmem_size; - unsigned gpgpu_shmem_sizeDefault; - unsigned gpgpu_shmem_sizePrefL1; - unsigned gpgpu_shmem_sizePrefShared; - unsigned mem_unit_ports; - - // texture and constant cache line sizes (used to determine number of memory accesses) - unsigned gpgpu_cache_texl1_linesize; - unsigned gpgpu_cache_constl1_linesize; - - unsigned gpgpu_max_insn_issue_per_warp; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - - bool adaptive_volta_cache_config; + public: + core_config(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_valid = false; + num_shmem_bank = 16; + shmem_limited_broadcast = false; + gpgpu_shmem_sizeDefault = (unsigned)-1; + gpgpu_shmem_sizePrefL1 = (unsigned)-1; + gpgpu_shmem_sizePrefShared = (unsigned)-1; + } + virtual void init() = 0; + + bool m_valid; + unsigned warp_size; + // backward pointer + class gpgpu_context *gpgpu_ctx; + + // off-chip memory request architecture parameters + int gpgpu_coalesce_arch; + + // shared memory bank conflict checking parameters + bool shmem_limited_broadcast; + static const address_type WORD_SIZE = 4; + unsigned num_shmem_bank; + unsigned shmem_bank_func(address_type addr) const { + return ((addr / WORD_SIZE) % num_shmem_bank); + } + unsigned mem_warp_parts; + mutable unsigned gpgpu_shmem_size; + unsigned gpgpu_shmem_sizeDefault; + unsigned gpgpu_shmem_sizePrefL1; + unsigned gpgpu_shmem_sizePrefShared; + unsigned mem_unit_ports; + + // texture and constant cache line sizes (used to determine number of memory + // accesses) + unsigned gpgpu_cache_texl1_linesize; + unsigned gpgpu_cache_constl1_linesize; + + unsigned gpgpu_max_insn_issue_per_warp; + bool gmem_skip_L1D; // on = global memory access always skip the L1 cache + + bool adaptive_volta_cache_config; }; -// bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper +// bounded stack that implements simt reconvergence using pdom mechanism from +// MICRO'07 paper const unsigned MAX_WARP_SIZE = 32; typedef std::bitset active_mask_t; -#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE +#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE typedef std::bitset simt_mask_t; typedef std::vector addr_vector_t; class simt_stack { -public: - simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu); - - void reset(); - void launch( address_type start_pc, const simt_mask_t &active_mask ); - void update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ); - - const simt_mask_t &get_active_mask() const; - void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; - unsigned get_rp() const; - void print(FILE *fp) const; - void resume(char * fname) ; - void print_checkpoint (FILE *fout) const; - -protected: - unsigned m_warp_id; - unsigned m_warp_size; - - - enum stack_entry_type { - STACK_ENTRY_TYPE_NORMAL = 0, - STACK_ENTRY_TYPE_CALL - }; - - struct simt_stack_entry { - address_type m_pc; - unsigned int m_calldepth; - simt_mask_t m_active_mask; - address_type m_recvg_pc; - unsigned long long m_branch_div_cycle; - stack_entry_type m_type; - simt_stack_entry() : - m_pc(-1), m_calldepth(0), m_active_mask(), m_recvg_pc(-1), m_branch_div_cycle(0), m_type(STACK_ENTRY_TYPE_NORMAL) { }; - }; - - std::deque m_stack; - - class gpgpu_sim * m_gpu; + public: + simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu); + + void reset(); + void launch(address_type start_pc, const simt_mask_t &active_mask); + void update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc); + + const simt_mask_t &get_active_mask() const; + void get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const; + unsigned get_rp() const; + void print(FILE *fp) const; + void resume(char *fname); + void print_checkpoint(FILE *fout) const; + + protected: + unsigned m_warp_id; + unsigned m_warp_size; + + enum stack_entry_type { STACK_ENTRY_TYPE_NORMAL = 0, STACK_ENTRY_TYPE_CALL }; + + struct simt_stack_entry { + address_type m_pc; + unsigned int m_calldepth; + simt_mask_t m_active_mask; + address_type m_recvg_pc; + unsigned long long m_branch_div_cycle; + stack_entry_type m_type; + simt_stack_entry() + : m_pc(-1), + m_calldepth(0), + m_active_mask(), + m_recvg_pc(-1), + m_branch_div_cycle(0), + m_type(STACK_ENTRY_TYPE_NORMAL){}; + }; + + std::deque m_stack; + + class gpgpu_sim *m_gpu; }; // Let's just upgrade to C++11 so we can use constexpr here... -// start allocating from this address (lower values used for allocating globals in .ptx file) +// start allocating from this address (lower values used for allocating globals +// in .ptx file) const unsigned long long GLOBAL_HEAP_START = 0xC0000000; // Volta max shmem size is 96kB const unsigned long long SHARED_MEM_SIZE_MAX = 96 * (1 << 10); @@ -455,873 +439,931 @@ const unsigned MAX_STREAMING_MULTIPROCESSORS = 80; const unsigned MAX_THREAD_PER_SM = 1 << 11; // MAX 64 warps / SM const unsigned MAX_WARP_PER_SM = 1 << 6; -const unsigned long long TOTAL_LOCAL_MEM_PER_SM = MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long TOTAL_SHARED_MEM = MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; -const unsigned long long TOTAL_LOCAL_MEM = MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long SHARED_GENERIC_START = GLOBAL_HEAP_START - TOTAL_SHARED_MEM; -const unsigned long long LOCAL_GENERIC_START = SHARED_GENERIC_START - TOTAL_LOCAL_MEM; -const unsigned long long STATIC_ALLOC_LIMIT = GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); +const unsigned long long TOTAL_LOCAL_MEM_PER_SM = + MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long TOTAL_SHARED_MEM = + MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; +const unsigned long long TOTAL_LOCAL_MEM = + MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long SHARED_GENERIC_START = + GLOBAL_HEAP_START - TOTAL_SHARED_MEM; +const unsigned long long LOCAL_GENERIC_START = + SHARED_GENERIC_START - TOTAL_LOCAL_MEM; +const unsigned long long STATIC_ALLOC_LIMIT = + GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); #if !defined(__CUDA_RUNTIME_API_H__) #include "builtin_types.h" struct cudaArray { - void *devPtr; - int devPtr32; - struct cudaChannelFormatDesc desc; - int width; - int height; - int size; //in bytes - unsigned dimensions; + void *devPtr; + int devPtr32; + struct cudaChannelFormatDesc desc; + int width; + int height; + int size; // in bytes + unsigned dimensions; }; #endif -// Struct that record other attributes in the textureReference declaration +// Struct that record other attributes in the textureReference declaration // - These attributes are passed thru __cudaRegisterTexture() struct textureReferenceAttr { - const struct textureReference *m_texref; - int m_dim; - enum cudaTextureReadMode m_readmode; - int m_ext; - textureReferenceAttr(const struct textureReference *texref, - int dim, - enum cudaTextureReadMode readmode, - int ext) - : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) - { } + const struct textureReference *m_texref; + int m_dim; + enum cudaTextureReadMode m_readmode; + int m_ext; + textureReferenceAttr(const struct textureReference *texref, int dim, + enum cudaTextureReadMode readmode, int ext) + : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) {} }; -class gpgpu_functional_sim_config -{ -public: - void reg_options(class OptionParser * opp); - - void ptx_set_tex_cache_linesize(unsigned linesize); - - unsigned get_forced_max_capability() const { return m_ptx_force_max_capability; } - bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } - bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } - bool experimental_lib_support() const { return m_experimental_lib_support; } - - int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } - const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } - int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } - unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const {return checkpoint_option; } - int get_checkpoint_kernel() const {return checkpoint_kernel; } - int get_checkpoint_CTA() const {return checkpoint_CTA; } - int get_resume_option() const {return resume_option; } - int get_resume_kernel() const {return resume_kernel; } - int get_resume_CTA() const {return resume_CTA; } - int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } -private: - // PTX options - int m_ptx_convert_to_ptxplus; - int m_ptx_use_cuobjdump; - int m_experimental_lib_support; - unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - int g_ptx_inst_debug_to_file; - char* g_ptx_inst_debug_file; - int g_ptx_inst_debug_thread_uid; - - unsigned m_texcache_linesize; +class gpgpu_functional_sim_config { + public: + void reg_options(class OptionParser *opp); + + void ptx_set_tex_cache_linesize(unsigned linesize); + + unsigned get_forced_max_capability() const { + return m_ptx_force_max_capability; + } + bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } + bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } + bool experimental_lib_support() const { return m_experimental_lib_support; } + + int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } + const char *get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } + int get_ptx_inst_debug_thread_uid() const { + return g_ptx_inst_debug_thread_uid; + } + unsigned get_texcache_linesize() const { return m_texcache_linesize; } + int get_checkpoint_option() const { return checkpoint_option; } + int get_checkpoint_kernel() const { return checkpoint_kernel; } + int get_checkpoint_CTA() const { return checkpoint_CTA; } + int get_resume_option() const { return resume_option; } + int get_resume_kernel() const { return resume_kernel; } + int get_resume_CTA() const { return resume_CTA; } + int get_checkpoint_CTA_t() const { return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const { return checkpoint_insn_Y; } + + private: + // PTX options + int m_ptx_convert_to_ptxplus; + int m_ptx_use_cuobjdump; + int m_experimental_lib_support; + unsigned m_ptx_force_max_capability; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; + char *g_ptx_inst_debug_file; + int g_ptx_inst_debug_thread_uid; + + unsigned m_texcache_linesize; }; - class gpgpu_t { -public: - gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ); - // backward pointer - class gpgpu_context* gpgpu_ctx; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - - //Move some cycle core stats here instead of being global - unsigned long long gpu_sim_cycle; - unsigned long long gpu_tot_sim_cycle; - - - void* gpu_malloc( size_t size ); - void* gpu_mallocarray( size_t count ); - void gpu_memset( size_t dst_start_addr, int c, size_t count ); - void memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ); - void memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ); - void memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ); - - class memory_space *get_global_memory() { return m_global_mem; } - class memory_space *get_tex_memory() { return m_tex_mem; } - class memory_space *get_surf_memory() { return m_surf_mem; } - - void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array); - void gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext); - void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); - const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); - - const struct textureReference* get_texref( const std::string &texname ) const - { - std::map >::const_iterator t=m_NameToTextureRef.find(texname); - assert( t != m_NameToTextureRef.end() ); - return *(t->second.begin()); - } - - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - - const struct textureReferenceAttr* get_texattr( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToAttribute.find(texname); - assert(t != m_NameToAttribute.end()); - return t->second; - } - - const gpgpu_functional_sim_config &get_config() const { return m_function_model_config; } - FILE* get_ptx_inst_debug_file() { return ptx_inst_debug_file; } - - // These maps return the current texture mappings for the GPU at any given time. - std::map getNameArrayMapping() {return m_NameToCudaArray;} - std::map getNameInfoMapping() {return m_NameToTextureInfo;} - -protected: - const gpgpu_functional_sim_config &m_function_model_config; - FILE* ptx_inst_debug_file; - - class memory_space *m_global_mem; - class memory_space *m_tex_mem; - class memory_space *m_surf_mem; - - unsigned long long m_dev_malloc; - // These maps contain the current texture mappings for the GPU at any given time. - std::map > m_NameToTextureRef; - std::map m_TextureRefToName; - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - std::map m_NameToAttribute; + public: + gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx); + // backward pointer + class gpgpu_context *gpgpu_ctx; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + + // Move some cycle core stats here instead of being global + unsigned long long gpu_sim_cycle; + unsigned long long gpu_tot_sim_cycle; + + void *gpu_malloc(size_t size); + void *gpu_mallocarray(size_t count); + void gpu_memset(size_t dst_start_addr, int c, size_t count); + void memcpy_to_gpu(size_t dst_start_addr, const void *src, size_t count); + void memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count); + void memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count); + + class memory_space *get_global_memory() { + return m_global_mem; + } + class memory_space *get_tex_memory() { + return m_tex_mem; + } + class memory_space *get_surf_memory() { + return m_surf_mem; + } + + void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference *texref, + const struct cudaArray *array); + void gpgpu_ptx_sim_bindNameToTexture(const char *name, + const struct textureReference *texref, + int dim, int readmode, int ext); + void gpgpu_ptx_sim_unbindTexture(const struct textureReference *texref); + const char *gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref); + + const struct textureReference *get_texref(const std::string &texname) const { + std::map >::const_iterator t = + m_NameToTextureRef.find(texname); + assert(t != m_NameToTextureRef.end()); + return *(t->second.begin()); + } + + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + + const struct textureReferenceAttr *get_texattr( + const std::string &texname) const { + std::map::const_iterator + t = m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); + return t->second; + } + + const gpgpu_functional_sim_config &get_config() const { + return m_function_model_config; + } + FILE *get_ptx_inst_debug_file() { return ptx_inst_debug_file; } + + // These maps return the current texture mappings for the GPU at any given + // time. + std::map getNameArrayMapping() { + return m_NameToCudaArray; + } + std::map getNameInfoMapping() { + return m_NameToTextureInfo; + } + + protected: + const gpgpu_functional_sim_config &m_function_model_config; + FILE *ptx_inst_debug_file; + + class memory_space *m_global_mem; + class memory_space *m_tex_mem; + class memory_space *m_surf_mem; + + unsigned long long m_dev_malloc; + // These maps contain the current texture mappings for the GPU at any given + // time. + std::map > + m_NameToTextureRef; + std::map m_TextureRefToName; + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + std::map m_NameToAttribute; }; -struct gpgpu_ptx_sim_info -{ - // Holds properties of the kernel (Kernel's resource use). - // These will be set to zero if a ptxinfo file is not present. - int lmem; - int smem; - int cmem; - int gmem; - int regs; - unsigned maxthreads; - unsigned ptx_version; - unsigned sm_target; +struct gpgpu_ptx_sim_info { + // Holds properties of the kernel (Kernel's resource use). + // These will be set to zero if a ptxinfo file is not present. + int lmem; + int smem; + int cmem; + int gmem; + int regs; + unsigned maxthreads; + unsigned ptx_version; + unsigned sm_target; }; - struct gpgpu_ptx_sim_arg { - gpgpu_ptx_sim_arg() { m_start=NULL; } - gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) - { - m_start=arg; - m_nbytes=size; - m_offset=offset; - } - const void *m_start; - size_t m_nbytes; - size_t m_offset; + gpgpu_ptx_sim_arg() { m_start = NULL; } + gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) { + m_start = arg; + m_nbytes = size; + m_offset = offset; + } + const void *m_start; + size_t m_nbytes; + size_t m_offset; }; typedef std::list gpgpu_ptx_sim_arg_list_t; class memory_space_t { -public: - memory_space_t() { m_type = undefined_space; m_bank=0; } - memory_space_t( const enum _memory_space_t &from ) { m_type = from; m_bank = 0; } - bool operator==( const memory_space_t &x ) const { return (m_bank == x.m_bank) && (m_type == x.m_type); } - bool operator!=( const memory_space_t &x ) const { return !(*this == x); } - bool operator<( const memory_space_t &x ) const - { - if(m_type < x.m_type) - return true; - else if(m_type > x.m_type) - return false; - else if( m_bank < x.m_bank ) - return true; + public: + memory_space_t() { + m_type = undefined_space; + m_bank = 0; + } + memory_space_t(const enum _memory_space_t &from) { + m_type = from; + m_bank = 0; + } + bool operator==(const memory_space_t &x) const { + return (m_bank == x.m_bank) && (m_type == x.m_type); + } + bool operator!=(const memory_space_t &x) const { return !(*this == x); } + bool operator<(const memory_space_t &x) const { + if (m_type < x.m_type) + return true; + else if (m_type > x.m_type) return false; - } - enum _memory_space_t get_type() const { return m_type; } - void set_type( enum _memory_space_t t ) { m_type = t; } - unsigned get_bank() const { return m_bank; } - void set_bank( unsigned b ) { m_bank = b; } - bool is_const() const { return (m_type == const_space) || (m_type == param_space_kernel); } - bool is_local() const { return (m_type == local_space) || (m_type == param_space_local); } - bool is_global() const { return (m_type == global_space); } - -private: - enum _memory_space_t m_type; - unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 manual, sec. 5.1.3) + else if (m_bank < x.m_bank) + return true; + return false; + } + enum _memory_space_t get_type() const { return m_type; } + void set_type(enum _memory_space_t t) { m_type = t; } + unsigned get_bank() const { return m_bank; } + void set_bank(unsigned b) { m_bank = b; } + bool is_const() const { + return (m_type == const_space) || (m_type == param_space_kernel); + } + bool is_local() const { + return (m_type == local_space) || (m_type == param_space_local); + } + bool is_global() const { return (m_type == global_space); } + + private: + enum _memory_space_t m_type; + unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 + // manual, sec. 5.1.3) }; const unsigned MAX_MEMORY_ACCESS_SIZE = 128; typedef std::bitset mem_access_byte_mask_t; -const unsigned SECTOR_CHUNCK_SIZE = 4; //four sectors -const unsigned SECTOR_SIZE = 32 ; //sector is 32 bytes width +const unsigned SECTOR_CHUNCK_SIZE = 4; // four sectors +const unsigned SECTOR_SIZE = 32; // sector is 32 bytes width typedef std::bitset mem_access_sector_mask_t; #define NO_PARTIAL_WRITE (mem_access_byte_mask_t()) -#define MEM_ACCESS_TYPE_TUP_DEF \ -MA_TUP_BEGIN( mem_access_type ) \ - MA_TUP( GLOBAL_ACC_R ), \ - MA_TUP( LOCAL_ACC_R ), \ - MA_TUP( CONST_ACC_R ), \ - MA_TUP( TEXTURE_ACC_R ), \ - MA_TUP( GLOBAL_ACC_W ), \ - MA_TUP( LOCAL_ACC_W ), \ - MA_TUP( L1_WRBK_ACC ), \ - MA_TUP( L2_WRBK_ACC ), \ - MA_TUP( INST_ACC_R ), \ - MA_TUP( L1_WR_ALLOC_R ), \ - MA_TUP( L2_WR_ALLOC_R ), \ - MA_TUP( NUM_MEM_ACCESS_TYPE ) \ -MA_TUP_END( mem_access_type ) +#define MEM_ACCESS_TYPE_TUP_DEF \ + MA_TUP_BEGIN(mem_access_type) \ + MA_TUP(GLOBAL_ACC_R) \ + , MA_TUP(LOCAL_ACC_R), MA_TUP(CONST_ACC_R), MA_TUP(TEXTURE_ACC_R), \ + MA_TUP(GLOBAL_ACC_W), MA_TUP(LOCAL_ACC_W), MA_TUP(L1_WRBK_ACC), \ + MA_TUP(L2_WRBK_ACC), MA_TUP(INST_ACC_R), MA_TUP(L1_WR_ALLOC_R), \ + MA_TUP(L2_WR_ALLOC_R), \ + MA_TUP(NUM_MEM_ACCESS_TYPE) MA_TUP_END(mem_access_type) #define MA_TUP_BEGIN(X) enum X { #define MA_TUP(X) X -#define MA_TUP_END(X) }; +#define MA_TUP_END(X) \ + } \ + ; MEM_ACCESS_TYPE_TUP_DEF #undef MA_TUP_BEGIN #undef MA_TUP #undef MA_TUP_END -const char * mem_access_type_str(enum mem_access_type access_type); +const char *mem_access_type_str(enum mem_access_type access_type); enum cache_operator_type { - CACHE_UNDEFINED, - - // loads - CACHE_ALL, // .ca - CACHE_LAST_USE, // .lu - CACHE_VOLATILE, // .cv - CACHE_L1, // .nc - - // loads and stores - CACHE_STREAMING, // .cs - CACHE_GLOBAL, // .cg - - // stores - CACHE_WRITE_BACK, // .wb - CACHE_WRITE_THROUGH // .wt + CACHE_UNDEFINED, + + // loads + CACHE_ALL, // .ca + CACHE_LAST_USE, // .lu + CACHE_VOLATILE, // .cv + CACHE_L1, // .nc + + // loads and stores + CACHE_STREAMING, // .cs + CACHE_GLOBAL, // .cg + + // stores + CACHE_WRITE_BACK, // .wb + CACHE_WRITE_THROUGH // .wt }; class mem_access_t { -public: - mem_access_t(gpgpu_context* ctx) { init(ctx); } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - gpgpu_context* ctx) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - const active_mask_t &active_mask, - const mem_access_byte_mask_t &byte_mask, - const mem_access_sector_mask_t §or_mask, - gpgpu_context* ctx) - : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - - new_addr_type get_addr() const { return m_addr; } - void set_addr(new_addr_type addr) {m_addr=addr;} - unsigned get_size() const { return m_req_size; } - const active_mask_t &get_warp_mask() const { return m_warp_mask; } - bool is_write() const { return m_write; } - enum mem_access_type get_type() const { return m_type; } - mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } - mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } - - void print(FILE *fp) const - { - fprintf(fp,"addr=0x%llx, %s, size=%u, ", m_addr, m_write?"store":"load ", m_req_size ); - switch(m_type) { - case GLOBAL_ACC_R: fprintf(fp,"GLOBAL_R"); break; - case LOCAL_ACC_R: fprintf(fp,"LOCAL_R "); break; - case CONST_ACC_R: fprintf(fp,"CONST "); break; - case TEXTURE_ACC_R: fprintf(fp,"TEXTURE "); break; - case GLOBAL_ACC_W: fprintf(fp,"GLOBAL_W"); break; - case LOCAL_ACC_W: fprintf(fp,"LOCAL_W "); break; - case L2_WRBK_ACC: fprintf(fp,"L2_WRBK "); break; - case INST_ACC_R: fprintf(fp,"INST "); break; - case L1_WRBK_ACC: fprintf(fp,"L1_WRBK "); break; - default: fprintf(fp,"unknown "); break; - } - } - - gpgpu_context* gpgpu_ctx; -private: - void init(gpgpu_context* ctx); - - unsigned m_uid; - new_addr_type m_addr; // request address - bool m_write; - unsigned m_req_size; // bytes - mem_access_type m_type; - active_mask_t m_warp_mask; - mem_access_byte_mask_t m_byte_mask; - mem_access_sector_mask_t m_sector_mask; + public: + mem_access_t(gpgpu_context *ctx) { init(ctx); } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, gpgpu_context *ctx) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, const active_mask_t &active_mask, + const mem_access_byte_mask_t &byte_mask, + const mem_access_sector_mask_t §or_mask, gpgpu_context *ctx) + : m_warp_mask(active_mask), + m_byte_mask(byte_mask), + m_sector_mask(sector_mask) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + + new_addr_type get_addr() const { return m_addr; } + void set_addr(new_addr_type addr) { m_addr = addr; } + unsigned get_size() const { return m_req_size; } + const active_mask_t &get_warp_mask() const { return m_warp_mask; } + bool is_write() const { return m_write; } + enum mem_access_type get_type() const { return m_type; } + mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } + mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } + + void print(FILE *fp) const { + fprintf(fp, "addr=0x%llx, %s, size=%u, ", m_addr, + m_write ? "store" : "load ", m_req_size); + switch (m_type) { + case GLOBAL_ACC_R: + fprintf(fp, "GLOBAL_R"); + break; + case LOCAL_ACC_R: + fprintf(fp, "LOCAL_R "); + break; + case CONST_ACC_R: + fprintf(fp, "CONST "); + break; + case TEXTURE_ACC_R: + fprintf(fp, "TEXTURE "); + break; + case GLOBAL_ACC_W: + fprintf(fp, "GLOBAL_W"); + break; + case LOCAL_ACC_W: + fprintf(fp, "LOCAL_W "); + break; + case L2_WRBK_ACC: + fprintf(fp, "L2_WRBK "); + break; + case INST_ACC_R: + fprintf(fp, "INST "); + break; + case L1_WRBK_ACC: + fprintf(fp, "L1_WRBK "); + break; + default: + fprintf(fp, "unknown "); + break; + } + } + + gpgpu_context *gpgpu_ctx; + + private: + void init(gpgpu_context *ctx); + + unsigned m_uid; + new_addr_type m_addr; // request address + bool m_write; + unsigned m_req_size; // bytes + mem_access_type m_type; + active_mask_t m_warp_mask; + mem_access_byte_mask_t m_byte_mask; + mem_access_sector_mask_t m_sector_mask; }; class mem_fetch; class mem_fetch_interface { -public: - virtual bool full( unsigned size, bool write ) const = 0; - virtual void push( mem_fetch *mf ) = 0; + public: + virtual bool full(unsigned size, bool write) const = 0; + virtual void push(mem_fetch *mf) = 0; }; class mem_fetch_allocator { -public: - virtual mem_fetch *alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const = 0; - virtual mem_fetch *alloc( const class warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle ) const = 0; + public: + virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type, + unsigned size, bool wr, + unsigned long long cycle) const = 0; + virtual mem_fetch *alloc(const class warp_inst_t &inst, + const mem_access_t &access, + unsigned long long cycle) const = 0; }; -// the maximum number of destination, source, or address uarch operands in a instruction -#define MAX_REG_OPERANDS 32 +// the maximum number of destination, source, or address uarch operands in a +// instruction +#define MAX_REG_OPERANDS 32 struct dram_callback_t { - dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } - void (*function)(const class inst_t*, class ptx_thread_info*); - - const class inst_t* instruction; - class ptx_thread_info *thread; + dram_callback_t() { + function = NULL; + instruction = NULL; + thread = NULL; + } + void (*function)(const class inst_t *, class ptx_thread_info *); + + const class inst_t *instruction; + class ptx_thread_info *thread; }; class inst_t { -public: - inst_t() - { - m_decoded=false; - pc=(address_type)-1; - reconvergence_pc=(address_type)-1; - op=NO_OP; - bar_type=NOT_BAR; - red_type=NOT_RED; - bar_id=(unsigned)-1; - bar_count=(unsigned)-1; - oprnd_type=UN_OP; - sp_op=OTHER_OP; - op_pipe=UNKOWN_OP; - mem_op=NOT_TEX; - num_operands=0; - num_regs=0; - memset(out, 0, sizeof(unsigned)); - memset(in, 0, sizeof(unsigned)); - is_vectorin=0; - is_vectorout=0; - space = memory_space_t(); - cache_op = CACHE_UNDEFINED; - latency = 1; - initiation_interval = 1; - for( unsigned i=0; i < MAX_REG_OPERANDS; i++ ) { - arch_reg.src[i] = -1; - arch_reg.dst[i] = -1; - } - isize=0; - } - bool valid() const { return m_decoded; } - virtual void print_insn( FILE *fp ) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); + public: + inst_t() { + m_decoded = false; + pc = (address_type)-1; + reconvergence_pc = (address_type)-1; + op = NO_OP; + bar_type = NOT_BAR; + red_type = NOT_RED; + bar_id = (unsigned)-1; + bar_count = (unsigned)-1; + oprnd_type = UN_OP; + sp_op = OTHER_OP; + op_pipe = UNKOWN_OP; + mem_op = NOT_TEX; + num_operands = 0; + num_regs = 0; + memset(out, 0, sizeof(unsigned)); + memset(in, 0, sizeof(unsigned)); + is_vectorin = 0; + is_vectorout = 0; + space = memory_space_t(); + cache_op = CACHE_UNDEFINED; + latency = 1; + initiation_interval = 1; + for (unsigned i = 0; i < MAX_REG_OPERANDS; i++) { + arch_reg.src[i] = -1; + arch_reg.dst[i] = -1; } - bool is_load() const { return (op == LOAD_OP ||op==TENSOR_CORE_LOAD_OP || memory_op == memory_load); } - bool is_store() const { return (op == STORE_OP ||op==TENSOR_CORE_STORE_OP || memory_op == memory_store); } - unsigned get_num_operands() const {return num_operands;} - unsigned get_num_regs() const {return num_regs;} - void set_num_regs(unsigned num) {num_regs=num;} - void set_num_operands(unsigned num) {num_operands=num;} - void set_bar_id(unsigned id) {bar_id=id;} - void set_bar_count(unsigned count) {bar_count=count;} - - address_type pc; // program counter address of instruction - unsigned isize; // size of instruction in bytes - op_type op; // opcode (uarch visible) - - barrier_type bar_type; - reduction_type red_type; - unsigned bar_id; - unsigned bar_count; - - types_of_operands oprnd_type; // code (uarch visible) identify if the operation is an interger or a floating point - special_ops sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... - operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of the operation (SP, SFU or MEM) - mem_operation mem_op; // code (uarch visible) identify memory type - _memory_op_t memory_op; // memory_op used by ptxplus - unsigned num_operands; - unsigned num_regs; // count vector operand as one register operand - - address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address - - unsigned out[8]; - unsigned outcount; - unsigned in[24]; - unsigned incount; - unsigned char is_vectorin; - unsigned char is_vectorout; - int pred; // predicate register number - int ar1, ar2; - // register number for bank conflict evaluation - struct { - int dst[MAX_REG_OPERANDS]; - int src[MAX_REG_OPERANDS]; - } arch_reg; - //int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation - unsigned latency; // operation latency - unsigned initiation_interval; - - unsigned data_size; // what is the size of the word being operated on? - memory_space_t space; - cache_operator_type cache_op; - -protected: - bool m_decoded; - virtual void pre_decode() {} + isize = 0; + } + bool valid() const { return m_decoded; } + virtual void print_insn(FILE *fp) const { + fprintf(fp, " [inst @ pc=0x%04x] ", pc); + } + bool is_load() const { + return (op == LOAD_OP || op == TENSOR_CORE_LOAD_OP || + memory_op == memory_load); + } + bool is_store() const { + return (op == STORE_OP || op == TENSOR_CORE_STORE_OP || + memory_op == memory_store); + } + unsigned get_num_operands() const { return num_operands; } + unsigned get_num_regs() const { return num_regs; } + void set_num_regs(unsigned num) { num_regs = num; } + void set_num_operands(unsigned num) { num_operands = num; } + void set_bar_id(unsigned id) { bar_id = id; } + void set_bar_count(unsigned count) { bar_count = count; } + + address_type pc; // program counter address of instruction + unsigned isize; // size of instruction in bytes + op_type op; // opcode (uarch visible) + + barrier_type bar_type; + reduction_type red_type; + unsigned bar_id; + unsigned bar_count; + + types_of_operands oprnd_type; // code (uarch visible) identify if the + // operation is an interger or a floating point + special_ops + sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... + operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of + // the operation (SP, SFU or MEM) + mem_operation mem_op; // code (uarch visible) identify memory type + _memory_op_t memory_op; // memory_op used by ptxplus + unsigned num_operands; + unsigned num_regs; // count vector operand as one register operand + + address_type reconvergence_pc; // -1 => not a branch, -2 => use function + // return address + + unsigned out[8]; + unsigned outcount; + unsigned in[24]; + unsigned incount; + unsigned char is_vectorin; + unsigned char is_vectorout; + int pred; // predicate register number + int ar1, ar2; + // register number for bank conflict evaluation + struct { + int dst[MAX_REG_OPERANDS]; + int src[MAX_REG_OPERANDS]; + } arch_reg; + // int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict + // evaluation + unsigned latency; // operation latency + unsigned initiation_interval; + + unsigned data_size; // what is the size of the word being operated on? + memory_space_t space; + cache_operator_type cache_op; + + protected: + bool m_decoded; + virtual void pre_decode() {} }; -enum divergence_support_t { - POST_DOMINATOR = 1, - NUM_SIMD_MODEL -}; +enum divergence_support_t { POST_DOMINATOR = 1, NUM_SIMD_MODEL }; const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; -class warp_inst_t: public inst_t { -public: - // constructors - warp_inst_t() - { - m_uid=0; - m_empty=true; - m_config=NULL; - } - warp_inst_t( const core_config *config ) - { - m_uid=0; - assert(config->warp_size<=MAX_WARP_SIZE); - m_config=config; - m_empty=true; - m_isatomic=false; - m_per_scalar_thread_valid=false; - m_mem_accesses_created=false; - m_cache_hit=false; - m_is_printf=false; - m_is_cdp = 0; +class warp_inst_t : public inst_t { + public: + // constructors + warp_inst_t() { + m_uid = 0; + m_empty = true; + m_config = NULL; + } + warp_inst_t(const core_config *config) { + m_uid = 0; + assert(config->warp_size <= MAX_WARP_SIZE); + m_config = config; + m_empty = true; + m_isatomic = false; + m_per_scalar_thread_valid = false; + m_mem_accesses_created = false; + m_cache_hit = false; + m_is_printf = false; + m_is_cdp = 0; + } + virtual ~warp_inst_t() {} + + // modifiers + void broadcast_barrier_reduction(const active_mask_t &access_mask); + void do_atomic(bool forceDo = false); + void do_atomic(const active_mask_t &access_mask, bool forceDo = false); + void clear() { m_empty = true; } + + void issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, int sch_id); + + const active_mask_t &get_active_mask() const { return m_warp_active_mask; } + void completed(unsigned long long cycle) + const; // stat collection: called when the instruction is completed + + void set_addr(unsigned n, new_addr_type addr) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - virtual ~warp_inst_t(){ + m_per_scalar_thread[n].memreqaddr[0] = addr; + } + void set_addr(unsigned n, new_addr_type *addr, unsigned num_addrs) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - - // modifiers - void broadcast_barrier_reduction( const active_mask_t& access_mask); - void do_atomic(bool forceDo=false); - void do_atomic( const active_mask_t& access_mask, bool forceDo=false ); - void clear() - { - m_empty=true; + assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); + for (unsigned i = 0; i < num_addrs; i++) + m_per_scalar_thread[n].memreqaddr[i] = addr[i]; + } + void print_m_accessq() { + if (accessq_empty()) + return; + else { + printf("Printing mem access generated\n"); + std::list::iterator it; + for (it = m_accessq.begin(); it != m_accessq.end(); ++it) { + printf("MEM_TXN_GEN:%s:%llx, Size:%d \n", + mem_access_type_str(it->get_type()), it->get_addr(), + it->get_size()); + } } - - void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ); - - const active_mask_t & get_active_mask() const - { - return m_warp_active_mask; + } + struct transaction_info { + std::bitset<4> chunks; // bitmask: 32-byte chunks accessed + mem_access_byte_mask_t bytes; + active_mask_t active; // threads in this transaction + + bool test_bytes(unsigned start_bit, unsigned end_bit) { + for (unsigned i = start_bit; i <= end_bit; i++) + if (bytes.test(i)) return true; + return false; } - void completed( unsigned long long cycle ) const; // stat collection: called when the instruction is completed - - void set_addr( unsigned n, new_addr_type addr ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - m_per_scalar_thread[n].memreqaddr[0] = addr; + }; + + void generate_mem_accesses(); + void memory_coalescing_arch(bool is_write, mem_access_type access_type); + void memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type); + void memory_coalescing_arch_reduce_and_send(bool is_write, + mem_access_type access_type, + const transaction_info &info, + new_addr_type addr, + unsigned segment_size); + + void add_callback(unsigned lane_id, void (*function)(const class inst_t *, + class ptx_thread_info *), + const inst_t *inst, class ptx_thread_info *thread, + bool atomic) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; + if (atomic) m_isatomic = true; } - void set_addr( unsigned n, new_addr_type* addr, unsigned num_addrs ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); - for(unsigned i=0; iwarp_size - 1; i >= 0; i--) + fprintf(fp, "%c", ((m_warp_active_mask[i]) ? '1' : '0')); + } + bool active(unsigned thread) const { return m_warp_active_mask.test(thread); } + unsigned active_count() const { return m_warp_active_mask.count(); } + unsigned issued_count() const { + assert(m_empty == false); + return m_warp_issued_mask.count(); + } // for instruction counting + bool empty() const { return m_empty; } + unsigned warp_id() const { + assert(!m_empty); + return m_warp_id; + } + unsigned warp_id_func() const // to be used in functional simulations only + { + return m_warp_id; + } + unsigned dynamic_warp_id() const { + assert(!m_empty); + return m_dynamic_warp_id; + } + bool has_callback(unsigned n) const { + return m_warp_active_mask[n] && m_per_scalar_thread_valid && + (m_per_scalar_thread[n].callback.function != NULL); + } + new_addr_type get_addr(unsigned n) const { + assert(m_per_scalar_thread_valid); + return m_per_scalar_thread[n].memreqaddr[0]; + } + + bool isatomic() const { return m_isatomic; } + + unsigned warp_size() const { return m_config->warp_size; } + + bool accessq_empty() const { return m_accessq.empty(); } + unsigned accessq_count() const { return m_accessq.size(); } + const mem_access_t &accessq_back() { return m_accessq.back(); } + void accessq_pop_back() { m_accessq.pop_back(); } + + bool dispatch_delay() { + if (cycles > 0) cycles--; + return cycles > 0; + } + + bool has_dispatch_delay() { return cycles > 0; } + + void print(FILE *fout) const; + unsigned get_uid() const { return m_uid; } + unsigned get_schd_id() const { return m_scheduler_id; } + + protected: + unsigned m_uid; + bool m_empty; + bool m_cache_hit; + unsigned long long issue_cycle; + unsigned cycles; // used for implementing initiation interval delay + bool m_isatomic; + bool m_is_printf; + unsigned m_warp_id; + unsigned m_dynamic_warp_id; + const core_config *m_config; + active_mask_t m_warp_active_mask; // dynamic active mask for timing model + // (after predication) + active_mask_t m_warp_issued_mask; // active mask at issue (prior to + // predication test) -- for instruction + // counting + + struct per_thread_info { + per_thread_info() { + for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) + memreqaddr[i] = 0; } - void print_m_accessq(){ - - if(accessq_empty()) - return; - else{ - printf("Printing mem access generated\n"); - std::list::iterator it; - for (it = m_accessq.begin(); it != m_accessq.end(); ++it){ - printf("MEM_TXN_GEN:%s:%llx, Size:%d \n",mem_access_type_str(it->get_type()), it->get_addr(),it->get_size()); - } - } - } - struct transaction_info { - std::bitset<4> chunks; // bitmask: 32-byte chunks accessed - mem_access_byte_mask_t bytes; - active_mask_t active; // threads in this transaction - - bool test_bytes(unsigned start_bit, unsigned end_bit) { - for( unsigned i=start_bit; i<=end_bit; i++ ) - if(bytes.test(i)) - return true; - return false; - } - }; - - void generate_mem_accesses(); - void memory_coalescing_arch( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); - - void add_callback( unsigned lane_id, - void (*function)(const class inst_t*, class ptx_thread_info*), - const inst_t *inst, - class ptx_thread_info *thread, - bool atomic) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - if(atomic) m_isatomic=true; - } - m_per_scalar_thread[lane_id].callback.function = function; - m_per_scalar_thread[lane_id].callback.instruction = inst; - m_per_scalar_thread[lane_id].callback.thread = thread; - } - void set_active( const active_mask_t &active ); + dram_callback_t callback; + new_addr_type memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; // effective + // address, + // upto 8 + // different + // requests (to + // support 32B + // access in 8 + // chunks of 4B + // each) + }; + bool m_per_scalar_thread_valid; + std::vector m_per_scalar_thread; + bool m_mem_accesses_created; + std::list m_accessq; + + unsigned m_scheduler_id; // the scheduler that issues this inst + + // Jin: cdp support + public: + int m_is_cdp; +}; + +void move_warp(warp_inst_t *&dst, warp_inst_t *&src); - void clear_active( const active_mask_t &inactive ); - void set_not_active( unsigned lane_id ); +size_t get_kernel_code_size(class function_info *entry); +class checkpoint { + public: + checkpoint(); + ~checkpoint() { printf("clasfsfss destructed\n"); } - // accessors - virtual void print_insn(FILE *fp) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); - for (int i=(int)m_config->warp_size-1; i>=0; i--) - fprintf(fp, "%c", ((m_warp_active_mask[i])?'1':'0') ); + void load_global_mem(class memory_space *temp_mem, char *f1name); + void store_global_mem(class memory_space *mem, char *fname, char *format); + unsigned radnom; +}; +/* + * This abstract class used as a base for functional and performance and + * simulation, it has basic functional simulation + * data structures and procedures. + */ +class core_t { + public: + core_t(gpgpu_sim *gpu, kernel_info_t *kernel, unsigned warp_size, + unsigned threads_per_shader) + : m_gpu(gpu), + m_kernel(kernel), + m_simt_stack(NULL), + m_thread(NULL), + m_warp_size(warp_size) { + m_warp_count = threads_per_shader / m_warp_size; + // Handle the case where the number of threads is not a + // multiple of the warp size + if (threads_per_shader % m_warp_size != 0) { + m_warp_count += 1; } - bool active( unsigned thread ) const { return m_warp_active_mask.test(thread); } - unsigned active_count() const { return m_warp_active_mask.count(); } - unsigned issued_count() const { assert(m_empty == false); return m_warp_issued_mask.count(); } // for instruction counting - bool empty() const { return m_empty; } - unsigned warp_id() const - { - assert( !m_empty ); - return m_warp_id; + assert(m_warp_count * m_warp_size > 0); + m_thread = (ptx_thread_info **)calloc(m_warp_count * m_warp_size, + sizeof(ptx_thread_info *)); + initilizeSIMTStack(m_warp_count, m_warp_size); + + for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) { + for (unsigned j = 0; j < MAX_BARRIERS_PER_CTA; j++) { + reduction_storage[i][j] = 0; + } } - unsigned warp_id_func() const // to be used in functional simulations only - { - return m_warp_id; + } + virtual ~core_t() { free(m_thread); } + virtual void warp_exit(unsigned warp_id) = 0; + virtual bool warp_waiting_at_barrier(unsigned warp_id) const = 0; + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) = 0; + class gpgpu_sim *get_gpu() { + return m_gpu; + } + void execute_warp_inst_t(warp_inst_t &inst, unsigned warpId = (unsigned)-1); + bool ptx_thread_done(unsigned hw_thread_id) const; + void updateSIMTStack(unsigned warpId, warp_inst_t *inst); + void initilizeSIMTStack(unsigned warp_count, unsigned warps_size); + void deleteSIMTStack(); + warp_inst_t getExecuteWarp(unsigned warpId); + void get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const; + kernel_info_t *get_kernel_info() { return m_kernel; } + class ptx_thread_info **get_thread_info() { + return m_thread; + } + unsigned get_warp_size() const { return m_warp_size; } + void and_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] &= value; + } + void or_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] |= value; + } + void popc_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] += value; + } + unsigned get_reduction_value(unsigned ctaid, unsigned barid) { + return reduction_storage[ctaid][barid]; + } + + protected: + class gpgpu_sim *m_gpu; + kernel_info_t *m_kernel; + simt_stack **m_simt_stack; // pdom based reconvergence context for each warp + class ptx_thread_info **m_thread; + unsigned m_warp_size; + unsigned m_warp_count; + unsigned reduction_storage[MAX_CTA_PER_SHADER][MAX_BARRIERS_PER_CTA]; +}; + +// register that can hold multiple instructions. +class register_set { + public: + register_set(unsigned num, const char *name) { + for (unsigned i = 0; i < num; i++) { + regs.push_back(new warp_inst_t()); } - unsigned dynamic_warp_id() const - { - assert( !m_empty ); - return m_dynamic_warp_id; + m_name = name; + } + bool has_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return true; + } } - bool has_callback( unsigned n ) const - { - return m_warp_active_mask[n] && m_per_scalar_thread_valid && - (m_per_scalar_thread[n].callback.function!=NULL); + return false; + } + bool has_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return has_free(); + + assert(reg_id < regs.size()); + return regs[reg_id]->empty(); + } + bool has_ready() { + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + return true; + } } - new_addr_type get_addr( unsigned n ) const - { - assert( m_per_scalar_thread_valid ); - return m_per_scalar_thread[n].memreqaddr[0]; + return false; + } + + void move_in(warp_inst_t *&src) { + warp_inst_t **free = get_free(); + move_warp(*free, src); + } + // void copy_in( warp_inst_t* src ){ + // src->copy_contents_to(*get_free()); + //} + void move_out_to(warp_inst_t *&dest) { + warp_inst_t **ready = get_ready(); + move_warp(dest, *ready); + } + + warp_inst_t **get_ready() { + warp_inst_t **ready; + ready = NULL; + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + if (ready and (*ready)->get_uid() < regs[i]->get_uid()) { + // ready is oldest + } else { + ready = ®s[i]; + } + } } - - bool isatomic() const { return m_isatomic; } - - unsigned warp_size() const { return m_config->warp_size; } - - bool accessq_empty() const { return m_accessq.empty(); } - unsigned accessq_count() const { return m_accessq.size(); } - const mem_access_t &accessq_back() { return m_accessq.back(); } - void accessq_pop_back() { m_accessq.pop_back(); } - - bool dispatch_delay() - { - if( cycles > 0 ) - cycles--; - return cycles > 0; + return ready; + } + + void print(FILE *fp) const { + fprintf(fp, "%s : @%p\n", m_name, this); + for (unsigned i = 0; i < regs.size(); i++) { + fprintf(fp, " "); + regs[i]->print(fp); + fprintf(fp, "\n"); } + } - bool has_dispatch_delay(){ - return cycles > 0; + warp_inst_t **get_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return ®s[i]; + } } - - void print( FILE *fout ) const; - unsigned get_uid() const { return m_uid; } - unsigned get_schd_id() const { return m_scheduler_id; } - -protected: - - unsigned m_uid; - bool m_empty; - bool m_cache_hit; - unsigned long long issue_cycle; - unsigned cycles; // used for implementing initiation interval delay - bool m_isatomic; - bool m_is_printf; - unsigned m_warp_id; - unsigned m_dynamic_warp_id; - const core_config *m_config; - active_mask_t m_warp_active_mask; // dynamic active mask for timing model (after predication) - active_mask_t m_warp_issued_mask; // active mask at issue (prior to predication test) -- for instruction counting - - struct per_thread_info { - per_thread_info() { - for(unsigned i=0; i m_per_scalar_thread; - bool m_mem_accesses_created; - std::list m_accessq; - - unsigned m_scheduler_id; //the scheduler that issues this inst - - //Jin: cdp support -public: - int m_is_cdp; - -}; - -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ); - -size_t get_kernel_code_size( class function_info *entry ); -class checkpoint -{ -public: - - checkpoint(); - ~checkpoint(){ - printf("clasfsfss destructed\n"); + assert(0 && "No free registers found"); + return NULL; + } + + warp_inst_t **get_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return get_free(); + + assert(reg_id < regs.size()); + if (regs[reg_id]->empty()) { + return ®s[reg_id]; } + assert(0 && "No free register found"); + return NULL; + } - void load_global_mem(class memory_space *temp_mem, char * f1name); - void store_global_mem(class memory_space *mem, char * fname , char * format); - unsigned radnom; + unsigned get_size() { return regs.size(); } - -}; -/* - * This abstract class used as a base for functional and performance and simulation, it has basic functional simulation - * data structures and procedures. - */ -class core_t { - public: - core_t( gpgpu_sim *gpu, - kernel_info_t *kernel, - unsigned warp_size, - unsigned threads_per_shader ) - : m_gpu( gpu ), - m_kernel( kernel ), - m_simt_stack( NULL ), - m_thread( NULL ), - m_warp_size( warp_size ) - { - m_warp_count = threads_per_shader/m_warp_size; - // Handle the case where the number of threads is not a - // multiple of the warp size - if ( threads_per_shader % m_warp_size != 0 ) { - m_warp_count += 1; - } - assert( m_warp_count * m_warp_size > 0 ); - m_thread = ( ptx_thread_info** ) - calloc( m_warp_count * m_warp_size, - sizeof( ptx_thread_info* ) ); - initilizeSIMTStack(m_warp_count,m_warp_size); - - for(unsigned i=0; iempty() ) { - return true; - } - } - return false; - } - bool has_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return has_free(); - - assert(reg_id < regs.size()); - return regs[reg_id]->empty(); - } - bool has_ready(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - return true; - } - } - return false; - } - - void move_in( warp_inst_t *&src ){ - warp_inst_t** free = get_free(); - move_warp(*free, src); - } - //void copy_in( warp_inst_t* src ){ - // src->copy_contents_to(*get_free()); - //} - void move_out_to( warp_inst_t *&dest ){ - warp_inst_t **ready=get_ready(); - move_warp(dest, *ready); - } - - warp_inst_t** get_ready(){ - warp_inst_t** ready; - ready = NULL; - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - if( ready and (*ready)->get_uid() < regs[i]->get_uid() ) { - // ready is oldest - } else { - ready = ®s[i]; - } - } - } - return ready; - } - - void print(FILE* fp) const{ - fprintf(fp, "%s : @%p\n", m_name, this); - for( unsigned i = 0; i < regs.size(); i++ ) { - fprintf(fp, " "); - regs[i]->print(fp); - fprintf(fp, "\n"); - } - } - - warp_inst_t ** get_free(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( regs[i]->empty() ) { - return ®s[i]; - } - } - assert(0 && "No free registers found"); - return NULL; - } - - warp_inst_t ** get_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return get_free(); - - assert(reg_id < regs.size()); - if( regs[reg_id]->empty() ) { - return ®s[reg_id]; - } - assert(0 && "No free register found"); - return NULL; - } - - unsigned get_size(){ - return regs.size(); - } - -private: - std::vector regs; - const char* m_name; + private: + std::vector regs; + const char *m_name; }; -#endif // #ifdef __cplusplus +#endif // #ifdef __cplusplus -#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED +#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED diff --git a/src/debug.cc b/src/debug.cc index c00ff9e..eaf39a6 100644 --- a/src/debug.cc +++ b/src/debug.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -26,183 +28,197 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "debug.h" -#include "gpgpu-sim/shader.h" -#include "gpgpu-sim/gpu-sim.h" -#include "cuda-sim/ptx_sim.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" +#include "cuda-sim/ptx_sim.h" +#include "gpgpu-sim/gpu-sim.h" +#include "gpgpu-sim/shader.h" -#include #include #include +#include -void gpgpu_sim::hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI ) -{ - g_watchpoint_hits[watchpoint_num]=watchpoint_event(thd,pI); +void gpgpu_sim::hit_watchpoint(unsigned watchpoint_num, ptx_thread_info *thd, + const ptx_instruction *pI) { + g_watchpoint_hits[watchpoint_num] = watchpoint_event(thd, pI); } -/// interactive debugger - -void gpgpu_sim::gpgpu_debug() -{ - bool done=true; - - static bool single_step=true; - static unsigned next_brkpt=1; - static std::map breakpoints; - - /// if single stepping, go to interactive debugger - - if( single_step ) - done=false; - - /// check if we've reached a breakpoint - const ptx_thread_info *brk_thd = NULL; - const ptx_instruction *brk_inst = NULL; - - for( std::map::iterator i=breakpoints.begin(); i!=breakpoints.end(); i++) { - unsigned num=i->first; - brk_pt &b=i->second; - if( b.is_watchpoint() ) { - unsigned addr = b.get_addr(); - unsigned new_value; - m_global_mem->read(addr,4,&new_value); - if( new_value != b.get_value() || g_watchpoint_hits.find(num) != g_watchpoint_hits.end() ) { - printf( "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new value=%x)\n", - num,b.get_value(),new_value ); - std::map::iterator w=g_watchpoint_hits.find(num); - if( w==g_watchpoint_hits.end() ) - printf( "GPGPU-Sim PTX DBG: memory transfer modified value\n"); - else { - watchpoint_event wa = w->second; - brk_thd = wa.thread(); - brk_inst = wa.inst(); - printf( "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, hwtid=%u\n", - brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); - printf( "GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf( "\n" ); - g_watchpoint_hits.erase(w); - } - b.set_value(new_value); - done = false; - } - } else { - /* - for( unsigned sid=0; sid < m_n_shader; sid++ ) { - unsigned hw_thread_id = -1; - abort(); - ptx_thread_info *thread = m_sc[sid]->get_functional_thread(hw_thread_id); - if( thread_at_brkpt(thread, b) ) { - done = false; - printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s (sm=%u, hwtid=%u)\n", - num, b.location().c_str(), sid, hw_thread_id ); - brk_thd = thread; - brk_inst = brk_thd->get_inst(); - printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, hwtid=%u\n", - brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); - printf( "GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf( "\n" ); - } - } - */ +/// interactive debugger + +void gpgpu_sim::gpgpu_debug() { + bool done = true; + + static bool single_step = true; + static unsigned next_brkpt = 1; + static std::map breakpoints; + + /// if single stepping, go to interactive debugger + + if (single_step) done = false; + + /// check if we've reached a breakpoint + const ptx_thread_info *brk_thd = NULL; + const ptx_instruction *brk_inst = NULL; + + for (std::map::iterator i = breakpoints.begin(); + i != breakpoints.end(); i++) { + unsigned num = i->first; + brk_pt &b = i->second; + if (b.is_watchpoint()) { + unsigned addr = b.get_addr(); + unsigned new_value; + m_global_mem->read(addr, 4, &new_value); + if (new_value != b.get_value() || + g_watchpoint_hits.find(num) != g_watchpoint_hits.end()) { + printf( + "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new " + "value=%x)\n", + num, b.get_value(), new_value); + std::map::iterator w = + g_watchpoint_hits.find(num); + if (w == g_watchpoint_hits.end()) + printf("GPGPU-Sim PTX DBG: memory transfer modified value\n"); + else { + watchpoint_event wa = w->second; + brk_thd = wa.thread(); + brk_inst = wa.inst(); + printf( + "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, " + "hwtid=%u\n", + brk_thd->get_uid(), brk_thd->get_hw_sid(), brk_thd->get_hw_tid()); + printf("GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf("\n"); + g_watchpoint_hits.erase(w); + } + b.set_value(new_value); + done = false; } - } - - if( done ) - assert( g_watchpoint_hits.empty() ); - - /// enter interactive debugger loop - - while (!done) { - printf("(ptx debugger) "); + } else { + /* + for( unsigned sid=0; sid < m_n_shader; sid++ ) { + unsigned hw_thread_id = -1; + abort(); + ptx_thread_info *thread = + m_sc[sid]->get_functional_thread(hw_thread_id); + if( thread_at_brkpt(thread, b) ) { + done = false; + printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s (sm=%u, + hwtid=%u)\n", + num, b.location().c_str(), sid, hw_thread_id ); + brk_thd = thread; + brk_inst = brk_thd->get_inst(); + printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, + hwtid=%u\n", + brk_thd->get_uid(),brk_thd->get_hw_sid(), + brk_thd->get_hw_tid() ); + printf( "GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf( "\n" ); + } + } + */ + } + } + + if (done) assert(g_watchpoint_hits.empty()); + + /// enter interactive debugger loop + + while (!done) { + printf("(ptx debugger) "); + fflush(stdout); + + char line[1024]; + fgets(line, 1024, stdin); + + char *tok = strtok(line, " \t\n"); + if (!strcmp(tok, "dp")) { + int shader_num = 0; + tok = strtok(NULL, " \t\n"); + sscanf(tok, "%d", &shader_num); + dump_pipeline((0x40 | 0x4 | 0x1), shader_num, 0); + printf("\n"); fflush(stdout); - - char line[1024]; - fgets(line,1024,stdin); - - char *tok = strtok(line," \t\n"); - if( !strcmp(tok,"dp") ) { - int shader_num = 0; - tok = strtok(NULL," \t\n"); - sscanf(tok,"%d",&shader_num); - dump_pipeline((0x40|0x4|0x1),shader_num,0); - printf("\n"); - fflush(stdout); - } else if( !strcmp(tok,"q") || !strcmp(tok,"quit") ) { - printf("\nreally quit GPGPU-Sim (y/n)?\n"); - fgets(line,1024,stdin); - tok = strtok(line," \t\n"); - if( !strcmp(tok,"y") ) { - exit(0); - } else { - printf("not quiting.\n"); - } - } else if( !strcmp(tok,"b") ) { - tok = strtok(NULL," \t\n"); - char brkpt[1024]; - sscanf(tok,"%s",brkpt); - tok = strtok(NULL," \t\n"); - unsigned uid; - sscanf(tok,"%u",&uid); - breakpoints[next_brkpt++] = brk_pt(brkpt,uid); - } else if( !strcmp(tok,"d") ) { - tok = strtok(NULL," \t\n"); - unsigned uid; - sscanf(tok,"%u",&uid); - breakpoints.erase(uid); - } else if( !strcmp(tok,"s") ) { - done = true; - } else if( !strcmp(tok,"c") ) { - single_step=false; - done = true; - } else if( !strcmp(tok,"w") ) { - tok = strtok(NULL," \t\n"); - unsigned addr; - sscanf(tok,"%x",&addr); - unsigned value; - m_global_mem->read(addr,4,&value); - m_global_mem->set_watch(addr,next_brkpt); - breakpoints[next_brkpt++] = brk_pt(addr,value); - } else if( !strcmp(tok,"l") ) { - if( brk_thd == NULL ) { - printf("no thread selected\n"); - } else { - addr_t pc = brk_thd->get_pc(); - addr_t start_pc = (pc<5)?0:(pc-5); - for( addr_t p=start_pc; p <= pc+5; p++ ) { - const ptx_instruction *i = brk_thd->get_inst(p); - if( i ) { - if( p != pc ) - printf( " " ); - else - printf( "==> " ); - i->print_insn(stdout); - printf( "\n" ); - } - } - } - } else if( !strcmp(tok,"h") ) { - printf("commands:\n"); - printf(" q - quit GPGPU-Sim\n"); - printf(" b : - set breakpoint\n"); - printf(" w - set watchpoint\n"); - printf(" del - delete breakpoint\n"); - printf(" s - single step one shader cycle (all cores)\n"); - printf(" c - continue simulation without single stepping\n"); - printf(" l - list PTX around current breakpoint\n"); - printf(" dp - display pipeline contents on SM \n"); - printf(" h - print this message\n"); + } else if (!strcmp(tok, "q") || !strcmp(tok, "quit")) { + printf("\nreally quit GPGPU-Sim (y/n)?\n"); + fgets(line, 1024, stdin); + tok = strtok(line, " \t\n"); + if (!strcmp(tok, "y")) { + exit(0); } else { - printf("\ncommand not understood.\n"); + printf("not quiting.\n"); } - fflush(stdout); - } + } else if (!strcmp(tok, "b")) { + tok = strtok(NULL, " \t\n"); + char brkpt[1024]; + sscanf(tok, "%s", brkpt); + tok = strtok(NULL, " \t\n"); + unsigned uid; + sscanf(tok, "%u", &uid); + breakpoints[next_brkpt++] = brk_pt(brkpt, uid); + } else if (!strcmp(tok, "d")) { + tok = strtok(NULL, " \t\n"); + unsigned uid; + sscanf(tok, "%u", &uid); + breakpoints.erase(uid); + } else if (!strcmp(tok, "s")) { + done = true; + } else if (!strcmp(tok, "c")) { + single_step = false; + done = true; + } else if (!strcmp(tok, "w")) { + tok = strtok(NULL, " \t\n"); + unsigned addr; + sscanf(tok, "%x", &addr); + unsigned value; + m_global_mem->read(addr, 4, &value); + m_global_mem->set_watch(addr, next_brkpt); + breakpoints[next_brkpt++] = brk_pt(addr, value); + } else if (!strcmp(tok, "l")) { + if (brk_thd == NULL) { + printf("no thread selected\n"); + } else { + addr_t pc = brk_thd->get_pc(); + addr_t start_pc = (pc < 5) ? 0 : (pc - 5); + for (addr_t p = start_pc; p <= pc + 5; p++) { + const ptx_instruction *i = brk_thd->get_inst(p); + if (i) { + if (p != pc) + printf(" "); + else + printf("==> "); + i->print_insn(stdout); + printf("\n"); + } + } + } + } else if (!strcmp(tok, "h")) { + printf("commands:\n"); + printf(" q - quit GPGPU-Sim\n"); + printf(" b : - set breakpoint\n"); + printf(" w - set watchpoint\n"); + printf(" del - delete breakpoint\n"); + printf( + " s - single step one shader cycle (all " + "cores)\n"); + printf( + " c - continue simulation without single " + "stepping\n"); + printf( + " l - list PTX around current " + "breakpoint\n"); + printf( + " dp - display pipeline contents on SM " + "\n"); + printf(" h - print this message\n"); + } else { + printf("\ncommand not understood.\n"); + } + fflush(stdout); + } } -bool thread_at_brkpt( ptx_thread_info *thread, const class brk_pt &b ) -{ - return b.is_equal(thread->get_location(),thread->get_uid()); +bool thread_at_brkpt(ptx_thread_info *thread, const class brk_pt &b) { + return b.is_equal(thread->get_location(), thread->get_uid()); } - diff --git a/src/debug.h b/src/debug.h index 4e79a9f..2799efe 100644 --- a/src/debug.h +++ b/src/debug.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -33,58 +35,53 @@ #include class brk_pt { -public: - brk_pt() { m_valid=false; } - brk_pt( const char *fileline, unsigned uid ) - { - m_valid = true; - m_watch = false; - m_fileline = std::string(fileline); - m_thread_uid=uid; - } - brk_pt( unsigned addr, unsigned value ) - { - m_valid = true; - m_watch = true; - m_addr = addr; - m_value = value; - } + public: + brk_pt() { m_valid = false; } + brk_pt(const char *fileline, unsigned uid) { + m_valid = true; + m_watch = false; + m_fileline = std::string(fileline); + m_thread_uid = uid; + } + brk_pt(unsigned addr, unsigned value) { + m_valid = true; + m_watch = true; + m_addr = addr; + m_value = value; + } - unsigned get_value() const { return m_value; } - addr_t get_addr() const { return m_addr; } - bool is_valid() const { return m_valid; } - bool is_watchpoint() const { return m_watch; } - bool is_equal( const std::string &fileline, unsigned uid ) const - { - if( m_watch ) - return false; - if( (m_thread_uid != (unsigned)-1) && (uid != m_thread_uid) ) - return false; - return m_fileline == fileline; - } - std::string location() const - { - char buffer[1024]; - sprintf(buffer,"%s thread uid = %u", m_fileline.c_str(), m_thread_uid); - return buffer; - } + unsigned get_value() const { return m_value; } + addr_t get_addr() const { return m_addr; } + bool is_valid() const { return m_valid; } + bool is_watchpoint() const { return m_watch; } + bool is_equal(const std::string &fileline, unsigned uid) const { + if (m_watch) return false; + if ((m_thread_uid != (unsigned)-1) && (uid != m_thread_uid)) return false; + return m_fileline == fileline; + } + std::string location() const { + char buffer[1024]; + sprintf(buffer, "%s thread uid = %u", m_fileline.c_str(), m_thread_uid); + return buffer; + } - unsigned set_value( unsigned val ) { return m_value=val; } -private: - bool m_valid; - bool m_watch; + unsigned set_value(unsigned val) { return m_value = val; } - // break point - std::string m_fileline; - unsigned m_thread_uid; + private: + bool m_valid; + bool m_watch; - // watch point - unsigned m_addr; - unsigned m_value; + // break point + std::string m_fileline; + unsigned m_thread_uid; + + // watch point + unsigned m_addr; + unsigned m_value; }; class ptx_thread_info; class ptx_instruction; -bool thread_at_brkpt( ptx_thread_info *thd_info, const class brk_pt &b ); +bool thread_at_brkpt(ptx_thread_info *thd_info, const class brk_pt &b); #endif diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 846773d..04aa3f8 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,253 +30,266 @@ #include "gpgpusim_entrypoint.h" #include -#include "option_parser.h" +#include "../libcuda/gpgpu_context.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx_parser.h" #include "gpgpu-sim/gpu-sim.h" #include "gpgpu-sim/icnt_wrapper.h" +#include "option_parser.h" #include "stream_manager.h" -#include "../libcuda/gpgpu_context.h" -#define MAX(a,b) (((a)>(b))?(a):(b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) static int sg_argc = 3; -static const char *sg_argv[] = {"", "-config","gpgpusim.config"}; - - -void * gpgpu_sim_thread_sequential(void * ctx_ptr) -{ - gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; - // at most one kernel running at a time - bool done; - do { - sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); - done = true; - if( ctx->the_gpgpusim->g_the_gpu->get_more_cta_left() ) { - done = false; - ctx->the_gpgpusim->g_the_gpu->init(); - while( ctx->the_gpgpusim->g_the_gpu->active() ) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); - } - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); +static const char *sg_argv[] = {"", "-config", "gpgpusim.config"}; + +void *gpgpu_sim_thread_sequential(void *ctx_ptr) { + gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; + // at most one kernel running at a time + bool done; + do { + sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); + done = true; + if (ctx->the_gpgpusim->g_the_gpu->get_more_cta_left()) { + done = false; + ctx->the_gpgpusim->g_the_gpu->init(); + while (ctx->the_gpgpusim->g_the_gpu->active()) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); } - sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); - } while(!done); - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); + } + sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); + } while (!done); + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } - - -static void termination_callback() -{ - printf("GPGPU-Sim: *** exit detected ***\n"); - fflush(stdout); +static void termination_callback() { + printf("GPGPU-Sim: *** exit detected ***\n"); + fflush(stdout); } -void *gpgpu_sim_thread_concurrent(void * ctx_ptr) -{ - gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; - atexit(termination_callback); - // concurrent kernel execution simulation thread +void *gpgpu_sim_thread_concurrent(void *ctx_ptr) { + gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; + atexit(termination_callback); + // concurrent kernel execution simulation thread + do { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim: *** simulation thread starting and spinning waiting for " + "work ***\n"); + fflush(stdout); + } + while (ctx->the_gpgpusim->g_stream_manager->empty_protected() && + !ctx->the_gpgpusim->g_sim_done) + ; + if (g_debug_execution >= 3) { + printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); + ctx->the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = true; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + bool active = false; + bool sim_cycles = false; + ctx->the_gpgpusim->g_the_gpu->init(); do { - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: *** simulation thread starting and spinning waiting for work ***\n"); - fflush(stdout); - } - while( ctx->the_gpgpusim->g_stream_manager->empty_protected() && !ctx->the_gpgpusim->g_sim_done ) - ; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); - ctx->the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); - } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = true; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - bool active = false; - bool sim_cycles = false; - ctx->the_gpgpusim->g_the_gpu->init(); - do { - // check if a kernel has completed - // launch operation on device if one is pending and can be run - - // Need to break this loop when a kernel completes. This was a - // source of non-deterministic behaviour in GPGPU-Sim (bug 147). - // If another stream operation is available, g_the_gpu remains active, - // causing this loop to not break. If the next operation happens to be - // another kernel, the gpu is not re-initialized and the inter-kernel - // behaviour may be incorrect. Check that a kernel has finished and - // no other kernel is currently running. - if(ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && !ctx->the_gpgpusim->g_the_gpu->active()) - break; - - //functional simulation - if( ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { - kernel_info_t * kernel = ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); - assert(kernel); - ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func(*kernel); - ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); - } - - //performance simulation - if( ctx->the_gpgpusim->g_the_gpu->active() ) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - sim_cycles = true; - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); - }else { - if(ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()){ - ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); - ctx->the_gpgpusim->g_sim_done = true; - ctx->the_gpgpusim->break_limit = true; - } - } - - active=ctx->the_gpgpusim->g_the_gpu->active() || !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); + // check if a kernel has completed + // launch operation on device if one is pending and can be run + + // Need to break this loop when a kernel completes. This was a + // source of non-deterministic behaviour in GPGPU-Sim (bug 147). + // If another stream operation is available, g_the_gpu remains active, + // causing this loop to not break. If the next operation happens to be + // another kernel, the gpu is not re-initialized and the inter-kernel + // behaviour may be incorrect. Check that a kernel has finished and + // no other kernel is currently running. + if (ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && + !ctx->the_gpgpusim->g_the_gpu->active()) + break; + + // functional simulation + if (ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { + kernel_info_t *kernel = + ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); + assert(kernel); + ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func( + *kernel); + ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); + } - } while( active && !ctx->the_gpgpusim->g_sim_done); - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); - fflush(stdout); - } - if(sim_cycles) { - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); + // performance simulation + if (ctx->the_gpgpusim->g_the_gpu->active()) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + sim_cycles = true; + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); + } else { + if (ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()) { + ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); + ctx->the_gpgpusim->g_sim_done = true; + ctx->the_gpgpusim->break_limit = true; } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = false; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - } while( !ctx->the_gpgpusim->g_sim_done ); + } - printf("GPGPU-Sim: *** simulation thread exiting ***\n"); - fflush(stdout); + active = ctx->the_gpgpusim->g_the_gpu->active() || + !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); - if(ctx->the_gpgpusim->break_limit) { - printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n"); - exit(1); + } while (active && !ctx->the_gpgpusim->g_sim_done); + if (g_debug_execution >= 3) { + printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); + fflush(stdout); } - - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + if (sim_cycles) { + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = false; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + } while (!ctx->the_gpgpusim->g_sim_done); + + printf("GPGPU-Sim: *** simulation thread exiting ***\n"); + fflush(stdout); + + if (ctx->the_gpgpusim->break_limit) { + printf( + "GPGPU-Sim: ** break due to reaching the maximum cycles (or " + "instructions) **\n"); + exit(1); + } + + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } -void gpgpu_context::synchronize() -{ - printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); - the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); -// sem_wait(&g_sim_signal_finish); - bool done = false; - do { - pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); - done = ( the_gpgpusim->g_stream_manager->empty() && !the_gpgpusim->g_sim_active ) || the_gpgpusim->g_sim_done; - pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); - } while (!done); - printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); - fflush(stdout); -// sem_post(&g_sim_signal_start); +void gpgpu_context::synchronize() { + printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); + the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); + // sem_wait(&g_sim_signal_finish); + bool done = false; + do { + pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); + done = (the_gpgpusim->g_stream_manager->empty() && + !the_gpgpusim->g_sim_active) || + the_gpgpusim->g_sim_done; + pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); + } while (!done); + printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); + fflush(stdout); + // sem_post(&g_sim_signal_start); } -void gpgpu_context::exit_simulation() -{ - the_gpgpusim->g_sim_done=true; - printf("GPGPU-Sim: exit_simulation called\n"); - fflush(stdout); - sem_wait(&(the_gpgpusim->g_sim_signal_exit)); - printf("GPGPU-Sim: simulation thread signaled exit\n"); - fflush(stdout); +void gpgpu_context::exit_simulation() { + the_gpgpusim->g_sim_done = true; + printf("GPGPU-Sim: exit_simulation called\n"); + fflush(stdout); + sem_wait(&(the_gpgpusim->g_sim_signal_exit)); + printf("GPGPU-Sim: simulation thread signaled exit\n"); + fflush(stdout); } -gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() -{ - srand(1); - print_splash(); - func_sim->read_sim_environment_variables(); - ptx_parser->read_parser_environment_variables(); - option_parser_t opp = option_parser_create(); - - ptx_reg_options(opp); - func_sim->ptx_opcocde_latency_options(opp); - - icnt_reg_options(opp); - the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); - the_gpgpusim->g_the_gpu_config->reg_options(opp); // register GPU microrachitecture options - - option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options - fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); - option_parser_print(opp, stdout); - // Set the Numeric locale to a standard locale where a decimal point is a "dot" not a "comma" - // so it does the parsing correctly independent of the system environment variables - assert(setlocale(LC_NUMERIC,"C")); - the_gpgpusim->g_the_gpu_config->init(); - - the_gpgpusim->g_the_gpu = new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); - the_gpgpusim->g_stream_manager = new stream_manager((the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); - - the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); - - sem_init(&(the_gpgpusim->g_sim_signal_start),0,0); - sem_init(&(the_gpgpusim->g_sim_signal_finish),0,0); - sem_init(&(the_gpgpusim->g_sim_signal_exit),0,0); - - return the_gpgpusim->g_the_gpu; +gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() { + srand(1); + print_splash(); + func_sim->read_sim_environment_variables(); + ptx_parser->read_parser_environment_variables(); + option_parser_t opp = option_parser_create(); + + ptx_reg_options(opp); + func_sim->ptx_opcocde_latency_options(opp); + + icnt_reg_options(opp); + the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); + the_gpgpusim->g_the_gpu_config->reg_options( + opp); // register GPU microrachitecture options + + option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options + fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); + option_parser_print(opp, stdout); + // Set the Numeric locale to a standard locale where a decimal point is a + // "dot" not a "comma" + // so it does the parsing correctly independent of the system environment + // variables + assert(setlocale(LC_NUMERIC, "C")); + the_gpgpusim->g_the_gpu_config->init(); + + the_gpgpusim->g_the_gpu = + new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); + the_gpgpusim->g_stream_manager = new stream_manager( + (the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); + + the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); + + sem_init(&(the_gpgpusim->g_sim_signal_start), 0, 0); + sem_init(&(the_gpgpusim->g_sim_signal_finish), 0, 0); + sem_init(&(the_gpgpusim->g_sim_signal_exit), 0, 0); + + return the_gpgpusim->g_the_gpu; } -void gpgpu_context::start_sim_thread(int api) -{ - if( the_gpgpusim->g_sim_done ) { - the_gpgpusim->g_sim_done = false; - if( api == 1 ) { - pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_concurrent,(void *)this); - } else { - pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_sequential,(void *)this); - } +void gpgpu_context::start_sim_thread(int api) { + if (the_gpgpusim->g_sim_done) { + the_gpgpusim->g_sim_done = false; + if (api == 1) { + pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, + gpgpu_sim_thread_concurrent, (void *)this); + } else { + pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, + gpgpu_sim_thread_sequential, (void *)this); } + } } -void gpgpu_context::print_simulation_time() -{ - time_t current_time, difference, d, h, m, s; - current_time = time((time_t *)NULL); - difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); - - d = difference/(3600*24); - h = difference/3600 - 24*d; - m = difference/60 - 60*(h + 24*d); - s = difference - 60*(m + 60*(h + 24*d)); - - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference) ); - const unsigned cycles_per_sec = (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); - printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec ); - printf("gpgpu_silicon_slowdown = %ux\n", the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); - fflush(stdout); +void gpgpu_context::print_simulation_time() { + time_t current_time, difference, d, h, m, s; + current_time = time((time_t *)NULL); + difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); + + d = difference / (3600 * 24); + h = difference / 3600 - 24 * d; + m = difference / 60 - 60 * (h + 24 * d); + s = difference - 60 * (m + 60 * (h + 24 * d)); + + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference)); + const unsigned cycles_per_sec = + (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); + printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec); + printf("gpgpu_silicon_slowdown = %ux\n", + the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); + fflush(stdout); } -int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf( kernel_info_t *grid ) -{ - the_gpgpusim->g_the_gpu->launch(grid); - sem_post(&(the_gpgpusim->g_sim_signal_start)); - sem_wait(&(the_gpgpusim->g_sim_signal_finish)); - return 0; +int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf(kernel_info_t *grid) { + the_gpgpusim->g_the_gpu->launch(grid); + sem_post(&(the_gpgpusim->g_sim_signal_start)); + sem_wait(&(the_gpgpusim->g_sim_signal_finish)); + return 0; } //! Functional simulation of OpenCL /*! * This function call the CUDA PTX functional simulator */ -int cuda_sim::gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ) -{ - //calling the CUDA PTX simulator, sending the kernel by reference and a flag set to true, - //the flag used by the function to distinguish OpenCL calls from the CUDA simulation calls which - //it is needed by the called function to not register the exit the exit of OpenCL kernel as it doesn't register entering in the first place as the CUDA kernels does - gpgpu_cuda_ptx_sim_main_func( *grid, true ); - return 0; +int cuda_sim::gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid) { + // calling the CUDA PTX simulator, sending the kernel by reference and a flag + // set to true, + // the flag used by the function to distinguish OpenCL calls from the CUDA + // simulation calls which + // it is needed by the called function to not register the exit the exit of + // OpenCL kernel as it doesn't register entering in the first place as the + // CUDA kernels does + gpgpu_cuda_ptx_sim_main_func(*grid, true); + return 0; } diff --git a/src/gpgpusim_entrypoint.h b/src/gpgpusim_entrypoint.h index 9f408df..517885c 100644 --- a/src/gpgpusim_entrypoint.h +++ b/src/gpgpusim_entrypoint.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,52 +30,50 @@ #ifndef GPGPUSIM_ENTRYPOINT_H_INCLUDED #define GPGPUSIM_ENTRYPOINT_H_INCLUDED -#include "abstract_hardware_model.h" #include #include #include +#include "abstract_hardware_model.h" -//extern time_t g_simulation_starttime; +// extern time_t g_simulation_starttime; class gpgpu_context; class GPGPUsim_ctx { - public: - GPGPUsim_ctx(gpgpu_context* ctx) { - g_sim_active = false; - g_sim_done = true; - break_limit = false; - g_sim_lock = PTHREAD_MUTEX_INITIALIZER; - - g_the_gpu_config=NULL; - g_the_gpu=NULL; - g_stream_manager=NULL; - the_cude_device=NULL; - the_context=NULL; - gpgpu_ctx = ctx; - } - - //struct gpgpu_ptx_sim_arg *grid_params; + public: + GPGPUsim_ctx(gpgpu_context *ctx) { + g_sim_active = false; + g_sim_done = true; + break_limit = false; + g_sim_lock = PTHREAD_MUTEX_INITIALIZER; - sem_t g_sim_signal_start; - sem_t g_sim_signal_finish; - sem_t g_sim_signal_exit; - time_t g_simulation_starttime; - pthread_t g_simulation_thread; + g_the_gpu_config = NULL; + g_the_gpu = NULL; + g_stream_manager = NULL; + the_cude_device = NULL; + the_context = NULL; + gpgpu_ctx = ctx; + } - class gpgpu_sim_config *g_the_gpu_config; - class gpgpu_sim *g_the_gpu; - class stream_manager *g_stream_manager; + // struct gpgpu_ptx_sim_arg *grid_params; - struct _cuda_device_id *the_cude_device; - struct CUctx_st* the_context; - gpgpu_context* gpgpu_ctx; + sem_t g_sim_signal_start; + sem_t g_sim_signal_finish; + sem_t g_sim_signal_exit; + time_t g_simulation_starttime; + pthread_t g_simulation_thread; + class gpgpu_sim_config *g_the_gpu_config; + class gpgpu_sim *g_the_gpu; + class stream_manager *g_stream_manager; - pthread_mutex_t g_sim_lock; - bool g_sim_active; - bool g_sim_done; - bool break_limit; + struct _cuda_device_id *the_cude_device; + struct CUctx_st *the_context; + gpgpu_context *gpgpu_ctx; + pthread_mutex_t g_sim_lock; + bool g_sim_active; + bool g_sim_done; + bool break_limit; }; #endif diff --git a/src/option_parser.cc b/src/option_parser.cc index 7d747f0..511553d 100644 --- a/src/option_parser.cc +++ b/src/option_parser.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,523 +27,527 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "option_parser.h" +#include #include #include -#include -#include -#include -#include -#include +#include #include -#include +#include +#include #include #include -#include -#include "option_parser.h" - +#include +#include +#include using namespace std; // A generic option registry regardless of data type -class OptionRegistryInterface -{ -public: - OptionRegistryInterface(const string optionName, const string optionDesc) - : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) - {} - - virtual ~OptionRegistryInterface() {} - - const string& GetName() { return m_optionName; } - const string& GetDesc() { return m_optionDesc; } - const bool isParsed() { return m_isParsed; } - virtual string toString() = 0; - virtual bool fromString(const string str) = 0; - virtual bool isFlag() = 0; - virtual bool assignDefault(const char *str) = 0; - -protected: - string m_optionName; - string m_optionDesc; - bool m_isParsed; // true if the target variable has been updated by fromString() +class OptionRegistryInterface { + public: + OptionRegistryInterface(const string optionName, const string optionDesc) + : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) {} + + virtual ~OptionRegistryInterface() {} + + const string &GetName() { return m_optionName; } + const string &GetDesc() { return m_optionDesc; } + const bool isParsed() { return m_isParsed; } + virtual string toString() = 0; + virtual bool fromString(const string str) = 0; + virtual bool isFlag() = 0; + virtual bool assignDefault(const char *str) = 0; + + protected: + string m_optionName; + string m_optionDesc; + bool m_isParsed; // true if the target variable has been updated by + // fromString() }; // Template for option registry - class T = specify data type of the option template -class OptionRegistry : public OptionRegistryInterface -{ -public: - OptionRegistry(const string name, const string desc, T &variable) - : OptionRegistryInterface(name, desc), m_variable(variable) - {} - - virtual ~OptionRegistry() {} - - virtual string toString() - { - stringstream ss; - ss << m_variable; - return ss.str(); - } - - virtual bool fromString(const string str) - { - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - ss << setbase(10); - if (str.size() > 1 && str[0] == '0') { - if (str.size() > 2 && str[1] == 'x') { - ss.ignore(2); - ss << setbase(16); - } else { - ss.ignore(1); - ss << setbase(8); - } - } - try { - ss >> m_variable; - } catch (exception &e) { - return false; +class OptionRegistry : public OptionRegistryInterface { + public: + OptionRegistry(const string name, const string desc, T &variable) + : OptionRegistryInterface(name, desc), m_variable(variable) {} + + virtual ~OptionRegistry() {} + + virtual string toString() { + stringstream ss; + ss << m_variable; + return ss.str(); + } + + virtual bool fromString(const string str) { + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + ss << setbase(10); + if (str.size() > 1 && str[0] == '0') { + if (str.size() > 2 && str[1] == 'x') { + ss.ignore(2); + ss << setbase(16); + } else { + ss.ignore(1); + ss << setbase(8); } - m_isParsed = true; - return true; - } + } + try { + ss >> m_variable; + } catch (exception &e) { + return false; + } + m_isParsed = true; + return true; + } - virtual bool isFlag() { return false; } - virtual bool assignDefault(const char *str) { return fromString(str); } + virtual bool isFlag() { return false; } + virtual bool assignDefault(const char *str) { return fromString(str); } - operator T() - { - return m_variable; - } + operator T() { return m_variable; } -private: - T &m_variable; + private: + T &m_variable; }; // specialized parser for string-type options -template<> -bool OptionRegistry::fromString(const string str) -{ - m_variable = str; - m_isParsed = true; - return true; +template <> +bool OptionRegistry::fromString(const string str) { + m_variable = str; + m_isParsed = true; + return true; } // specialized parser for c-string type options -template<> -bool OptionRegistry::fromString(const string str) -{ - m_variable = new char[str.size() + 1]; - strcpy(m_variable, str.c_str()); - m_isParsed = true; - return true; +template <> +bool OptionRegistry::fromString(const string str) { + m_variable = new char[str.size() + 1]; + strcpy(m_variable, str.c_str()); + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template<> -bool OptionRegistry::assignDefault(const char *str) -{ - m_variable = const_cast(str); // c-string options are not meant to be edited anyway - m_isParsed = true; - return true; +template <> +bool OptionRegistry::assignDefault(const char *str) { + m_variable = const_cast( + str); // c-string options are not meant to be edited anyway + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template<> -string OptionRegistry::toString() -{ - stringstream ss; - if (m_variable != NULL) { - ss << m_variable; - } else { - ss << "NULL"; - } - return ss.str(); +template <> +string OptionRegistry::toString() { + stringstream ss; + if (m_variable != NULL) { + ss << m_variable; + } else { + ss << "NULL"; + } + return ss.str(); } -// specialized parser for boolean options -template<> -bool OptionRegistry::fromString(const string str) -{ - int value = 1; - bool parsed = true; - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - try { - ss >> value; - } catch (stringstream::failure &ep) { - parsed = false; - } - assert(value == 0 or value == 1); // sanity check for boolean options (it can only be 1 or 0) - m_variable = (value != 0); - m_isParsed = true; - return parsed; +// specialized parser for boolean options +template <> +bool OptionRegistry::fromString(const string str) { + int value = 1; + bool parsed = true; + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + try { + ss >> value; + } catch (stringstream::failure &ep) { + parsed = false; + } + assert(value == 0 or + value == + 1); // sanity check for boolean options (it can only be 1 or 0) + m_variable = (value != 0); + m_isParsed = true; + return parsed; } // specializing a flag query function to identify boolean option -template<> -bool OptionRegistry::isFlag() { return true; } - -// class holding a collection of options and parse them from command line/configfile -class OptionParser -{ -public: - OptionParser() {} - ~OptionParser() - { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { - delete (*i_option); - } - } - - template - void Register(const string optionName, const string optionDesc, T &optionVariable, const char *optionDefault) - { - OptionRegistry *p_option = new OptionRegistry(optionName, optionDesc, optionVariable); - m_optionReg.push_back(p_option); - m_optionMap[optionName] = p_option; - p_option->assignDefault(optionDefault); - } - - void ParseCommandLine(int argc, const char * const argv[]) - { - for (int i = 1; i < argc; i++) { - OptionMap::iterator i_option; - bool optionFound = false; - - i_option = m_optionMap.find(argv[i]); - if (i_option != m_optionMap.end()) { - const char *argstr = (i + 1 < argc)? argv[i + 1] : ""; - OptionRegistryInterface *p_option = i_option->second; - if (p_option->isFlag()) { - if (p_option->fromString(argstr) == true) { - i += 1; - } - } else { - if (p_option->fromString(argstr) == false) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for option '%s'.\n", argstr, argv[i]); - exit(1); - } - i += 1; - } - optionFound = true; - } else if (string(argv[i]) == "-config") { - if (i + 1 >= argc) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Missing filename for option '-config'.\n"); - exit(1); - } - - ParseFile(argv[i + 1]); +template <> +bool OptionRegistry::isFlag() { + return true; +} + +// class holding a collection of options and parse them from command +// line/configfile +class OptionParser { + public: + OptionParser() {} + ~OptionParser() { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); + ++i_option) { + delete (*i_option); + } + } + + template + void Register(const string optionName, const string optionDesc, + T &optionVariable, const char *optionDefault) { + OptionRegistry *p_option = + new OptionRegistry(optionName, optionDesc, optionVariable); + m_optionReg.push_back(p_option); + m_optionMap[optionName] = p_option; + p_option->assignDefault(optionDefault); + } + + void ParseCommandLine(int argc, const char *const argv[]) { + for (int i = 1; i < argc; i++) { + OptionMap::iterator i_option; + bool optionFound = false; + + i_option = m_optionMap.find(argv[i]); + if (i_option != m_optionMap.end()) { + const char *argstr = (i + 1 < argc) ? argv[i + 1] : ""; + OptionRegistryInterface *p_option = i_option->second; + if (p_option->isFlag()) { + if (p_option->fromString(argstr) == true) { i += 1; - optionFound = true; - } - if (optionFound == false) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", argv[i]); + } + } else { + if (p_option->fromString(argstr) == false) { + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for " + "option '%s'.\n", + argstr, argv[i]); exit(1); - } + } + i += 1; + } + optionFound = true; + } else if (string(argv[i]) == "-config") { + if (i + 1 >= argc) { + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: Missing filename for option " + "'-config'.\n"); + exit(1); + } + + ParseFile(argv[i + 1]); + i += 1; + optionFound = true; } - } - - - void ParseFile(const char *filename) { - ifstream inputFile; - stringstream args; - - // open config file, stream every line into a continuous buffer - // get rid of comments in the process - inputFile.open(filename); - if (!inputFile.good()) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", filename); - exit(1); + if (optionFound == false) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", + argv[i]); + exit(1); } - while (inputFile.good()) { - string line; - getline(inputFile, line); - size_t commentStart = line.find_first_of("#"); - if (commentStart != line.npos) { - line.erase(commentStart); - } - args << line << ' '; + } + } + + void ParseFile(const char *filename) { + ifstream inputFile; + stringstream args; + + // open config file, stream every line into a continuous buffer + // get rid of comments in the process + inputFile.open(filename); + if (!inputFile.good()) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", + filename); + exit(1); + } + while (inputFile.good()) { + string line; + getline(inputFile, line); + size_t commentStart = line.find_first_of("#"); + if (commentStart != line.npos) { + line.erase(commentStart); } - inputFile.close(); - - ParseStringStream(args); - } - - // parse the given string as tokens separated by a set of given delimiters - void ParseString(string inputString, const string delimiters = string(" ;")) { - // convert all delimiter characters into whitespaces - for (unsigned t = 0; t < inputString.size(); t++) { - for (unsigned d = 0; d < delimiters.size(); d++) { - if (inputString[t] == delimiters.at(d)) { - inputString[t] = ' '; - break; - } - } + args << line << ' '; + } + inputFile.close(); + + ParseStringStream(args); + } + + // parse the given string as tokens separated by a set of given delimiters + void ParseString(string inputString, const string delimiters = string(" ;")) { + // convert all delimiter characters into whitespaces + for (unsigned t = 0; t < inputString.size(); t++) { + for (unsigned d = 0; d < delimiters.size(); d++) { + if (inputString[t] == delimiters.at(d)) { + inputString[t] = ' '; + break; + } } - stringstream args(inputString); - ParseStringStream(args); - } - - // parse the given stringstream as whitespace-separated tokens. drain the stream in the process - void ParseStringStream(stringstream &args) { - // extract non-whitespace string tokens - vector argv; - argv.push_back(new char[6]); - strcpy(argv[0], "dummy"); - while (args.good()) { - string argNew; - args >> argNew; - - if (argNew.size() == 0) continue; // this is probably the last token - - if (argNew[0] == '"') { - while (args.good() && argNew[argNew.size()-1] != '"') { - string argCont; - args >> argCont; - argNew += " " + argCont; - } - argNew.erase(0,1); - argNew.erase(argNew.size()-1); - } - - char *c_argNew = new char[argNew.size() + 1]; - strcpy(c_argNew, argNew.c_str()); - argv.push_back(c_argNew); + } + stringstream args(inputString); + ParseStringStream(args); + } + + // parse the given stringstream as whitespace-separated tokens. drain the + // stream in the process + void ParseStringStream(stringstream &args) { + // extract non-whitespace string tokens + vector argv; + argv.push_back(new char[6]); + strcpy(argv[0], "dummy"); + while (args.good()) { + string argNew; + args >> argNew; + + if (argNew.size() == 0) continue; // this is probably the last token + + if (argNew[0] == '"') { + while (args.good() && argNew[argNew.size() - 1] != '"') { + string argCont; + args >> argCont; + argNew += " " + argCont; + } + argNew.erase(0, 1); + argNew.erase(argNew.size() - 1); } - // pass the string token into normal commandline parser - char **targv = (char**)calloc(argv.size(), sizeof(char*)); - for( unsigned k=0; k < argv.size(); k++ ) - targv[k] = argv[k]; - ParseCommandLine(argv.size(), targv); - free(targv); - for (size_t i = 0; i < argv.size(); i++) { - delete[] argv[i]; - } - } - - void Print(FILE *fout) - { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { - stringstream sout; - if ((*i_option)->isParsed() == false) { - cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" << (*i_option)->GetName() << "'\n"; - assert(0); - } - sout << setw(20) << left << (*i_option)->GetName() << " "; - sout << setw(20) << right << (*i_option)->toString() << " # "; - sout << left << (*i_option)->GetDesc(); - sout << std::endl; - fprintf(fout, "%s", sout.str().c_str()); + char *c_argNew = new char[argNew.size() + 1]; + strcpy(c_argNew, argNew.c_str()); + argv.push_back(c_argNew); + } + + // pass the string token into normal commandline parser + char **targv = (char **)calloc(argv.size(), sizeof(char *)); + for (unsigned k = 0; k < argv.size(); k++) targv[k] = argv[k]; + ParseCommandLine(argv.size(), targv); + free(targv); + for (size_t i = 0; i < argv.size(); i++) { + delete[] argv[i]; + } + } + + void Print(FILE *fout) { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); + ++i_option) { + stringstream sout; + if ((*i_option)->isParsed() == false) { + cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" + << (*i_option)->GetName() << "'\n"; + assert(0); } - } + sout << setw(20) << left << (*i_option)->GetName() << " "; + sout << setw(20) << right << (*i_option)->toString() << " # "; + sout << left << (*i_option)->GetDesc(); + sout << std::endl; + fprintf(fout, "%s", sout.str().c_str()); + } + } -private: - typedef list OptionCollection; - OptionCollection m_optionReg; - typedef map OptionMap; - OptionMap m_optionMap; + private: + typedef list OptionCollection; + OptionCollection m_optionReg; + typedef map OptionMap; + OptionMap m_optionMap; }; #include "option_parser.h" -option_parser_t option_parser_create() -{ - OptionParser *p_opr = new OptionParser(); - return reinterpret_cast(p_opr); +option_parser_t option_parser_create() { + OptionParser *p_opr = new OptionParser(); + return reinterpret_cast(p_opr); } -void option_parser_destroy(option_parser_t opp) -{ - OptionParser *p_opr = reinterpret_cast(opp); - delete p_opr; +void option_parser_destroy(option_parser_t opp) { + OptionParser *p_opr = reinterpret_cast(opp); + delete p_opr; } -void option_parser_register(option_parser_t opp, - const char *name, - enum option_dtype type, - void *variable, - const char *desc, - const char *defaultvalue) -{ - OptionParser *p_opr = reinterpret_cast(opp); - switch (type) { - case OPT_INT32: p_opr->Register(name, desc, *(int*)variable, defaultvalue); break; - case OPT_UINT32: p_opr->Register(name, desc, *(unsigned int*)variable, defaultvalue); break; - case OPT_INT64: p_opr->Register(name, desc, *(long long*)variable, defaultvalue); break; - case OPT_UINT64: p_opr->Register(name, desc, *(unsigned long long*)variable, defaultvalue); break; - case OPT_BOOL: p_opr->Register(name, desc, *(bool*)variable, defaultvalue); break; - case OPT_FLOAT: p_opr->Register(name, desc, *(float*)variable, defaultvalue); break; - case OPT_DOUBLE: p_opr->Register(name, desc, *(double*)variable, defaultvalue); break; - case OPT_CHAR: p_opr->Register(name, desc, *(char*)variable, defaultvalue); break; - case OPT_CSTR: p_opr->Register(name, desc, *(char**)variable, defaultvalue); break; - default: - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", type); - exit(1); - break; - } +void option_parser_register(option_parser_t opp, const char *name, + enum option_dtype type, void *variable, + const char *desc, const char *defaultvalue) { + OptionParser *p_opr = reinterpret_cast(opp); + switch (type) { + case OPT_INT32: + p_opr->Register(name, desc, *(int *)variable, defaultvalue); + break; + case OPT_UINT32: + p_opr->Register(name, desc, *(unsigned int *)variable, + defaultvalue); + break; + case OPT_INT64: + p_opr->Register(name, desc, *(long long *)variable, + defaultvalue); + break; + case OPT_UINT64: + p_opr->Register( + name, desc, *(unsigned long long *)variable, defaultvalue); + break; + case OPT_BOOL: + p_opr->Register(name, desc, *(bool *)variable, defaultvalue); + break; + case OPT_FLOAT: + p_opr->Register(name, desc, *(float *)variable, defaultvalue); + break; + case OPT_DOUBLE: + p_opr->Register(name, desc, *(double *)variable, defaultvalue); + break; + case OPT_CHAR: + p_opr->Register(name, desc, *(char *)variable, defaultvalue); + break; + case OPT_CSTR: + p_opr->Register(name, desc, *(char **)variable, defaultvalue); + break; + default: + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", + type); + exit(1); + break; + } } -void option_parser_cmdline(option_parser_t opp, - int argc, const char *argv[]) -{ - OptionParser *p_opr = reinterpret_cast(opp); - return p_opr->ParseCommandLine(argc,argv); - +void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]) { + OptionParser *p_opr = reinterpret_cast(opp); + return p_opr->ParseCommandLine(argc, argv); } - -void option_parser_cfgfile(option_parser_t opp, - const char *filename) -{ - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->ParseFile(filename); +void option_parser_cfgfile(option_parser_t opp, const char *filename) { + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->ParseFile(filename); } void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, - const char *delimiters) -{ - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->ParseString(inputstring, delimiters); + const char *inputstring, + const char *delimiters) { + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->ParseString(inputstring, delimiters); } -void option_parser_print(option_parser_t opp, - FILE *fout) -{ - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->Print(fout); +void option_parser_print(option_parser_t opp, FILE *fout) { + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->Print(fout); } - - // #define UNIT_TEST #ifdef UNIT_TEST -class testtype -{ -public: - int idata; - float fdata; - string sdata; - unsigned long long ulldata; - bool bdata; - unsigned int boolint; - char * coption; - - testtype() - : idata(0), - fdata(0.0f), - sdata(""), - ulldata(0), - bdata(false) - { } +class testtype { + public: + int idata; + float fdata; + string sdata; + unsigned long long ulldata; + bool bdata; + unsigned int boolint; + char *coption; + + testtype() : idata(0), fdata(0.0f), sdata(""), ulldata(0), bdata(false) {} }; - -int cppinterfacetest(int argc, const char *argv[]) -{ - testtype c; - OptionParser optionparser; - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - - optionparser.Register("-idata", "integer data", c.idata, "-456"); - optionparser.Register("-fdata", "floating point data", c.fdata, "0.001"); - optionparser.Register("-sdata", "first string data", c.sdata, "hellow"); - optionparser.Register("-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); - optionparser.Register("-someflag", "first flag", c.bdata, "0"); - optionparser.Register("-otherflag", "second flag", (bool&)c.boolint, "1"); - optionparser.Register("-coption", "char * data", c.coption, NULL); - - cout << "Default: \n"; - optionparser.Print(stdout); - - optionparser.ParseCommandLine(argc, argv); - - cout << "Commandline Parse Results: \n"; - optionparser.Print(stdout); - - optionparser.ParseFile("test.config"); - cout << "File Parse Results: \n"; - optionparser.Print(stdout); - cout << c.sdata << ' ' << c.idata << endl; - - return 0; +int cppinterfacetest(int argc, const char *argv[]) { + testtype c; + OptionParser optionparser; + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); + + optionparser.Register("-idata", "integer data", c.idata, "-456"); + optionparser.Register("-fdata", "floating point data", c.fdata, + "0.001"); + optionparser.Register("-sdata", "first string data", c.sdata, + "hellow"); + optionparser.Register( + "-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); + optionparser.Register("-someflag", "first flag", c.bdata, "0"); + optionparser.Register("-otherflag", "second flag", (bool &)c.boolint, + "1"); + optionparser.Register("-coption", "char * data", c.coption, NULL); + + cout << "Default: \n"; + optionparser.Print(stdout); + + optionparser.ParseCommandLine(argc, argv); + + cout << "Commandline Parse Results: \n"; + optionparser.Print(stdout); + + optionparser.ParseFile("test.config"); + cout << "File Parse Results: \n"; + optionparser.Print(stdout); + cout << c.sdata << ' ' << c.idata << endl; + + return 0; } -int cinterfacetest(int argc, const char *argv[]) -{ - testtype c; - option_parser_t opp = option_parser_create(); - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - char *otherstr; - - option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", "-456"); - option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, "floating point data", "0.001"); - option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, "first string data", "hellow"); - option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, "unsigend long long data", "0x123456789abcdef1"); - option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", "0"); - option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", "1"); - option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", NULL); - - printf("Default: \n"); - option_parser_print(opp, stdout); - - option_parser_cmdline(opp, argc, argv); - - printf("Commandline Parse Results: \n"); - option_parser_print(opp, stdout); - - option_parser_cfgfile(opp, "test.config"); - printf("File Parse Results: \n"); - option_parser_print(opp, stdout); - printf("%s %d\n", otherstr, c.idata); - - option_parser_destroy(opp); - - return 0; +int cinterfacetest(int argc, const char *argv[]) { + testtype c; + option_parser_t opp = option_parser_create(); + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); + char *otherstr; + + option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", + "-456"); + option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, + "floating point data", "0.001"); + option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, + "first string data", "hellow"); + option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, + "unsigend long long data", "0x123456789abcdef1"); + option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", + "0"); + option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", + "1"); + option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", + NULL); + + printf("Default: \n"); + option_parser_print(opp, stdout); + + option_parser_cmdline(opp, argc, argv); + + printf("Commandline Parse Results: \n"); + option_parser_print(opp, stdout); + + option_parser_cfgfile(opp, "test.config"); + printf("File Parse Results: \n"); + option_parser_print(opp, stdout); + printf("%s %d\n", otherstr, c.idata); + + option_parser_destroy(opp); + + return 0; } -int stringparsertest() -{ - int tABC; - int tDEF; - char tMode; - char *tName; - - option_parser_t opp = option_parser_create(); - option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); - option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); - option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); - option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); - - option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", " ;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); - - option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", " =;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); - - return 0; +int stringparsertest() { + int tABC; + int tDEF; + char tMode; + char *tName; + + option_parser_t opp = option_parser_create(); + option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); + option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); + option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); + option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); + + option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", + " ;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); + + option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", + " =;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); + + return 0; } -int main(int argc, const char *argv[]) -{ - cppinterfacetest(argc,argv); - cinterfacetest(argc,argv); - stringparsertest(); +int main(int argc, const char *argv[]) { + cppinterfacetest(argc, argv); + cinterfacetest(argc, argv); + stringparsertest(); - return 0; + return 0; } #endif - diff --git a/src/option_parser.h b/src/option_parser.h index 1f7f96c..180bc9d 100644 --- a/src/option_parser.h +++ b/src/option_parser.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,26 +27,25 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include #include - // pointer to C++ class typedef class OptionParser *option_parser_t; // data type of the option enum option_dtype { - OPT_INT32, - OPT_UINT32, - OPT_INT64, - OPT_UINT64, - OPT_BOOL, - OPT_FLOAT, - OPT_DOUBLE, - OPT_CHAR, - OPT_CSTR + OPT_INT32, + OPT_UINT32, + OPT_INT64, + OPT_UINT64, + OPT_BOOL, + OPT_FLOAT, + OPT_DOUBLE, + OPT_CHAR, + OPT_CSTR }; // create and destroy option parser @@ -52,28 +53,19 @@ option_parser_t option_parser_create(); void option_parser_destroy(option_parser_t opp); // register new option -void option_parser_register(option_parser_t opp, - const char *name, - enum option_dtype type, - void *variable, - const char *desc, - const char *defaultvalue); +void option_parser_register(option_parser_t opp, const char *name, + enum option_dtype type, void *variable, + const char *desc, const char *defaultvalue); // parse command line -void option_parser_cmdline(option_parser_t opp, - int argc, const char *argv[]); - - +void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]); // parse config file -void option_parser_cfgfile(option_parser_t opp, - const char *filename); +void option_parser_cfgfile(option_parser_t opp, const char *filename); // parse a delimited string void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, + const char *inputstring, const char *delimiters); // print options -void option_parser_print(option_parser_t opp, - FILE *fout); - +void option_parser_print(option_parser_t opp, FILE *fout); diff --git a/src/statwrapper.cc b/src/statwrapper.cc index e273e78..cda25b6 100644 --- a/src/statwrapper.cc +++ b/src/statwrapper.cc @@ -1,48 +1,33 @@ -//a Wraper function for stats class -#include "intersim2/stats.hpp" +// a Wraper function for stats class #include +#include "intersim2/stats.hpp" -Stats* StatCreate (const char * name, double bin_size, int num_bins) { - Stats* newstat = new Stats(NULL,name,bin_size,num_bins); - newstat->Clear (); - return newstat; +Stats *StatCreate(const char *name, double bin_size, int num_bins) { + Stats *newstat = new Stats(NULL, name, bin_size, num_bins); + newstat->Clear(); + return newstat; } -void StatClear(void * st) -{ - ((Stats *)st)->Clear(); -} +void StatClear(void *st) { ((Stats *)st)->Clear(); } -void StatAddSample (void * st, int val) -{ - ((Stats *)st)->AddSample(val); -} +void StatAddSample(void *st, int val) { ((Stats *)st)->AddSample(val); } -double StatAverage(void * st) -{ - return((Stats *)st)->Average(); -} +double StatAverage(void *st) { return ((Stats *)st)->Average(); } -double StatMax(void * st) -{ - return((Stats *)st)->Max(); -} +double StatMax(void *st) { return ((Stats *)st)->Max(); } -double StatMin(void * st) -{ - return((Stats *)st)->Min(); -} +double StatMin(void *st) { return ((Stats *)st)->Min(); } -void StatDisp (void * st) -{ - printf ("Stats for "); - ((Stats *)st)->DisplayHierarchy(); -// if (((Stats *)st)->NeverUsed()) { -// printf (" was never updated!\n"); -// } else { - printf("Min %f Max %f Average %f \n",((Stats *)st)->Min(),((Stats *)st)->Max(),StatAverage(st)); - ((Stats *)st)->Display(); -// } +void StatDisp(void *st) { + printf("Stats for "); + ((Stats *)st)->DisplayHierarchy(); + // if (((Stats *)st)->NeverUsed()) { + // printf (" was never updated!\n"); + // } else { + printf("Min %f Max %f Average %f \n", ((Stats *)st)->Min(), + ((Stats *)st)->Max(), StatAverage(st)); + ((Stats *)st)->Display(); + // } } #if 0 @@ -55,5 +40,3 @@ int main () StatDisp(mytest); } #endif - - diff --git a/src/statwrapper.h b/src/statwrapper.h index 65c2ab9..4c00447 100644 --- a/src/statwrapper.h +++ b/src/statwrapper.h @@ -1,12 +1,12 @@ #ifndef STAT_WRAPER_H #define STAT_WRAPER_H -class Stats* StatCreate (const char * name, double bin_size, int num_bins) ; -void StatClear(void * st); -void StatAddSample (void * st, int val); -double StatAverage(void * st) ; -double StatMax(void * st) ; -double StatMin(void * st) ; -void StatDisp (void * st); +class Stats* StatCreate(const char* name, double bin_size, int num_bins); +void StatClear(void* st); +void StatAddSample(void* st, int val); +double StatAverage(void* st); +double StatMax(void* st); +double StatMin(void* st); +void StatDisp(void* st); #endif diff --git a/src/stream_manager.cc b/src/stream_manager.cc index be3dd71..f813ed4 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -26,463 +28,445 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "stream_manager.h" -#include "gpgpusim_entrypoint.h" +#include "../libcuda/gpgpu_context.h" #include "cuda-sim/cuda-sim.h" #include "gpgpu-sim/gpu-sim.h" -#include "../libcuda/gpgpu_context.h" +#include "gpgpusim_entrypoint.h" unsigned CUstream_st::sm_next_stream_uid = 0; -CUstream_st::CUstream_st() -{ - m_pending = false; - m_uid = sm_next_stream_uid++; - pthread_mutex_init(&m_lock,NULL); -} - -bool CUstream_st::empty() -{ - pthread_mutex_lock(&m_lock); - bool empty = m_operations.empty(); - pthread_mutex_unlock(&m_lock); - return empty; +CUstream_st::CUstream_st() { + m_pending = false; + m_uid = sm_next_stream_uid++; + pthread_mutex_init(&m_lock, NULL); } -bool CUstream_st::busy() -{ - pthread_mutex_lock(&m_lock); - bool pending = m_pending; - pthread_mutex_unlock(&m_lock); - return pending; +bool CUstream_st::empty() { + pthread_mutex_lock(&m_lock); + bool empty = m_operations.empty(); + pthread_mutex_unlock(&m_lock); + return empty; } -void CUstream_st::synchronize() -{ - // called by host thread - bool done=false; - do{ - pthread_mutex_lock(&m_lock); - done = m_operations.empty(); - pthread_mutex_unlock(&m_lock); - } while ( !done ); +bool CUstream_st::busy() { + pthread_mutex_lock(&m_lock); + bool pending = m_pending; + pthread_mutex_unlock(&m_lock); + return pending; } -void CUstream_st::push( const stream_operation &op ) -{ - // called by host thread +void CUstream_st::synchronize() { + // called by host thread + bool done = false; + do { pthread_mutex_lock(&m_lock); - m_operations.push_back( op ); + done = m_operations.empty(); pthread_mutex_unlock(&m_lock); + } while (!done); } -void CUstream_st::record_next_done() -{ - // called by gpu thread - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_operations.pop_front(); - m_pending=false; - pthread_mutex_unlock(&m_lock); +void CUstream_st::push(const stream_operation &op) { + // called by host thread + pthread_mutex_lock(&m_lock); + m_operations.push_back(op); + pthread_mutex_unlock(&m_lock); } - -stream_operation CUstream_st::next() -{ - // called by gpu thread - pthread_mutex_lock(&m_lock); - m_pending = true; - stream_operation result = m_operations.front(); - pthread_mutex_unlock(&m_lock); - return result; +void CUstream_st::record_next_done() { + // called by gpu thread + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_operations.pop_front(); + m_pending = false; + pthread_mutex_unlock(&m_lock); } -void CUstream_st::cancel_front() -{ - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_pending = false; - pthread_mutex_unlock(&m_lock); - +stream_operation CUstream_st::next() { + // called by gpu thread + pthread_mutex_lock(&m_lock); + m_pending = true; + stream_operation result = m_operations.front(); + pthread_mutex_unlock(&m_lock); + return result; } -void CUstream_st::print(FILE *fp) -{ - pthread_mutex_lock(&m_lock); - fprintf(fp,"GPGPU-Sim API: stream %u has %zu operations\n", m_uid, m_operations.size() ); - std::list::iterator i; - unsigned n=0; - for( i=m_operations.begin(); i!=m_operations.end(); i++ ) { - stream_operation &op = *i; - fprintf(fp,"GPGPU-Sim API: %u : ", n++); - op.print(fp); - fprintf(fp,"\n"); - } - pthread_mutex_unlock(&m_lock); +void CUstream_st::cancel_front() { + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_pending = false; + pthread_mutex_unlock(&m_lock); } +void CUstream_st::print(FILE *fp) { + pthread_mutex_lock(&m_lock); + fprintf(fp, "GPGPU-Sim API: stream %u has %zu operations\n", m_uid, + m_operations.size()); + std::list::iterator i; + unsigned n = 0; + for (i = m_operations.begin(); i != m_operations.end(); i++) { + stream_operation &op = *i; + fprintf(fp, "GPGPU-Sim API: %u : ", n++); + op.print(fp); + fprintf(fp, "\n"); + } + pthread_mutex_unlock(&m_lock); +} -bool stream_operation::do_operation( gpgpu_sim *gpu ) -{ - if( is_noop() ) - return true; +bool stream_operation::do_operation(gpgpu_sim *gpu) { + if (is_noop()) return true; - assert(!m_done && m_stream); - if(g_debug_execution >= 3) - printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid() ); - switch( m_type ) { + assert(!m_done && m_stream); + if (g_debug_execution >= 3) + printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid()); + switch (m_type) { case stream_memcpy_host_to_device: - if(g_debug_execution >= 3) - printf("memcpy host-to-device\n"); - gpu->memcpy_to_gpu(m_device_address_dst,m_host_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy host-to-device\n"); + gpu->memcpy_to_gpu(m_device_address_dst, m_host_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_host: - if(g_debug_execution >= 3) - printf("memcpy device-to-host\n"); - gpu->memcpy_from_gpu(m_host_address_dst,m_device_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy device-to-host\n"); + gpu->memcpy_from_gpu(m_host_address_dst, m_device_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_device: - if(g_debug_execution >= 3) - printf("memcpy device-to-device\n"); - gpu->memcpy_gpu_to_gpu(m_device_address_dst,m_device_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy device-to-device\n"); + gpu->memcpy_gpu_to_gpu(m_device_address_dst, m_device_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_to_symbol: - if(g_debug_execution >= 3) - printf("memcpy to symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_src,m_cnt,m_offset,1,gpu); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy to symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( + m_symbol, m_host_address_src, m_cnt, m_offset, 1, gpu); + m_stream->record_next_done(); + break; case stream_memcpy_from_symbol: - if(g_debug_execution >= 3) - printf("memcpy from symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_dst,m_cnt,m_offset,0,gpu); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy from symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( + m_symbol, m_host_address_dst, m_cnt, m_offset, 0, gpu); + m_stream->record_next_done(); + break; case stream_kernel_launch: - if( m_sim_mode ) { //Functional Sim - if(g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); - m_kernel->print_parent_info(); - } - gpu->set_cache_config(m_kernel->name()); - gpu->functional_launch( m_kernel ); + if (m_sim_mode) { // Functional Sim + if (g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str()); + m_kernel->print_parent_info(); } - else { //Performance Sim - if( gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { - if(g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); - m_kernel->print_parent_info(); - } - gpu->set_cache_config(m_kernel->name()); - gpu->launch( m_kernel ); - } - else { - if(m_kernel->m_launch_latency) - m_kernel->m_launch_latency--; - if(g_debug_execution >= 3) - printf("kernel %d: \'%s\', latency %u not ready to transfer to GPU hardware scheduler\n", - m_kernel->get_uid(), m_kernel->name().c_str(), m_kernel->m_launch_latency); - return false; - } + gpu->set_cache_config(m_kernel->name()); + gpu->functional_launch(m_kernel); + } else { // Performance Sim + if (gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { + if (g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str()); + m_kernel->print_parent_info(); + } + gpu->set_cache_config(m_kernel->name()); + gpu->launch(m_kernel); + } else { + if (m_kernel->m_launch_latency) m_kernel->m_launch_latency--; + if (g_debug_execution >= 3) + printf( + "kernel %d: \'%s\', latency %u not ready to transfer to GPU " + "hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str(), + m_kernel->m_launch_latency); + return false; } - break; + } + break; case stream_event: { - printf("event update\n"); - time_t wallclock = time((time_t *)NULL); - m_event->update( gpu->gpu_tot_sim_cycle, wallclock ); - m_stream->record_next_done(); - } - break; + printf("event update\n"); + time_t wallclock = time((time_t *)NULL); + m_event->update(gpu->gpu_tot_sim_cycle, wallclock); + m_stream->record_next_done(); + } break; case stream_wait_event: { - //only allows next op to go if event is done - //otherwise stays in the stream queue - printf("stream wait event processing...\n"); - if(m_event->done()) - printf("stream wait event done\n"); - m_stream->record_next_done(); - } - break; + // only allows next op to go if event is done + // otherwise stays in the stream queue + printf("stream wait event processing...\n"); + if (m_event->done()) printf("stream wait event done\n"); + m_stream->record_next_done(); + } break; default: - abort(); - } - m_done=true; - fflush(stdout); - return true; + abort(); + } + m_done = true; + fflush(stdout); + return true; } -void stream_operation::print( FILE *fp ) const -{ - fprintf(fp," stream operation " ); - switch( m_type ) { - case stream_event: fprintf(fp,"event"); break; - case stream_kernel_launch: fprintf(fp,"kernel"); break; - case stream_memcpy_device_to_device: fprintf(fp,"memcpy device-to-device"); break; - case stream_memcpy_device_to_host: fprintf(fp,"memcpy device-to-host"); break; - case stream_memcpy_host_to_device: fprintf(fp,"memcpy host-to-device"); break; - case stream_memcpy_to_symbol: fprintf(fp,"memcpy to symbol"); break; - case stream_memcpy_from_symbol: fprintf(fp,"memcpy from symbol"); break; - case stream_no_op: fprintf(fp,"no-op"); break; - } +void stream_operation::print(FILE *fp) const { + fprintf(fp, " stream operation "); + switch (m_type) { + case stream_event: + fprintf(fp, "event"); + break; + case stream_kernel_launch: + fprintf(fp, "kernel"); + break; + case stream_memcpy_device_to_device: + fprintf(fp, "memcpy device-to-device"); + break; + case stream_memcpy_device_to_host: + fprintf(fp, "memcpy device-to-host"); + break; + case stream_memcpy_host_to_device: + fprintf(fp, "memcpy host-to-device"); + break; + case stream_memcpy_to_symbol: + fprintf(fp, "memcpy to symbol"); + break; + case stream_memcpy_from_symbol: + fprintf(fp, "memcpy from symbol"); + break; + case stream_no_op: + fprintf(fp, "no-op"); + break; + } } -stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) -{ - m_gpu = gpu; - m_service_stream_zero = false; - m_cuda_launch_blocking = cuda_launch_blocking; - pthread_mutex_init(&m_lock,NULL); +stream_manager::stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking) { + m_gpu = gpu; + m_service_stream_zero = false; + m_cuda_launch_blocking = cuda_launch_blocking; + pthread_mutex_init(&m_lock, NULL); } -bool stream_manager::operation( bool * sim) -{ - bool check=check_finished_kernel(); - pthread_mutex_lock(&m_lock); -// if(check)m_gpu->print_stats(); - stream_operation op =front(); - if(!op.do_operation( m_gpu )) //not ready to execute - { - //cancel operation - if( op.is_kernel() ) { - unsigned grid_uid = op.get_kernel()->get_uid(); - m_grid_id_to_stream.erase(grid_uid); - } - op.get_stream()->cancel_front(); - +bool stream_manager::operation(bool *sim) { + bool check = check_finished_kernel(); + pthread_mutex_lock(&m_lock); + // if(check)m_gpu->print_stats(); + stream_operation op = front(); + if (!op.do_operation(m_gpu)) // not ready to execute + { + // cancel operation + if (op.is_kernel()) { + unsigned grid_uid = op.get_kernel()->get_uid(); + m_grid_id_to_stream.erase(grid_uid); } - pthread_mutex_unlock(&m_lock); - //pthread_mutex_lock(&m_lock); - // simulate a clock cycle on the GPU - return check; + op.get_stream()->cancel_front(); + } + pthread_mutex_unlock(&m_lock); + // pthread_mutex_lock(&m_lock); + // simulate a clock cycle on the GPU + return check; } -bool stream_manager::check_finished_kernel() -{ - unsigned grid_uid = m_gpu->finished_kernel(); - bool check=register_finished_kernel(grid_uid); - return check; +bool stream_manager::check_finished_kernel() { + unsigned grid_uid = m_gpu->finished_kernel(); + bool check = register_finished_kernel(grid_uid); + return check; } -bool stream_manager::register_finished_kernel(unsigned grid_uid) -{ - // called by gpu simulation thread - if(grid_uid > 0){ - CUstream_st *stream = m_grid_id_to_stream[grid_uid]; - kernel_info_t *kernel = stream->front().get_kernel(); - assert( grid_uid == kernel->get_uid() ); - - //Jin: should check children kernels for CDP - if(kernel->is_finished()) { -// std::ofstream kernel_stat("kernel_stat.txt", std::ofstream::out | std::ofstream::app); -// kernel_stat<< " kernel " << grid_uid << ": " << kernel->name(); -// if(kernel->get_parent()) -// kernel_stat << ", parent " << kernel->get_parent()->get_uid() << -// ", launch " << kernel->launch_cycle; -// kernel_stat<< ", start " << kernel->start_cycle << -// ", end " << kernel->end_cycle << ", retire " << gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; -// printf("kernel %d finishes, retires from stream %d\n", grid_uid, stream->get_uid()); -// kernel_stat.flush(); -// kernel_stat.close(); - stream->record_next_done(); - m_grid_id_to_stream.erase(grid_uid); - kernel->notify_parent_finished(); - delete kernel; - return true; - } +bool stream_manager::register_finished_kernel(unsigned grid_uid) { + // called by gpu simulation thread + if (grid_uid > 0) { + CUstream_st *stream = m_grid_id_to_stream[grid_uid]; + kernel_info_t *kernel = stream->front().get_kernel(); + assert(grid_uid == kernel->get_uid()); + + // Jin: should check children kernels for CDP + if (kernel->is_finished()) { + // std::ofstream kernel_stat("kernel_stat.txt", + // std::ofstream::out | std::ofstream::app); + // kernel_stat<< " kernel " << grid_uid << ": " << + // kernel->name(); + // if(kernel->get_parent()) + // kernel_stat << ", parent " << + // kernel->get_parent()->get_uid() << + // ", launch " << kernel->launch_cycle; + // kernel_stat<< ", start " << kernel->start_cycle << + // ", end " << kernel->end_cycle << ", retire " << + // gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; + // printf("kernel %d finishes, retires from stream %d\n", + // grid_uid, stream->get_uid()); + // kernel_stat.flush(); + // kernel_stat.close(); + stream->record_next_done(); + m_grid_id_to_stream.erase(grid_uid); + kernel->notify_parent_finished(); + delete kernel; + return true; } + } - return false; + return false; } -void stream_manager::stop_all_running_kernels(){ - pthread_mutex_lock(&m_lock); +void stream_manager::stop_all_running_kernels() { + pthread_mutex_lock(&m_lock); - // Signal m_gpu to stop all running kernels - m_gpu->stop_all_running_kernels(); + // Signal m_gpu to stop all running kernels + m_gpu->stop_all_running_kernels(); - // Clean up all streams waiting on running kernels - int count=0; - while(check_finished_kernel()){ - count++; - } + // Clean up all streams waiting on running kernels + int count = 0; + while (check_finished_kernel()) { + count++; + } - // If any kernels completed, print out the current stats - if(count > 0) - m_gpu->print_stats(); + // If any kernels completed, print out the current stats + if (count > 0) m_gpu->print_stats(); - pthread_mutex_unlock(&m_lock); + pthread_mutex_unlock(&m_lock); } -stream_operation stream_manager::front() -{ - // called by gpu simulation thread - stream_operation result; -// if( concurrent_streams_empty() ) - m_service_stream_zero = true; - if( m_service_stream_zero ) { - if( !m_stream_zero.empty() && !m_stream_zero.busy() ) { - result = m_stream_zero.next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = &m_stream_zero; - } - } else { - m_service_stream_zero = false; - } +stream_operation stream_manager::front() { + // called by gpu simulation thread + stream_operation result; + // if( concurrent_streams_empty() ) + m_service_stream_zero = true; + if (m_service_stream_zero) { + if (!m_stream_zero.empty() && !m_stream_zero.busy()) { + result = m_stream_zero.next(); + if (result.is_kernel()) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = &m_stream_zero; + } + } else { + m_service_stream_zero = false; } - - if(!m_service_stream_zero) - { - std::list::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++) { - CUstream_st *stream = *s; - if( !stream->busy() && !stream->empty() ) { - result = stream->next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = stream; - } - break; - } + } + + if (!m_service_stream_zero) { + std::list::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + CUstream_st *stream = *s; + if (!stream->busy() && !stream->empty()) { + result = stream->next(); + if (result.is_kernel()) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = stream; } + break; + } } - return result; + } + return result; } -void stream_manager::add_stream( struct CUstream_st *stream ) -{ - // called by host thread - pthread_mutex_lock(&m_lock); - m_streams.push_back(stream); - pthread_mutex_unlock(&m_lock); +void stream_manager::add_stream(struct CUstream_st *stream) { + // called by host thread + pthread_mutex_lock(&m_lock); + m_streams.push_back(stream); + pthread_mutex_unlock(&m_lock); } -void stream_manager::destroy_stream( CUstream_st *stream ) -{ - // called by host thread - pthread_mutex_lock(&m_lock); - while( !stream->empty() ) - ; - std::list::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++ ) { - if( *s == stream ) { - m_streams.erase(s); - break; - } +void stream_manager::destroy_stream(CUstream_st *stream) { + // called by host thread + pthread_mutex_lock(&m_lock); + while (!stream->empty()) + ; + std::list::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + if (*s == stream) { + m_streams.erase(s); + break; } - delete stream; - pthread_mutex_unlock(&m_lock); + } + delete stream; + pthread_mutex_unlock(&m_lock); } -bool stream_manager::concurrent_streams_empty() -{ - bool result = true; - if (m_streams.empty()) - return true; - // called by gpu simulation thread - std::list::iterator s; - for( s=m_streams.begin(); s!=m_streams.end();++s ) { - struct CUstream_st *stream = *s; - if( !stream->empty() ) { - //stream->print(stdout); - result = false; - break; - } +bool stream_manager::concurrent_streams_empty() { + bool result = true; + if (m_streams.empty()) return true; + // called by gpu simulation thread + std::list::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); ++s) { + struct CUstream_st *stream = *s; + if (!stream->empty()) { + // stream->print(stdout); + result = false; + break; } - return result; + } + return result; } -bool stream_manager::empty_protected() -{ - bool result = true; - pthread_mutex_lock(&m_lock); - if( !concurrent_streams_empty() ) - result = false; - if( !m_stream_zero.empty() ) - result = false; - pthread_mutex_unlock(&m_lock); - return result; +bool stream_manager::empty_protected() { + bool result = true; + pthread_mutex_lock(&m_lock); + if (!concurrent_streams_empty()) result = false; + if (!m_stream_zero.empty()) result = false; + pthread_mutex_unlock(&m_lock); + return result; } -bool stream_manager::empty() -{ - bool result = true; - if( !concurrent_streams_empty() ) - result = false; - if( !m_stream_zero.empty() ) - result = false; - return result; +bool stream_manager::empty() { + bool result = true; + if (!concurrent_streams_empty()) result = false; + if (!m_stream_zero.empty()) result = false; + return result; } - -void stream_manager::print( FILE *fp) -{ - pthread_mutex_lock(&m_lock); - print_impl(fp); - pthread_mutex_unlock(&m_lock); +void stream_manager::print(FILE *fp) { + pthread_mutex_lock(&m_lock); + print_impl(fp); + pthread_mutex_unlock(&m_lock); } -void stream_manager::print_impl( FILE *fp) -{ - fprintf(fp,"GPGPU-Sim API: Stream Manager State\n"); - std::list::iterator s; - for( s=m_streams.begin(); s!=m_streams.end();++s ) { - struct CUstream_st *stream = *s; - if( !stream->empty() ) - stream->print(fp); - } - if( !m_stream_zero.empty() ) - m_stream_zero.print(fp); +void stream_manager::print_impl(FILE *fp) { + fprintf(fp, "GPGPU-Sim API: Stream Manager State\n"); + std::list::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); ++s) { + struct CUstream_st *stream = *s; + if (!stream->empty()) stream->print(fp); + } + if (!m_stream_zero.empty()) m_stream_zero.print(fp); } -void stream_manager::push( stream_operation op ) -{ - struct CUstream_st *stream = op.get_stream(); - - // block if stream 0 (or concurrency disabled) and pending concurrent operations exist - bool block= !stream || m_cuda_launch_blocking; - while(block) { - pthread_mutex_lock(&m_lock); - block = !concurrent_streams_empty(); - pthread_mutex_unlock(&m_lock); - }; +void stream_manager::push(stream_operation op) { + struct CUstream_st *stream = op.get_stream(); + // block if stream 0 (or concurrency disabled) and pending concurrent + // operations exist + bool block = !stream || m_cuda_launch_blocking; + while (block) { pthread_mutex_lock(&m_lock); - if(!m_gpu->cycle_insn_cta_max_hit()) { - // Accept the stream operation if the maximum cycle/instruction/cta counts are not triggered - if( stream && !m_cuda_launch_blocking ) { - stream->push(op); - } else { - op.set_stream(&m_stream_zero); - m_stream_zero.push(op); - } - }else { - // Otherwise, ignore operation and continue - printf("GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. Skipping:"); - op.print(stdout); - printf("\n"); - } - if(g_debug_execution >= 3) - print_impl(stdout); + block = !concurrent_streams_empty(); pthread_mutex_unlock(&m_lock); - if( m_cuda_launch_blocking || stream == NULL ) { - unsigned int wait_amount = 100; - unsigned int wait_cap = 100000; // 100ms - while( !empty() ) { - // sleep to prevent CPU hog by empty spin - // sleep time increased exponentially ensure fast response when needed - usleep(wait_amount); - wait_amount *= 2; - if (wait_amount > wait_cap) - wait_amount = wait_cap; - } + }; + + pthread_mutex_lock(&m_lock); + if (!m_gpu->cycle_insn_cta_max_hit()) { + // Accept the stream operation if the maximum cycle/instruction/cta counts + // are not triggered + if (stream && !m_cuda_launch_blocking) { + stream->push(op); + } else { + op.set_stream(&m_stream_zero); + m_stream_zero.push(op); } + } else { + // Otherwise, ignore operation and continue + printf( + "GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. " + "Skipping:"); + op.print(stdout); + printf("\n"); + } + if (g_debug_execution >= 3) print_impl(stdout); + pthread_mutex_unlock(&m_lock); + if (m_cuda_launch_blocking || stream == NULL) { + unsigned int wait_amount = 100; + unsigned int wait_cap = 100000; // 100ms + while (!empty()) { + // sleep to prevent CPU hog by empty spin + // sleep time increased exponentially ensure fast response when needed + usleep(wait_amount); + wait_amount *= 2; + if (wait_amount > wait_cap) wait_amount = wait_cap; + } + } } -void stream_manager::pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ){ - std::list::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++ ) { - stream_operation op(*s,e,flags); - push(op); - } +void stream_manager::pushCudaStreamWaitEventToAllStreams(CUevent_st *e, + unsigned int flags) { + std::list::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + stream_operation op(*s, e, flags); + push(op); + } } diff --git a/src/stream_manager.h b/src/stream_manager.h index 3fbdbaf..88e28c8 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,248 +30,249 @@ #ifndef STREAM_MANAGER_H_INCLUDED #define STREAM_MANAGER_H_INCLUDED -#include "abstract_hardware_model.h" -#include #include #include +#include +#include "abstract_hardware_model.h" -//class stream_barrier { -//public: +// class stream_barrier { +// public: // stream_barrier() { m_pending_streams=0; } // void inc() { m_pending_streams++; } // void dec() { assert(m_pending_streams); m_pending_streams--; } // unsigned value() const { return m_pending_streams; } -//private: +// private: // unsigned m_pending_streams; //}; enum stream_operation_type { - stream_no_op, - stream_memcpy_host_to_device, - stream_memcpy_device_to_host, - stream_memcpy_device_to_device, - stream_memcpy_to_symbol, - stream_memcpy_from_symbol, - stream_kernel_launch, - stream_event, - stream_wait_event + stream_no_op, + stream_memcpy_host_to_device, + stream_memcpy_device_to_host, + stream_memcpy_device_to_device, + stream_memcpy_to_symbol, + stream_memcpy_from_symbol, + stream_kernel_launch, + stream_event, + stream_wait_event }; class stream_operation { -public: - stream_operation() - { - m_kernel=NULL; - m_type = stream_no_op; - m_stream = NULL; - m_done=true; - } - stream_operation( const void *src, const char *symbol, size_t count, size_t offset, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_stream = stream; - m_type=stream_memcpy_to_symbol; - m_host_address_src=src; - m_symbol=symbol; - m_cnt=count; - m_offset=offset; - m_done=false; - } - stream_operation( const char *symbol, void *dst, size_t count, size_t offset, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_stream = stream; - m_type=stream_memcpy_from_symbol; - m_host_address_dst=dst; - m_symbol=symbol; - m_cnt=count; - m_offset=offset; - m_done=false; - } - stream_operation( kernel_info_t *kernel, bool sim_mode, struct CUstream_st *stream ) - { - m_type=stream_kernel_launch; - m_kernel=kernel; - m_sim_mode=sim_mode; - m_stream=stream; - m_done=false; - } - stream_operation( struct CUevent_st *e, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_event; - m_event=e; - m_stream=stream; - m_done=false; - } - stream_operation( struct CUstream_st *stream, class CUevent_st *e, unsigned int flags ) - { - m_kernel=NULL; - m_type=stream_wait_event; - m_event=e; - m_stream=stream; - m_done=false; - } - stream_operation( const void *host_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_host_to_device; - m_host_address_src =host_address_src; - m_device_address_dst=device_address_dst; - m_host_address_dst=NULL; - m_device_address_src=0; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } - stream_operation( size_t device_address_src, void *host_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_device_to_host; - m_device_address_src=device_address_src; - m_host_address_dst=host_address_dst; - m_device_address_dst=0; - m_host_address_src=NULL; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } - stream_operation( size_t device_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_device_to_device; - m_device_address_src=device_address_src; - m_device_address_dst=device_address_dst; - m_host_address_src=NULL; - m_host_address_dst=NULL; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } + public: + stream_operation() { + m_kernel = NULL; + m_type = stream_no_op; + m_stream = NULL; + m_done = true; + } + stream_operation(const void *src, const char *symbol, size_t count, + size_t offset, struct CUstream_st *stream) { + m_kernel = NULL; + m_stream = stream; + m_type = stream_memcpy_to_symbol; + m_host_address_src = src; + m_symbol = symbol; + m_cnt = count; + m_offset = offset; + m_done = false; + } + stream_operation(const char *symbol, void *dst, size_t count, size_t offset, + struct CUstream_st *stream) { + m_kernel = NULL; + m_stream = stream; + m_type = stream_memcpy_from_symbol; + m_host_address_dst = dst; + m_symbol = symbol; + m_cnt = count; + m_offset = offset; + m_done = false; + } + stream_operation(kernel_info_t *kernel, bool sim_mode, + struct CUstream_st *stream) { + m_type = stream_kernel_launch; + m_kernel = kernel; + m_sim_mode = sim_mode; + m_stream = stream; + m_done = false; + } + stream_operation(struct CUevent_st *e, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_event; + m_event = e; + m_stream = stream; + m_done = false; + } + stream_operation(struct CUstream_st *stream, class CUevent_st *e, + unsigned int flags) { + m_kernel = NULL; + m_type = stream_wait_event; + m_event = e; + m_stream = stream; + m_done = false; + } + stream_operation(const void *host_address_src, size_t device_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_host_to_device; + m_host_address_src = host_address_src; + m_device_address_dst = device_address_dst; + m_host_address_dst = NULL; + m_device_address_src = 0; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } + stream_operation(size_t device_address_src, void *host_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_device_to_host; + m_device_address_src = device_address_src; + m_host_address_dst = host_address_dst; + m_device_address_dst = 0; + m_host_address_src = NULL; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } + stream_operation(size_t device_address_src, size_t device_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_device_to_device; + m_device_address_src = device_address_src; + m_device_address_dst = device_address_dst; + m_host_address_src = NULL; + m_host_address_dst = NULL; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } - bool is_kernel() const { return m_type == stream_kernel_launch; } - bool is_mem() const { - return m_type == stream_memcpy_host_to_device || - m_type == stream_memcpy_device_to_host || - m_type == stream_memcpy_host_to_device; - } - bool is_noop() const { return m_type == stream_no_op; } - bool is_done() const { return m_done; } - kernel_info_t *get_kernel() { return m_kernel; } - bool do_operation( gpgpu_sim *gpu ); - void print( FILE *fp ) const; - struct CUstream_st *get_stream() { return m_stream; } - void set_stream( CUstream_st *stream ) { m_stream = stream; } + bool is_kernel() const { return m_type == stream_kernel_launch; } + bool is_mem() const { + return m_type == stream_memcpy_host_to_device || + m_type == stream_memcpy_device_to_host || + m_type == stream_memcpy_host_to_device; + } + bool is_noop() const { return m_type == stream_no_op; } + bool is_done() const { return m_done; } + kernel_info_t *get_kernel() { return m_kernel; } + bool do_operation(gpgpu_sim *gpu); + void print(FILE *fp) const; + struct CUstream_st *get_stream() { + return m_stream; + } + void set_stream(CUstream_st *stream) { m_stream = stream; } -private: - struct CUstream_st *m_stream; + private: + struct CUstream_st *m_stream; - bool m_done; + bool m_done; - stream_operation_type m_type; - size_t m_device_address_dst; - size_t m_device_address_src; - void *m_host_address_dst; - const void *m_host_address_src; - size_t m_cnt; + stream_operation_type m_type; + size_t m_device_address_dst; + size_t m_device_address_src; + void *m_host_address_dst; + const void *m_host_address_src; + size_t m_cnt; - const char *m_symbol; - size_t m_offset; + const char *m_symbol; + size_t m_offset; - bool m_sim_mode; - kernel_info_t *m_kernel; - struct CUevent_st *m_event; + bool m_sim_mode; + kernel_info_t *m_kernel; + struct CUevent_st *m_event; }; struct CUevent_st { -public: - CUevent_st( bool blocking ) - { - m_uid = ++m_next_event_uid; - m_blocking = blocking; - m_updates = 0; - m_wallclock = 0; - m_gpu_tot_sim_cycle = 0; - m_done = false; - } - void update( double cycle, time_t clk ) - { - m_updates++; - m_wallclock=clk; - m_gpu_tot_sim_cycle=cycle; - m_done = true; - } - //void set_done() { assert(!m_done); m_done=true; } - int get_uid() const { return m_uid; } - unsigned num_updates() const { return m_updates; } - bool done() const { return m_done; } - time_t clock() const { return m_wallclock; } -private: - int m_uid; - bool m_blocking; - bool m_done; - int m_updates; - time_t m_wallclock; - double m_gpu_tot_sim_cycle; + public: + CUevent_st(bool blocking) { + m_uid = ++m_next_event_uid; + m_blocking = blocking; + m_updates = 0; + m_wallclock = 0; + m_gpu_tot_sim_cycle = 0; + m_done = false; + } + void update(double cycle, time_t clk) { + m_updates++; + m_wallclock = clk; + m_gpu_tot_sim_cycle = cycle; + m_done = true; + } + // void set_done() { assert(!m_done); m_done=true; } + int get_uid() const { return m_uid; } + unsigned num_updates() const { return m_updates; } + bool done() const { return m_done; } + time_t clock() const { return m_wallclock; } + + private: + int m_uid; + bool m_blocking; + bool m_done; + int m_updates; + time_t m_wallclock; + double m_gpu_tot_sim_cycle; - static int m_next_event_uid; + static int m_next_event_uid; }; struct CUstream_st { -public: - CUstream_st(); - bool empty(); - bool busy(); - void synchronize(); - void push( const stream_operation &op ); - void record_next_done(); - stream_operation next(); - void cancel_front(); //front operation fails, cancle the pending status - stream_operation &front() { return m_operations.front(); } - void print( FILE *fp ); - unsigned get_uid() const { return m_uid; } + public: + CUstream_st(); + bool empty(); + bool busy(); + void synchronize(); + void push(const stream_operation &op); + void record_next_done(); + stream_operation next(); + void cancel_front(); // front operation fails, cancle the pending status + stream_operation &front() { return m_operations.front(); } + void print(FILE *fp); + unsigned get_uid() const { return m_uid; } -private: - unsigned m_uid; - static unsigned sm_next_stream_uid; + private: + unsigned m_uid; + static unsigned sm_next_stream_uid; - std::list m_operations; - bool m_pending; // front operation has started but not yet completed + std::list m_operations; + bool m_pending; // front operation has started but not yet completed - pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream operation at one time + pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream + // operation at one time }; class stream_manager { -public: - stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ); - bool register_finished_kernel(unsigned grid_uid ); - bool check_finished_kernel( ); - stream_operation front(); - void add_stream( CUstream_st *stream ); - void destroy_stream( CUstream_st *stream ); - bool concurrent_streams_empty(); - bool empty_protected(); - bool empty(); - void print( FILE *fp); - void push( stream_operation op ); - void pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ); - bool operation(bool * sim); - void stop_all_running_kernels(); - unsigned size() {return m_streams.size(); }; - bool is_blocking() {return m_cuda_launch_blocking; }; -private: - void print_impl( FILE *fp); + public: + stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking); + bool register_finished_kernel(unsigned grid_uid); + bool check_finished_kernel(); + stream_operation front(); + void add_stream(CUstream_st *stream); + void destroy_stream(CUstream_st *stream); + bool concurrent_streams_empty(); + bool empty_protected(); + bool empty(); + void print(FILE *fp); + void push(stream_operation op); + void pushCudaStreamWaitEventToAllStreams(CUevent_st *e, unsigned int flags); + bool operation(bool *sim); + void stop_all_running_kernels(); + unsigned size() { return m_streams.size(); }; + bool is_blocking() { return m_cuda_launch_blocking; }; + + private: + void print_impl(FILE *fp); - bool m_cuda_launch_blocking; - gpgpu_sim *m_gpu; - std::list m_streams; - std::map m_grid_id_to_stream; - CUstream_st m_stream_zero; - bool m_service_stream_zero; - pthread_mutex_t m_lock; + bool m_cuda_launch_blocking; + gpgpu_sim *m_gpu; + std::list m_streams; + std::map m_grid_id_to_stream; + CUstream_st m_stream_zero; + bool m_service_stream_zero; + pthread_mutex_t m_lock; }; #endif diff --git a/src/tr1_hash_map.h b/src/tr1_hash_map.h index 8c7513e..0672ef8 100644 --- a/src/tr1_hash_map.h +++ b/src/tr1_hash_map.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -25,26 +27,27 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once // detection and fallback for unordered_map in C++0x #ifdef __cplusplus - // detect GCC 4.3 or later and use unordered map (part of C++0x) - // unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its enabled. - #if defined( __GNUC__ ) and not defined( _GLIBCXX_DEBUG ) - #if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 - #include - #define tr1_hash_map std::unordered_map - #define tr1_hash_map_ismap 0 - #else - #include - #define tr1_hash_map std::map - #define tr1_hash_map_ismap 1 - #endif - #else - #include - #define tr1_hash_map std::map - #define tr1_hash_map_ismap 1 - #endif +// detect GCC 4.3 or later and use unordered map (part of C++0x) +// unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its +// enabled. +#if defined(__GNUC__) and not defined(_GLIBCXX_DEBUG) +#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 +#include +#define tr1_hash_map std::unordered_map +#define tr1_hash_map_ismap 0 +#else +#include +#define tr1_hash_map std::map +#define tr1_hash_map_ismap 1 +#endif +#else +#include +#define tr1_hash_map std::map +#define tr1_hash_map_ismap 1 +#endif #endif diff --git a/src/trace.cc b/src/trace.cc index 5171e46..b5c2524 100644 --- a/src/trace.cc +++ b/src/trace.cc @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,27 +32,27 @@ namespace Trace { - #define TS_TUP_BEGIN(X) const char* trace_streams_str[] = { #define TS_TUP(X) #X -#define TS_TUP_END(X) }; +#define TS_TUP_END(X) \ + } \ + ; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END - bool enabled = false; - int sampling_core = 0; - int sampling_memory_partition = -1; - bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; - const char* config_str; +bool enabled = false; +int sampling_core = 0; +int sampling_memory_partition = -1; +bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; +const char* config_str; - void init() - { - for ( unsigned i = 0; i < NUM_TRACE_STREAMS; ++i ) { - if ( strstr( config_str, trace_streams_str[i] ) != NULL ) { - trace_streams_enabled[ i ] = true; - } - } +void init() { + for (unsigned i = 0; i < NUM_TRACE_STREAMS; ++i) { + if (strstr(config_str, trace_streams_str[i]) != NULL) { + trace_streams_enabled[i] = true; } -} + } +} +} diff --git a/src/trace.h b/src/trace.h index 0b96dcf..c4e0001 100644 --- a/src/trace.h +++ b/src/trace.h @@ -7,14 +7,16 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this +// Redistributions in binary form must reproduce the above copyright notice, +// this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -35,52 +37,57 @@ namespace Trace { #define TS_TUP_BEGIN(X) enum X { #define TS_TUP(X) X -#define TS_TUP_END(X) }; +#define TS_TUP_END(X) \ + } \ + ; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END - extern bool enabled; - extern int sampling_core; - extern int sampling_memory_partition; - extern const char* trace_streams_str[]; - extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; - extern const char* config_str; +extern bool enabled; +extern int sampling_core; +extern int sampling_memory_partition; +extern const char* trace_streams_str[]; +extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; +extern const char* config_str; - void init(); - -} // namespace Trace +void init(); +} // namespace Trace #if TRACING_ON #define SIM_PRINT_STR "GPGPU-Sim Cycle %llu: %s - " #define DTRACE(x) ((Trace::trace_streams_enabled[Trace::x]) && Trace::enabled) -#define DPRINTF(x, ...) do {\ - if (DTRACE(x)) {\ - printf( SIM_PRINT_STR,\ - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::x] );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define DPRINTF(x, ...) \ + do { \ + if (DTRACE(x)) { \ + printf(SIM_PRINT_STR, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::x]); \ + printf(__VA_ARGS__); \ + } \ + } while (0) -#define DPRINTFG(x, ...) do {\ - if (DTRACE(x)) {\ - printf( SIM_PRINT_STR,\ - gpu_sim_cycle + gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::x] );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define DPRINTFG(x, ...) \ + do { \ + if (DTRACE(x)) { \ + printf(SIM_PRINT_STR, gpu_sim_cycle + gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::x]); \ + printf(__VA_ARGS__); \ + } \ + } while (0) -#else +#else #define DTRACE(x) (false) -#define DPRINTF(x, ...) do {} while (0) -#define DPRINTFG(x, ...) do {} while (0) +#define DPRINTF(x, ...) \ + do { \ + } while (0) +#define DPRINTFG(x, ...) \ + do { \ + } while (0) -#endif +#endif -#endif +#endif -- cgit v1.3 From ae63df6d86f995876f38ab3cdf47eac61ccda8cd Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 07:48:08 -0400 Subject: Revert "Add src/ director formatting" This reverts commit 26ca8de4a6ec9bfe422a14cbe325a5f257df453b. --- src/abstract_hardware_model.cc | 2033 ++++++++++++++++++----------------- src/abstract_hardware_model.h | 2276 ++++++++++++++++++++-------------------- src/debug.cc | 350 +++--- src/debug.h | 91 +- src/gpgpusim_entrypoint.cc | 439 ++++---- src/gpgpusim_entrypoint.h | 70 +- src/option_parser.cc | 894 ++++++++-------- src/option_parser.h | 50 +- src/statwrapper.cc | 59 +- src/statwrapper.h | 14 +- src/stream_manager.cc | 740 ++++++------- src/stream_manager.h | 423 ++++---- src/tr1_hash_map.h | 43 +- src/trace.cc | 36 +- src/trace.h | 73 +- 15 files changed, 3766 insertions(+), 3825 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 33a7080..07232ee 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,1192 +25,1185 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + #include "abstract_hardware_model.h" -#include -#include -#include -#include -#include "../libcuda/gpgpu_context.h" -#include "cuda-sim/cuda-sim.h" #include "cuda-sim/memory.h" -#include "cuda-sim/ptx-stats.h" #include "cuda-sim/ptx_ir.h" +#include "cuda-sim/ptx-stats.h" +#include "cuda-sim/cuda-sim.h" #include "gpgpu-sim/gpu-sim.h" -#include "gpgpusim_entrypoint.h" #include "option_parser.h" +#include "gpgpusim_entrypoint.h" +#include +#include +#include +#include +#include "../libcuda/gpgpu_context.h" -void mem_access_t::init(gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_uid = ++(gpgpu_ctx->sm_next_access_uid); - m_addr = 0; - m_req_size = 0; +void mem_access_t::init(gpgpu_context* ctx) +{ + gpgpu_ctx = ctx; + m_uid=++(gpgpu_ctx->sm_next_access_uid); + m_addr=0; + m_req_size=0; } -void warp_inst_t::issue(const active_mask_t &mask, unsigned warp_id, - unsigned long long cycle, int dynamic_warp_id, - int sch_id) { - m_warp_active_mask = mask; - m_warp_issued_mask = mask; - m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); - m_warp_id = warp_id; - m_dynamic_warp_id = dynamic_warp_id; - issue_cycle = cycle; - cycles = initiation_interval; - m_cache_hit = false; - m_empty = false; - m_scheduler_id = sch_id; +void warp_inst_t::issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ) +{ + m_warp_active_mask = mask; + m_warp_issued_mask = mask; + m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); + m_warp_id = warp_id; + m_dynamic_warp_id = dynamic_warp_id; + issue_cycle = cycle; + cycles = initiation_interval; + m_cache_hit=false; + m_empty=false; + m_scheduler_id=sch_id; } -checkpoint::checkpoint() { - struct stat st = {0}; +checkpoint::checkpoint() +{ - if (stat("checkpoint_files", &st) == -1) { - mkdir("checkpoint_files", 0777); - } -} -void checkpoint::load_global_mem(class memory_space *temp_mem, char *f1name) { - FILE *fp2 = fopen(f1name, "r"); - assert(fp2 != NULL); - char line[128]; /* or other suitable maximum line size */ - unsigned int offset; - while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ - { - unsigned int index; - char *pch; - pch = strtok(line, " "); - if (pch[0] == 'g' || pch[0] == 's' || pch[0] == 'l') { - pch = strtok(NULL, " "); - - std::stringstream ss; - ss << std::hex << pch; - ss >> index; - - offset = 0; - } else { - unsigned int data; - std::stringstream ss; - ss << std::hex << pch; - ss >> data; - temp_mem->write_only(offset, index, 4, &data); - offset = offset + 4; + struct stat st = {0}; + + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); } - // fputs ( line, stdout ); /* write the line */ - } - fclose(fp2); + +} +void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) +{ + + FILE * fp2 = fopen(f1name, "r"); + assert(fp2!=NULL); + char line [ 128 ]; /* or other suitable maximum line size */ + unsigned int offset ; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + unsigned int index; + char * pch; + pch = strtok (line," "); + if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') + { + + pch = strtok (NULL, " "); + + std::stringstream ss; + ss << std::hex << pch; + ss >> index; + + offset=0; + } + else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset,index, 4,&data); + offset= offset+4; + } + //fputs ( line, stdout ); /* write the line */ + } + fclose ( fp2 ); } -void checkpoint::store_global_mem(class memory_space *mem, char *fname, - char *format) { - FILE *fp3 = fopen(fname, "w"); - assert(fp3 != NULL); - mem->print(format, fp3); - fclose(fp3); +void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) +{ + + FILE * fp3 = fopen(fname, "w"); + assert(fp3!=NULL); + mem->print(format,fp3); + fclose(fp3); } -void move_warp(warp_inst_t *&dst, warp_inst_t *&src) { - assert(dst->empty()); - warp_inst_t *temp = dst; - dst = src; - src = temp; - src->clear(); +void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) +{ + assert( dst->empty() ); + warp_inst_t* temp = dst; + dst = src; + src = temp; + src->clear(); } -void gpgpu_functional_sim_config::reg_options(class OptionParser *opp) { - option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, - &m_ptx_use_cuobjdump, - "Use cuobjdump to extract ptx and sass from binaries", + +void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) +{ + option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, + &m_ptx_use_cuobjdump, + "Use cuobjdump to extract ptx and sass from binaries", #if (CUDART_VERSION >= 4000) - "1" + "1" #else - "0" + "0" #endif - ); - option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, - &m_experimental_lib_support, - "Try to extract code from cuda libraries [Broken " - "because of unknown cudaGetExportTable]", - "0"); - option_parser_register(opp, "-checkpoint_option", OPT_INT32, - &checkpoint_option, - " checkpointing flag (0 = no checkpoint)", "0"); - option_parser_register( - opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, - " checkpointing during execution of which kernel (1- 1st kernel)", "1"); - option_parser_register( - opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, - " checkpointing after # of CTA (< less than total CTA)", "0"); - option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, - " resume flag (0 = no resume)", "0"); - option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, - " Resume from which kernel (1= 1st kernel)", "0"); - option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, - " resume from which CTA ", "0"); - option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, - " resume from which CTA ", "0"); - option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, - &checkpoint_insn_Y, " resume from which CTA ", "0"); - - option_parser_register( - opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, - "Convert SASS (native ISA) to ptxplus and run ptxplus", "0"); - option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, - &m_ptx_force_max_capability, - "Force maximum compute capability", "0"); - option_parser_register( - opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, &g_ptx_inst_debug_to_file, - "Dump executed instructions' debug information to file", "0"); - option_parser_register( - opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, - "Executed instructions' debug output file", "inst_debug.txt"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, - &g_ptx_inst_debug_thread_uid, - "Thread UID for executed instructions' debug output", - "1"); + ); + option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, + &m_experimental_lib_support, + "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", + "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", + "0"); + option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", + "1"); + option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", + "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", + "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", + "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, + " resume from which CTA ", + "0"); + + option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, + &m_ptx_convert_to_ptxplus, + "Convert SASS (native ISA) to ptxplus and run ptxplus", + "0"); + option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, + &m_ptx_force_max_capability, + "Force maximum compute capability", + "0"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, + &g_ptx_inst_debug_to_file, + "Dump executed instructions' debug information to file", + "0"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, + "Executed instructions' debug output file", + "inst_debug.txt"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, &g_ptx_inst_debug_thread_uid, + "Thread UID for executed instructions' debug output", + "1"); } -void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize( - unsigned linesize) { - m_texcache_linesize = linesize; +void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize(unsigned linesize) +{ + m_texcache_linesize = linesize; } -gpgpu_t::gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx) - : m_function_model_config(config) { - gpgpu_ctx = ctx; - m_global_mem = new memory_space_impl<8192>("global", 64 * 1024); - - m_tex_mem = new memory_space_impl<8192>("tex", 64 * 1024); - m_surf_mem = new memory_space_impl<8192>("surf", 64 * 1024); - - m_dev_malloc = GLOBAL_HEAP_START; - checkpoint_option = m_function_model_config.get_checkpoint_option(); - checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); - checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); - resume_option = m_function_model_config.get_resume_option(); - resume_kernel = m_function_model_config.get_resume_kernel(); - resume_CTA = m_function_model_config.get_resume_CTA(); - checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); - checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); - - // initialize texture mappings to empty - m_NameToTextureInfo.clear(); - m_NameToCudaArray.clear(); - m_TextureRefToName.clear(); - m_NameToAttribute.clear(); - - if (m_function_model_config.get_ptx_inst_debug_to_file() != 0) - ptx_inst_debug_file = - fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); - - gpu_sim_cycle = 0; - gpu_tot_sim_cycle = 0; +gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ) + : m_function_model_config(config) +{ + gpgpu_ctx = ctx; + m_global_mem = new memory_space_impl<8192>("global",64*1024); + + m_tex_mem = new memory_space_impl<8192>("tex",64*1024); + m_surf_mem = new memory_space_impl<8192>("surf",64*1024); + + m_dev_malloc=GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + + // initialize texture mappings to empty + m_NameToTextureInfo.clear(); + m_NameToCudaArray.clear(); + m_TextureRefToName.clear(); + m_NameToAttribute.clear(); + + if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) + ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); + + gpu_sim_cycle=0; + gpu_tot_sim_cycle=0; } -address_type line_size_based_tag_func(new_addr_type address, - new_addr_type line_size) { - // gives the tag for an address based on a given line size - return address & ~(line_size - 1); +address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_size) +{ + //gives the tag for an address based on a given line size + return address & ~(line_size-1); } -const char *mem_access_type_str(enum mem_access_type access_type) { -#define MA_TUP_BEGIN(X) static const char *access_type_str[] = { -#define MA_TUP(X) #X -#define MA_TUP_END(X) \ - } \ - ; - MEM_ACCESS_TYPE_TUP_DEF -#undef MA_TUP_BEGIN -#undef MA_TUP -#undef MA_TUP_END +const char * mem_access_type_str(enum mem_access_type access_type) +{ + #define MA_TUP_BEGIN(X) static const char* access_type_str[] = { + #define MA_TUP(X) #X + #define MA_TUP_END(X) }; + MEM_ACCESS_TYPE_TUP_DEF + #undef MA_TUP_BEGIN + #undef MA_TUP + #undef MA_TUP_END - assert(access_type < NUM_MEM_ACCESS_TYPE); + assert(access_type < NUM_MEM_ACCESS_TYPE); - return access_type_str[access_type]; + return access_type_str[access_type]; } -void warp_inst_t::clear_active(const active_mask_t &inactive) { - active_mask_t test = m_warp_active_mask; - test &= inactive; - assert(test == inactive); // verify threads being disabled were active - m_warp_active_mask &= ~inactive; + +void warp_inst_t::clear_active( const active_mask_t &inactive ) { + active_mask_t test = m_warp_active_mask; + test &= inactive; + assert( test == inactive ); // verify threads being disabled were active + m_warp_active_mask &= ~inactive; } -void warp_inst_t::set_not_active(unsigned lane_id) { - m_warp_active_mask.reset(lane_id); +void warp_inst_t::set_not_active( unsigned lane_id ) { + m_warp_active_mask.reset(lane_id); } -void warp_inst_t::set_active(const active_mask_t &active) { - m_warp_active_mask = active; - if (m_isatomic) { - for (unsigned i = 0; i < m_config->warp_size; i++) { - if (!m_warp_active_mask.test(i)) { - m_per_scalar_thread[i].callback.function = NULL; - m_per_scalar_thread[i].callback.instruction = NULL; - m_per_scalar_thread[i].callback.thread = NULL; +void warp_inst_t::set_active( const active_mask_t &active ) { + m_warp_active_mask = active; + if( m_isatomic ) { + for( unsigned i=0; i < m_config->warp_size; i++ ) { + if( !m_warp_active_mask.test(i) ) { + m_per_scalar_thread[i].callback.function = NULL; + m_per_scalar_thread[i].callback.instruction = NULL; + m_per_scalar_thread[i].callback.thread = NULL; + } } - } - } + } } void warp_inst_t::do_atomic(bool forceDo) { - do_atomic(m_warp_active_mask, forceDo); + do_atomic( m_warp_active_mask,forceDo ); } -void warp_inst_t::do_atomic(const active_mask_t &access_mask, bool forceDo) { - assert(m_isatomic && (!m_empty || forceDo)); - for (unsigned i = 0; i < m_config->warp_size; i++) { - if (access_mask.test(i)) { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if (cb.thread) cb.function(cb.instruction, cb.thread); + +void warp_inst_t::do_atomic( const active_mask_t& access_mask,bool forceDo ) { + assert( m_isatomic && (!m_empty||forceDo) ); + for( unsigned i=0; i < m_config->warp_size; i++ ) + { + if( access_mask.test(i) ) + { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if( cb.thread ) + cb.function(cb.instruction, cb.thread); + } } - } } -void warp_inst_t::broadcast_barrier_reduction( - const active_mask_t &access_mask) { - for (unsigned i = 0; i < m_config->warp_size; i++) { - if (access_mask.test(i)) { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if (cb.thread) { - cb.function(cb.instruction, cb.thread); - } +void warp_inst_t::broadcast_barrier_reduction(const active_mask_t& access_mask) +{ + for( unsigned i=0; i < m_config->warp_size; i++ ) + { + if( access_mask.test(i) ) + { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if( cb.thread ){ + cb.function(cb.instruction, cb.thread); + } + } } - } } -void warp_inst_t::generate_mem_accesses() { - if (empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created) return; - if (!((op == LOAD_OP) || (op == TENSOR_CORE_LOAD_OP) || (op == STORE_OP) || - (op == TENSOR_CORE_STORE_OP))) - return; - if (m_warp_active_mask.count() == 0) return; // predicated off +void warp_inst_t::generate_mem_accesses() +{ + if( empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created ) + return; + if (!((op == LOAD_OP) || (op==TENSOR_CORE_LOAD_OP) || (op == STORE_OP)||(op==TENSOR_CORE_STORE_OP))) + return; + if( m_warp_active_mask.count() == 0 ) + return; // predicated off - const size_t starting_queue_size = m_accessq.size(); + const size_t starting_queue_size = m_accessq.size(); - assert(is_load() || is_store()); - assert(m_per_scalar_thread_valid); // need address information per thread + assert( is_load() || is_store() ); + assert( m_per_scalar_thread_valid ); // need address information per thread - bool is_write = is_store(); + bool is_write = is_store(); - mem_access_type access_type; - switch (space.get_type()) { + mem_access_type access_type; + switch (space.get_type()) { case const_space: - case param_space_kernel: - access_type = CONST_ACC_R; - break; - case tex_space: - access_type = TEXTURE_ACC_R; - break; - case global_space: - access_type = is_write ? GLOBAL_ACC_W : GLOBAL_ACC_R; - break; + case param_space_kernel: + access_type = CONST_ACC_R; + break; + case tex_space: + access_type = TEXTURE_ACC_R; + break; + case global_space: + access_type = is_write? GLOBAL_ACC_W: GLOBAL_ACC_R; + break; case local_space: - case param_space_local: - access_type = is_write ? LOCAL_ACC_W : LOCAL_ACC_R; - break; - case shared_space: - break; - case sstarr_space: - break; - default: - assert(0); - break; - } + case param_space_local: + access_type = is_write? LOCAL_ACC_W: LOCAL_ACC_R; + break; + case shared_space: break; + case sstarr_space: break; + default: assert(0); break; + } - // Calculate memory accesses generated by this warp - new_addr_type cache_block_size = 0; // in bytes + // Calculate memory accesses generated by this warp + new_addr_type cache_block_size = 0; // in bytes - switch (space.get_type()) { + switch( space.get_type() ) { case shared_space: case sstarr_space: { - unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; - unsigned total_accesses = 0; - for (unsigned subwarp = 0; subwarp < m_config->mem_warp_parts; - subwarp++) { - // data structures used per part warp - std::map > - bank_accs; // bank -> word address -> access count - - // step 1: compute accesses to words in banks - for (unsigned thread = subwarp * subwarp_size; - thread < (subwarp + 1) * subwarp_size; thread++) { - if (!active(thread)) continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - // FIXME: deferred allocation of shared memory should not accumulate - // across kernel launches - // assert( addr < m_config->gpgpu_shmem_size ); - unsigned bank = m_config->shmem_bank_func(addr); - new_addr_type word = - line_size_based_tag_func(addr, m_config->WORD_SIZE); - bank_accs[bank][word]++; - } - - if (m_config->shmem_limited_broadcast) { - // step 2: look for and select a broadcast bank/word if one occurs - bool broadcast_detected = false; - new_addr_type broadcast_word = (new_addr_type)-1; - unsigned broadcast_bank = (unsigned)-1; - std::map >::iterator b; - for (b = bank_accs.begin(); b != bank_accs.end(); b++) { - unsigned bank = b->first; - std::map &access_set = b->second; - std::map::iterator w; - for (w = access_set.begin(); w != access_set.end(); ++w) { - if (w->second > 1) { - // found a broadcast - broadcast_detected = true; - broadcast_bank = bank; - broadcast_word = w->first; - break; - } + unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; + unsigned total_accesses=0; + for( unsigned subwarp=0; subwarp < m_config->mem_warp_parts; subwarp++ ) { + + // data structures used per part warp + std::map > bank_accs; // bank -> word address -> access count + + // step 1: compute accesses to words in banks + for( unsigned thread=subwarp*subwarp_size; thread < (subwarp+1)*subwarp_size; thread++ ) { + if( !active(thread) ) + continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + //FIXME: deferred allocation of shared memory should not accumulate across kernel launches + //assert( addr < m_config->gpgpu_shmem_size ); + unsigned bank = m_config->shmem_bank_func(addr); + new_addr_type word = line_size_based_tag_func(addr,m_config->WORD_SIZE); + bank_accs[bank][word]++; } - if (broadcast_detected) break; - } - - // step 3: figure out max bank accesses performed, taking account of - // broadcast case - unsigned max_bank_accesses = 0; - for (b = bank_accs.begin(); b != bank_accs.end(); b++) { - unsigned bank_accesses = 0; - std::map &access_set = b->second; - std::map::iterator w; - for (w = access_set.begin(); w != access_set.end(); ++w) - bank_accesses += w->second; - if (broadcast_detected && broadcast_bank == b->first) { - for (w = access_set.begin(); w != access_set.end(); ++w) { - if (w->first == broadcast_word) { - unsigned n = w->second; - assert(n > 1); // or this wasn't a broadcast - assert(bank_accesses >= (n - 1)); - bank_accesses -= (n - 1); - break; + + if (m_config->shmem_limited_broadcast) { + // step 2: look for and select a broadcast bank/word if one occurs + bool broadcast_detected = false; + new_addr_type broadcast_word=(new_addr_type)-1; + unsigned broadcast_bank=(unsigned)-1; + std::map >::iterator b; + for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { + unsigned bank = b->first; + std::map &access_set = b->second; + std::map::iterator w; + for( w=access_set.begin(); w != access_set.end(); ++w ) { + if( w->second > 1 ) { + // found a broadcast + broadcast_detected=true; + broadcast_bank=bank; + broadcast_word=w->first; + break; + } + } + if( broadcast_detected ) + break; + } + + // step 3: figure out max bank accesses performed, taking account of broadcast case + unsigned max_bank_accesses=0; + for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { + unsigned bank_accesses=0; + std::map &access_set = b->second; + std::map::iterator w; + for( w=access_set.begin(); w != access_set.end(); ++w ) + bank_accesses += w->second; + if( broadcast_detected && broadcast_bank == b->first ) { + for( w=access_set.begin(); w != access_set.end(); ++w ) { + if( w->first == broadcast_word ) { + unsigned n = w->second; + assert(n > 1); // or this wasn't a broadcast + assert(bank_accesses >= (n-1)); + bank_accesses -= (n-1); + break; + } + } + } + if( bank_accesses > max_bank_accesses ) + max_bank_accesses = bank_accesses; } - } - } - if (bank_accesses > max_bank_accesses) - max_bank_accesses = bank_accesses; - } - // step 4: accumulate - total_accesses += max_bank_accesses; - } else { - // step 2: look for the bank with the maximum number of access to - // different words - unsigned max_bank_accesses = 0; - std::map >::iterator b; - for (b = bank_accs.begin(); b != bank_accs.end(); b++) { - max_bank_accesses = - std::max(max_bank_accesses, (unsigned)b->second.size()); - } - - // step 3: accumulate - total_accesses += max_bank_accesses; + // step 4: accumulate + total_accesses+= max_bank_accesses; + } else { + // step 2: look for the bank with the maximum number of access to different words + unsigned max_bank_accesses=0; + std::map >::iterator b; + for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { + max_bank_accesses = std::max(max_bank_accesses, (unsigned)b->second.size()); + } + + // step 3: accumulate + total_accesses+= max_bank_accesses; + } } - } - assert(total_accesses > 0 && total_accesses <= m_config->warp_size); - cycles = total_accesses; // shared memory conflicts modeled as larger - // initiation interval - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( - pc, total_accesses); - break; + assert( total_accesses > 0 && total_accesses <= m_config->warp_size ); + cycles = total_accesses; // shared memory conflicts modeled as larger initiation interval + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); + break; } - case tex_space: - cache_block_size = m_config->gpgpu_cache_texl1_linesize; - break; - case const_space: - case param_space_kernel: - cache_block_size = m_config->gpgpu_cache_constl1_linesize; - break; + case tex_space: + cache_block_size = m_config->gpgpu_cache_texl1_linesize; + break; + case const_space: case param_space_kernel: + cache_block_size = m_config->gpgpu_cache_constl1_linesize; + break; - case global_space: - case local_space: - case param_space_local: - if (m_config->gpgpu_coalesce_arch >= 13) { - if (isatomic()) - memory_coalescing_arch_atomic(is_write, access_type); - else - memory_coalescing_arch(is_write, access_type); - } else - abort(); + case global_space: case local_space: case param_space_local: + if( m_config->gpgpu_coalesce_arch >= 13) { + if(isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else abort(); - break; + break; default: - abort(); - } - - if (cache_block_size) { - assert(m_accessq.empty()); - mem_access_byte_mask_t byte_mask; - std::map - accesses; // block address -> set of thread offsets in warp - std::map::iterator a; - for (unsigned thread = 0; thread < m_config->warp_size; thread++) { - if (!active(thread)) continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr, cache_block_size); - accesses[block_address].set(thread); - unsigned idx = addr - block_address; - for (unsigned i = 0; i < data_size; i++) byte_mask.set(idx + i); + abort(); } - for (a = accesses.begin(); a != accesses.end(); ++a) - m_accessq.push_back(mem_access_t( - access_type, a->first, cache_block_size, is_write, a->second, - byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); - } - - if (space.get_type() == global_space) { - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( - pc, m_accessq.size() - starting_queue_size); - } - m_mem_accesses_created = true; -} -void warp_inst_t::memory_coalescing_arch(bool is_write, - mem_access_type access_type) { - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; - - if (m_config->gpgpu_coalesce_arch >= 20 && - m_config->gpgpu_coalesce_arch < 39) { - // Fermi and Kepler, L1 is normal and L2 is sector - if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } else if (m_config->gpgpu_coalesce_arch >= 40) { - // Maxwell, Pascal and Volta, L1 and L2 are sectors - // all requests should be 32 bytes - sector_segment_size = true; - } - - switch (data_size) { - case 1: - segment_size = 32; - break; - case 2: - segment_size = sector_segment_size ? 32 : 64; - break; - case 4: - case 8: - case 16: - segment_size = sector_segment_size ? 32 : 128; - break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { - std::map subwarp_transactions; - - // step 1: find all transactions generated by this subwarp - for (unsigned thread = subwarp * subwarp_size; - thread < subwarp_size * (subwarp + 1); thread++) { - if (!active(thread)) continue; - - unsigned data_size_coales = data_size; - unsigned num_accesses = 1; - - if (space.get_type() == local_space || - space.get_type() == param_space_local) { - // Local memory accesses >4B were split into 4B chunks - if (data_size >= 4) { - data_size_coales = 4; - num_accesses = data_size / 4; + if( cache_block_size ) { + assert( m_accessq.empty() ); + mem_access_byte_mask_t byte_mask; + std::map accesses; // block address -> set of thread offsets in warp + std::map::iterator a; + for( unsigned thread=0; thread < m_config->warp_size; thread++ ) { + if( !active(thread) ) + continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr,cache_block_size); + accesses[block_address].set(thread); + unsigned idx = addr-block_address; + for( unsigned i=0; i < data_size; i++ ) + byte_mask.set(idx+i); } - // Otherwise keep the same data_size for sub-4B access to local memory - } + for( a=accesses.begin(); a != accesses.end(); ++a ) + m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); + } - assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); - - // for(unsigned access=0; accessgpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); } + m_mem_accesses_created=true; +} - // step 2: reduce each transaction size, if possible - std::map::iterator t; - for (t = subwarp_transactions.begin(); t != subwarp_transactions.end(); - t++) { - new_addr_type addr = t->first; - const transaction_info &info = t->second; +void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_type ) +{ + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) + { + //Fermi and Kepler, L1 is normal and L2 is sector + if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } + else if(m_config->gpgpu_coalesce_arch >= 40) + { + //Maxwell, Pascal and Volta, L1 and L2 are sectors + //all requests should be 32 bytes + sector_segment_size = true; + } - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, - segment_size); + switch( data_size ) { + case 1: segment_size = 32; break; + case 2: segment_size = sector_segment_size? 32 : 64; break; + case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; } - } -} + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { + std::map subwarp_transactions; + + // step 1: find all transactions generated by this subwarp + for( unsigned thread=subwarp*subwarp_size; thread4B were split into 4B chunks + if(data_size >= 4) { + data_size_coales = 4; + num_accesses = data_size/4; + } + // Otherwise keep the same data_size for sub-4B access to local memory + } -void warp_inst_t::memory_coalescing_arch_atomic(bool is_write, - mem_access_type access_type) { - assert(space.get_type() == - global_space); // Atomics allowed only for global memory - - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; - - if (m_config->gpgpu_coalesce_arch >= 20 && - m_config->gpgpu_coalesce_arch < 39) { - // Fermi and Kepler, L1 is normal and L2 is sector - if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } else if (m_config->gpgpu_coalesce_arch >= 40) { - // Maxwell, Pascal and Volta, L1 and L2 are sectors - // all requests should be 32 bytes - sector_segment_size = true; - } - - switch (data_size) { - case 1: - segment_size = 32; - break; - case 2: - segment_size = sector_segment_size ? 32 : 64; - break; - case 4: - case 8: - case 16: - segment_size = sector_segment_size ? 32 : 128; - break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { - std::map > - subwarp_transactions; // each block addr maps to a list of transactions - - // step 1: find all transactions generated by this subwarp - for (unsigned thread = subwarp * subwarp_size; - thread < subwarp_size * (subwarp + 1); thread++) { - if (!active(thread)) continue; - - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr, segment_size); - unsigned chunk = (addr & 127) / 32; // which 32-byte chunk within in a - // 128-byte chunk does this thread - // access? - - // can only write to one segment - assert(block_address == - line_size_based_tag_func(addr + data_size - 1, segment_size)); - - // Find a transaction that does not conflict with this thread's accesses - bool new_transaction = true; - std::list::iterator it; - transaction_info *info; - for (it = subwarp_transactions[block_address].begin(); - it != subwarp_transactions[block_address].end(); it++) { - unsigned idx = (addr & 127); - if (not it->test_bytes(idx, idx + data_size - 1)) { - new_transaction = false; - info = &(*it); - break; + + assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); + +// for(unsigned access=0; accesschunks.set(chunk); - info->active.set(thread); - unsigned idx = (addr & 127); - for (unsigned i = 0; i < data_size; i++) { - assert(!info->bytes.test(idx + i)); - info->bytes.set(idx + i); - } - } - // step 2: reduce each transaction size, if possible - std::map >::iterator t_list; - for (t_list = subwarp_transactions.begin(); - t_list != subwarp_transactions.end(); t_list++) { - // For each block addr - new_addr_type addr = t_list->first; - const std::list &transaction_list = t_list->second; - - std::list::const_iterator t; - for (t = transaction_list.begin(); t != transaction_list.end(); t++) { - // For each transaction - const transaction_info &info = *t; - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, - addr, segment_size); - } + // step 2: reduce each transaction size, if possible + std::map< new_addr_type, transaction_info >::iterator t; + for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { + new_addr_type addr = t->first; + const transaction_info &info = t->second; + + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); + + } } - } } -void warp_inst_t::memory_coalescing_arch_reduce_and_send( - bool is_write, mem_access_type access_type, const transaction_info &info, - new_addr_type addr, unsigned segment_size) { - assert((addr & (segment_size - 1)) == 0); - - const std::bitset<4> &q = info.chunks; - assert(q.count() >= 1); - std::bitset<2> h; // halves (used to check if 64 byte segment can be - // compressed into a single 32 byte segment) - - unsigned size = segment_size; - if (segment_size == 128) { - bool lower_half_used = q[0] || q[1]; - bool upper_half_used = q[2] || q[3]; - if (lower_half_used && !upper_half_used) { - // only lower 64 bytes used - size = 64; - if (q[0]) h.set(0); - if (q[1]) h.set(1); - } else if ((!lower_half_used) && upper_half_used) { - // only upper 64 bytes used - addr = addr + 64; - size = 64; - if (q[2]) h.set(0); - if (q[3]) h.set(1); - } else { - assert(lower_half_used && upper_half_used); - } - } else if (segment_size == 64) { - // need to set halves - if ((addr % 128) == 0) { - if (q[0]) h.set(0); - if (q[1]) h.set(1); - } else { - assert((addr % 128) == 64); - if (q[2]) h.set(0); - if (q[3]) h.set(1); - } - } - if (size == 64) { - bool lower_half_used = h[0]; - bool upper_half_used = h[1]; - if (lower_half_used && !upper_half_used) { - size = 32; - } else if ((!lower_half_used) && upper_half_used) { - addr = addr + 32; - size = 32; - } else { - assert(lower_half_used && upper_half_used); - } - } - m_accessq.push_back(mem_access_t(access_type, addr, size, is_write, - info.active, info.bytes, info.chunks, - m_config->gpgpu_ctx)); +void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) +{ + + assert(space.get_type() == global_space); // Atomics allowed only for global memory + + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) + { + //Fermi and Kepler, L1 is normal and L2 is sector + if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } + else if(m_config->gpgpu_coalesce_arch >= 40) + { + //Maxwell, Pascal and Volta, L1 and L2 are sectors + //all requests should be 32 bytes + sector_segment_size = true; + } + + switch( data_size ) { + case 1: segment_size = 32; break; + case 2: segment_size = sector_segment_size? 32 : 64; break; + case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { + std::map > subwarp_transactions; // each block addr maps to a list of transactions + + // step 1: find all transactions generated by this subwarp + for( unsigned thread=subwarp*subwarp_size; thread::iterator it; + transaction_info* info; + for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) { + unsigned idx = (addr&127); + if(not it->test_bytes(idx,idx+data_size-1)) { + new_transaction = false; + info = &(*it); + break; + } + } + if(new_transaction) { + // Need a new transaction + subwarp_transactions[block_address].push_back(transaction_info()); + info = &subwarp_transactions[block_address].back(); + } + assert(info); + + info->chunks.set(chunk); + info->active.set(thread); + unsigned idx = (addr&127); + for( unsigned i=0; i < data_size; i++ ) { + assert(!info->bytes.test(idx+i)); + info->bytes.set(idx+i); + } + } + + // step 2: reduce each transaction size, if possible + std::map< new_addr_type, std::list >::iterator t_list; + for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) { + // For each block addr + new_addr_type addr = t_list->first; + const std::list& transaction_list = t_list->second; + + std::list::const_iterator t; + for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { + // For each transaction + const transaction_info &info = *t; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); + } + } + } +} + +void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) +{ + assert( (addr & (segment_size-1)) == 0 ); + + const std::bitset<4> &q = info.chunks; + assert( q.count() >= 1 ); + std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) + + unsigned size=segment_size; + if( segment_size == 128 ) { + bool lower_half_used = q[0] || q[1]; + bool upper_half_used = q[2] || q[3]; + if( lower_half_used && !upper_half_used ) { + // only lower 64 bytes used + size = 64; + if(q[0]) h.set(0); + if(q[1]) h.set(1); + } else if ( (!lower_half_used) && upper_half_used ) { + // only upper 64 bytes used + addr = addr+64; + size = 64; + if(q[2]) h.set(0); + if(q[3]) h.set(1); + } else { + assert(lower_half_used && upper_half_used); + } + } else if( segment_size == 64 ) { + // need to set halves + if( (addr % 128) == 0 ) { + if(q[0]) h.set(0); + if(q[1]) h.set(1); + } else { + assert( (addr % 128) == 64 ); + if(q[2]) h.set(0); + if(q[3]) h.set(1); + } + } + if( size == 64 ) { + bool lower_half_used = h[0]; + bool upper_half_used = h[1]; + if( lower_half_used && !upper_half_used ) { + size = 32; + } else if ( (!lower_half_used) && upper_half_used ) { + addr = addr+32; + size = 32; + } else { + assert(lower_half_used && upper_half_used); + } + } + m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks,m_config->gpgpu_ctx) ); } -void warp_inst_t::completed(unsigned long long cycle) const { - unsigned long long latency = cycle - issue_cycle; - assert(latency <= cycle); // underflow detection - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency( - pc, latency * active_count()); +void warp_inst_t::completed( unsigned long long cycle ) const +{ + unsigned long long latency = cycle - issue_cycle; + assert(latency <= cycle); // underflow detection + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency(pc, latency * active_count()); } -kernel_info_t::kernel_info_t(dim3 gridDim, dim3 blockDim, - class function_info *entry) { - m_kernel_entry = entry; - m_grid_dim = gridDim; - m_block_dim = blockDim; - m_next_cta.x = 0; - m_next_cta.y = 0; - m_next_cta.z = 0; - m_next_tid = m_next_cta; - m_num_cores_running = 0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); - - // Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - // Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set = false; + +kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry) +{ + m_kernel_entry=entry; + m_grid_dim=gridDim; + m_block_dim=blockDim; + m_next_cta.x=0; + m_next_cta.y=0; + m_next_cta.z=0; + m_next_tid=m_next_cta; + m_num_cores_running=0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param",64*1024); + + //Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + //Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set=false; } -/*A snapshot of the texture mappings needs to be stored in the kernel's info as +/*A snapshot of the texture mappings needs to be stored in the kernel's info as kernels should use the texture bindings seen at the time of launch and textures can be bound/unbound asynchronously with respect to streams. */ -kernel_info_t::kernel_info_t( - dim3 gridDim, dim3 blockDim, class function_info *entry, - std::map nameToCudaArray, - std::map nameToTextureInfo) { - m_kernel_entry = entry; - m_grid_dim = gridDim; - m_block_dim = blockDim; - m_next_cta.x = 0; - m_next_cta.y = 0; - m_next_cta.z = 0; - m_next_tid = m_next_cta; - m_num_cores_running = 0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); - - // Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - // Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set = false; - m_NameToCudaArray = nameToCudaArray; - m_NameToTextureInfo = nameToTextureInfo; +kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo) +{ + m_kernel_entry=entry; + m_grid_dim=gridDim; + m_block_dim=blockDim; + m_next_cta.x=0; + m_next_cta.y=0; + m_next_cta.z=0; + m_next_tid=m_next_cta; + m_num_cores_running=0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param",64*1024); + + //Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + //Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set=false; + m_NameToCudaArray = nameToCudaArray; + m_NameToTextureInfo = nameToTextureInfo; } -kernel_info_t::~kernel_info_t() { - assert(m_active_threads.empty()); - destroy_cta_streams(); - delete m_param_mem; +kernel_info_t::~kernel_info_t() +{ + assert( m_active_threads.empty() ); + destroy_cta_streams(); + delete m_param_mem; } -std::string kernel_info_t::name() const { return m_kernel_entry->get_name(); } +std::string kernel_info_t::name() const +{ + return m_kernel_entry->get_name(); +} -// Jin: parent and child kernel management for CDP -void kernel_info_t::set_parent(kernel_info_t *parent, dim3 parent_ctaid, - dim3 parent_tid) { - m_parent_kernel = parent; - m_parent_ctaid = parent_ctaid; - m_parent_tid = parent_tid; - parent->set_child(this); +//Jin: parent and child kernel management for CDP +void kernel_info_t::set_parent(kernel_info_t * parent, + dim3 parent_ctaid, dim3 parent_tid) { + m_parent_kernel = parent; + m_parent_ctaid = parent_ctaid; + m_parent_tid = parent_tid; + parent->set_child(this); } -void kernel_info_t::set_child(kernel_info_t *child) { - m_child_kernels.push_back(child); +void kernel_info_t::set_child(kernel_info_t * child) { + m_child_kernels.push_back(child); } -void kernel_info_t::remove_child(kernel_info_t *child) { - assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) != - m_child_kernels.end()); - m_child_kernels.remove(child); +void kernel_info_t::remove_child(kernel_info_t * child) { + assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) + != m_child_kernels.end()); + m_child_kernels.remove(child); } bool kernel_info_t::is_finished() { - if (done() && children_all_finished()) - return true; + if(done() && children_all_finished()) + return true; else - return false; + return false; } bool kernel_info_t::children_all_finished() { - if (!m_child_kernels.empty()) return false; - - return true; + if(!m_child_kernels.empty()) + return false; + + return true; } void kernel_info_t::notify_parent_finished() { - if (m_parent_kernel) { - m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= - ((m_kernel_entry->get_args_aligned_size() + 255) / 256 * 256); - m_parent_kernel->remove_child(this); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager - ->register_finished_kernel(m_parent_kernel->get_uid()); - } + if(m_parent_kernel) { + m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= ((m_kernel_entry->get_args_aligned_size() + 255)/256*256); + m_parent_kernel->remove_child(this); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(m_parent_kernel->get_uid()); + } } -CUstream_st *kernel_info_t::create_stream_cta(dim3 ctaid) { - assert(get_default_stream_cta(ctaid)); - CUstream_st *stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); - assert(m_cta_streams[ctaid].size() >= 1); // must have default stream - m_cta_streams[ctaid].push_back(stream); +CUstream_st * kernel_info_t::create_stream_cta(dim3 ctaid) { + assert(get_default_stream_cta(ctaid)); + CUstream_st * stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); + assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); + assert(m_cta_streams[ctaid].size() >= 1); //must have default stream + m_cta_streams[ctaid].push_back(stream); - return stream; + return stream; } -CUstream_st *kernel_info_t::get_default_stream_cta(dim3 ctaid) { - if (m_cta_streams.find(ctaid) != m_cta_streams.end()) { - assert(m_cta_streams[ctaid].size() >= - 1); // already created, must have default stream - return *(m_cta_streams[ctaid].begin()); - } else { - m_cta_streams[ctaid] = std::list(); - CUstream_st *stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream( - stream); - m_cta_streams[ctaid].push_back(stream); - return stream; - } +CUstream_st * kernel_info_t::get_default_stream_cta(dim3 ctaid) { + if(m_cta_streams.find(ctaid) != m_cta_streams.end()) { + assert(m_cta_streams[ctaid].size() >= 1); //already created, must have default stream + return *(m_cta_streams[ctaid].begin()); + } + else { + m_cta_streams[ctaid] = std::list(); + CUstream_st * stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); + m_cta_streams[ctaid].push_back(stream); + return stream; + } } -bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st *stream) { - if (m_cta_streams.find(ctaid) == m_cta_streams.end()) return false; +bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st* stream) { + if(m_cta_streams.find(ctaid) == m_cta_streams.end()) + return false; - std::list &stream_list = m_cta_streams[ctaid]; - if (std::find(stream_list.begin(), stream_list.end(), stream) == - stream_list.end()) - return false; - else - return true; + std::list &stream_list = m_cta_streams[ctaid]; + if(std::find(stream_list.begin(), stream_list.end(), stream) + == stream_list.end()) + return false; + else + return true; } void kernel_info_t::print_parent_info() { - if (m_parent_kernel) { - printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", - m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), - m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, m_parent_tid.x, - m_parent_tid.y, m_parent_tid.z); - } + if(m_parent_kernel) { + printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", + m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), + m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, + m_parent_tid.x, m_parent_tid.y, m_parent_tid.z); + } } void kernel_info_t::destroy_cta_streams() { - printf("Destroy streams for kernel %d: ", get_uid()); - size_t stream_size = 0; - for (auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { - stream_size += s->second.size(); - for (auto ss = s->second.begin(); ss != s->second.end(); ss++) - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream( - *ss); - s->second.clear(); - } - printf("size %lu\n", stream_size); - m_cta_streams.clear(); + printf("Destroy streams for kernel %d: ", get_uid()); size_t stream_size = 0; + for(auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { + stream_size += s->second.size(); + for(auto ss = s->second.begin(); ss != s->second.end(); ss++) + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream(*ss); + s->second.clear(); + } + printf("size %lu\n", stream_size); + m_cta_streams.clear(); } -simt_stack::simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu) { - m_warp_id = wid; - m_warp_size = warpSize; - m_gpu = gpu; - reset(); +simt_stack::simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu) +{ + m_warp_id=wid; + m_warp_size = warpSize; + m_gpu=gpu; + reset(); } -void simt_stack::reset() { m_stack.clear(); } - -void simt_stack::launch(address_type start_pc, const simt_mask_t &active_mask) { - reset(); - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = start_pc; - new_stack_entry.m_calldepth = 1; - new_stack_entry.m_active_mask = active_mask; - new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; - m_stack.push_back(new_stack_entry); +void simt_stack::reset() +{ + m_stack.clear(); } -void simt_stack::resume(char *fname) { - reset(); - - FILE *fp2 = fopen(fname, "r"); - assert(fp2 != NULL); - - char line[200]; /* or other suitable maximum line size */ - - while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ - { +void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) +{ + reset(); simt_stack_entry new_stack_entry; - char *pch; - pch = strtok(line, " "); - for (unsigned j = 0; j < m_warp_size; j++) { - if (pch[0] == '1') - new_stack_entry.m_active_mask.set(j); - else - new_stack_entry.m_active_mask.reset(j); - pch = strtok(NULL, " "); - } - - new_stack_entry.m_pc = atoi(pch); - pch = strtok(NULL, " "); - new_stack_entry.m_calldepth = atoi(pch); - pch = strtok(NULL, " "); - new_stack_entry.m_recvg_pc = atoi(pch); - pch = strtok(NULL, " "); - new_stack_entry.m_branch_div_cycle = atoi(pch); - pch = strtok(NULL, " "); - if (pch[0] == '0') - new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; - else - new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + new_stack_entry.m_pc = start_pc; + new_stack_entry.m_calldepth = 1; + new_stack_entry.m_active_mask = active_mask; + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; m_stack.push_back(new_stack_entry); - } - fclose(fp2); } -const simt_mask_t &simt_stack::get_active_mask() const { - assert(m_stack.size() > 0); - return m_stack.back().m_active_mask; -} +void simt_stack::resume( char * fname ) +{ + reset(); + + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + + char line [ 200 ]; /* or other suitable maximum line size */ + + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + simt_stack_entry new_stack_entry; + char * pch; + pch = strtok (line," "); + for (unsigned j=0; j 0); - *pc = m_stack.back().m_pc; - *rpc = m_stack.back().m_recvg_pc; + } -unsigned simt_stack::get_rp() const { - assert(m_stack.size() > 0); - return m_stack.back().m_recvg_pc; +const simt_mask_t &simt_stack::get_active_mask() const +{ + assert(m_stack.size() > 0); + return m_stack.back().m_active_mask; } -void simt_stack::print(FILE *fout) const { - for (unsigned k = 0; k < m_stack.size(); k++) { - simt_stack_entry stack_entry = m_stack[k]; - if (k == 0) { - fprintf(fout, "w%02d %1u ", m_warp_id, k); - } else { - fprintf(fout, " %1u ", k); - } - for (unsigned j = 0; j < m_warp_size; j++) - fprintf(fout, "%c", (stack_entry.m_active_mask.test(j) ? '1' : '0')); - fprintf(fout, " pc: 0x%03x", stack_entry.m_pc); - if (stack_entry.m_recvg_pc == (unsigned)-1) { - fprintf(fout, " rp: ---- tp: %s cd: %2u ", - (stack_entry.m_type == STACK_ENTRY_TYPE_CALL ? "C" : "N"), - stack_entry.m_calldepth); - } else { - fprintf(fout, " rp: %4u tp: %s cd: %2u ", stack_entry.m_recvg_pc, - (stack_entry.m_type == STACK_ENTRY_TYPE_CALL ? "C" : "N"), - stack_entry.m_calldepth); - } - if (stack_entry.m_branch_div_cycle != 0) { - fprintf(fout, " bd@%6u ", (unsigned)stack_entry.m_branch_div_cycle); - } else { - fprintf(fout, " "); - } - m_gpu->gpgpu_ctx->func_sim->ptx_print_insn(stack_entry.m_pc, fout); - fprintf(fout, "\n"); - } +void simt_stack::get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const +{ + assert(m_stack.size() > 0); + *pc = m_stack.back().m_pc; + *rpc = m_stack.back().m_recvg_pc; } -void simt_stack::print_checkpoint(FILE *fout) const { - for (unsigned k = 0; k < m_stack.size(); k++) { - simt_stack_entry stack_entry = m_stack[k]; - - for (unsigned j = 0; j < m_warp_size; j++) - fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j) ? '1' : '0')); - fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc, - stack_entry.m_calldepth, stack_entry.m_recvg_pc, - stack_entry.m_branch_div_cycle, stack_entry.m_type); - fprintf(fout, "%d %d\n", m_warp_id, m_warp_size); - } +unsigned simt_stack::get_rp() const +{ + assert(m_stack.size() > 0); + return m_stack.back().m_recvg_pc; } -void simt_stack::update(simt_mask_t &thread_done, addr_vector_t &next_pc, - address_type recvg_pc, op_type next_inst_op, - unsigned next_inst_size, address_type next_inst_pc) { - assert(m_stack.size() > 0); - - assert(next_pc.size() == m_warp_size); - - simt_mask_t top_active_mask = m_stack.back().m_active_mask; - address_type top_recvg_pc = m_stack.back().m_recvg_pc; - address_type top_pc = - m_stack.back().m_pc; // the pc of the instruction just executed - stack_entry_type top_type = m_stack.back().m_type; - assert(top_pc == next_inst_pc); - assert(top_active_mask.any()); - - const address_type null_pc = -1; - bool warp_diverged = false; - address_type new_recvg_pc = null_pc; - unsigned num_divergent_paths = 0; - - std::map divergent_paths; - while (top_active_mask.any()) { - // extract a group of threads with the same next PC among the active threads - // in the warp - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - for (int i = m_warp_size - 1; i >= 0; i--) { - if (top_active_mask.test(i)) { // is this thread active? - if (thread_done.test(i)) { - top_active_mask.reset(i); // remove completed thread from active mask - } else if (tmp_next_pc == null_pc) { - tmp_next_pc = next_pc[i]; - tmp_active_mask.set(i); - top_active_mask.reset(i); - } else if (tmp_next_pc == next_pc[i]) { - tmp_active_mask.set(i); - top_active_mask.reset(i); +void simt_stack::print (FILE *fout) const +{ + for ( unsigned k=0; k < m_stack.size(); k++ ) { + simt_stack_entry stack_entry = m_stack[k]; + if ( k==0 ) { + fprintf(fout, "w%02d %1u ", m_warp_id, k ); + } else { + fprintf(fout, " %1u ", k ); } - } + for (unsigned j=0; jgpgpu_ctx->func_sim->ptx_print_insn( stack_entry.m_pc, fout ); + fprintf(fout,"\n"); } - if (tmp_next_pc == null_pc) { - assert(!top_active_mask.any()); // all threads done - continue; - } +} - divergent_paths[tmp_next_pc] = tmp_active_mask; - num_divergent_paths++; - } - - address_type not_taken_pc = next_inst_pc + next_inst_size; - assert(num_divergent_paths <= 2); - for (unsigned i = 0; i < num_divergent_paths; i++) { - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - tmp_active_mask.reset(); - if (divergent_paths.find(not_taken_pc) != divergent_paths.end()) { - assert(i == 0); - tmp_next_pc = not_taken_pc; - tmp_active_mask = divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); - } else { - std::map::iterator it = - divergent_paths.begin(); - tmp_next_pc = it->first; - tmp_active_mask = divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); +void simt_stack::print_checkpoint (FILE *fout) const +{ + for ( unsigned k=0; k < m_stack.size(); k++ ) { + simt_stack_entry stack_entry = m_stack[k]; + + for (unsigned j=0; jgpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; - new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; - m_stack.push_back(new_stack_entry); - return; - } else if (next_inst_op == RET_OPS && top_type == STACK_ENTRY_TYPE_CALL) { - // pop the CALL Entry - assert(num_divergent_paths == 1); - m_stack.pop_back(); - - assert(m_stack.size() > 0); - m_stack.back().m_pc = tmp_next_pc; // set the PC of the stack top entry - // to return PC from the call stack; - // Check if the New top of the stack is reconverging - if (tmp_next_pc == m_stack.back().m_recvg_pc && - m_stack.back().m_type != STACK_ENTRY_TYPE_CALL) { - assert(m_stack.back().m_type == STACK_ENTRY_TYPE_NORMAL); - m_stack.pop_back(); - } - return; - } +void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ) +{ + assert(m_stack.size() > 0); + + assert( next_pc.size() == m_warp_size ); + + simt_mask_t top_active_mask = m_stack.back().m_active_mask; + address_type top_recvg_pc = m_stack.back().m_recvg_pc; + address_type top_pc = m_stack.back().m_pc; // the pc of the instruction just executed + stack_entry_type top_type = m_stack.back().m_type; + assert(top_pc==next_inst_pc); + assert(top_active_mask.any()); + + const address_type null_pc = -1; + bool warp_diverged = false; + address_type new_recvg_pc = null_pc; + unsigned num_divergent_paths=0; + + std::map divergent_paths; + while (top_active_mask.any()) { + + // extract a group of threads with the same next PC among the active threads in the warp + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + for (int i = m_warp_size - 1; i >= 0; i--) { + if ( top_active_mask.test(i) ) { // is this thread active? + if (thread_done.test(i)) { + top_active_mask.reset(i); // remove completed thread from active mask + } else if (tmp_next_pc == null_pc) { + tmp_next_pc = next_pc[i]; + tmp_active_mask.set(i); + top_active_mask.reset(i); + } else if (tmp_next_pc == next_pc[i]) { + tmp_active_mask.set(i); + top_active_mask.reset(i); + } + } + } - // discard the new entry if its PC matches with reconvergence PC - // that automatically reconverges the entry - // If the top stack entry is CALL, dont reconverge. - if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) - continue; - - // this new entry is not converging - // if this entry does not include thread from the warp, divergence occurs - if ((num_divergent_paths > 1) && !warp_diverged) { - warp_diverged = true; - // modify the existing top entry into a reconvergence entry in the pdom - // stack - new_recvg_pc = recvg_pc; - if (new_recvg_pc != top_recvg_pc) { - m_stack.back().m_pc = new_recvg_pc; - m_stack.back().m_branch_div_cycle = - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; + if(tmp_next_pc == null_pc) { + assert(!top_active_mask.any()); // all threads done + continue; + } - m_stack.push_back(simt_stack_entry()); - } + divergent_paths[tmp_next_pc]=tmp_active_mask; + num_divergent_paths++; } - // discard the new entry if its PC matches with reconvergence PC - if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; - // update the current top of pdom stack - m_stack.back().m_pc = tmp_next_pc; - m_stack.back().m_active_mask = tmp_active_mask; - if (warp_diverged) { - m_stack.back().m_calldepth = 0; - m_stack.back().m_recvg_pc = new_recvg_pc; - } else { - m_stack.back().m_recvg_pc = top_recvg_pc; - } + address_type not_taken_pc = next_inst_pc+next_inst_size; + assert(num_divergent_paths<=2); + for(unsigned i=0; i:: iterator it=divergent_paths.begin(); + tmp_next_pc=it->first; + tmp_active_mask=divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } + + // HANDLE THE SPECIAL CASES FIRST + if (next_inst_op== CALL_OPS){ + // Since call is not a divergent instruction, all threads should have executed a call instruction + assert(num_divergent_paths == 1); + + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = tmp_next_pc; + new_stack_entry.m_active_mask = tmp_active_mask; + new_stack_entry.m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + return; + }else if(next_inst_op == RET_OPS && top_type==STACK_ENTRY_TYPE_CALL){ + // pop the CALL Entry + assert(num_divergent_paths == 1); + m_stack.pop_back(); + + assert(m_stack.size() > 0); + m_stack.back().m_pc=tmp_next_pc;// set the PC of the stack top entry to return PC from the call stack; + // Check if the New top of the stack is reconverging + if (tmp_next_pc == m_stack.back().m_recvg_pc && m_stack.back().m_type!=STACK_ENTRY_TYPE_CALL){ + assert(m_stack.back().m_type==STACK_ENTRY_TYPE_NORMAL); + m_stack.pop_back(); + } + return; + } + + // discard the new entry if its PC matches with reconvergence PC + // that automatically reconverges the entry + // If the top stack entry is CALL, dont reconverge. + if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) continue; + + // this new entry is not converging + // if this entry does not include thread from the warp, divergence occurs + if ((num_divergent_paths>1) && !warp_diverged ) { + warp_diverged = true; + // modify the existing top entry into a reconvergence entry in the pdom stack + new_recvg_pc = recvg_pc; + if (new_recvg_pc != top_recvg_pc) { + m_stack.back().m_pc = new_recvg_pc; + m_stack.back().m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; + + m_stack.push_back(simt_stack_entry()); + } + } - m_stack.push_back(simt_stack_entry()); - } - assert(m_stack.size() > 0); - m_stack.pop_back(); + // discard the new entry if its PC matches with reconvergence PC + if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; - if (warp_diverged) { - m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); - } -} + // update the current top of pdom stack + m_stack.back().m_pc = tmp_next_pc; + m_stack.back().m_active_mask = tmp_active_mask; + if (warp_diverged) { + m_stack.back().m_calldepth = 0; + m_stack.back().m_recvg_pc = new_recvg_pc; + } else { + m_stack.back().m_recvg_pc = top_recvg_pc; + } -void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) { - for (unsigned t = 0; t < m_warp_size; t++) { - if (inst.active(t)) { - if (warpId == (unsigned(-1))) warpId = inst.warp_id(); - unsigned tid = m_warp_size * warpId + t; - m_thread[tid]->ptx_exec_inst(inst, t); + m_stack.push_back(simt_stack_entry()); + } + assert(m_stack.size() > 0); + m_stack.pop_back(); - // virtual function - checkExecutionStatusAndUpdate(inst, t, tid); + + if (warp_diverged) { + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); } - } } -bool core_t::ptx_thread_done(unsigned hw_thread_id) const { - return ((m_thread[hw_thread_id] == NULL) || - m_thread[hw_thread_id]->is_done()); +void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) +{ + for ( unsigned t=0; t < m_warp_size; t++ ) { + if( inst.active(t) ) { + if(warpId==(unsigned (-1))) + warpId = inst.warp_id(); + unsigned tid=m_warp_size*warpId+t; + m_thread[tid]->ptx_exec_inst(inst,t); + + //virtual function + checkExecutionStatusAndUpdate(inst,t,tid); + } + } } - -void core_t::updateSIMTStack(unsigned warpId, warp_inst_t *inst) { - simt_mask_t thread_done; - addr_vector_t next_pc; - unsigned wtid = warpId * m_warp_size; - for (unsigned i = 0; i < m_warp_size; i++) { - if (ptx_thread_done(wtid + i)) { - thread_done.set(i); - next_pc.push_back((address_type)-1); - } else { - if (inst->reconvergence_pc == RECONVERGE_RETURN_PC) - inst->reconvergence_pc = get_return_pc(m_thread[wtid + i]); - next_pc.push_back(m_thread[wtid + i]->get_pc()); + +bool core_t::ptx_thread_done( unsigned hw_thread_id ) const +{ + return ((m_thread[ hw_thread_id ]==NULL) || m_thread[ hw_thread_id ]->is_done()); +} + +void core_t::updateSIMTStack(unsigned warpId, warp_inst_t * inst) +{ + simt_mask_t thread_done; + addr_vector_t next_pc; + unsigned wtid = warpId * m_warp_size; + for (unsigned i = 0; i < m_warp_size; i++) { + if( ptx_thread_done(wtid+i) ) { + thread_done.set(i); + next_pc.push_back( (address_type)-1 ); + } else { + if( inst->reconvergence_pc == RECONVERGE_RETURN_PC ) + inst->reconvergence_pc = get_return_pc(m_thread[wtid+i]); + next_pc.push_back( m_thread[wtid+i]->get_pc() ); + } } - } - m_simt_stack[warpId]->update(thread_done, next_pc, inst->reconvergence_pc, - inst->op, inst->isize, inst->pc); + m_simt_stack[warpId]->update(thread_done,next_pc,inst->reconvergence_pc, inst->op,inst->isize,inst->pc); } //! Get the warp to be executed using the data taken form the SIMT stack -warp_inst_t core_t::getExecuteWarp(unsigned warpId) { - unsigned pc, rpc; - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); - warp_inst_t wi = *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); - wi.set_active(m_simt_stack[warpId]->get_active_mask()); - return wi; +warp_inst_t core_t::getExecuteWarp(unsigned warpId) +{ + unsigned pc,rpc; + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + warp_inst_t wi= *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); + wi.set_active(m_simt_stack[warpId]->get_active_mask()); + return wi; } -void core_t::deleteSIMTStack() { - if (m_simt_stack) { - for (unsigned i = 0; i < m_warp_count; ++i) delete m_simt_stack[i]; - delete[] m_simt_stack; - m_simt_stack = NULL; - } +void core_t::deleteSIMTStack() +{ + if ( m_simt_stack ) { + for (unsigned i = 0; i < m_warp_count; ++i) + delete m_simt_stack[i]; + delete[] m_simt_stack; + m_simt_stack = NULL; + } } -void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) { - m_simt_stack = new simt_stack *[warp_count]; - for (unsigned i = 0; i < warp_count; ++i) - m_simt_stack[i] = new simt_stack(i, warp_size, m_gpu); - m_warp_size = warp_size; - m_warp_count = warp_count; +void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) +{ + m_simt_stack = new simt_stack*[warp_count]; + for (unsigned i = 0; i < warp_count; ++i) + m_simt_stack[i] = new simt_stack(i,warp_size,m_gpu); + m_warp_size = warp_size; + m_warp_count = warp_count; } -void core_t::get_pdom_stack_top_info(unsigned warpId, unsigned *pc, - unsigned *rpc) const { - m_simt_stack[warpId]->get_pdom_stack_top_info(pc, rpc); +void core_t::get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const +{ + m_simt_stack[warpId]->get_pdom_stack_top_info(pc,rpc); } diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 206ae06..29e4a30 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -35,41 +33,45 @@ class gpgpu_sim; class kernel_info_t; class gpgpu_context; -// Set a hard limit of 32 CTAs per shader [cuda only has 8] + +//Set a hard limit of 32 CTAs per shader [cuda only has 8] #define MAX_CTA_PER_SHADER 32 #define MAX_BARRIERS_PER_CTA 16 -// After expanding the vector input and output operands +//After expanding the vector input and output operands #define MAX_INPUT_VALUES 24 #define MAX_OUTPUT_VALUES 8 enum _memory_space_t { - undefined_space = 0, - reg_space, - local_space, - shared_space, - sstarr_space, - param_space_unclassified, - param_space_kernel, /* global to all threads in a kernel : read-only */ - param_space_local, /* local to a thread : read-writable */ - const_space, - tex_space, - surf_space, - global_space, - generic_space, - instruction_space + undefined_space=0, + reg_space, + local_space, + shared_space, + sstarr_space, + param_space_unclassified, + param_space_kernel, /* global to all threads in a kernel : read-only */ + param_space_local, /* local to a thread : read-writable */ + const_space, + tex_space, + surf_space, + global_space, + generic_space, + instruction_space }; -enum FuncCache { + +enum FuncCache +{ FuncCachePreferNone = 0, FuncCachePreferShared = 1, FuncCachePreferL1 = 2 }; + #ifdef __cplusplus -#include #include +#include #include typedef unsigned long long new_addr_type; @@ -77,357 +79,371 @@ typedef unsigned long long cudaTextureObject_t; typedef unsigned address_type; typedef unsigned addr_t; -// the following are operations the timing model can see +// the following are operations the timing model can see enum uarch_op_t { - NO_OP = -1, - ALU_OP = 1, - SFU_OP, - TENSOR_CORE_OP, - DP_OP, - SP_OP, - INTP_OP, - ALU_SFU_OP, - LOAD_OP, - TENSOR_CORE_LOAD_OP, - TENSOR_CORE_STORE_OP, - STORE_OP, - BRANCH_OP, - BARRIER_OP, - MEMORY_BARRIER_OP, - CALL_OPS, - RET_OPS + NO_OP=-1, + ALU_OP=1, + SFU_OP, + TENSOR_CORE_OP, + DP_OP, + SP_OP, + INTP_OP, + ALU_SFU_OP, + LOAD_OP, + TENSOR_CORE_LOAD_OP, + TENSOR_CORE_STORE_OP, + STORE_OP, + BRANCH_OP, + BARRIER_OP, + MEMORY_BARRIER_OP, + CALL_OPS, + RET_OPS }; typedef enum uarch_op_t op_type; -enum uarch_bar_t { NOT_BAR = -1, SYNC = 1, ARRIVE, RED }; + +enum uarch_bar_t { + NOT_BAR=-1, + SYNC=1, + ARRIVE, + RED +}; typedef enum uarch_bar_t barrier_type; -enum uarch_red_t { NOT_RED = -1, POPC_RED = 1, AND_RED, OR_RED }; +enum uarch_red_t { + NOT_RED=-1, + POPC_RED=1, + AND_RED, + OR_RED +}; typedef enum uarch_red_t reduction_type; -enum uarch_operand_type_t { UN_OP = -1, INT_OP, FP_OP }; + +enum uarch_operand_type_t { + UN_OP=-1, + INT_OP, + FP_OP +}; typedef enum uarch_operand_type_t types_of_operands; enum special_operations_t { - OTHER_OP, - INT__OP, - INT_MUL24_OP, - INT_MUL32_OP, - INT_MUL_OP, - INT_DIV_OP, - FP_MUL_OP, - FP_DIV_OP, - FP__OP, - FP_SQRT_OP, - FP_LG_OP, - FP_SIN_OP, - FP_EXP_OP + OTHER_OP, + INT__OP, + INT_MUL24_OP, + INT_MUL32_OP, + INT_MUL_OP, + INT_DIV_OP, + FP_MUL_OP, + FP_DIV_OP, + FP__OP, + FP_SQRT_OP, + FP_LG_OP, + FP_SIN_OP, + FP_EXP_OP }; -typedef enum special_operations_t - special_ops; // Required to identify for the power model +typedef enum special_operations_t special_ops; // Required to identify for the power model enum operation_pipeline_t { - UNKOWN_OP, - SP__OP, - DP__OP, - INTP__OP, - SFU__OP, - TENSOR_CORE__OP, - MEM__OP + UNKOWN_OP, + SP__OP, + DP__OP, + INTP__OP, + SFU__OP, + TENSOR_CORE__OP, + MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; -enum mem_operation_t { NOT_TEX, TEX }; +enum mem_operation_t { + NOT_TEX, + TEX +}; typedef enum mem_operation_t mem_operation; -enum _memory_op_t { no_memory_op = 0, memory_load, memory_store }; +enum _memory_op_t { + no_memory_op = 0, + memory_load, + memory_store +}; -#include -#include -#include #include -#include #include -#include #include +#include +#include +#include +#include +#include #if !defined(__VECTOR_TYPES_H__) #include "vector_types.h" #endif struct dim3comp { - bool operator()(const dim3 &a, const dim3 &b) const { - if (a.z < b.z) - return true; - else if (a.y < b.y) - return true; - else if (a.x < b.x) - return true; - else - return false; - } + bool operator() (const dim3 & a, const dim3 & b) const + { + if(a.z < b.z) + return true; + else if(a.y < b.y) + return true; + else if (a.x < b.x) + return true; + else + return false; + } }; -void increment_x_then_y_then_z(dim3 &i, const dim3 &bound); +void increment_x_then_y_then_z( dim3 &i, const dim3 &bound); -// Jin: child kernel information for CDP +//Jin: child kernel information for CDP #include "stream_manager.h" class stream_manager; struct CUstream_st; -// extern stream_manager * g_stream_manager; -// support for pinned memories added -extern std::map pinned_memory; +//extern stream_manager * g_stream_manager; +//support for pinned memories added +extern std::map pinned_memory; extern std::map pinned_memory_size; class kernel_info_t { - public: - // kernel_info_t() - // { - // m_valid=false; - // m_kernel_entry=NULL; - // m_uid=0; - // m_num_cores_running=0; - // m_param_mem=NULL; - // } - kernel_info_t(dim3 gridDim, dim3 blockDim, class function_info *entry); - kernel_info_t( - dim3 gridDim, dim3 blockDim, class function_info *entry, - std::map nameToCudaArray, - std::map nameToTextureInfo); - ~kernel_info_t(); - - void inc_running() { m_num_cores_running++; } - void dec_running() { - assert(m_num_cores_running > 0); - m_num_cores_running--; - } - bool running() const { return m_num_cores_running > 0; } - bool done() const { return no_more_ctas_to_run() && !running(); } - class function_info *entry() { - return m_kernel_entry; - } - const class function_info *entry() const { return m_kernel_entry; } - - size_t num_blocks() const { - return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; - } - - size_t threads_per_cta() const { - return m_block_dim.x * m_block_dim.y * m_block_dim.z; - } - - dim3 get_grid_dim() const { return m_grid_dim; } - dim3 get_cta_dim() const { return m_block_dim; } - - void increment_cta_id() { - increment_x_then_y_then_z(m_next_cta, m_grid_dim); - m_next_tid.x = 0; - m_next_tid.y = 0; - m_next_tid.z = 0; - } - dim3 get_next_cta_id() const { return m_next_cta; } - unsigned get_next_cta_id_single() const { - return m_next_cta.x + m_grid_dim.x * m_next_cta.y + - m_grid_dim.x * m_grid_dim.y * m_next_cta.z; - } - bool no_more_ctas_to_run() const { - return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || - m_next_cta.z >= m_grid_dim.z); - } - - void increment_thread_id() { - increment_x_then_y_then_z(m_next_tid, m_block_dim); - } - dim3 get_next_thread_id_3d() const { return m_next_tid; } - unsigned get_next_thread_id() const { - return m_next_tid.x + m_block_dim.x * m_next_tid.y + - m_block_dim.x * m_block_dim.y * m_next_tid.z; - } - bool more_threads_in_cta() const { - return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && - m_next_tid.x < m_block_dim.x; - } - unsigned get_uid() const { return m_uid; } - std::string name() const; - - std::list &active_threads() { - return m_active_threads; - } - class memory_space *get_param_memory() { - return m_param_mem; - } - - // The following functions access texture bindings present at the kernel's - // launch - - const struct cudaArray *get_texarray(const std::string &texname) const { - std::map::const_iterator t = - m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo *get_texinfo(const std::string &texname) const { - std::map::const_iterator t = - m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - - private: - kernel_info_t(const kernel_info_t &); // disable copy constructor - void operator=(const kernel_info_t &); // disable copy operator - - class function_info *m_kernel_entry; - - unsigned m_uid; - - // These maps contain the snapshot of the texture mappings at kernel launch - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - - dim3 m_grid_dim; - dim3 m_block_dim; - dim3 m_next_cta; - dim3 m_next_tid; - - unsigned m_num_cores_running; - - std::list m_active_threads; - class memory_space *m_param_mem; - - public: - // Jin: parent and child kernel management for CDP - void set_parent(kernel_info_t *parent, dim3 parent_ctaid, dim3 parent_tid); - void set_child(kernel_info_t *child); - void remove_child(kernel_info_t *child); - bool is_finished(); - bool children_all_finished(); - void notify_parent_finished(); - CUstream_st *create_stream_cta(dim3 ctaid); - CUstream_st *get_default_stream_cta(dim3 ctaid); - bool cta_has_stream(dim3 ctaid, CUstream_st *stream); - void destroy_cta_streams(); - void print_parent_info(); - kernel_info_t *get_parent() { return m_parent_kernel; } - - private: - kernel_info_t *m_parent_kernel; - dim3 m_parent_ctaid; - dim3 m_parent_tid; - std::list m_child_kernels; // child kernel launched - std::map, dim3comp> - m_cta_streams; // streams created in each CTA - - // Jin: kernel timing - public: - unsigned long long launch_cycle; - unsigned long long start_cycle; - unsigned long long end_cycle; - unsigned m_launch_latency; - - mutable bool volta_cache_config_set; +public: +// kernel_info_t() +// { +// m_valid=false; +// m_kernel_entry=NULL; +// m_uid=0; +// m_num_cores_running=0; +// m_param_mem=NULL; +// } + kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry); + kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo); + ~kernel_info_t(); + + void inc_running() { m_num_cores_running++; } + void dec_running() + { + assert( m_num_cores_running > 0 ); + m_num_cores_running--; + } + bool running() const { return m_num_cores_running>0; } + bool done() const + { + return no_more_ctas_to_run() && !running(); + } + class function_info *entry() { return m_kernel_entry; } + const class function_info *entry() const { return m_kernel_entry; } + + size_t num_blocks() const + { + return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; + } + + size_t threads_per_cta() const + { + return m_block_dim.x * m_block_dim.y * m_block_dim.z; + } + + dim3 get_grid_dim() const { return m_grid_dim; } + dim3 get_cta_dim() const { return m_block_dim; } + + void increment_cta_id() + { + increment_x_then_y_then_z(m_next_cta,m_grid_dim); + m_next_tid.x=0; + m_next_tid.y=0; + m_next_tid.z=0; + } + dim3 get_next_cta_id() const { return m_next_cta; } + unsigned get_next_cta_id_single() const + { + return m_next_cta.x + m_grid_dim.x*m_next_cta.y + m_grid_dim.x*m_grid_dim.y*m_next_cta.z; + } + bool no_more_ctas_to_run() const + { + return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || m_next_cta.z >= m_grid_dim.z ); + } + + void increment_thread_id() { increment_x_then_y_then_z(m_next_tid,m_block_dim); } + dim3 get_next_thread_id_3d() const { return m_next_tid; } + unsigned get_next_thread_id() const + { + return m_next_tid.x + m_block_dim.x*m_next_tid.y + m_block_dim.x*m_block_dim.y*m_next_tid.z; + } + bool more_threads_in_cta() const + { + return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && m_next_tid.x < m_block_dim.x; + } + unsigned get_uid() const { return m_uid; } + std::string name() const; + + std::list &active_threads() { return m_active_threads; } + class memory_space *get_param_memory() { return m_param_mem; } + + + //The following functions access texture bindings present at the kernel's launch + + const struct cudaArray* get_texarray( const std::string &texname ) const + { + std::map::const_iterator t=m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo* get_texinfo( const std::string &texname ) const + { + std::map::const_iterator t=m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + +private: + kernel_info_t( const kernel_info_t & ); // disable copy constructor + void operator=( const kernel_info_t & ); // disable copy operator + + class function_info *m_kernel_entry; + + unsigned m_uid; + + //These maps contain the snapshot of the texture mappings at kernel launch + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + + dim3 m_grid_dim; + dim3 m_block_dim; + dim3 m_next_cta; + dim3 m_next_tid; + + unsigned m_num_cores_running; + + std::list m_active_threads; + class memory_space *m_param_mem; + +public: + //Jin: parent and child kernel management for CDP + void set_parent(kernel_info_t * parent, dim3 parent_ctaid, dim3 parent_tid); + void set_child(kernel_info_t * child); + void remove_child(kernel_info_t * child); + bool is_finished(); + bool children_all_finished(); + void notify_parent_finished(); + CUstream_st * create_stream_cta(dim3 ctaid); + CUstream_st * get_default_stream_cta(dim3 ctaid); + bool cta_has_stream(dim3 ctaid, CUstream_st* stream); + void destroy_cta_streams(); + void print_parent_info(); + kernel_info_t * get_parent() { return m_parent_kernel; } + +private: + kernel_info_t * m_parent_kernel; + dim3 m_parent_ctaid; + dim3 m_parent_tid; + std::list m_child_kernels; //child kernel launched + std::map< dim3, std::list, dim3comp > m_cta_streams; //streams created in each CTA + +//Jin: kernel timing +public: + unsigned long long launch_cycle; + unsigned long long start_cycle; + unsigned long long end_cycle; + unsigned m_launch_latency; + + mutable bool volta_cache_config_set; }; class core_config { - public: - core_config(gpgpu_context *ctx) { - gpgpu_ctx = ctx; - m_valid = false; - num_shmem_bank = 16; - shmem_limited_broadcast = false; - gpgpu_shmem_sizeDefault = (unsigned)-1; - gpgpu_shmem_sizePrefL1 = (unsigned)-1; - gpgpu_shmem_sizePrefShared = (unsigned)-1; - } - virtual void init() = 0; - - bool m_valid; - unsigned warp_size; - // backward pointer - class gpgpu_context *gpgpu_ctx; - - // off-chip memory request architecture parameters - int gpgpu_coalesce_arch; - - // shared memory bank conflict checking parameters - bool shmem_limited_broadcast; - static const address_type WORD_SIZE = 4; - unsigned num_shmem_bank; - unsigned shmem_bank_func(address_type addr) const { - return ((addr / WORD_SIZE) % num_shmem_bank); - } - unsigned mem_warp_parts; - mutable unsigned gpgpu_shmem_size; - unsigned gpgpu_shmem_sizeDefault; - unsigned gpgpu_shmem_sizePrefL1; - unsigned gpgpu_shmem_sizePrefShared; - unsigned mem_unit_ports; - - // texture and constant cache line sizes (used to determine number of memory - // accesses) - unsigned gpgpu_cache_texl1_linesize; - unsigned gpgpu_cache_constl1_linesize; - - unsigned gpgpu_max_insn_issue_per_warp; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - - bool adaptive_volta_cache_config; + public: + core_config(gpgpu_context* ctx) + { + gpgpu_ctx = ctx; + m_valid = false; + num_shmem_bank=16; + shmem_limited_broadcast = false; + gpgpu_shmem_sizeDefault=(unsigned)-1; + gpgpu_shmem_sizePrefL1=(unsigned)-1; + gpgpu_shmem_sizePrefShared=(unsigned)-1; + } + virtual void init() = 0; + + bool m_valid; + unsigned warp_size; + // backward pointer + class gpgpu_context* gpgpu_ctx; + + // off-chip memory request architecture parameters + int gpgpu_coalesce_arch; + + // shared memory bank conflict checking parameters + bool shmem_limited_broadcast; + static const address_type WORD_SIZE=4; + unsigned num_shmem_bank; + unsigned shmem_bank_func(address_type addr) const + { + return ((addr/WORD_SIZE) % num_shmem_bank); + } + unsigned mem_warp_parts; + mutable unsigned gpgpu_shmem_size; + unsigned gpgpu_shmem_sizeDefault; + unsigned gpgpu_shmem_sizePrefL1; + unsigned gpgpu_shmem_sizePrefShared; + unsigned mem_unit_ports; + + // texture and constant cache line sizes (used to determine number of memory accesses) + unsigned gpgpu_cache_texl1_linesize; + unsigned gpgpu_cache_constl1_linesize; + + unsigned gpgpu_max_insn_issue_per_warp; + bool gmem_skip_L1D; // on = global memory access always skip the L1 cache + + bool adaptive_volta_cache_config; }; -// bounded stack that implements simt reconvergence using pdom mechanism from -// MICRO'07 paper +// bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper const unsigned MAX_WARP_SIZE = 32; typedef std::bitset active_mask_t; -#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE +#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE typedef std::bitset simt_mask_t; typedef std::vector addr_vector_t; class simt_stack { - public: - simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu); - - void reset(); - void launch(address_type start_pc, const simt_mask_t &active_mask); - void update(simt_mask_t &thread_done, addr_vector_t &next_pc, - address_type recvg_pc, op_type next_inst_op, - unsigned next_inst_size, address_type next_inst_pc); - - const simt_mask_t &get_active_mask() const; - void get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const; - unsigned get_rp() const; - void print(FILE *fp) const; - void resume(char *fname); - void print_checkpoint(FILE *fout) const; - - protected: - unsigned m_warp_id; - unsigned m_warp_size; - - enum stack_entry_type { STACK_ENTRY_TYPE_NORMAL = 0, STACK_ENTRY_TYPE_CALL }; - - struct simt_stack_entry { - address_type m_pc; - unsigned int m_calldepth; - simt_mask_t m_active_mask; - address_type m_recvg_pc; - unsigned long long m_branch_div_cycle; - stack_entry_type m_type; - simt_stack_entry() - : m_pc(-1), - m_calldepth(0), - m_active_mask(), - m_recvg_pc(-1), - m_branch_div_cycle(0), - m_type(STACK_ENTRY_TYPE_NORMAL){}; - }; - - std::deque m_stack; - - class gpgpu_sim *m_gpu; +public: + simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu); + + void reset(); + void launch( address_type start_pc, const simt_mask_t &active_mask ); + void update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ); + + const simt_mask_t &get_active_mask() const; + void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; + unsigned get_rp() const; + void print(FILE *fp) const; + void resume(char * fname) ; + void print_checkpoint (FILE *fout) const; + +protected: + unsigned m_warp_id; + unsigned m_warp_size; + + + enum stack_entry_type { + STACK_ENTRY_TYPE_NORMAL = 0, + STACK_ENTRY_TYPE_CALL + }; + + struct simt_stack_entry { + address_type m_pc; + unsigned int m_calldepth; + simt_mask_t m_active_mask; + address_type m_recvg_pc; + unsigned long long m_branch_div_cycle; + stack_entry_type m_type; + simt_stack_entry() : + m_pc(-1), m_calldepth(0), m_active_mask(), m_recvg_pc(-1), m_branch_div_cycle(0), m_type(STACK_ENTRY_TYPE_NORMAL) { }; + }; + + std::deque m_stack; + + class gpgpu_sim * m_gpu; }; // Let's just upgrade to C++11 so we can use constexpr here... -// start allocating from this address (lower values used for allocating globals -// in .ptx file) +// start allocating from this address (lower values used for allocating globals in .ptx file) const unsigned long long GLOBAL_HEAP_START = 0xC0000000; // Volta max shmem size is 96kB const unsigned long long SHARED_MEM_SIZE_MAX = 96 * (1 << 10); @@ -439,931 +455,873 @@ const unsigned MAX_STREAMING_MULTIPROCESSORS = 80; const unsigned MAX_THREAD_PER_SM = 1 << 11; // MAX 64 warps / SM const unsigned MAX_WARP_PER_SM = 1 << 6; -const unsigned long long TOTAL_LOCAL_MEM_PER_SM = - MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long TOTAL_SHARED_MEM = - MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; -const unsigned long long TOTAL_LOCAL_MEM = - MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long SHARED_GENERIC_START = - GLOBAL_HEAP_START - TOTAL_SHARED_MEM; -const unsigned long long LOCAL_GENERIC_START = - SHARED_GENERIC_START - TOTAL_LOCAL_MEM; -const unsigned long long STATIC_ALLOC_LIMIT = - GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); +const unsigned long long TOTAL_LOCAL_MEM_PER_SM = MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long TOTAL_SHARED_MEM = MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; +const unsigned long long TOTAL_LOCAL_MEM = MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long SHARED_GENERIC_START = GLOBAL_HEAP_START - TOTAL_SHARED_MEM; +const unsigned long long LOCAL_GENERIC_START = SHARED_GENERIC_START - TOTAL_LOCAL_MEM; +const unsigned long long STATIC_ALLOC_LIMIT = GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); #if !defined(__CUDA_RUNTIME_API_H__) #include "builtin_types.h" struct cudaArray { - void *devPtr; - int devPtr32; - struct cudaChannelFormatDesc desc; - int width; - int height; - int size; // in bytes - unsigned dimensions; + void *devPtr; + int devPtr32; + struct cudaChannelFormatDesc desc; + int width; + int height; + int size; //in bytes + unsigned dimensions; }; #endif -// Struct that record other attributes in the textureReference declaration +// Struct that record other attributes in the textureReference declaration // - These attributes are passed thru __cudaRegisterTexture() struct textureReferenceAttr { - const struct textureReference *m_texref; - int m_dim; - enum cudaTextureReadMode m_readmode; - int m_ext; - textureReferenceAttr(const struct textureReference *texref, int dim, - enum cudaTextureReadMode readmode, int ext) - : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) {} + const struct textureReference *m_texref; + int m_dim; + enum cudaTextureReadMode m_readmode; + int m_ext; + textureReferenceAttr(const struct textureReference *texref, + int dim, + enum cudaTextureReadMode readmode, + int ext) + : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) + { } }; -class gpgpu_functional_sim_config { - public: - void reg_options(class OptionParser *opp); - - void ptx_set_tex_cache_linesize(unsigned linesize); - - unsigned get_forced_max_capability() const { - return m_ptx_force_max_capability; - } - bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } - bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } - bool experimental_lib_support() const { return m_experimental_lib_support; } - - int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } - const char *get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } - int get_ptx_inst_debug_thread_uid() const { - return g_ptx_inst_debug_thread_uid; - } - unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const { return checkpoint_option; } - int get_checkpoint_kernel() const { return checkpoint_kernel; } - int get_checkpoint_CTA() const { return checkpoint_CTA; } - int get_resume_option() const { return resume_option; } - int get_resume_kernel() const { return resume_kernel; } - int get_resume_CTA() const { return resume_CTA; } - int get_checkpoint_CTA_t() const { return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const { return checkpoint_insn_Y; } - - private: - // PTX options - int m_ptx_convert_to_ptxplus; - int m_ptx_use_cuobjdump; - int m_experimental_lib_support; - unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - int g_ptx_inst_debug_to_file; - char *g_ptx_inst_debug_file; - int g_ptx_inst_debug_thread_uid; - - unsigned m_texcache_linesize; +class gpgpu_functional_sim_config +{ +public: + void reg_options(class OptionParser * opp); + + void ptx_set_tex_cache_linesize(unsigned linesize); + + unsigned get_forced_max_capability() const { return m_ptx_force_max_capability; } + bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } + bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } + bool experimental_lib_support() const { return m_experimental_lib_support; } + + int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } + const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } + int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } + unsigned get_texcache_linesize() const { return m_texcache_linesize; } + int get_checkpoint_option() const {return checkpoint_option; } + int get_checkpoint_kernel() const {return checkpoint_kernel; } + int get_checkpoint_CTA() const {return checkpoint_CTA; } + int get_resume_option() const {return resume_option; } + int get_resume_kernel() const {return resume_kernel; } + int get_resume_CTA() const {return resume_CTA; } + int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } +private: + // PTX options + int m_ptx_convert_to_ptxplus; + int m_ptx_use_cuobjdump; + int m_experimental_lib_support; + unsigned m_ptx_force_max_capability; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; + char* g_ptx_inst_debug_file; + int g_ptx_inst_debug_thread_uid; + + unsigned m_texcache_linesize; }; + class gpgpu_t { - public: - gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx); - // backward pointer - class gpgpu_context *gpgpu_ctx; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - - // Move some cycle core stats here instead of being global - unsigned long long gpu_sim_cycle; - unsigned long long gpu_tot_sim_cycle; - - void *gpu_malloc(size_t size); - void *gpu_mallocarray(size_t count); - void gpu_memset(size_t dst_start_addr, int c, size_t count); - void memcpy_to_gpu(size_t dst_start_addr, const void *src, size_t count); - void memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count); - void memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count); - - class memory_space *get_global_memory() { - return m_global_mem; - } - class memory_space *get_tex_memory() { - return m_tex_mem; - } - class memory_space *get_surf_memory() { - return m_surf_mem; - } - - void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference *texref, - const struct cudaArray *array); - void gpgpu_ptx_sim_bindNameToTexture(const char *name, - const struct textureReference *texref, - int dim, int readmode, int ext); - void gpgpu_ptx_sim_unbindTexture(const struct textureReference *texref); - const char *gpgpu_ptx_sim_findNamefromTexture( - const struct textureReference *texref); - - const struct textureReference *get_texref(const std::string &texname) const { - std::map >::const_iterator t = - m_NameToTextureRef.find(texname); - assert(t != m_NameToTextureRef.end()); - return *(t->second.begin()); - } - - const struct cudaArray *get_texarray(const std::string &texname) const { - std::map::const_iterator t = - m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo *get_texinfo(const std::string &texname) const { - std::map::const_iterator t = - m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - - const struct textureReferenceAttr *get_texattr( - const std::string &texname) const { - std::map::const_iterator - t = m_NameToAttribute.find(texname); - assert(t != m_NameToAttribute.end()); - return t->second; - } - - const gpgpu_functional_sim_config &get_config() const { - return m_function_model_config; - } - FILE *get_ptx_inst_debug_file() { return ptx_inst_debug_file; } - - // These maps return the current texture mappings for the GPU at any given - // time. - std::map getNameArrayMapping() { - return m_NameToCudaArray; - } - std::map getNameInfoMapping() { - return m_NameToTextureInfo; - } - - protected: - const gpgpu_functional_sim_config &m_function_model_config; - FILE *ptx_inst_debug_file; - - class memory_space *m_global_mem; - class memory_space *m_tex_mem; - class memory_space *m_surf_mem; - - unsigned long long m_dev_malloc; - // These maps contain the current texture mappings for the GPU at any given - // time. - std::map > - m_NameToTextureRef; - std::map m_TextureRefToName; - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - std::map m_NameToAttribute; +public: + gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ); + // backward pointer + class gpgpu_context* gpgpu_ctx; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + + //Move some cycle core stats here instead of being global + unsigned long long gpu_sim_cycle; + unsigned long long gpu_tot_sim_cycle; + + + void* gpu_malloc( size_t size ); + void* gpu_mallocarray( size_t count ); + void gpu_memset( size_t dst_start_addr, int c, size_t count ); + void memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ); + void memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ); + void memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ); + + class memory_space *get_global_memory() { return m_global_mem; } + class memory_space *get_tex_memory() { return m_tex_mem; } + class memory_space *get_surf_memory() { return m_surf_mem; } + + void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array); + void gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext); + void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); + const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); + + const struct textureReference* get_texref( const std::string &texname ) const + { + std::map >::const_iterator t=m_NameToTextureRef.find(texname); + assert( t != m_NameToTextureRef.end() ); + return *(t->second.begin()); + } + + const struct cudaArray* get_texarray( const std::string &texname ) const + { + std::map::const_iterator t=m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo* get_texinfo( const std::string &texname ) const + { + std::map::const_iterator t=m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + + const struct textureReferenceAttr* get_texattr( const std::string &texname ) const + { + std::map::const_iterator t=m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); + return t->second; + } + + const gpgpu_functional_sim_config &get_config() const { return m_function_model_config; } + FILE* get_ptx_inst_debug_file() { return ptx_inst_debug_file; } + + // These maps return the current texture mappings for the GPU at any given time. + std::map getNameArrayMapping() {return m_NameToCudaArray;} + std::map getNameInfoMapping() {return m_NameToTextureInfo;} + +protected: + const gpgpu_functional_sim_config &m_function_model_config; + FILE* ptx_inst_debug_file; + + class memory_space *m_global_mem; + class memory_space *m_tex_mem; + class memory_space *m_surf_mem; + + unsigned long long m_dev_malloc; + // These maps contain the current texture mappings for the GPU at any given time. + std::map > m_NameToTextureRef; + std::map m_TextureRefToName; + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + std::map m_NameToAttribute; }; -struct gpgpu_ptx_sim_info { - // Holds properties of the kernel (Kernel's resource use). - // These will be set to zero if a ptxinfo file is not present. - int lmem; - int smem; - int cmem; - int gmem; - int regs; - unsigned maxthreads; - unsigned ptx_version; - unsigned sm_target; +struct gpgpu_ptx_sim_info +{ + // Holds properties of the kernel (Kernel's resource use). + // These will be set to zero if a ptxinfo file is not present. + int lmem; + int smem; + int cmem; + int gmem; + int regs; + unsigned maxthreads; + unsigned ptx_version; + unsigned sm_target; }; + struct gpgpu_ptx_sim_arg { - gpgpu_ptx_sim_arg() { m_start = NULL; } - gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) { - m_start = arg; - m_nbytes = size; - m_offset = offset; - } - const void *m_start; - size_t m_nbytes; - size_t m_offset; + gpgpu_ptx_sim_arg() { m_start=NULL; } + gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) + { + m_start=arg; + m_nbytes=size; + m_offset=offset; + } + const void *m_start; + size_t m_nbytes; + size_t m_offset; }; typedef std::list gpgpu_ptx_sim_arg_list_t; class memory_space_t { - public: - memory_space_t() { - m_type = undefined_space; - m_bank = 0; - } - memory_space_t(const enum _memory_space_t &from) { - m_type = from; - m_bank = 0; - } - bool operator==(const memory_space_t &x) const { - return (m_bank == x.m_bank) && (m_type == x.m_type); - } - bool operator!=(const memory_space_t &x) const { return !(*this == x); } - bool operator<(const memory_space_t &x) const { - if (m_type < x.m_type) - return true; - else if (m_type > x.m_type) +public: + memory_space_t() { m_type = undefined_space; m_bank=0; } + memory_space_t( const enum _memory_space_t &from ) { m_type = from; m_bank = 0; } + bool operator==( const memory_space_t &x ) const { return (m_bank == x.m_bank) && (m_type == x.m_type); } + bool operator!=( const memory_space_t &x ) const { return !(*this == x); } + bool operator<( const memory_space_t &x ) const + { + if(m_type < x.m_type) + return true; + else if(m_type > x.m_type) + return false; + else if( m_bank < x.m_bank ) + return true; return false; - else if (m_bank < x.m_bank) - return true; - return false; - } - enum _memory_space_t get_type() const { return m_type; } - void set_type(enum _memory_space_t t) { m_type = t; } - unsigned get_bank() const { return m_bank; } - void set_bank(unsigned b) { m_bank = b; } - bool is_const() const { - return (m_type == const_space) || (m_type == param_space_kernel); - } - bool is_local() const { - return (m_type == local_space) || (m_type == param_space_local); - } - bool is_global() const { return (m_type == global_space); } - - private: - enum _memory_space_t m_type; - unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 - // manual, sec. 5.1.3) + } + enum _memory_space_t get_type() const { return m_type; } + void set_type( enum _memory_space_t t ) { m_type = t; } + unsigned get_bank() const { return m_bank; } + void set_bank( unsigned b ) { m_bank = b; } + bool is_const() const { return (m_type == const_space) || (m_type == param_space_kernel); } + bool is_local() const { return (m_type == local_space) || (m_type == param_space_local); } + bool is_global() const { return (m_type == global_space); } + +private: + enum _memory_space_t m_type; + unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 manual, sec. 5.1.3) }; const unsigned MAX_MEMORY_ACCESS_SIZE = 128; typedef std::bitset mem_access_byte_mask_t; -const unsigned SECTOR_CHUNCK_SIZE = 4; // four sectors -const unsigned SECTOR_SIZE = 32; // sector is 32 bytes width +const unsigned SECTOR_CHUNCK_SIZE = 4; //four sectors +const unsigned SECTOR_SIZE = 32 ; //sector is 32 bytes width typedef std::bitset mem_access_sector_mask_t; #define NO_PARTIAL_WRITE (mem_access_byte_mask_t()) -#define MEM_ACCESS_TYPE_TUP_DEF \ - MA_TUP_BEGIN(mem_access_type) \ - MA_TUP(GLOBAL_ACC_R) \ - , MA_TUP(LOCAL_ACC_R), MA_TUP(CONST_ACC_R), MA_TUP(TEXTURE_ACC_R), \ - MA_TUP(GLOBAL_ACC_W), MA_TUP(LOCAL_ACC_W), MA_TUP(L1_WRBK_ACC), \ - MA_TUP(L2_WRBK_ACC), MA_TUP(INST_ACC_R), MA_TUP(L1_WR_ALLOC_R), \ - MA_TUP(L2_WR_ALLOC_R), \ - MA_TUP(NUM_MEM_ACCESS_TYPE) MA_TUP_END(mem_access_type) +#define MEM_ACCESS_TYPE_TUP_DEF \ +MA_TUP_BEGIN( mem_access_type ) \ + MA_TUP( GLOBAL_ACC_R ), \ + MA_TUP( LOCAL_ACC_R ), \ + MA_TUP( CONST_ACC_R ), \ + MA_TUP( TEXTURE_ACC_R ), \ + MA_TUP( GLOBAL_ACC_W ), \ + MA_TUP( LOCAL_ACC_W ), \ + MA_TUP( L1_WRBK_ACC ), \ + MA_TUP( L2_WRBK_ACC ), \ + MA_TUP( INST_ACC_R ), \ + MA_TUP( L1_WR_ALLOC_R ), \ + MA_TUP( L2_WR_ALLOC_R ), \ + MA_TUP( NUM_MEM_ACCESS_TYPE ) \ +MA_TUP_END( mem_access_type ) #define MA_TUP_BEGIN(X) enum X { #define MA_TUP(X) X -#define MA_TUP_END(X) \ - } \ - ; +#define MA_TUP_END(X) }; MEM_ACCESS_TYPE_TUP_DEF #undef MA_TUP_BEGIN #undef MA_TUP #undef MA_TUP_END -const char *mem_access_type_str(enum mem_access_type access_type); +const char * mem_access_type_str(enum mem_access_type access_type); enum cache_operator_type { - CACHE_UNDEFINED, - - // loads - CACHE_ALL, // .ca - CACHE_LAST_USE, // .lu - CACHE_VOLATILE, // .cv - CACHE_L1, // .nc - - // loads and stores - CACHE_STREAMING, // .cs - CACHE_GLOBAL, // .cg - - // stores - CACHE_WRITE_BACK, // .wb - CACHE_WRITE_THROUGH // .wt + CACHE_UNDEFINED, + + // loads + CACHE_ALL, // .ca + CACHE_LAST_USE, // .lu + CACHE_VOLATILE, // .cv + CACHE_L1, // .nc + + // loads and stores + CACHE_STREAMING, // .cs + CACHE_GLOBAL, // .cg + + // stores + CACHE_WRITE_BACK, // .wb + CACHE_WRITE_THROUGH // .wt }; class mem_access_t { - public: - mem_access_t(gpgpu_context *ctx) { init(ctx); } - mem_access_t(mem_access_type type, new_addr_type address, unsigned size, - bool wr, gpgpu_context *ctx) { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - mem_access_t(mem_access_type type, new_addr_type address, unsigned size, - bool wr, const active_mask_t &active_mask, - const mem_access_byte_mask_t &byte_mask, - const mem_access_sector_mask_t §or_mask, gpgpu_context *ctx) - : m_warp_mask(active_mask), - m_byte_mask(byte_mask), - m_sector_mask(sector_mask) { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - - new_addr_type get_addr() const { return m_addr; } - void set_addr(new_addr_type addr) { m_addr = addr; } - unsigned get_size() const { return m_req_size; } - const active_mask_t &get_warp_mask() const { return m_warp_mask; } - bool is_write() const { return m_write; } - enum mem_access_type get_type() const { return m_type; } - mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } - mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } - - void print(FILE *fp) const { - fprintf(fp, "addr=0x%llx, %s, size=%u, ", m_addr, - m_write ? "store" : "load ", m_req_size); - switch (m_type) { - case GLOBAL_ACC_R: - fprintf(fp, "GLOBAL_R"); - break; - case LOCAL_ACC_R: - fprintf(fp, "LOCAL_R "); - break; - case CONST_ACC_R: - fprintf(fp, "CONST "); - break; - case TEXTURE_ACC_R: - fprintf(fp, "TEXTURE "); - break; - case GLOBAL_ACC_W: - fprintf(fp, "GLOBAL_W"); - break; - case LOCAL_ACC_W: - fprintf(fp, "LOCAL_W "); - break; - case L2_WRBK_ACC: - fprintf(fp, "L2_WRBK "); - break; - case INST_ACC_R: - fprintf(fp, "INST "); - break; - case L1_WRBK_ACC: - fprintf(fp, "L1_WRBK "); - break; - default: - fprintf(fp, "unknown "); - break; - } - } - - gpgpu_context *gpgpu_ctx; - - private: - void init(gpgpu_context *ctx); - - unsigned m_uid; - new_addr_type m_addr; // request address - bool m_write; - unsigned m_req_size; // bytes - mem_access_type m_type; - active_mask_t m_warp_mask; - mem_access_byte_mask_t m_byte_mask; - mem_access_sector_mask_t m_sector_mask; +public: + mem_access_t(gpgpu_context* ctx) { init(ctx); } + mem_access_t( mem_access_type type, + new_addr_type address, + unsigned size, + bool wr, + gpgpu_context* ctx) + { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + mem_access_t( mem_access_type type, + new_addr_type address, + unsigned size, + bool wr, + const active_mask_t &active_mask, + const mem_access_byte_mask_t &byte_mask, + const mem_access_sector_mask_t §or_mask, + gpgpu_context* ctx) + : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) + { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + + new_addr_type get_addr() const { return m_addr; } + void set_addr(new_addr_type addr) {m_addr=addr;} + unsigned get_size() const { return m_req_size; } + const active_mask_t &get_warp_mask() const { return m_warp_mask; } + bool is_write() const { return m_write; } + enum mem_access_type get_type() const { return m_type; } + mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } + mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } + + void print(FILE *fp) const + { + fprintf(fp,"addr=0x%llx, %s, size=%u, ", m_addr, m_write?"store":"load ", m_req_size ); + switch(m_type) { + case GLOBAL_ACC_R: fprintf(fp,"GLOBAL_R"); break; + case LOCAL_ACC_R: fprintf(fp,"LOCAL_R "); break; + case CONST_ACC_R: fprintf(fp,"CONST "); break; + case TEXTURE_ACC_R: fprintf(fp,"TEXTURE "); break; + case GLOBAL_ACC_W: fprintf(fp,"GLOBAL_W"); break; + case LOCAL_ACC_W: fprintf(fp,"LOCAL_W "); break; + case L2_WRBK_ACC: fprintf(fp,"L2_WRBK "); break; + case INST_ACC_R: fprintf(fp,"INST "); break; + case L1_WRBK_ACC: fprintf(fp,"L1_WRBK "); break; + default: fprintf(fp,"unknown "); break; + } + } + + gpgpu_context* gpgpu_ctx; +private: + void init(gpgpu_context* ctx); + + unsigned m_uid; + new_addr_type m_addr; // request address + bool m_write; + unsigned m_req_size; // bytes + mem_access_type m_type; + active_mask_t m_warp_mask; + mem_access_byte_mask_t m_byte_mask; + mem_access_sector_mask_t m_sector_mask; }; class mem_fetch; class mem_fetch_interface { - public: - virtual bool full(unsigned size, bool write) const = 0; - virtual void push(mem_fetch *mf) = 0; +public: + virtual bool full( unsigned size, bool write ) const = 0; + virtual void push( mem_fetch *mf ) = 0; }; class mem_fetch_allocator { - public: - virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type, - unsigned size, bool wr, - unsigned long long cycle) const = 0; - virtual mem_fetch *alloc(const class warp_inst_t &inst, - const mem_access_t &access, - unsigned long long cycle) const = 0; +public: + virtual mem_fetch *alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const = 0; + virtual mem_fetch *alloc( const class warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle ) const = 0; }; -// the maximum number of destination, source, or address uarch operands in a -// instruction -#define MAX_REG_OPERANDS 32 +// the maximum number of destination, source, or address uarch operands in a instruction +#define MAX_REG_OPERANDS 32 struct dram_callback_t { - dram_callback_t() { - function = NULL; - instruction = NULL; - thread = NULL; - } - void (*function)(const class inst_t *, class ptx_thread_info *); - - const class inst_t *instruction; - class ptx_thread_info *thread; + dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } + void (*function)(const class inst_t*, class ptx_thread_info*); + + const class inst_t* instruction; + class ptx_thread_info *thread; }; class inst_t { - public: - inst_t() { - m_decoded = false; - pc = (address_type)-1; - reconvergence_pc = (address_type)-1; - op = NO_OP; - bar_type = NOT_BAR; - red_type = NOT_RED; - bar_id = (unsigned)-1; - bar_count = (unsigned)-1; - oprnd_type = UN_OP; - sp_op = OTHER_OP; - op_pipe = UNKOWN_OP; - mem_op = NOT_TEX; - num_operands = 0; - num_regs = 0; - memset(out, 0, sizeof(unsigned)); - memset(in, 0, sizeof(unsigned)); - is_vectorin = 0; - is_vectorout = 0; - space = memory_space_t(); - cache_op = CACHE_UNDEFINED; - latency = 1; - initiation_interval = 1; - for (unsigned i = 0; i < MAX_REG_OPERANDS; i++) { - arch_reg.src[i] = -1; - arch_reg.dst[i] = -1; +public: + inst_t() + { + m_decoded=false; + pc=(address_type)-1; + reconvergence_pc=(address_type)-1; + op=NO_OP; + bar_type=NOT_BAR; + red_type=NOT_RED; + bar_id=(unsigned)-1; + bar_count=(unsigned)-1; + oprnd_type=UN_OP; + sp_op=OTHER_OP; + op_pipe=UNKOWN_OP; + mem_op=NOT_TEX; + num_operands=0; + num_regs=0; + memset(out, 0, sizeof(unsigned)); + memset(in, 0, sizeof(unsigned)); + is_vectorin=0; + is_vectorout=0; + space = memory_space_t(); + cache_op = CACHE_UNDEFINED; + latency = 1; + initiation_interval = 1; + for( unsigned i=0; i < MAX_REG_OPERANDS; i++ ) { + arch_reg.src[i] = -1; + arch_reg.dst[i] = -1; + } + isize=0; + } + bool valid() const { return m_decoded; } + virtual void print_insn( FILE *fp ) const + { + fprintf(fp," [inst @ pc=0x%04x] ", pc ); } - isize = 0; - } - bool valid() const { return m_decoded; } - virtual void print_insn(FILE *fp) const { - fprintf(fp, " [inst @ pc=0x%04x] ", pc); - } - bool is_load() const { - return (op == LOAD_OP || op == TENSOR_CORE_LOAD_OP || - memory_op == memory_load); - } - bool is_store() const { - return (op == STORE_OP || op == TENSOR_CORE_STORE_OP || - memory_op == memory_store); - } - unsigned get_num_operands() const { return num_operands; } - unsigned get_num_regs() const { return num_regs; } - void set_num_regs(unsigned num) { num_regs = num; } - void set_num_operands(unsigned num) { num_operands = num; } - void set_bar_id(unsigned id) { bar_id = id; } - void set_bar_count(unsigned count) { bar_count = count; } - - address_type pc; // program counter address of instruction - unsigned isize; // size of instruction in bytes - op_type op; // opcode (uarch visible) - - barrier_type bar_type; - reduction_type red_type; - unsigned bar_id; - unsigned bar_count; - - types_of_operands oprnd_type; // code (uarch visible) identify if the - // operation is an interger or a floating point - special_ops - sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... - operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of - // the operation (SP, SFU or MEM) - mem_operation mem_op; // code (uarch visible) identify memory type - _memory_op_t memory_op; // memory_op used by ptxplus - unsigned num_operands; - unsigned num_regs; // count vector operand as one register operand - - address_type reconvergence_pc; // -1 => not a branch, -2 => use function - // return address - - unsigned out[8]; - unsigned outcount; - unsigned in[24]; - unsigned incount; - unsigned char is_vectorin; - unsigned char is_vectorout; - int pred; // predicate register number - int ar1, ar2; - // register number for bank conflict evaluation - struct { - int dst[MAX_REG_OPERANDS]; - int src[MAX_REG_OPERANDS]; - } arch_reg; - // int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict - // evaluation - unsigned latency; // operation latency - unsigned initiation_interval; - - unsigned data_size; // what is the size of the word being operated on? - memory_space_t space; - cache_operator_type cache_op; - - protected: - bool m_decoded; - virtual void pre_decode() {} + bool is_load() const { return (op == LOAD_OP ||op==TENSOR_CORE_LOAD_OP || memory_op == memory_load); } + bool is_store() const { return (op == STORE_OP ||op==TENSOR_CORE_STORE_OP || memory_op == memory_store); } + unsigned get_num_operands() const {return num_operands;} + unsigned get_num_regs() const {return num_regs;} + void set_num_regs(unsigned num) {num_regs=num;} + void set_num_operands(unsigned num) {num_operands=num;} + void set_bar_id(unsigned id) {bar_id=id;} + void set_bar_count(unsigned count) {bar_count=count;} + + address_type pc; // program counter address of instruction + unsigned isize; // size of instruction in bytes + op_type op; // opcode (uarch visible) + + barrier_type bar_type; + reduction_type red_type; + unsigned bar_id; + unsigned bar_count; + + types_of_operands oprnd_type; // code (uarch visible) identify if the operation is an interger or a floating point + special_ops sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... + operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of the operation (SP, SFU or MEM) + mem_operation mem_op; // code (uarch visible) identify memory type + _memory_op_t memory_op; // memory_op used by ptxplus + unsigned num_operands; + unsigned num_regs; // count vector operand as one register operand + + address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address + + unsigned out[8]; + unsigned outcount; + unsigned in[24]; + unsigned incount; + unsigned char is_vectorin; + unsigned char is_vectorout; + int pred; // predicate register number + int ar1, ar2; + // register number for bank conflict evaluation + struct { + int dst[MAX_REG_OPERANDS]; + int src[MAX_REG_OPERANDS]; + } arch_reg; + //int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation + unsigned latency; // operation latency + unsigned initiation_interval; + + unsigned data_size; // what is the size of the word being operated on? + memory_space_t space; + cache_operator_type cache_op; + +protected: + bool m_decoded; + virtual void pre_decode() {} }; -enum divergence_support_t { POST_DOMINATOR = 1, NUM_SIMD_MODEL }; +enum divergence_support_t { + POST_DOMINATOR = 1, + NUM_SIMD_MODEL +}; const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; -class warp_inst_t : public inst_t { - public: - // constructors - warp_inst_t() { - m_uid = 0; - m_empty = true; - m_config = NULL; - } - warp_inst_t(const core_config *config) { - m_uid = 0; - assert(config->warp_size <= MAX_WARP_SIZE); - m_config = config; - m_empty = true; - m_isatomic = false; - m_per_scalar_thread_valid = false; - m_mem_accesses_created = false; - m_cache_hit = false; - m_is_printf = false; - m_is_cdp = 0; - } - virtual ~warp_inst_t() {} - - // modifiers - void broadcast_barrier_reduction(const active_mask_t &access_mask); - void do_atomic(bool forceDo = false); - void do_atomic(const active_mask_t &access_mask, bool forceDo = false); - void clear() { m_empty = true; } - - void issue(const active_mask_t &mask, unsigned warp_id, - unsigned long long cycle, int dynamic_warp_id, int sch_id); - - const active_mask_t &get_active_mask() const { return m_warp_active_mask; } - void completed(unsigned long long cycle) - const; // stat collection: called when the instruction is completed - - void set_addr(unsigned n, new_addr_type addr) { - if (!m_per_scalar_thread_valid) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid = true; - } - m_per_scalar_thread[n].memreqaddr[0] = addr; - } - void set_addr(unsigned n, new_addr_type *addr, unsigned num_addrs) { - if (!m_per_scalar_thread_valid) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid = true; - } - assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); - for (unsigned i = 0; i < num_addrs; i++) - m_per_scalar_thread[n].memreqaddr[i] = addr[i]; - } - void print_m_accessq() { - if (accessq_empty()) - return; - else { - printf("Printing mem access generated\n"); - std::list::iterator it; - for (it = m_accessq.begin(); it != m_accessq.end(); ++it) { - printf("MEM_TXN_GEN:%s:%llx, Size:%d \n", - mem_access_type_str(it->get_type()), it->get_addr(), - it->get_size()); - } +class warp_inst_t: public inst_t { +public: + // constructors + warp_inst_t() + { + m_uid=0; + m_empty=true; + m_config=NULL; } - } - struct transaction_info { - std::bitset<4> chunks; // bitmask: 32-byte chunks accessed - mem_access_byte_mask_t bytes; - active_mask_t active; // threads in this transaction - - bool test_bytes(unsigned start_bit, unsigned end_bit) { - for (unsigned i = start_bit; i <= end_bit; i++) - if (bytes.test(i)) return true; - return false; + warp_inst_t( const core_config *config ) + { + m_uid=0; + assert(config->warp_size<=MAX_WARP_SIZE); + m_config=config; + m_empty=true; + m_isatomic=false; + m_per_scalar_thread_valid=false; + m_mem_accesses_created=false; + m_cache_hit=false; + m_is_printf=false; + m_is_cdp = 0; } - }; - - void generate_mem_accesses(); - void memory_coalescing_arch(bool is_write, mem_access_type access_type); - void memory_coalescing_arch_atomic(bool is_write, - mem_access_type access_type); - void memory_coalescing_arch_reduce_and_send(bool is_write, - mem_access_type access_type, - const transaction_info &info, - new_addr_type addr, - unsigned segment_size); - - void add_callback(unsigned lane_id, void (*function)(const class inst_t *, - class ptx_thread_info *), - const inst_t *inst, class ptx_thread_info *thread, - bool atomic) { - if (!m_per_scalar_thread_valid) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid = true; - if (atomic) m_isatomic = true; + virtual ~warp_inst_t(){ } - m_per_scalar_thread[lane_id].callback.function = function; - m_per_scalar_thread[lane_id].callback.instruction = inst; - m_per_scalar_thread[lane_id].callback.thread = thread; - } - void set_active(const active_mask_t &active); - - void clear_active(const active_mask_t &inactive); - void set_not_active(unsigned lane_id); - - // accessors - virtual void print_insn(FILE *fp) const { - fprintf(fp, " [inst @ pc=0x%04x] ", pc); - for (int i = (int)m_config->warp_size - 1; i >= 0; i--) - fprintf(fp, "%c", ((m_warp_active_mask[i]) ? '1' : '0')); - } - bool active(unsigned thread) const { return m_warp_active_mask.test(thread); } - unsigned active_count() const { return m_warp_active_mask.count(); } - unsigned issued_count() const { - assert(m_empty == false); - return m_warp_issued_mask.count(); - } // for instruction counting - bool empty() const { return m_empty; } - unsigned warp_id() const { - assert(!m_empty); - return m_warp_id; - } - unsigned warp_id_func() const // to be used in functional simulations only - { - return m_warp_id; - } - unsigned dynamic_warp_id() const { - assert(!m_empty); - return m_dynamic_warp_id; - } - bool has_callback(unsigned n) const { - return m_warp_active_mask[n] && m_per_scalar_thread_valid && - (m_per_scalar_thread[n].callback.function != NULL); - } - new_addr_type get_addr(unsigned n) const { - assert(m_per_scalar_thread_valid); - return m_per_scalar_thread[n].memreqaddr[0]; - } - - bool isatomic() const { return m_isatomic; } - - unsigned warp_size() const { return m_config->warp_size; } - - bool accessq_empty() const { return m_accessq.empty(); } - unsigned accessq_count() const { return m_accessq.size(); } - const mem_access_t &accessq_back() { return m_accessq.back(); } - void accessq_pop_back() { m_accessq.pop_back(); } - - bool dispatch_delay() { - if (cycles > 0) cycles--; - return cycles > 0; - } - - bool has_dispatch_delay() { return cycles > 0; } - - void print(FILE *fout) const; - unsigned get_uid() const { return m_uid; } - unsigned get_schd_id() const { return m_scheduler_id; } - - protected: - unsigned m_uid; - bool m_empty; - bool m_cache_hit; - unsigned long long issue_cycle; - unsigned cycles; // used for implementing initiation interval delay - bool m_isatomic; - bool m_is_printf; - unsigned m_warp_id; - unsigned m_dynamic_warp_id; - const core_config *m_config; - active_mask_t m_warp_active_mask; // dynamic active mask for timing model - // (after predication) - active_mask_t m_warp_issued_mask; // active mask at issue (prior to - // predication test) -- for instruction - // counting - - struct per_thread_info { - per_thread_info() { - for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) - memreqaddr[i] = 0; + + // modifiers + void broadcast_barrier_reduction( const active_mask_t& access_mask); + void do_atomic(bool forceDo=false); + void do_atomic( const active_mask_t& access_mask, bool forceDo=false ); + void clear() + { + m_empty=true; } - dram_callback_t callback; - new_addr_type memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; // effective - // address, - // upto 8 - // different - // requests (to - // support 32B - // access in 8 - // chunks of 4B - // each) - }; - bool m_per_scalar_thread_valid; - std::vector m_per_scalar_thread; - bool m_mem_accesses_created; - std::list m_accessq; - - unsigned m_scheduler_id; // the scheduler that issues this inst - - // Jin: cdp support - public: - int m_is_cdp; -}; -void move_warp(warp_inst_t *&dst, warp_inst_t *&src); + void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ); -size_t get_kernel_code_size(class function_info *entry); -class checkpoint { - public: - checkpoint(); - ~checkpoint() { printf("clasfsfss destructed\n"); } + const active_mask_t & get_active_mask() const + { + return m_warp_active_mask; + } + void completed( unsigned long long cycle ) const; // stat collection: called when the instruction is completed - void load_global_mem(class memory_space *temp_mem, char *f1name); - void store_global_mem(class memory_space *mem, char *fname, char *format); - unsigned radnom; -}; -/* - * This abstract class used as a base for functional and performance and - * simulation, it has basic functional simulation - * data structures and procedures. - */ -class core_t { - public: - core_t(gpgpu_sim *gpu, kernel_info_t *kernel, unsigned warp_size, - unsigned threads_per_shader) - : m_gpu(gpu), - m_kernel(kernel), - m_simt_stack(NULL), - m_thread(NULL), - m_warp_size(warp_size) { - m_warp_count = threads_per_shader / m_warp_size; - // Handle the case where the number of threads is not a - // multiple of the warp size - if (threads_per_shader % m_warp_size != 0) { - m_warp_count += 1; + void set_addr( unsigned n, new_addr_type addr ) + { + if( !m_per_scalar_thread_valid ) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid=true; + } + m_per_scalar_thread[n].memreqaddr[0] = addr; } - assert(m_warp_count * m_warp_size > 0); - m_thread = (ptx_thread_info **)calloc(m_warp_count * m_warp_size, - sizeof(ptx_thread_info *)); - initilizeSIMTStack(m_warp_count, m_warp_size); - - for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) { - for (unsigned j = 0; j < MAX_BARRIERS_PER_CTA; j++) { - reduction_storage[i][j] = 0; - } + void set_addr( unsigned n, new_addr_type* addr, unsigned num_addrs ) + { + if( !m_per_scalar_thread_valid ) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid=true; + } + assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); + for(unsigned i=0; i::iterator it; + for (it = m_accessq.begin(); it != m_accessq.end(); ++it){ + printf("MEM_TXN_GEN:%s:%llx, Size:%d \n",mem_access_type_str(it->get_type()), it->get_addr(),it->get_size()); + } + } + } + struct transaction_info { + std::bitset<4> chunks; // bitmask: 32-byte chunks accessed + mem_access_byte_mask_t bytes; + active_mask_t active; // threads in this transaction + + bool test_bytes(unsigned start_bit, unsigned end_bit) { + for( unsigned i=start_bit; i<=end_bit; i++ ) + if(bytes.test(i)) + return true; + return false; + } + }; + + void generate_mem_accesses(); + void memory_coalescing_arch( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ); + void memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); + + void add_callback( unsigned lane_id, + void (*function)(const class inst_t*, class ptx_thread_info*), + const inst_t *inst, + class ptx_thread_info *thread, + bool atomic) + { + if( !m_per_scalar_thread_valid ) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid=true; + if(atomic) m_isatomic=true; + } + m_per_scalar_thread[lane_id].callback.function = function; + m_per_scalar_thread[lane_id].callback.instruction = inst; + m_per_scalar_thread[lane_id].callback.thread = thread; + } + void set_active( const active_mask_t &active ); -// register that can hold multiple instructions. -class register_set { - public: - register_set(unsigned num, const char *name) { - for (unsigned i = 0; i < num; i++) { - regs.push_back(new warp_inst_t()); + void clear_active( const active_mask_t &inactive ); + void set_not_active( unsigned lane_id ); + + // accessors + virtual void print_insn(FILE *fp) const + { + fprintf(fp," [inst @ pc=0x%04x] ", pc ); + for (int i=(int)m_config->warp_size-1; i>=0; i--) + fprintf(fp, "%c", ((m_warp_active_mask[i])?'1':'0') ); } - m_name = name; - } - bool has_free() { - for (unsigned i = 0; i < regs.size(); i++) { - if (regs[i]->empty()) { - return true; - } + bool active( unsigned thread ) const { return m_warp_active_mask.test(thread); } + unsigned active_count() const { return m_warp_active_mask.count(); } + unsigned issued_count() const { assert(m_empty == false); return m_warp_issued_mask.count(); } // for instruction counting + bool empty() const { return m_empty; } + unsigned warp_id() const + { + assert( !m_empty ); + return m_warp_id; } - return false; - } - bool has_free(bool sub_core_model, unsigned reg_id) { - // in subcore model, each sched has a one specific reg to use (based on - // sched id) - if (!sub_core_model) return has_free(); - - assert(reg_id < regs.size()); - return regs[reg_id]->empty(); - } - bool has_ready() { - for (unsigned i = 0; i < regs.size(); i++) { - if (not regs[i]->empty()) { - return true; - } + unsigned warp_id_func() const // to be used in functional simulations only + { + return m_warp_id; } - return false; - } - - void move_in(warp_inst_t *&src) { - warp_inst_t **free = get_free(); - move_warp(*free, src); - } - // void copy_in( warp_inst_t* src ){ - // src->copy_contents_to(*get_free()); - //} - void move_out_to(warp_inst_t *&dest) { - warp_inst_t **ready = get_ready(); - move_warp(dest, *ready); - } - - warp_inst_t **get_ready() { - warp_inst_t **ready; - ready = NULL; - for (unsigned i = 0; i < regs.size(); i++) { - if (not regs[i]->empty()) { - if (ready and (*ready)->get_uid() < regs[i]->get_uid()) { - // ready is oldest - } else { - ready = ®s[i]; - } - } + unsigned dynamic_warp_id() const + { + assert( !m_empty ); + return m_dynamic_warp_id; } - return ready; - } - - void print(FILE *fp) const { - fprintf(fp, "%s : @%p\n", m_name, this); - for (unsigned i = 0; i < regs.size(); i++) { - fprintf(fp, " "); - regs[i]->print(fp); - fprintf(fp, "\n"); + bool has_callback( unsigned n ) const + { + return m_warp_active_mask[n] && m_per_scalar_thread_valid && + (m_per_scalar_thread[n].callback.function!=NULL); + } + new_addr_type get_addr( unsigned n ) const + { + assert( m_per_scalar_thread_valid ); + return m_per_scalar_thread[n].memreqaddr[0]; + } + + bool isatomic() const { return m_isatomic; } + + unsigned warp_size() const { return m_config->warp_size; } + + bool accessq_empty() const { return m_accessq.empty(); } + unsigned accessq_count() const { return m_accessq.size(); } + const mem_access_t &accessq_back() { return m_accessq.back(); } + void accessq_pop_back() { m_accessq.pop_back(); } + + bool dispatch_delay() + { + if( cycles > 0 ) + cycles--; + return cycles > 0; } - } - warp_inst_t **get_free() { - for (unsigned i = 0; i < regs.size(); i++) { - if (regs[i]->empty()) { - return ®s[i]; - } + bool has_dispatch_delay(){ + return cycles > 0; } - assert(0 && "No free registers found"); - return NULL; - } - - warp_inst_t **get_free(bool sub_core_model, unsigned reg_id) { - // in subcore model, each sched has a one specific reg to use (based on - // sched id) - if (!sub_core_model) return get_free(); - - assert(reg_id < regs.size()); - if (regs[reg_id]->empty()) { - return ®s[reg_id]; + + void print( FILE *fout ) const; + unsigned get_uid() const { return m_uid; } + unsigned get_schd_id() const { return m_scheduler_id; } + +protected: + + unsigned m_uid; + bool m_empty; + bool m_cache_hit; + unsigned long long issue_cycle; + unsigned cycles; // used for implementing initiation interval delay + bool m_isatomic; + bool m_is_printf; + unsigned m_warp_id; + unsigned m_dynamic_warp_id; + const core_config *m_config; + active_mask_t m_warp_active_mask; // dynamic active mask for timing model (after predication) + active_mask_t m_warp_issued_mask; // active mask at issue (prior to predication test) -- for instruction counting + + struct per_thread_info { + per_thread_info() { + for(unsigned i=0; i m_per_scalar_thread; + bool m_mem_accesses_created; + std::list m_accessq; + + unsigned m_scheduler_id; //the scheduler that issues this inst + + //Jin: cdp support +public: + int m_is_cdp; + +}; + +void move_warp( warp_inst_t *&dst, warp_inst_t *&src ); + +size_t get_kernel_code_size( class function_info *entry ); +class checkpoint +{ +public: + + checkpoint(); + ~checkpoint(){ + printf("clasfsfss destructed\n"); } - assert(0 && "No free register found"); - return NULL; - } - unsigned get_size() { return regs.size(); } + void load_global_mem(class memory_space *temp_mem, char * f1name); + void store_global_mem(class memory_space *mem, char * fname , char * format); + unsigned radnom; - private: - std::vector regs; - const char *m_name; + +}; +/* + * This abstract class used as a base for functional and performance and simulation, it has basic functional simulation + * data structures and procedures. + */ +class core_t { + public: + core_t( gpgpu_sim *gpu, + kernel_info_t *kernel, + unsigned warp_size, + unsigned threads_per_shader ) + : m_gpu( gpu ), + m_kernel( kernel ), + m_simt_stack( NULL ), + m_thread( NULL ), + m_warp_size( warp_size ) + { + m_warp_count = threads_per_shader/m_warp_size; + // Handle the case where the number of threads is not a + // multiple of the warp size + if ( threads_per_shader % m_warp_size != 0 ) { + m_warp_count += 1; + } + assert( m_warp_count * m_warp_size > 0 ); + m_thread = ( ptx_thread_info** ) + calloc( m_warp_count * m_warp_size, + sizeof( ptx_thread_info* ) ); + initilizeSIMTStack(m_warp_count,m_warp_size); + + for(unsigned i=0; iempty() ) { + return true; + } + } + return false; + } + bool has_free(bool sub_core_model, unsigned reg_id){ + //in subcore model, each sched has a one specific reg to use (based on sched id) + if(!sub_core_model) + return has_free(); + + assert(reg_id < regs.size()); + return regs[reg_id]->empty(); + } + bool has_ready(){ + for( unsigned i = 0; i < regs.size(); i++ ) { + if( not regs[i]->empty() ) { + return true; + } + } + return false; + } + + void move_in( warp_inst_t *&src ){ + warp_inst_t** free = get_free(); + move_warp(*free, src); + } + //void copy_in( warp_inst_t* src ){ + // src->copy_contents_to(*get_free()); + //} + void move_out_to( warp_inst_t *&dest ){ + warp_inst_t **ready=get_ready(); + move_warp(dest, *ready); + } + + warp_inst_t** get_ready(){ + warp_inst_t** ready; + ready = NULL; + for( unsigned i = 0; i < regs.size(); i++ ) { + if( not regs[i]->empty() ) { + if( ready and (*ready)->get_uid() < regs[i]->get_uid() ) { + // ready is oldest + } else { + ready = ®s[i]; + } + } + } + return ready; + } + + void print(FILE* fp) const{ + fprintf(fp, "%s : @%p\n", m_name, this); + for( unsigned i = 0; i < regs.size(); i++ ) { + fprintf(fp, " "); + regs[i]->print(fp); + fprintf(fp, "\n"); + } + } + + warp_inst_t ** get_free(){ + for( unsigned i = 0; i < regs.size(); i++ ) { + if( regs[i]->empty() ) { + return ®s[i]; + } + } + assert(0 && "No free registers found"); + return NULL; + } + + warp_inst_t ** get_free(bool sub_core_model, unsigned reg_id){ + //in subcore model, each sched has a one specific reg to use (based on sched id) + if(!sub_core_model) + return get_free(); + + assert(reg_id < regs.size()); + if( regs[reg_id]->empty() ) { + return ®s[reg_id]; + } + assert(0 && "No free register found"); + return NULL; + } + + unsigned get_size(){ + return regs.size(); + } + +private: + std::vector regs; + const char* m_name; }; -#endif // #ifdef __cplusplus +#endif // #ifdef __cplusplus -#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED +#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED diff --git a/src/debug.cc b/src/debug.cc index eaf39a6..c00ff9e 100644 --- a/src/debug.cc +++ b/src/debug.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,197 +26,183 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "debug.h" +#include "gpgpu-sim/shader.h" +#include "gpgpu-sim/gpu-sim.h" +#include "cuda-sim/ptx_sim.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" -#include "cuda-sim/ptx_sim.h" -#include "gpgpu-sim/gpu-sim.h" -#include "gpgpu-sim/shader.h" +#include #include #include -#include -void gpgpu_sim::hit_watchpoint(unsigned watchpoint_num, ptx_thread_info *thd, - const ptx_instruction *pI) { - g_watchpoint_hits[watchpoint_num] = watchpoint_event(thd, pI); +void gpgpu_sim::hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI ) +{ + g_watchpoint_hits[watchpoint_num]=watchpoint_event(thd,pI); } -/// interactive debugger - -void gpgpu_sim::gpgpu_debug() { - bool done = true; - - static bool single_step = true; - static unsigned next_brkpt = 1; - static std::map breakpoints; - - /// if single stepping, go to interactive debugger - - if (single_step) done = false; - - /// check if we've reached a breakpoint - const ptx_thread_info *brk_thd = NULL; - const ptx_instruction *brk_inst = NULL; - - for (std::map::iterator i = breakpoints.begin(); - i != breakpoints.end(); i++) { - unsigned num = i->first; - brk_pt &b = i->second; - if (b.is_watchpoint()) { - unsigned addr = b.get_addr(); - unsigned new_value; - m_global_mem->read(addr, 4, &new_value); - if (new_value != b.get_value() || - g_watchpoint_hits.find(num) != g_watchpoint_hits.end()) { - printf( - "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new " - "value=%x)\n", - num, b.get_value(), new_value); - std::map::iterator w = - g_watchpoint_hits.find(num); - if (w == g_watchpoint_hits.end()) - printf("GPGPU-Sim PTX DBG: memory transfer modified value\n"); - else { - watchpoint_event wa = w->second; - brk_thd = wa.thread(); - brk_inst = wa.inst(); - printf( - "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, " - "hwtid=%u\n", - brk_thd->get_uid(), brk_thd->get_hw_sid(), brk_thd->get_hw_tid()); - printf("GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf("\n"); - g_watchpoint_hits.erase(w); - } - b.set_value(new_value); - done = false; - } - } else { - /* - for( unsigned sid=0; sid < m_n_shader; sid++ ) { - unsigned hw_thread_id = -1; - abort(); - ptx_thread_info *thread = - m_sc[sid]->get_functional_thread(hw_thread_id); - if( thread_at_brkpt(thread, b) ) { - done = false; - printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s (sm=%u, - hwtid=%u)\n", - num, b.location().c_str(), sid, hw_thread_id ); - brk_thd = thread; - brk_inst = brk_thd->get_inst(); - printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, - hwtid=%u\n", - brk_thd->get_uid(),brk_thd->get_hw_sid(), - brk_thd->get_hw_tid() ); - printf( "GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf( "\n" ); - } - } - */ - } - } - - if (done) assert(g_watchpoint_hits.empty()); - - /// enter interactive debugger loop - - while (!done) { - printf("(ptx debugger) "); - fflush(stdout); - - char line[1024]; - fgets(line, 1024, stdin); - - char *tok = strtok(line, " \t\n"); - if (!strcmp(tok, "dp")) { - int shader_num = 0; - tok = strtok(NULL, " \t\n"); - sscanf(tok, "%d", &shader_num); - dump_pipeline((0x40 | 0x4 | 0x1), shader_num, 0); - printf("\n"); - fflush(stdout); - } else if (!strcmp(tok, "q") || !strcmp(tok, "quit")) { - printf("\nreally quit GPGPU-Sim (y/n)?\n"); - fgets(line, 1024, stdin); - tok = strtok(line, " \t\n"); - if (!strcmp(tok, "y")) { - exit(0); +/// interactive debugger + +void gpgpu_sim::gpgpu_debug() +{ + bool done=true; + + static bool single_step=true; + static unsigned next_brkpt=1; + static std::map breakpoints; + + /// if single stepping, go to interactive debugger + + if( single_step ) + done=false; + + /// check if we've reached a breakpoint + const ptx_thread_info *brk_thd = NULL; + const ptx_instruction *brk_inst = NULL; + + for( std::map::iterator i=breakpoints.begin(); i!=breakpoints.end(); i++) { + unsigned num=i->first; + brk_pt &b=i->second; + if( b.is_watchpoint() ) { + unsigned addr = b.get_addr(); + unsigned new_value; + m_global_mem->read(addr,4,&new_value); + if( new_value != b.get_value() || g_watchpoint_hits.find(num) != g_watchpoint_hits.end() ) { + printf( "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new value=%x)\n", + num,b.get_value(),new_value ); + std::map::iterator w=g_watchpoint_hits.find(num); + if( w==g_watchpoint_hits.end() ) + printf( "GPGPU-Sim PTX DBG: memory transfer modified value\n"); + else { + watchpoint_event wa = w->second; + brk_thd = wa.thread(); + brk_inst = wa.inst(); + printf( "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, hwtid=%u\n", + brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); + printf( "GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf( "\n" ); + g_watchpoint_hits.erase(w); + } + b.set_value(new_value); + done = false; + } } else { - printf("not quiting.\n"); + /* + for( unsigned sid=0; sid < m_n_shader; sid++ ) { + unsigned hw_thread_id = -1; + abort(); + ptx_thread_info *thread = m_sc[sid]->get_functional_thread(hw_thread_id); + if( thread_at_brkpt(thread, b) ) { + done = false; + printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s (sm=%u, hwtid=%u)\n", + num, b.location().c_str(), sid, hw_thread_id ); + brk_thd = thread; + brk_inst = brk_thd->get_inst(); + printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, hwtid=%u\n", + brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); + printf( "GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf( "\n" ); + } + } + */ } - } else if (!strcmp(tok, "b")) { - tok = strtok(NULL, " \t\n"); - char brkpt[1024]; - sscanf(tok, "%s", brkpt); - tok = strtok(NULL, " \t\n"); - unsigned uid; - sscanf(tok, "%u", &uid); - breakpoints[next_brkpt++] = brk_pt(brkpt, uid); - } else if (!strcmp(tok, "d")) { - tok = strtok(NULL, " \t\n"); - unsigned uid; - sscanf(tok, "%u", &uid); - breakpoints.erase(uid); - } else if (!strcmp(tok, "s")) { - done = true; - } else if (!strcmp(tok, "c")) { - single_step = false; - done = true; - } else if (!strcmp(tok, "w")) { - tok = strtok(NULL, " \t\n"); - unsigned addr; - sscanf(tok, "%x", &addr); - unsigned value; - m_global_mem->read(addr, 4, &value); - m_global_mem->set_watch(addr, next_brkpt); - breakpoints[next_brkpt++] = brk_pt(addr, value); - } else if (!strcmp(tok, "l")) { - if (brk_thd == NULL) { - printf("no thread selected\n"); + } + + if( done ) + assert( g_watchpoint_hits.empty() ); + + /// enter interactive debugger loop + + while (!done) { + printf("(ptx debugger) "); + fflush(stdout); + + char line[1024]; + fgets(line,1024,stdin); + + char *tok = strtok(line," \t\n"); + if( !strcmp(tok,"dp") ) { + int shader_num = 0; + tok = strtok(NULL," \t\n"); + sscanf(tok,"%d",&shader_num); + dump_pipeline((0x40|0x4|0x1),shader_num,0); + printf("\n"); + fflush(stdout); + } else if( !strcmp(tok,"q") || !strcmp(tok,"quit") ) { + printf("\nreally quit GPGPU-Sim (y/n)?\n"); + fgets(line,1024,stdin); + tok = strtok(line," \t\n"); + if( !strcmp(tok,"y") ) { + exit(0); + } else { + printf("not quiting.\n"); + } + } else if( !strcmp(tok,"b") ) { + tok = strtok(NULL," \t\n"); + char brkpt[1024]; + sscanf(tok,"%s",brkpt); + tok = strtok(NULL," \t\n"); + unsigned uid; + sscanf(tok,"%u",&uid); + breakpoints[next_brkpt++] = brk_pt(brkpt,uid); + } else if( !strcmp(tok,"d") ) { + tok = strtok(NULL," \t\n"); + unsigned uid; + sscanf(tok,"%u",&uid); + breakpoints.erase(uid); + } else if( !strcmp(tok,"s") ) { + done = true; + } else if( !strcmp(tok,"c") ) { + single_step=false; + done = true; + } else if( !strcmp(tok,"w") ) { + tok = strtok(NULL," \t\n"); + unsigned addr; + sscanf(tok,"%x",&addr); + unsigned value; + m_global_mem->read(addr,4,&value); + m_global_mem->set_watch(addr,next_brkpt); + breakpoints[next_brkpt++] = brk_pt(addr,value); + } else if( !strcmp(tok,"l") ) { + if( brk_thd == NULL ) { + printf("no thread selected\n"); + } else { + addr_t pc = brk_thd->get_pc(); + addr_t start_pc = (pc<5)?0:(pc-5); + for( addr_t p=start_pc; p <= pc+5; p++ ) { + const ptx_instruction *i = brk_thd->get_inst(p); + if( i ) { + if( p != pc ) + printf( " " ); + else + printf( "==> " ); + i->print_insn(stdout); + printf( "\n" ); + } + } + } + } else if( !strcmp(tok,"h") ) { + printf("commands:\n"); + printf(" q - quit GPGPU-Sim\n"); + printf(" b : - set breakpoint\n"); + printf(" w - set watchpoint\n"); + printf(" del - delete breakpoint\n"); + printf(" s - single step one shader cycle (all cores)\n"); + printf(" c - continue simulation without single stepping\n"); + printf(" l - list PTX around current breakpoint\n"); + printf(" dp - display pipeline contents on SM \n"); + printf(" h - print this message\n"); } else { - addr_t pc = brk_thd->get_pc(); - addr_t start_pc = (pc < 5) ? 0 : (pc - 5); - for (addr_t p = start_pc; p <= pc + 5; p++) { - const ptx_instruction *i = brk_thd->get_inst(p); - if (i) { - if (p != pc) - printf(" "); - else - printf("==> "); - i->print_insn(stdout); - printf("\n"); - } - } + printf("\ncommand not understood.\n"); } - } else if (!strcmp(tok, "h")) { - printf("commands:\n"); - printf(" q - quit GPGPU-Sim\n"); - printf(" b : - set breakpoint\n"); - printf(" w - set watchpoint\n"); - printf(" del - delete breakpoint\n"); - printf( - " s - single step one shader cycle (all " - "cores)\n"); - printf( - " c - continue simulation without single " - "stepping\n"); - printf( - " l - list PTX around current " - "breakpoint\n"); - printf( - " dp - display pipeline contents on SM " - "\n"); - printf(" h - print this message\n"); - } else { - printf("\ncommand not understood.\n"); - } - fflush(stdout); - } + fflush(stdout); + } } -bool thread_at_brkpt(ptx_thread_info *thread, const class brk_pt &b) { - return b.is_equal(thread->get_location(), thread->get_uid()); +bool thread_at_brkpt( ptx_thread_info *thread, const class brk_pt &b ) +{ + return b.is_equal(thread->get_location(),thread->get_uid()); } + diff --git a/src/debug.h b/src/debug.h index 2799efe..4e79a9f 100644 --- a/src/debug.h +++ b/src/debug.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -35,53 +33,58 @@ #include class brk_pt { - public: - brk_pt() { m_valid = false; } - brk_pt(const char *fileline, unsigned uid) { - m_valid = true; - m_watch = false; - m_fileline = std::string(fileline); - m_thread_uid = uid; - } - brk_pt(unsigned addr, unsigned value) { - m_valid = true; - m_watch = true; - m_addr = addr; - m_value = value; - } +public: + brk_pt() { m_valid=false; } + brk_pt( const char *fileline, unsigned uid ) + { + m_valid = true; + m_watch = false; + m_fileline = std::string(fileline); + m_thread_uid=uid; + } + brk_pt( unsigned addr, unsigned value ) + { + m_valid = true; + m_watch = true; + m_addr = addr; + m_value = value; + } - unsigned get_value() const { return m_value; } - addr_t get_addr() const { return m_addr; } - bool is_valid() const { return m_valid; } - bool is_watchpoint() const { return m_watch; } - bool is_equal(const std::string &fileline, unsigned uid) const { - if (m_watch) return false; - if ((m_thread_uid != (unsigned)-1) && (uid != m_thread_uid)) return false; - return m_fileline == fileline; - } - std::string location() const { - char buffer[1024]; - sprintf(buffer, "%s thread uid = %u", m_fileline.c_str(), m_thread_uid); - return buffer; - } + unsigned get_value() const { return m_value; } + addr_t get_addr() const { return m_addr; } + bool is_valid() const { return m_valid; } + bool is_watchpoint() const { return m_watch; } + bool is_equal( const std::string &fileline, unsigned uid ) const + { + if( m_watch ) + return false; + if( (m_thread_uid != (unsigned)-1) && (uid != m_thread_uid) ) + return false; + return m_fileline == fileline; + } + std::string location() const + { + char buffer[1024]; + sprintf(buffer,"%s thread uid = %u", m_fileline.c_str(), m_thread_uid); + return buffer; + } - unsigned set_value(unsigned val) { return m_value = val; } + unsigned set_value( unsigned val ) { return m_value=val; } +private: + bool m_valid; + bool m_watch; - private: - bool m_valid; - bool m_watch; + // break point + std::string m_fileline; + unsigned m_thread_uid; - // break point - std::string m_fileline; - unsigned m_thread_uid; - - // watch point - unsigned m_addr; - unsigned m_value; + // watch point + unsigned m_addr; + unsigned m_value; }; class ptx_thread_info; class ptx_instruction; -bool thread_at_brkpt(ptx_thread_info *thd_info, const class brk_pt &b); +bool thread_at_brkpt( ptx_thread_info *thd_info, const class brk_pt &b ); #endif diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 04aa3f8..846773d 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,266 +28,253 @@ #include "gpgpusim_entrypoint.h" #include -#include "../libcuda/gpgpu_context.h" +#include "option_parser.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx_parser.h" #include "gpgpu-sim/gpu-sim.h" #include "gpgpu-sim/icnt_wrapper.h" -#include "option_parser.h" #include "stream_manager.h" +#include "../libcuda/gpgpu_context.h" -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) static int sg_argc = 3; -static const char *sg_argv[] = {"", "-config", "gpgpusim.config"}; - -void *gpgpu_sim_thread_sequential(void *ctx_ptr) { - gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; - // at most one kernel running at a time - bool done; - do { - sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); - done = true; - if (ctx->the_gpgpusim->g_the_gpu->get_more_cta_left()) { - done = false; - ctx->the_gpgpusim->g_the_gpu->init(); - while (ctx->the_gpgpusim->g_the_gpu->active()) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); +static const char *sg_argv[] = {"", "-config","gpgpusim.config"}; + + +void * gpgpu_sim_thread_sequential(void * ctx_ptr) +{ + gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; + // at most one kernel running at a time + bool done; + do { + sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); + done = true; + if( ctx->the_gpgpusim->g_the_gpu->get_more_cta_left() ) { + done = false; + ctx->the_gpgpusim->g_the_gpu->init(); + while( ctx->the_gpgpusim->g_the_gpu->active() ) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); + } + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); } - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); - } - sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); - } while (!done); - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); + } while(!done); + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } -static void termination_callback() { - printf("GPGPU-Sim: *** exit detected ***\n"); - fflush(stdout); + + +static void termination_callback() +{ + printf("GPGPU-Sim: *** exit detected ***\n"); + fflush(stdout); } -void *gpgpu_sim_thread_concurrent(void *ctx_ptr) { - gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; - atexit(termination_callback); - // concurrent kernel execution simulation thread - do { - if (g_debug_execution >= 3) { - printf( - "GPGPU-Sim: *** simulation thread starting and spinning waiting for " - "work ***\n"); - fflush(stdout); - } - while (ctx->the_gpgpusim->g_stream_manager->empty_protected() && - !ctx->the_gpgpusim->g_sim_done) - ; - if (g_debug_execution >= 3) { - printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); - ctx->the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); - } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = true; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - bool active = false; - bool sim_cycles = false; - ctx->the_gpgpusim->g_the_gpu->init(); +void *gpgpu_sim_thread_concurrent(void * ctx_ptr) +{ + gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; + atexit(termination_callback); + // concurrent kernel execution simulation thread do { - // check if a kernel has completed - // launch operation on device if one is pending and can be run - - // Need to break this loop when a kernel completes. This was a - // source of non-deterministic behaviour in GPGPU-Sim (bug 147). - // If another stream operation is available, g_the_gpu remains active, - // causing this loop to not break. If the next operation happens to be - // another kernel, the gpu is not re-initialized and the inter-kernel - // behaviour may be incorrect. Check that a kernel has finished and - // no other kernel is currently running. - if (ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && - !ctx->the_gpgpusim->g_the_gpu->active()) - break; - - // functional simulation - if (ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { - kernel_info_t *kernel = - ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); - assert(kernel); - ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func( - *kernel); - ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); - } + if(g_debug_execution >= 3) { + printf("GPGPU-Sim: *** simulation thread starting and spinning waiting for work ***\n"); + fflush(stdout); + } + while( ctx->the_gpgpusim->g_stream_manager->empty_protected() && !ctx->the_gpgpusim->g_sim_done ) + ; + if(g_debug_execution >= 3) { + printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); + ctx->the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = true; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + bool active = false; + bool sim_cycles = false; + ctx->the_gpgpusim->g_the_gpu->init(); + do { + // check if a kernel has completed + // launch operation on device if one is pending and can be run + + // Need to break this loop when a kernel completes. This was a + // source of non-deterministic behaviour in GPGPU-Sim (bug 147). + // If another stream operation is available, g_the_gpu remains active, + // causing this loop to not break. If the next operation happens to be + // another kernel, the gpu is not re-initialized and the inter-kernel + // behaviour may be incorrect. Check that a kernel has finished and + // no other kernel is currently running. + if(ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && !ctx->the_gpgpusim->g_the_gpu->active()) + break; + + //functional simulation + if( ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { + kernel_info_t * kernel = ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); + assert(kernel); + ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func(*kernel); + ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); + } + + //performance simulation + if( ctx->the_gpgpusim->g_the_gpu->active() ) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + sim_cycles = true; + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); + }else { + if(ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()){ + ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); + ctx->the_gpgpusim->g_sim_done = true; + ctx->the_gpgpusim->break_limit = true; + } + } - // performance simulation - if (ctx->the_gpgpusim->g_the_gpu->active()) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - sim_cycles = true; - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); - } else { - if (ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()) { - ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); - ctx->the_gpgpusim->g_sim_done = true; - ctx->the_gpgpusim->break_limit = true; + active=ctx->the_gpgpusim->g_the_gpu->active() || !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); + + } while( active && !ctx->the_gpgpusim->g_sim_done); + if(g_debug_execution >= 3) { + printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); + fflush(stdout); } - } + if(sim_cycles) { + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = false; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + } while( !ctx->the_gpgpusim->g_sim_done ); - active = ctx->the_gpgpusim->g_the_gpu->active() || - !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); + printf("GPGPU-Sim: *** simulation thread exiting ***\n"); + fflush(stdout); - } while (active && !ctx->the_gpgpusim->g_sim_done); - if (g_debug_execution >= 3) { - printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); - fflush(stdout); - } - if (sim_cycles) { - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); + if(ctx->the_gpgpusim->break_limit) { + printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n"); + exit(1); } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = false; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - } while (!ctx->the_gpgpusim->g_sim_done); - - printf("GPGPU-Sim: *** simulation thread exiting ***\n"); - fflush(stdout); - - if (ctx->the_gpgpusim->break_limit) { - printf( - "GPGPU-Sim: ** break due to reaching the maximum cycles (or " - "instructions) **\n"); - exit(1); - } - - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } -void gpgpu_context::synchronize() { - printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); - the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); - // sem_wait(&g_sim_signal_finish); - bool done = false; - do { - pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); - done = (the_gpgpusim->g_stream_manager->empty() && - !the_gpgpusim->g_sim_active) || - the_gpgpusim->g_sim_done; - pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); - } while (!done); - printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); - fflush(stdout); - // sem_post(&g_sim_signal_start); +void gpgpu_context::synchronize() +{ + printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); + the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); +// sem_wait(&g_sim_signal_finish); + bool done = false; + do { + pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); + done = ( the_gpgpusim->g_stream_manager->empty() && !the_gpgpusim->g_sim_active ) || the_gpgpusim->g_sim_done; + pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); + } while (!done); + printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); + fflush(stdout); +// sem_post(&g_sim_signal_start); } -void gpgpu_context::exit_simulation() { - the_gpgpusim->g_sim_done = true; - printf("GPGPU-Sim: exit_simulation called\n"); - fflush(stdout); - sem_wait(&(the_gpgpusim->g_sim_signal_exit)); - printf("GPGPU-Sim: simulation thread signaled exit\n"); - fflush(stdout); +void gpgpu_context::exit_simulation() +{ + the_gpgpusim->g_sim_done=true; + printf("GPGPU-Sim: exit_simulation called\n"); + fflush(stdout); + sem_wait(&(the_gpgpusim->g_sim_signal_exit)); + printf("GPGPU-Sim: simulation thread signaled exit\n"); + fflush(stdout); } -gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() { - srand(1); - print_splash(); - func_sim->read_sim_environment_variables(); - ptx_parser->read_parser_environment_variables(); - option_parser_t opp = option_parser_create(); - - ptx_reg_options(opp); - func_sim->ptx_opcocde_latency_options(opp); - - icnt_reg_options(opp); - the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); - the_gpgpusim->g_the_gpu_config->reg_options( - opp); // register GPU microrachitecture options - - option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options - fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); - option_parser_print(opp, stdout); - // Set the Numeric locale to a standard locale where a decimal point is a - // "dot" not a "comma" - // so it does the parsing correctly independent of the system environment - // variables - assert(setlocale(LC_NUMERIC, "C")); - the_gpgpusim->g_the_gpu_config->init(); - - the_gpgpusim->g_the_gpu = - new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); - the_gpgpusim->g_stream_manager = new stream_manager( - (the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); - - the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); - - sem_init(&(the_gpgpusim->g_sim_signal_start), 0, 0); - sem_init(&(the_gpgpusim->g_sim_signal_finish), 0, 0); - sem_init(&(the_gpgpusim->g_sim_signal_exit), 0, 0); - - return the_gpgpusim->g_the_gpu; +gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() +{ + srand(1); + print_splash(); + func_sim->read_sim_environment_variables(); + ptx_parser->read_parser_environment_variables(); + option_parser_t opp = option_parser_create(); + + ptx_reg_options(opp); + func_sim->ptx_opcocde_latency_options(opp); + + icnt_reg_options(opp); + the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); + the_gpgpusim->g_the_gpu_config->reg_options(opp); // register GPU microrachitecture options + + option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options + fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); + option_parser_print(opp, stdout); + // Set the Numeric locale to a standard locale where a decimal point is a "dot" not a "comma" + // so it does the parsing correctly independent of the system environment variables + assert(setlocale(LC_NUMERIC,"C")); + the_gpgpusim->g_the_gpu_config->init(); + + the_gpgpusim->g_the_gpu = new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); + the_gpgpusim->g_stream_manager = new stream_manager((the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); + + the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); + + sem_init(&(the_gpgpusim->g_sim_signal_start),0,0); + sem_init(&(the_gpgpusim->g_sim_signal_finish),0,0); + sem_init(&(the_gpgpusim->g_sim_signal_exit),0,0); + + return the_gpgpusim->g_the_gpu; } -void gpgpu_context::start_sim_thread(int api) { - if (the_gpgpusim->g_sim_done) { - the_gpgpusim->g_sim_done = false; - if (api == 1) { - pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, - gpgpu_sim_thread_concurrent, (void *)this); - } else { - pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, - gpgpu_sim_thread_sequential, (void *)this); +void gpgpu_context::start_sim_thread(int api) +{ + if( the_gpgpusim->g_sim_done ) { + the_gpgpusim->g_sim_done = false; + if( api == 1 ) { + pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_concurrent,(void *)this); + } else { + pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_sequential,(void *)this); + } } - } } -void gpgpu_context::print_simulation_time() { - time_t current_time, difference, d, h, m, s; - current_time = time((time_t *)NULL); - difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); - - d = difference / (3600 * 24); - h = difference / 3600 - 24 * d; - m = difference / 60 - 60 * (h + 24 * d); - s = difference - 60 * (m + 60 * (h + 24 * d)); - - fflush(stderr); - printf( - "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", - (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference)); - const unsigned cycles_per_sec = - (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); - printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec); - printf("gpgpu_silicon_slowdown = %ux\n", - the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); - fflush(stdout); +void gpgpu_context::print_simulation_time() +{ + time_t current_time, difference, d, h, m, s; + current_time = time((time_t *)NULL); + difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); + + d = difference/(3600*24); + h = difference/3600 - 24*d; + m = difference/60 - 60*(h + 24*d); + s = difference - 60*(m + 60*(h + 24*d)); + + fflush(stderr); + printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference ); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference) ); + const unsigned cycles_per_sec = (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); + printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec ); + printf("gpgpu_silicon_slowdown = %ux\n", the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); + fflush(stdout); } -int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf(kernel_info_t *grid) { - the_gpgpusim->g_the_gpu->launch(grid); - sem_post(&(the_gpgpusim->g_sim_signal_start)); - sem_wait(&(the_gpgpusim->g_sim_signal_finish)); - return 0; +int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf( kernel_info_t *grid ) +{ + the_gpgpusim->g_the_gpu->launch(grid); + sem_post(&(the_gpgpusim->g_sim_signal_start)); + sem_wait(&(the_gpgpusim->g_sim_signal_finish)); + return 0; } //! Functional simulation of OpenCL /*! * This function call the CUDA PTX functional simulator */ -int cuda_sim::gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid) { - // calling the CUDA PTX simulator, sending the kernel by reference and a flag - // set to true, - // the flag used by the function to distinguish OpenCL calls from the CUDA - // simulation calls which - // it is needed by the called function to not register the exit the exit of - // OpenCL kernel as it doesn't register entering in the first place as the - // CUDA kernels does - gpgpu_cuda_ptx_sim_main_func(*grid, true); - return 0; +int cuda_sim::gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ) +{ + //calling the CUDA PTX simulator, sending the kernel by reference and a flag set to true, + //the flag used by the function to distinguish OpenCL calls from the CUDA simulation calls which + //it is needed by the called function to not register the exit the exit of OpenCL kernel as it doesn't register entering in the first place as the CUDA kernels does + gpgpu_cuda_ptx_sim_main_func( *grid, true ); + return 0; } diff --git a/src/gpgpusim_entrypoint.h b/src/gpgpusim_entrypoint.h index 517885c..9f408df 100644 --- a/src/gpgpusim_entrypoint.h +++ b/src/gpgpusim_entrypoint.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,50 +28,52 @@ #ifndef GPGPUSIM_ENTRYPOINT_H_INCLUDED #define GPGPUSIM_ENTRYPOINT_H_INCLUDED +#include "abstract_hardware_model.h" #include #include #include -#include "abstract_hardware_model.h" -// extern time_t g_simulation_starttime; +//extern time_t g_simulation_starttime; class gpgpu_context; class GPGPUsim_ctx { - public: - GPGPUsim_ctx(gpgpu_context *ctx) { - g_sim_active = false; - g_sim_done = true; - break_limit = false; - g_sim_lock = PTHREAD_MUTEX_INITIALIZER; + public: + GPGPUsim_ctx(gpgpu_context* ctx) { + g_sim_active = false; + g_sim_done = true; + break_limit = false; + g_sim_lock = PTHREAD_MUTEX_INITIALIZER; + + g_the_gpu_config=NULL; + g_the_gpu=NULL; + g_stream_manager=NULL; + the_cude_device=NULL; + the_context=NULL; + gpgpu_ctx = ctx; + } + + //struct gpgpu_ptx_sim_arg *grid_params; - g_the_gpu_config = NULL; - g_the_gpu = NULL; - g_stream_manager = NULL; - the_cude_device = NULL; - the_context = NULL; - gpgpu_ctx = ctx; - } + sem_t g_sim_signal_start; + sem_t g_sim_signal_finish; + sem_t g_sim_signal_exit; + time_t g_simulation_starttime; + pthread_t g_simulation_thread; - // struct gpgpu_ptx_sim_arg *grid_params; + class gpgpu_sim_config *g_the_gpu_config; + class gpgpu_sim *g_the_gpu; + class stream_manager *g_stream_manager; - sem_t g_sim_signal_start; - sem_t g_sim_signal_finish; - sem_t g_sim_signal_exit; - time_t g_simulation_starttime; - pthread_t g_simulation_thread; + struct _cuda_device_id *the_cude_device; + struct CUctx_st* the_context; + gpgpu_context* gpgpu_ctx; - class gpgpu_sim_config *g_the_gpu_config; - class gpgpu_sim *g_the_gpu; - class stream_manager *g_stream_manager; - struct _cuda_device_id *the_cude_device; - struct CUctx_st *the_context; - gpgpu_context *gpgpu_ctx; + pthread_mutex_t g_sim_lock; + bool g_sim_active; + bool g_sim_done; + bool break_limit; - pthread_mutex_t g_sim_lock; - bool g_sim_active; - bool g_sim_done; - bool break_limit; }; #endif diff --git a/src/option_parser.cc b/src/option_parser.cc index 511553d..7d747f0 100644 --- a/src/option_parser.cc +++ b/src/option_parser.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,527 +25,523 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "option_parser.h" -#include #include #include -#include -#include -#include +#include +#include #include -#include -#include +#include #include -#include +#include #include +#include +#include +#include +#include "option_parser.h" + using namespace std; // A generic option registry regardless of data type -class OptionRegistryInterface { - public: - OptionRegistryInterface(const string optionName, const string optionDesc) - : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) {} - - virtual ~OptionRegistryInterface() {} - - const string &GetName() { return m_optionName; } - const string &GetDesc() { return m_optionDesc; } - const bool isParsed() { return m_isParsed; } - virtual string toString() = 0; - virtual bool fromString(const string str) = 0; - virtual bool isFlag() = 0; - virtual bool assignDefault(const char *str) = 0; - - protected: - string m_optionName; - string m_optionDesc; - bool m_isParsed; // true if the target variable has been updated by - // fromString() +class OptionRegistryInterface +{ +public: + OptionRegistryInterface(const string optionName, const string optionDesc) + : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) + {} + + virtual ~OptionRegistryInterface() {} + + const string& GetName() { return m_optionName; } + const string& GetDesc() { return m_optionDesc; } + const bool isParsed() { return m_isParsed; } + virtual string toString() = 0; + virtual bool fromString(const string str) = 0; + virtual bool isFlag() = 0; + virtual bool assignDefault(const char *str) = 0; + +protected: + string m_optionName; + string m_optionDesc; + bool m_isParsed; // true if the target variable has been updated by fromString() }; // Template for option registry - class T = specify data type of the option template -class OptionRegistry : public OptionRegistryInterface { - public: - OptionRegistry(const string name, const string desc, T &variable) - : OptionRegistryInterface(name, desc), m_variable(variable) {} - - virtual ~OptionRegistry() {} - - virtual string toString() { - stringstream ss; - ss << m_variable; - return ss.str(); - } - - virtual bool fromString(const string str) { - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - ss << setbase(10); - if (str.size() > 1 && str[0] == '0') { - if (str.size() > 2 && str[1] == 'x') { - ss.ignore(2); - ss << setbase(16); - } else { - ss.ignore(1); - ss << setbase(8); +class OptionRegistry : public OptionRegistryInterface +{ +public: + OptionRegistry(const string name, const string desc, T &variable) + : OptionRegistryInterface(name, desc), m_variable(variable) + {} + + virtual ~OptionRegistry() {} + + virtual string toString() + { + stringstream ss; + ss << m_variable; + return ss.str(); + } + + virtual bool fromString(const string str) + { + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + ss << setbase(10); + if (str.size() > 1 && str[0] == '0') { + if (str.size() > 2 && str[1] == 'x') { + ss.ignore(2); + ss << setbase(16); + } else { + ss.ignore(1); + ss << setbase(8); + } } - } - try { - ss >> m_variable; - } catch (exception &e) { - return false; - } - m_isParsed = true; - return true; - } + try { + ss >> m_variable; + } catch (exception &e) { + return false; + } + m_isParsed = true; + return true; + } - virtual bool isFlag() { return false; } - virtual bool assignDefault(const char *str) { return fromString(str); } + virtual bool isFlag() { return false; } + virtual bool assignDefault(const char *str) { return fromString(str); } - operator T() { return m_variable; } + operator T() + { + return m_variable; + } - private: - T &m_variable; +private: + T &m_variable; }; // specialized parser for string-type options -template <> -bool OptionRegistry::fromString(const string str) { - m_variable = str; - m_isParsed = true; - return true; +template<> +bool OptionRegistry::fromString(const string str) +{ + m_variable = str; + m_isParsed = true; + return true; } // specialized parser for c-string type options -template <> -bool OptionRegistry::fromString(const string str) { - m_variable = new char[str.size() + 1]; - strcpy(m_variable, str.c_str()); - m_isParsed = true; - return true; +template<> +bool OptionRegistry::fromString(const string str) +{ + m_variable = new char[str.size() + 1]; + strcpy(m_variable, str.c_str()); + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template <> -bool OptionRegistry::assignDefault(const char *str) { - m_variable = const_cast( - str); // c-string options are not meant to be edited anyway - m_isParsed = true; - return true; +template<> +bool OptionRegistry::assignDefault(const char *str) +{ + m_variable = const_cast(str); // c-string options are not meant to be edited anyway + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template <> -string OptionRegistry::toString() { - stringstream ss; - if (m_variable != NULL) { - ss << m_variable; - } else { - ss << "NULL"; - } - return ss.str(); +template<> +string OptionRegistry::toString() +{ + stringstream ss; + if (m_variable != NULL) { + ss << m_variable; + } else { + ss << "NULL"; + } + return ss.str(); } -// specialized parser for boolean options -template <> -bool OptionRegistry::fromString(const string str) { - int value = 1; - bool parsed = true; - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - try { - ss >> value; - } catch (stringstream::failure &ep) { - parsed = false; - } - assert(value == 0 or - value == - 1); // sanity check for boolean options (it can only be 1 or 0) - m_variable = (value != 0); - m_isParsed = true; - return parsed; +// specialized parser for boolean options +template<> +bool OptionRegistry::fromString(const string str) +{ + int value = 1; + bool parsed = true; + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + try { + ss >> value; + } catch (stringstream::failure &ep) { + parsed = false; + } + assert(value == 0 or value == 1); // sanity check for boolean options (it can only be 1 or 0) + m_variable = (value != 0); + m_isParsed = true; + return parsed; } // specializing a flag query function to identify boolean option -template <> -bool OptionRegistry::isFlag() { - return true; -} - -// class holding a collection of options and parse them from command -// line/configfile -class OptionParser { - public: - OptionParser() {} - ~OptionParser() { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); - ++i_option) { - delete (*i_option); - } - } - - template - void Register(const string optionName, const string optionDesc, - T &optionVariable, const char *optionDefault) { - OptionRegistry *p_option = - new OptionRegistry(optionName, optionDesc, optionVariable); - m_optionReg.push_back(p_option); - m_optionMap[optionName] = p_option; - p_option->assignDefault(optionDefault); - } - - void ParseCommandLine(int argc, const char *const argv[]) { - for (int i = 1; i < argc; i++) { - OptionMap::iterator i_option; - bool optionFound = false; - - i_option = m_optionMap.find(argv[i]); - if (i_option != m_optionMap.end()) { - const char *argstr = (i + 1 < argc) ? argv[i + 1] : ""; - OptionRegistryInterface *p_option = i_option->second; - if (p_option->isFlag()) { - if (p_option->fromString(argstr) == true) { +template<> +bool OptionRegistry::isFlag() { return true; } + +// class holding a collection of options and parse them from command line/configfile +class OptionParser +{ +public: + OptionParser() {} + ~OptionParser() + { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { + delete (*i_option); + } + } + + template + void Register(const string optionName, const string optionDesc, T &optionVariable, const char *optionDefault) + { + OptionRegistry *p_option = new OptionRegistry(optionName, optionDesc, optionVariable); + m_optionReg.push_back(p_option); + m_optionMap[optionName] = p_option; + p_option->assignDefault(optionDefault); + } + + void ParseCommandLine(int argc, const char * const argv[]) + { + for (int i = 1; i < argc; i++) { + OptionMap::iterator i_option; + bool optionFound = false; + + i_option = m_optionMap.find(argv[i]); + if (i_option != m_optionMap.end()) { + const char *argstr = (i + 1 < argc)? argv[i + 1] : ""; + OptionRegistryInterface *p_option = i_option->second; + if (p_option->isFlag()) { + if (p_option->fromString(argstr) == true) { + i += 1; + } + } else { + if (p_option->fromString(argstr) == false) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for option '%s'.\n", argstr, argv[i]); + exit(1); + } + i += 1; + } + optionFound = true; + } else if (string(argv[i]) == "-config") { + if (i + 1 >= argc) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Missing filename for option '-config'.\n"); + exit(1); + } + + ParseFile(argv[i + 1]); i += 1; - } - } else { - if (p_option->fromString(argstr) == false) { - fprintf(stderr, - "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for " - "option '%s'.\n", - argstr, argv[i]); + optionFound = true; + } + if (optionFound == false) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", argv[i]); exit(1); - } - i += 1; - } - optionFound = true; - } else if (string(argv[i]) == "-config") { - if (i + 1 >= argc) { - fprintf(stderr, - "\n\nGPGPU-Sim ** ERROR: Missing filename for option " - "'-config'.\n"); - exit(1); - } - - ParseFile(argv[i + 1]); - i += 1; - optionFound = true; + } } - if (optionFound == false) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", - argv[i]); - exit(1); + } + + + void ParseFile(const char *filename) { + ifstream inputFile; + stringstream args; + + // open config file, stream every line into a continuous buffer + // get rid of comments in the process + inputFile.open(filename); + if (!inputFile.good()) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", filename); + exit(1); } - } - } - - void ParseFile(const char *filename) { - ifstream inputFile; - stringstream args; - - // open config file, stream every line into a continuous buffer - // get rid of comments in the process - inputFile.open(filename); - if (!inputFile.good()) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", - filename); - exit(1); - } - while (inputFile.good()) { - string line; - getline(inputFile, line); - size_t commentStart = line.find_first_of("#"); - if (commentStart != line.npos) { - line.erase(commentStart); + while (inputFile.good()) { + string line; + getline(inputFile, line); + size_t commentStart = line.find_first_of("#"); + if (commentStart != line.npos) { + line.erase(commentStart); + } + args << line << ' '; } - args << line << ' '; - } - inputFile.close(); - - ParseStringStream(args); - } - - // parse the given string as tokens separated by a set of given delimiters - void ParseString(string inputString, const string delimiters = string(" ;")) { - // convert all delimiter characters into whitespaces - for (unsigned t = 0; t < inputString.size(); t++) { - for (unsigned d = 0; d < delimiters.size(); d++) { - if (inputString[t] == delimiters.at(d)) { - inputString[t] = ' '; - break; - } + inputFile.close(); + + ParseStringStream(args); + } + + // parse the given string as tokens separated by a set of given delimiters + void ParseString(string inputString, const string delimiters = string(" ;")) { + // convert all delimiter characters into whitespaces + for (unsigned t = 0; t < inputString.size(); t++) { + for (unsigned d = 0; d < delimiters.size(); d++) { + if (inputString[t] == delimiters.at(d)) { + inputString[t] = ' '; + break; + } + } } - } - stringstream args(inputString); - ParseStringStream(args); - } - - // parse the given stringstream as whitespace-separated tokens. drain the - // stream in the process - void ParseStringStream(stringstream &args) { - // extract non-whitespace string tokens - vector argv; - argv.push_back(new char[6]); - strcpy(argv[0], "dummy"); - while (args.good()) { - string argNew; - args >> argNew; - - if (argNew.size() == 0) continue; // this is probably the last token - - if (argNew[0] == '"') { - while (args.good() && argNew[argNew.size() - 1] != '"') { - string argCont; - args >> argCont; - argNew += " " + argCont; - } - argNew.erase(0, 1); - argNew.erase(argNew.size() - 1); + stringstream args(inputString); + ParseStringStream(args); + } + + // parse the given stringstream as whitespace-separated tokens. drain the stream in the process + void ParseStringStream(stringstream &args) { + // extract non-whitespace string tokens + vector argv; + argv.push_back(new char[6]); + strcpy(argv[0], "dummy"); + while (args.good()) { + string argNew; + args >> argNew; + + if (argNew.size() == 0) continue; // this is probably the last token + + if (argNew[0] == '"') { + while (args.good() && argNew[argNew.size()-1] != '"') { + string argCont; + args >> argCont; + argNew += " " + argCont; + } + argNew.erase(0,1); + argNew.erase(argNew.size()-1); + } + + char *c_argNew = new char[argNew.size() + 1]; + strcpy(c_argNew, argNew.c_str()); + argv.push_back(c_argNew); } - char *c_argNew = new char[argNew.size() + 1]; - strcpy(c_argNew, argNew.c_str()); - argv.push_back(c_argNew); - } - - // pass the string token into normal commandline parser - char **targv = (char **)calloc(argv.size(), sizeof(char *)); - for (unsigned k = 0; k < argv.size(); k++) targv[k] = argv[k]; - ParseCommandLine(argv.size(), targv); - free(targv); - for (size_t i = 0; i < argv.size(); i++) { - delete[] argv[i]; - } - } - - void Print(FILE *fout) { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); - ++i_option) { - stringstream sout; - if ((*i_option)->isParsed() == false) { - cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" - << (*i_option)->GetName() << "'\n"; - assert(0); + // pass the string token into normal commandline parser + char **targv = (char**)calloc(argv.size(), sizeof(char*)); + for( unsigned k=0; k < argv.size(); k++ ) + targv[k] = argv[k]; + ParseCommandLine(argv.size(), targv); + free(targv); + for (size_t i = 0; i < argv.size(); i++) { + delete[] argv[i]; } - sout << setw(20) << left << (*i_option)->GetName() << " "; - sout << setw(20) << right << (*i_option)->toString() << " # "; - sout << left << (*i_option)->GetDesc(); - sout << std::endl; - fprintf(fout, "%s", sout.str().c_str()); - } - } + } + + void Print(FILE *fout) + { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { + stringstream sout; + if ((*i_option)->isParsed() == false) { + cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" << (*i_option)->GetName() << "'\n"; + assert(0); + } + sout << setw(20) << left << (*i_option)->GetName() << " "; + sout << setw(20) << right << (*i_option)->toString() << " # "; + sout << left << (*i_option)->GetDesc(); + sout << std::endl; + fprintf(fout, "%s", sout.str().c_str()); + } + } - private: - typedef list OptionCollection; - OptionCollection m_optionReg; - typedef map OptionMap; - OptionMap m_optionMap; +private: + typedef list OptionCollection; + OptionCollection m_optionReg; + typedef map OptionMap; + OptionMap m_optionMap; }; #include "option_parser.h" -option_parser_t option_parser_create() { - OptionParser *p_opr = new OptionParser(); - return reinterpret_cast(p_opr); +option_parser_t option_parser_create() +{ + OptionParser *p_opr = new OptionParser(); + return reinterpret_cast(p_opr); } -void option_parser_destroy(option_parser_t opp) { - OptionParser *p_opr = reinterpret_cast(opp); - delete p_opr; +void option_parser_destroy(option_parser_t opp) +{ + OptionParser *p_opr = reinterpret_cast(opp); + delete p_opr; } -void option_parser_register(option_parser_t opp, const char *name, - enum option_dtype type, void *variable, - const char *desc, const char *defaultvalue) { - OptionParser *p_opr = reinterpret_cast(opp); - switch (type) { - case OPT_INT32: - p_opr->Register(name, desc, *(int *)variable, defaultvalue); - break; - case OPT_UINT32: - p_opr->Register(name, desc, *(unsigned int *)variable, - defaultvalue); - break; - case OPT_INT64: - p_opr->Register(name, desc, *(long long *)variable, - defaultvalue); - break; - case OPT_UINT64: - p_opr->Register( - name, desc, *(unsigned long long *)variable, defaultvalue); - break; - case OPT_BOOL: - p_opr->Register(name, desc, *(bool *)variable, defaultvalue); - break; - case OPT_FLOAT: - p_opr->Register(name, desc, *(float *)variable, defaultvalue); - break; - case OPT_DOUBLE: - p_opr->Register(name, desc, *(double *)variable, defaultvalue); - break; - case OPT_CHAR: - p_opr->Register(name, desc, *(char *)variable, defaultvalue); - break; - case OPT_CSTR: - p_opr->Register(name, desc, *(char **)variable, defaultvalue); - break; - default: - fprintf(stderr, - "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", - type); - exit(1); - break; - } +void option_parser_register(option_parser_t opp, + const char *name, + enum option_dtype type, + void *variable, + const char *desc, + const char *defaultvalue) +{ + OptionParser *p_opr = reinterpret_cast(opp); + switch (type) { + case OPT_INT32: p_opr->Register(name, desc, *(int*)variable, defaultvalue); break; + case OPT_UINT32: p_opr->Register(name, desc, *(unsigned int*)variable, defaultvalue); break; + case OPT_INT64: p_opr->Register(name, desc, *(long long*)variable, defaultvalue); break; + case OPT_UINT64: p_opr->Register(name, desc, *(unsigned long long*)variable, defaultvalue); break; + case OPT_BOOL: p_opr->Register(name, desc, *(bool*)variable, defaultvalue); break; + case OPT_FLOAT: p_opr->Register(name, desc, *(float*)variable, defaultvalue); break; + case OPT_DOUBLE: p_opr->Register(name, desc, *(double*)variable, defaultvalue); break; + case OPT_CHAR: p_opr->Register(name, desc, *(char*)variable, defaultvalue); break; + case OPT_CSTR: p_opr->Register(name, desc, *(char**)variable, defaultvalue); break; + default: + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", type); + exit(1); + break; + } } -void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]) { - OptionParser *p_opr = reinterpret_cast(opp); - return p_opr->ParseCommandLine(argc, argv); +void option_parser_cmdline(option_parser_t opp, + int argc, const char *argv[]) +{ + OptionParser *p_opr = reinterpret_cast(opp); + return p_opr->ParseCommandLine(argc,argv); + } -void option_parser_cfgfile(option_parser_t opp, const char *filename) { - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->ParseFile(filename); + +void option_parser_cfgfile(option_parser_t opp, + const char *filename) +{ + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->ParseFile(filename); } void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, - const char *delimiters) { - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->ParseString(inputstring, delimiters); + const char *inputstring, + const char *delimiters) +{ + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->ParseString(inputstring, delimiters); } -void option_parser_print(option_parser_t opp, FILE *fout) { - OptionParser *p_opr = reinterpret_cast(opp); - p_opr->Print(fout); +void option_parser_print(option_parser_t opp, + FILE *fout) +{ + OptionParser *p_opr = reinterpret_cast(opp); + p_opr->Print(fout); } + + // #define UNIT_TEST #ifdef UNIT_TEST -class testtype { - public: - int idata; - float fdata; - string sdata; - unsigned long long ulldata; - bool bdata; - unsigned int boolint; - char *coption; - - testtype() : idata(0), fdata(0.0f), sdata(""), ulldata(0), bdata(false) {} +class testtype +{ +public: + int idata; + float fdata; + string sdata; + unsigned long long ulldata; + bool bdata; + unsigned int boolint; + char * coption; + + testtype() + : idata(0), + fdata(0.0f), + sdata(""), + ulldata(0), + bdata(false) + { } }; -int cppinterfacetest(int argc, const char *argv[]) { - testtype c; - OptionParser optionparser; - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - - optionparser.Register("-idata", "integer data", c.idata, "-456"); - optionparser.Register("-fdata", "floating point data", c.fdata, - "0.001"); - optionparser.Register("-sdata", "first string data", c.sdata, - "hellow"); - optionparser.Register( - "-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); - optionparser.Register("-someflag", "first flag", c.bdata, "0"); - optionparser.Register("-otherflag", "second flag", (bool &)c.boolint, - "1"); - optionparser.Register("-coption", "char * data", c.coption, NULL); - - cout << "Default: \n"; - optionparser.Print(stdout); - - optionparser.ParseCommandLine(argc, argv); - - cout << "Commandline Parse Results: \n"; - optionparser.Print(stdout); - - optionparser.ParseFile("test.config"); - cout << "File Parse Results: \n"; - optionparser.Print(stdout); - cout << c.sdata << ' ' << c.idata << endl; - - return 0; + +int cppinterfacetest(int argc, const char *argv[]) +{ + testtype c; + OptionParser optionparser; + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); + + optionparser.Register("-idata", "integer data", c.idata, "-456"); + optionparser.Register("-fdata", "floating point data", c.fdata, "0.001"); + optionparser.Register("-sdata", "first string data", c.sdata, "hellow"); + optionparser.Register("-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); + optionparser.Register("-someflag", "first flag", c.bdata, "0"); + optionparser.Register("-otherflag", "second flag", (bool&)c.boolint, "1"); + optionparser.Register("-coption", "char * data", c.coption, NULL); + + cout << "Default: \n"; + optionparser.Print(stdout); + + optionparser.ParseCommandLine(argc, argv); + + cout << "Commandline Parse Results: \n"; + optionparser.Print(stdout); + + optionparser.ParseFile("test.config"); + cout << "File Parse Results: \n"; + optionparser.Print(stdout); + cout << c.sdata << ' ' << c.idata << endl; + + return 0; } -int cinterfacetest(int argc, const char *argv[]) { - testtype c; - option_parser_t opp = option_parser_create(); - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - char *otherstr; - - option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", - "-456"); - option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, - "floating point data", "0.001"); - option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, - "first string data", "hellow"); - option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, - "unsigend long long data", "0x123456789abcdef1"); - option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", - "0"); - option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", - "1"); - option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", - NULL); - - printf("Default: \n"); - option_parser_print(opp, stdout); - - option_parser_cmdline(opp, argc, argv); - - printf("Commandline Parse Results: \n"); - option_parser_print(opp, stdout); - - option_parser_cfgfile(opp, "test.config"); - printf("File Parse Results: \n"); - option_parser_print(opp, stdout); - printf("%s %d\n", otherstr, c.idata); - - option_parser_destroy(opp); - - return 0; +int cinterfacetest(int argc, const char *argv[]) +{ + testtype c; + option_parser_t opp = option_parser_create(); + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); + char *otherstr; + + option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", "-456"); + option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, "floating point data", "0.001"); + option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, "first string data", "hellow"); + option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, "unsigend long long data", "0x123456789abcdef1"); + option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", "0"); + option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", "1"); + option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", NULL); + + printf("Default: \n"); + option_parser_print(opp, stdout); + + option_parser_cmdline(opp, argc, argv); + + printf("Commandline Parse Results: \n"); + option_parser_print(opp, stdout); + + option_parser_cfgfile(opp, "test.config"); + printf("File Parse Results: \n"); + option_parser_print(opp, stdout); + printf("%s %d\n", otherstr, c.idata); + + option_parser_destroy(opp); + + return 0; } -int stringparsertest() { - int tABC; - int tDEF; - char tMode; - char *tName; - - option_parser_t opp = option_parser_create(); - option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); - option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); - option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); - option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); - - option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", - " ;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); - - option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", - " =;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); - - return 0; +int stringparsertest() +{ + int tABC; + int tDEF; + char tMode; + char *tName; + + option_parser_t opp = option_parser_create(); + option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); + option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); + option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); + option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); + + option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", " ;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); + + option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", " =;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); + + return 0; } -int main(int argc, const char *argv[]) { - cppinterfacetest(argc, argv); - cinterfacetest(argc, argv); - stringparsertest(); +int main(int argc, const char *argv[]) +{ + cppinterfacetest(argc,argv); + cinterfacetest(argc,argv); + stringparsertest(); - return 0; + return 0; } #endif + diff --git a/src/option_parser.h b/src/option_parser.h index 180bc9d..1f7f96c 100644 --- a/src/option_parser.h +++ b/src/option_parser.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,25 +25,26 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include #include + // pointer to C++ class typedef class OptionParser *option_parser_t; // data type of the option enum option_dtype { - OPT_INT32, - OPT_UINT32, - OPT_INT64, - OPT_UINT64, - OPT_BOOL, - OPT_FLOAT, - OPT_DOUBLE, - OPT_CHAR, - OPT_CSTR + OPT_INT32, + OPT_UINT32, + OPT_INT64, + OPT_UINT64, + OPT_BOOL, + OPT_FLOAT, + OPT_DOUBLE, + OPT_CHAR, + OPT_CSTR }; // create and destroy option parser @@ -53,19 +52,28 @@ option_parser_t option_parser_create(); void option_parser_destroy(option_parser_t opp); // register new option -void option_parser_register(option_parser_t opp, const char *name, - enum option_dtype type, void *variable, - const char *desc, const char *defaultvalue); +void option_parser_register(option_parser_t opp, + const char *name, + enum option_dtype type, + void *variable, + const char *desc, + const char *defaultvalue); // parse command line -void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]); +void option_parser_cmdline(option_parser_t opp, + int argc, const char *argv[]); + + // parse config file -void option_parser_cfgfile(option_parser_t opp, const char *filename); +void option_parser_cfgfile(option_parser_t opp, + const char *filename); // parse a delimited string void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, + const char *inputstring, const char *delimiters); // print options -void option_parser_print(option_parser_t opp, FILE *fout); +void option_parser_print(option_parser_t opp, + FILE *fout); + diff --git a/src/statwrapper.cc b/src/statwrapper.cc index cda25b6..e273e78 100644 --- a/src/statwrapper.cc +++ b/src/statwrapper.cc @@ -1,33 +1,48 @@ -// a Wraper function for stats class -#include +//a Wraper function for stats class #include "intersim2/stats.hpp" +#include -Stats *StatCreate(const char *name, double bin_size, int num_bins) { - Stats *newstat = new Stats(NULL, name, bin_size, num_bins); - newstat->Clear(); - return newstat; +Stats* StatCreate (const char * name, double bin_size, int num_bins) { + Stats* newstat = new Stats(NULL,name,bin_size,num_bins); + newstat->Clear (); + return newstat; } -void StatClear(void *st) { ((Stats *)st)->Clear(); } +void StatClear(void * st) +{ + ((Stats *)st)->Clear(); +} -void StatAddSample(void *st, int val) { ((Stats *)st)->AddSample(val); } +void StatAddSample (void * st, int val) +{ + ((Stats *)st)->AddSample(val); +} -double StatAverage(void *st) { return ((Stats *)st)->Average(); } +double StatAverage(void * st) +{ + return((Stats *)st)->Average(); +} -double StatMax(void *st) { return ((Stats *)st)->Max(); } +double StatMax(void * st) +{ + return((Stats *)st)->Max(); +} -double StatMin(void *st) { return ((Stats *)st)->Min(); } +double StatMin(void * st) +{ + return((Stats *)st)->Min(); +} -void StatDisp(void *st) { - printf("Stats for "); - ((Stats *)st)->DisplayHierarchy(); - // if (((Stats *)st)->NeverUsed()) { - // printf (" was never updated!\n"); - // } else { - printf("Min %f Max %f Average %f \n", ((Stats *)st)->Min(), - ((Stats *)st)->Max(), StatAverage(st)); - ((Stats *)st)->Display(); - // } +void StatDisp (void * st) +{ + printf ("Stats for "); + ((Stats *)st)->DisplayHierarchy(); +// if (((Stats *)st)->NeverUsed()) { +// printf (" was never updated!\n"); +// } else { + printf("Min %f Max %f Average %f \n",((Stats *)st)->Min(),((Stats *)st)->Max(),StatAverage(st)); + ((Stats *)st)->Display(); +// } } #if 0 @@ -40,3 +55,5 @@ int main () StatDisp(mytest); } #endif + + diff --git a/src/statwrapper.h b/src/statwrapper.h index 4c00447..65c2ab9 100644 --- a/src/statwrapper.h +++ b/src/statwrapper.h @@ -1,12 +1,12 @@ #ifndef STAT_WRAPER_H #define STAT_WRAPER_H -class Stats* StatCreate(const char* name, double bin_size, int num_bins); -void StatClear(void* st); -void StatAddSample(void* st, int val); -double StatAverage(void* st); -double StatMax(void* st); -double StatMin(void* st); -void StatDisp(void* st); +class Stats* StatCreate (const char * name, double bin_size, int num_bins) ; +void StatClear(void * st); +void StatAddSample (void * st, int val); +double StatAverage(void * st) ; +double StatMax(void * st) ; +double StatMin(void * st) ; +void StatDisp (void * st); #endif diff --git a/src/stream_manager.cc b/src/stream_manager.cc index f813ed4..be3dd71 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -28,445 +26,463 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "stream_manager.h" -#include "../libcuda/gpgpu_context.h" +#include "gpgpusim_entrypoint.h" #include "cuda-sim/cuda-sim.h" #include "gpgpu-sim/gpu-sim.h" -#include "gpgpusim_entrypoint.h" +#include "../libcuda/gpgpu_context.h" unsigned CUstream_st::sm_next_stream_uid = 0; -CUstream_st::CUstream_st() { - m_pending = false; - m_uid = sm_next_stream_uid++; - pthread_mutex_init(&m_lock, NULL); +CUstream_st::CUstream_st() +{ + m_pending = false; + m_uid = sm_next_stream_uid++; + pthread_mutex_init(&m_lock,NULL); +} + +bool CUstream_st::empty() +{ + pthread_mutex_lock(&m_lock); + bool empty = m_operations.empty(); + pthread_mutex_unlock(&m_lock); + return empty; } -bool CUstream_st::empty() { - pthread_mutex_lock(&m_lock); - bool empty = m_operations.empty(); - pthread_mutex_unlock(&m_lock); - return empty; +bool CUstream_st::busy() +{ + pthread_mutex_lock(&m_lock); + bool pending = m_pending; + pthread_mutex_unlock(&m_lock); + return pending; } -bool CUstream_st::busy() { - pthread_mutex_lock(&m_lock); - bool pending = m_pending; - pthread_mutex_unlock(&m_lock); - return pending; +void CUstream_st::synchronize() +{ + // called by host thread + bool done=false; + do{ + pthread_mutex_lock(&m_lock); + done = m_operations.empty(); + pthread_mutex_unlock(&m_lock); + } while ( !done ); } -void CUstream_st::synchronize() { - // called by host thread - bool done = false; - do { +void CUstream_st::push( const stream_operation &op ) +{ + // called by host thread pthread_mutex_lock(&m_lock); - done = m_operations.empty(); + m_operations.push_back( op ); pthread_mutex_unlock(&m_lock); - } while (!done); } -void CUstream_st::push(const stream_operation &op) { - // called by host thread - pthread_mutex_lock(&m_lock); - m_operations.push_back(op); - pthread_mutex_unlock(&m_lock); +void CUstream_st::record_next_done() +{ + // called by gpu thread + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_operations.pop_front(); + m_pending=false; + pthread_mutex_unlock(&m_lock); } -void CUstream_st::record_next_done() { - // called by gpu thread - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_operations.pop_front(); - m_pending = false; - pthread_mutex_unlock(&m_lock); -} -stream_operation CUstream_st::next() { - // called by gpu thread - pthread_mutex_lock(&m_lock); - m_pending = true; - stream_operation result = m_operations.front(); - pthread_mutex_unlock(&m_lock); - return result; +stream_operation CUstream_st::next() +{ + // called by gpu thread + pthread_mutex_lock(&m_lock); + m_pending = true; + stream_operation result = m_operations.front(); + pthread_mutex_unlock(&m_lock); + return result; } -void CUstream_st::cancel_front() { - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_pending = false; - pthread_mutex_unlock(&m_lock); +void CUstream_st::cancel_front() +{ + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_pending = false; + pthread_mutex_unlock(&m_lock); + } -void CUstream_st::print(FILE *fp) { - pthread_mutex_lock(&m_lock); - fprintf(fp, "GPGPU-Sim API: stream %u has %zu operations\n", m_uid, - m_operations.size()); - std::list::iterator i; - unsigned n = 0; - for (i = m_operations.begin(); i != m_operations.end(); i++) { - stream_operation &op = *i; - fprintf(fp, "GPGPU-Sim API: %u : ", n++); - op.print(fp); - fprintf(fp, "\n"); - } - pthread_mutex_unlock(&m_lock); +void CUstream_st::print(FILE *fp) +{ + pthread_mutex_lock(&m_lock); + fprintf(fp,"GPGPU-Sim API: stream %u has %zu operations\n", m_uid, m_operations.size() ); + std::list::iterator i; + unsigned n=0; + for( i=m_operations.begin(); i!=m_operations.end(); i++ ) { + stream_operation &op = *i; + fprintf(fp,"GPGPU-Sim API: %u : ", n++); + op.print(fp); + fprintf(fp,"\n"); + } + pthread_mutex_unlock(&m_lock); } -bool stream_operation::do_operation(gpgpu_sim *gpu) { - if (is_noop()) return true; - assert(!m_done && m_stream); - if (g_debug_execution >= 3) - printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid()); - switch (m_type) { +bool stream_operation::do_operation( gpgpu_sim *gpu ) +{ + if( is_noop() ) + return true; + + assert(!m_done && m_stream); + if(g_debug_execution >= 3) + printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid() ); + switch( m_type ) { case stream_memcpy_host_to_device: - if (g_debug_execution >= 3) printf("memcpy host-to-device\n"); - gpu->memcpy_to_gpu(m_device_address_dst, m_host_address_src, m_cnt); - m_stream->record_next_done(); - break; + if(g_debug_execution >= 3) + printf("memcpy host-to-device\n"); + gpu->memcpy_to_gpu(m_device_address_dst,m_host_address_src,m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_host: - if (g_debug_execution >= 3) printf("memcpy device-to-host\n"); - gpu->memcpy_from_gpu(m_host_address_dst, m_device_address_src, m_cnt); - m_stream->record_next_done(); - break; + if(g_debug_execution >= 3) + printf("memcpy device-to-host\n"); + gpu->memcpy_from_gpu(m_host_address_dst,m_device_address_src,m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_device: - if (g_debug_execution >= 3) printf("memcpy device-to-device\n"); - gpu->memcpy_gpu_to_gpu(m_device_address_dst, m_device_address_src, m_cnt); - m_stream->record_next_done(); - break; + if(g_debug_execution >= 3) + printf("memcpy device-to-device\n"); + gpu->memcpy_gpu_to_gpu(m_device_address_dst,m_device_address_src,m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_to_symbol: - if (g_debug_execution >= 3) printf("memcpy to symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( - m_symbol, m_host_address_src, m_cnt, m_offset, 1, gpu); - m_stream->record_next_done(); - break; + if(g_debug_execution >= 3) + printf("memcpy to symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_src,m_cnt,m_offset,1,gpu); + m_stream->record_next_done(); + break; case stream_memcpy_from_symbol: - if (g_debug_execution >= 3) printf("memcpy from symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( - m_symbol, m_host_address_dst, m_cnt, m_offset, 0, gpu); - m_stream->record_next_done(); - break; + if(g_debug_execution >= 3) + printf("memcpy from symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_dst,m_cnt,m_offset,0,gpu); + m_stream->record_next_done(); + break; case stream_kernel_launch: - if (m_sim_mode) { // Functional Sim - if (g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", - m_kernel->get_uid(), m_kernel->name().c_str()); - m_kernel->print_parent_info(); + if( m_sim_mode ) { //Functional Sim + if(g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); + m_kernel->print_parent_info(); + } + gpu->set_cache_config(m_kernel->name()); + gpu->functional_launch( m_kernel ); } - gpu->set_cache_config(m_kernel->name()); - gpu->functional_launch(m_kernel); - } else { // Performance Sim - if (gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { - if (g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", - m_kernel->get_uid(), m_kernel->name().c_str()); - m_kernel->print_parent_info(); - } - gpu->set_cache_config(m_kernel->name()); - gpu->launch(m_kernel); - } else { - if (m_kernel->m_launch_latency) m_kernel->m_launch_latency--; - if (g_debug_execution >= 3) - printf( - "kernel %d: \'%s\', latency %u not ready to transfer to GPU " - "hardware scheduler\n", - m_kernel->get_uid(), m_kernel->name().c_str(), - m_kernel->m_launch_latency); - return false; + else { //Performance Sim + if( gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { + if(g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); + m_kernel->print_parent_info(); + } + gpu->set_cache_config(m_kernel->name()); + gpu->launch( m_kernel ); + } + else { + if(m_kernel->m_launch_latency) + m_kernel->m_launch_latency--; + if(g_debug_execution >= 3) + printf("kernel %d: \'%s\', latency %u not ready to transfer to GPU hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str(), m_kernel->m_launch_latency); + return false; + } } - } - break; + break; case stream_event: { - printf("event update\n"); - time_t wallclock = time((time_t *)NULL); - m_event->update(gpu->gpu_tot_sim_cycle, wallclock); - m_stream->record_next_done(); - } break; + printf("event update\n"); + time_t wallclock = time((time_t *)NULL); + m_event->update( gpu->gpu_tot_sim_cycle, wallclock ); + m_stream->record_next_done(); + } + break; case stream_wait_event: { - // only allows next op to go if event is done - // otherwise stays in the stream queue - printf("stream wait event processing...\n"); - if (m_event->done()) printf("stream wait event done\n"); - m_stream->record_next_done(); - } break; + //only allows next op to go if event is done + //otherwise stays in the stream queue + printf("stream wait event processing...\n"); + if(m_event->done()) + printf("stream wait event done\n"); + m_stream->record_next_done(); + } + break; default: - abort(); - } - m_done = true; - fflush(stdout); - return true; + abort(); + } + m_done=true; + fflush(stdout); + return true; } -void stream_operation::print(FILE *fp) const { - fprintf(fp, " stream operation "); - switch (m_type) { - case stream_event: - fprintf(fp, "event"); - break; - case stream_kernel_launch: - fprintf(fp, "kernel"); - break; - case stream_memcpy_device_to_device: - fprintf(fp, "memcpy device-to-device"); - break; - case stream_memcpy_device_to_host: - fprintf(fp, "memcpy device-to-host"); - break; - case stream_memcpy_host_to_device: - fprintf(fp, "memcpy host-to-device"); - break; - case stream_memcpy_to_symbol: - fprintf(fp, "memcpy to symbol"); - break; - case stream_memcpy_from_symbol: - fprintf(fp, "memcpy from symbol"); - break; - case stream_no_op: - fprintf(fp, "no-op"); - break; - } +void stream_operation::print( FILE *fp ) const +{ + fprintf(fp," stream operation " ); + switch( m_type ) { + case stream_event: fprintf(fp,"event"); break; + case stream_kernel_launch: fprintf(fp,"kernel"); break; + case stream_memcpy_device_to_device: fprintf(fp,"memcpy device-to-device"); break; + case stream_memcpy_device_to_host: fprintf(fp,"memcpy device-to-host"); break; + case stream_memcpy_host_to_device: fprintf(fp,"memcpy host-to-device"); break; + case stream_memcpy_to_symbol: fprintf(fp,"memcpy to symbol"); break; + case stream_memcpy_from_symbol: fprintf(fp,"memcpy from symbol"); break; + case stream_no_op: fprintf(fp,"no-op"); break; + } } -stream_manager::stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking) { - m_gpu = gpu; - m_service_stream_zero = false; - m_cuda_launch_blocking = cuda_launch_blocking; - pthread_mutex_init(&m_lock, NULL); +stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) +{ + m_gpu = gpu; + m_service_stream_zero = false; + m_cuda_launch_blocking = cuda_launch_blocking; + pthread_mutex_init(&m_lock,NULL); } -bool stream_manager::operation(bool *sim) { - bool check = check_finished_kernel(); - pthread_mutex_lock(&m_lock); - // if(check)m_gpu->print_stats(); - stream_operation op = front(); - if (!op.do_operation(m_gpu)) // not ready to execute - { - // cancel operation - if (op.is_kernel()) { - unsigned grid_uid = op.get_kernel()->get_uid(); - m_grid_id_to_stream.erase(grid_uid); +bool stream_manager::operation( bool * sim) +{ + bool check=check_finished_kernel(); + pthread_mutex_lock(&m_lock); +// if(check)m_gpu->print_stats(); + stream_operation op =front(); + if(!op.do_operation( m_gpu )) //not ready to execute + { + //cancel operation + if( op.is_kernel() ) { + unsigned grid_uid = op.get_kernel()->get_uid(); + m_grid_id_to_stream.erase(grid_uid); + } + op.get_stream()->cancel_front(); + } - op.get_stream()->cancel_front(); - } - pthread_mutex_unlock(&m_lock); - // pthread_mutex_lock(&m_lock); - // simulate a clock cycle on the GPU - return check; + pthread_mutex_unlock(&m_lock); + //pthread_mutex_lock(&m_lock); + // simulate a clock cycle on the GPU + return check; } -bool stream_manager::check_finished_kernel() { - unsigned grid_uid = m_gpu->finished_kernel(); - bool check = register_finished_kernel(grid_uid); - return check; +bool stream_manager::check_finished_kernel() +{ + unsigned grid_uid = m_gpu->finished_kernel(); + bool check=register_finished_kernel(grid_uid); + return check; } -bool stream_manager::register_finished_kernel(unsigned grid_uid) { - // called by gpu simulation thread - if (grid_uid > 0) { - CUstream_st *stream = m_grid_id_to_stream[grid_uid]; - kernel_info_t *kernel = stream->front().get_kernel(); - assert(grid_uid == kernel->get_uid()); - - // Jin: should check children kernels for CDP - if (kernel->is_finished()) { - // std::ofstream kernel_stat("kernel_stat.txt", - // std::ofstream::out | std::ofstream::app); - // kernel_stat<< " kernel " << grid_uid << ": " << - // kernel->name(); - // if(kernel->get_parent()) - // kernel_stat << ", parent " << - // kernel->get_parent()->get_uid() << - // ", launch " << kernel->launch_cycle; - // kernel_stat<< ", start " << kernel->start_cycle << - // ", end " << kernel->end_cycle << ", retire " << - // gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; - // printf("kernel %d finishes, retires from stream %d\n", - // grid_uid, stream->get_uid()); - // kernel_stat.flush(); - // kernel_stat.close(); - stream->record_next_done(); - m_grid_id_to_stream.erase(grid_uid); - kernel->notify_parent_finished(); - delete kernel; - return true; +bool stream_manager::register_finished_kernel(unsigned grid_uid) +{ + // called by gpu simulation thread + if(grid_uid > 0){ + CUstream_st *stream = m_grid_id_to_stream[grid_uid]; + kernel_info_t *kernel = stream->front().get_kernel(); + assert( grid_uid == kernel->get_uid() ); + + //Jin: should check children kernels for CDP + if(kernel->is_finished()) { +// std::ofstream kernel_stat("kernel_stat.txt", std::ofstream::out | std::ofstream::app); +// kernel_stat<< " kernel " << grid_uid << ": " << kernel->name(); +// if(kernel->get_parent()) +// kernel_stat << ", parent " << kernel->get_parent()->get_uid() << +// ", launch " << kernel->launch_cycle; +// kernel_stat<< ", start " << kernel->start_cycle << +// ", end " << kernel->end_cycle << ", retire " << gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; +// printf("kernel %d finishes, retires from stream %d\n", grid_uid, stream->get_uid()); +// kernel_stat.flush(); +// kernel_stat.close(); + stream->record_next_done(); + m_grid_id_to_stream.erase(grid_uid); + kernel->notify_parent_finished(); + delete kernel; + return true; + } } - } - return false; + return false; } -void stream_manager::stop_all_running_kernels() { - pthread_mutex_lock(&m_lock); +void stream_manager::stop_all_running_kernels(){ + pthread_mutex_lock(&m_lock); - // Signal m_gpu to stop all running kernels - m_gpu->stop_all_running_kernels(); + // Signal m_gpu to stop all running kernels + m_gpu->stop_all_running_kernels(); - // Clean up all streams waiting on running kernels - int count = 0; - while (check_finished_kernel()) { - count++; - } + // Clean up all streams waiting on running kernels + int count=0; + while(check_finished_kernel()){ + count++; + } - // If any kernels completed, print out the current stats - if (count > 0) m_gpu->print_stats(); + // If any kernels completed, print out the current stats + if(count > 0) + m_gpu->print_stats(); - pthread_mutex_unlock(&m_lock); + pthread_mutex_unlock(&m_lock); } -stream_operation stream_manager::front() { - // called by gpu simulation thread - stream_operation result; - // if( concurrent_streams_empty() ) - m_service_stream_zero = true; - if (m_service_stream_zero) { - if (!m_stream_zero.empty() && !m_stream_zero.busy()) { - result = m_stream_zero.next(); - if (result.is_kernel()) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = &m_stream_zero; - } - } else { - m_service_stream_zero = false; +stream_operation stream_manager::front() +{ + // called by gpu simulation thread + stream_operation result; +// if( concurrent_streams_empty() ) + m_service_stream_zero = true; + if( m_service_stream_zero ) { + if( !m_stream_zero.empty() && !m_stream_zero.busy() ) { + result = m_stream_zero.next(); + if( result.is_kernel() ) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = &m_stream_zero; + } + } else { + m_service_stream_zero = false; + } } - } - - if (!m_service_stream_zero) { - std::list::iterator s; - for (s = m_streams.begin(); s != m_streams.end(); s++) { - CUstream_st *stream = *s; - if (!stream->busy() && !stream->empty()) { - result = stream->next(); - if (result.is_kernel()) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = stream; + + if(!m_service_stream_zero) + { + std::list::iterator s; + for( s=m_streams.begin(); s != m_streams.end(); s++) { + CUstream_st *stream = *s; + if( !stream->busy() && !stream->empty() ) { + result = stream->next(); + if( result.is_kernel() ) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = stream; + } + break; + } } - break; - } } - } - return result; + return result; } -void stream_manager::add_stream(struct CUstream_st *stream) { - // called by host thread - pthread_mutex_lock(&m_lock); - m_streams.push_back(stream); - pthread_mutex_unlock(&m_lock); +void stream_manager::add_stream( struct CUstream_st *stream ) +{ + // called by host thread + pthread_mutex_lock(&m_lock); + m_streams.push_back(stream); + pthread_mutex_unlock(&m_lock); } -void stream_manager::destroy_stream(CUstream_st *stream) { - // called by host thread - pthread_mutex_lock(&m_lock); - while (!stream->empty()) - ; - std::list::iterator s; - for (s = m_streams.begin(); s != m_streams.end(); s++) { - if (*s == stream) { - m_streams.erase(s); - break; +void stream_manager::destroy_stream( CUstream_st *stream ) +{ + // called by host thread + pthread_mutex_lock(&m_lock); + while( !stream->empty() ) + ; + std::list::iterator s; + for( s=m_streams.begin(); s != m_streams.end(); s++ ) { + if( *s == stream ) { + m_streams.erase(s); + break; + } } - } - delete stream; - pthread_mutex_unlock(&m_lock); + delete stream; + pthread_mutex_unlock(&m_lock); } -bool stream_manager::concurrent_streams_empty() { - bool result = true; - if (m_streams.empty()) return true; - // called by gpu simulation thread - std::list::iterator s; - for (s = m_streams.begin(); s != m_streams.end(); ++s) { - struct CUstream_st *stream = *s; - if (!stream->empty()) { - // stream->print(stdout); - result = false; - break; +bool stream_manager::concurrent_streams_empty() +{ + bool result = true; + if (m_streams.empty()) + return true; + // called by gpu simulation thread + std::list::iterator s; + for( s=m_streams.begin(); s!=m_streams.end();++s ) { + struct CUstream_st *stream = *s; + if( !stream->empty() ) { + //stream->print(stdout); + result = false; + break; + } } - } - return result; + return result; } -bool stream_manager::empty_protected() { - bool result = true; - pthread_mutex_lock(&m_lock); - if (!concurrent_streams_empty()) result = false; - if (!m_stream_zero.empty()) result = false; - pthread_mutex_unlock(&m_lock); - return result; +bool stream_manager::empty_protected() +{ + bool result = true; + pthread_mutex_lock(&m_lock); + if( !concurrent_streams_empty() ) + result = false; + if( !m_stream_zero.empty() ) + result = false; + pthread_mutex_unlock(&m_lock); + return result; } -bool stream_manager::empty() { - bool result = true; - if (!concurrent_streams_empty()) result = false; - if (!m_stream_zero.empty()) result = false; - return result; +bool stream_manager::empty() +{ + bool result = true; + if( !concurrent_streams_empty() ) + result = false; + if( !m_stream_zero.empty() ) + result = false; + return result; } -void stream_manager::print(FILE *fp) { - pthread_mutex_lock(&m_lock); - print_impl(fp); - pthread_mutex_unlock(&m_lock); + +void stream_manager::print( FILE *fp) +{ + pthread_mutex_lock(&m_lock); + print_impl(fp); + pthread_mutex_unlock(&m_lock); } -void stream_manager::print_impl(FILE *fp) { - fprintf(fp, "GPGPU-Sim API: Stream Manager State\n"); - std::list::iterator s; - for (s = m_streams.begin(); s != m_streams.end(); ++s) { - struct CUstream_st *stream = *s; - if (!stream->empty()) stream->print(fp); - } - if (!m_stream_zero.empty()) m_stream_zero.print(fp); +void stream_manager::print_impl( FILE *fp) +{ + fprintf(fp,"GPGPU-Sim API: Stream Manager State\n"); + std::list::iterator s; + for( s=m_streams.begin(); s!=m_streams.end();++s ) { + struct CUstream_st *stream = *s; + if( !stream->empty() ) + stream->print(fp); + } + if( !m_stream_zero.empty() ) + m_stream_zero.print(fp); } -void stream_manager::push(stream_operation op) { - struct CUstream_st *stream = op.get_stream(); +void stream_manager::push( stream_operation op ) +{ + struct CUstream_st *stream = op.get_stream(); + + // block if stream 0 (or concurrency disabled) and pending concurrent operations exist + bool block= !stream || m_cuda_launch_blocking; + while(block) { + pthread_mutex_lock(&m_lock); + block = !concurrent_streams_empty(); + pthread_mutex_unlock(&m_lock); + }; - // block if stream 0 (or concurrency disabled) and pending concurrent - // operations exist - bool block = !stream || m_cuda_launch_blocking; - while (block) { pthread_mutex_lock(&m_lock); - block = !concurrent_streams_empty(); - pthread_mutex_unlock(&m_lock); - }; - - pthread_mutex_lock(&m_lock); - if (!m_gpu->cycle_insn_cta_max_hit()) { - // Accept the stream operation if the maximum cycle/instruction/cta counts - // are not triggered - if (stream && !m_cuda_launch_blocking) { - stream->push(op); - } else { - op.set_stream(&m_stream_zero); - m_stream_zero.push(op); + if(!m_gpu->cycle_insn_cta_max_hit()) { + // Accept the stream operation if the maximum cycle/instruction/cta counts are not triggered + if( stream && !m_cuda_launch_blocking ) { + stream->push(op); + } else { + op.set_stream(&m_stream_zero); + m_stream_zero.push(op); + } + }else { + // Otherwise, ignore operation and continue + printf("GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. Skipping:"); + op.print(stdout); + printf("\n"); } - } else { - // Otherwise, ignore operation and continue - printf( - "GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. " - "Skipping:"); - op.print(stdout); - printf("\n"); - } - if (g_debug_execution >= 3) print_impl(stdout); - pthread_mutex_unlock(&m_lock); - if (m_cuda_launch_blocking || stream == NULL) { - unsigned int wait_amount = 100; - unsigned int wait_cap = 100000; // 100ms - while (!empty()) { - // sleep to prevent CPU hog by empty spin - // sleep time increased exponentially ensure fast response when needed - usleep(wait_amount); - wait_amount *= 2; - if (wait_amount > wait_cap) wait_amount = wait_cap; + if(g_debug_execution >= 3) + print_impl(stdout); + pthread_mutex_unlock(&m_lock); + if( m_cuda_launch_blocking || stream == NULL ) { + unsigned int wait_amount = 100; + unsigned int wait_cap = 100000; // 100ms + while( !empty() ) { + // sleep to prevent CPU hog by empty spin + // sleep time increased exponentially ensure fast response when needed + usleep(wait_amount); + wait_amount *= 2; + if (wait_amount > wait_cap) + wait_amount = wait_cap; + } } - } } -void stream_manager::pushCudaStreamWaitEventToAllStreams(CUevent_st *e, - unsigned int flags) { - std::list::iterator s; - for (s = m_streams.begin(); s != m_streams.end(); s++) { - stream_operation op(*s, e, flags); - push(op); - } +void stream_manager::pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ){ + std::list::iterator s; + for( s=m_streams.begin(); s != m_streams.end(); s++ ) { + stream_operation op(*s,e,flags); + push(op); + } } diff --git a/src/stream_manager.h b/src/stream_manager.h index 88e28c8..3fbdbaf 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -30,249 +28,248 @@ #ifndef STREAM_MANAGER_H_INCLUDED #define STREAM_MANAGER_H_INCLUDED +#include "abstract_hardware_model.h" +#include #include #include -#include -#include "abstract_hardware_model.h" -// class stream_barrier { -// public: +//class stream_barrier { +//public: // stream_barrier() { m_pending_streams=0; } // void inc() { m_pending_streams++; } // void dec() { assert(m_pending_streams); m_pending_streams--; } // unsigned value() const { return m_pending_streams; } -// private: +//private: // unsigned m_pending_streams; //}; enum stream_operation_type { - stream_no_op, - stream_memcpy_host_to_device, - stream_memcpy_device_to_host, - stream_memcpy_device_to_device, - stream_memcpy_to_symbol, - stream_memcpy_from_symbol, - stream_kernel_launch, - stream_event, - stream_wait_event + stream_no_op, + stream_memcpy_host_to_device, + stream_memcpy_device_to_host, + stream_memcpy_device_to_device, + stream_memcpy_to_symbol, + stream_memcpy_from_symbol, + stream_kernel_launch, + stream_event, + stream_wait_event }; class stream_operation { - public: - stream_operation() { - m_kernel = NULL; - m_type = stream_no_op; - m_stream = NULL; - m_done = true; - } - stream_operation(const void *src, const char *symbol, size_t count, - size_t offset, struct CUstream_st *stream) { - m_kernel = NULL; - m_stream = stream; - m_type = stream_memcpy_to_symbol; - m_host_address_src = src; - m_symbol = symbol; - m_cnt = count; - m_offset = offset; - m_done = false; - } - stream_operation(const char *symbol, void *dst, size_t count, size_t offset, - struct CUstream_st *stream) { - m_kernel = NULL; - m_stream = stream; - m_type = stream_memcpy_from_symbol; - m_host_address_dst = dst; - m_symbol = symbol; - m_cnt = count; - m_offset = offset; - m_done = false; - } - stream_operation(kernel_info_t *kernel, bool sim_mode, - struct CUstream_st *stream) { - m_type = stream_kernel_launch; - m_kernel = kernel; - m_sim_mode = sim_mode; - m_stream = stream; - m_done = false; - } - stream_operation(struct CUevent_st *e, struct CUstream_st *stream) { - m_kernel = NULL; - m_type = stream_event; - m_event = e; - m_stream = stream; - m_done = false; - } - stream_operation(struct CUstream_st *stream, class CUevent_st *e, - unsigned int flags) { - m_kernel = NULL; - m_type = stream_wait_event; - m_event = e; - m_stream = stream; - m_done = false; - } - stream_operation(const void *host_address_src, size_t device_address_dst, - size_t cnt, struct CUstream_st *stream) { - m_kernel = NULL; - m_type = stream_memcpy_host_to_device; - m_host_address_src = host_address_src; - m_device_address_dst = device_address_dst; - m_host_address_dst = NULL; - m_device_address_src = 0; - m_cnt = cnt; - m_stream = stream; - m_sim_mode = false; - m_done = false; - } - stream_operation(size_t device_address_src, void *host_address_dst, - size_t cnt, struct CUstream_st *stream) { - m_kernel = NULL; - m_type = stream_memcpy_device_to_host; - m_device_address_src = device_address_src; - m_host_address_dst = host_address_dst; - m_device_address_dst = 0; - m_host_address_src = NULL; - m_cnt = cnt; - m_stream = stream; - m_sim_mode = false; - m_done = false; - } - stream_operation(size_t device_address_src, size_t device_address_dst, - size_t cnt, struct CUstream_st *stream) { - m_kernel = NULL; - m_type = stream_memcpy_device_to_device; - m_device_address_src = device_address_src; - m_device_address_dst = device_address_dst; - m_host_address_src = NULL; - m_host_address_dst = NULL; - m_cnt = cnt; - m_stream = stream; - m_sim_mode = false; - m_done = false; - } +public: + stream_operation() + { + m_kernel=NULL; + m_type = stream_no_op; + m_stream = NULL; + m_done=true; + } + stream_operation( const void *src, const char *symbol, size_t count, size_t offset, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_stream = stream; + m_type=stream_memcpy_to_symbol; + m_host_address_src=src; + m_symbol=symbol; + m_cnt=count; + m_offset=offset; + m_done=false; + } + stream_operation( const char *symbol, void *dst, size_t count, size_t offset, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_stream = stream; + m_type=stream_memcpy_from_symbol; + m_host_address_dst=dst; + m_symbol=symbol; + m_cnt=count; + m_offset=offset; + m_done=false; + } + stream_operation( kernel_info_t *kernel, bool sim_mode, struct CUstream_st *stream ) + { + m_type=stream_kernel_launch; + m_kernel=kernel; + m_sim_mode=sim_mode; + m_stream=stream; + m_done=false; + } + stream_operation( struct CUevent_st *e, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_type=stream_event; + m_event=e; + m_stream=stream; + m_done=false; + } + stream_operation( struct CUstream_st *stream, class CUevent_st *e, unsigned int flags ) + { + m_kernel=NULL; + m_type=stream_wait_event; + m_event=e; + m_stream=stream; + m_done=false; + } + stream_operation( const void *host_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_type=stream_memcpy_host_to_device; + m_host_address_src =host_address_src; + m_device_address_dst=device_address_dst; + m_host_address_dst=NULL; + m_device_address_src=0; + m_cnt=cnt; + m_stream=stream; + m_sim_mode=false; + m_done=false; + } + stream_operation( size_t device_address_src, void *host_address_dst, size_t cnt, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_type=stream_memcpy_device_to_host; + m_device_address_src=device_address_src; + m_host_address_dst=host_address_dst; + m_device_address_dst=0; + m_host_address_src=NULL; + m_cnt=cnt; + m_stream=stream; + m_sim_mode=false; + m_done=false; + } + stream_operation( size_t device_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) + { + m_kernel=NULL; + m_type=stream_memcpy_device_to_device; + m_device_address_src=device_address_src; + m_device_address_dst=device_address_dst; + m_host_address_src=NULL; + m_host_address_dst=NULL; + m_cnt=cnt; + m_stream=stream; + m_sim_mode=false; + m_done=false; + } - bool is_kernel() const { return m_type == stream_kernel_launch; } - bool is_mem() const { - return m_type == stream_memcpy_host_to_device || - m_type == stream_memcpy_device_to_host || - m_type == stream_memcpy_host_to_device; - } - bool is_noop() const { return m_type == stream_no_op; } - bool is_done() const { return m_done; } - kernel_info_t *get_kernel() { return m_kernel; } - bool do_operation(gpgpu_sim *gpu); - void print(FILE *fp) const; - struct CUstream_st *get_stream() { - return m_stream; - } - void set_stream(CUstream_st *stream) { m_stream = stream; } + bool is_kernel() const { return m_type == stream_kernel_launch; } + bool is_mem() const { + return m_type == stream_memcpy_host_to_device || + m_type == stream_memcpy_device_to_host || + m_type == stream_memcpy_host_to_device; + } + bool is_noop() const { return m_type == stream_no_op; } + bool is_done() const { return m_done; } + kernel_info_t *get_kernel() { return m_kernel; } + bool do_operation( gpgpu_sim *gpu ); + void print( FILE *fp ) const; + struct CUstream_st *get_stream() { return m_stream; } + void set_stream( CUstream_st *stream ) { m_stream = stream; } - private: - struct CUstream_st *m_stream; +private: + struct CUstream_st *m_stream; - bool m_done; + bool m_done; - stream_operation_type m_type; - size_t m_device_address_dst; - size_t m_device_address_src; - void *m_host_address_dst; - const void *m_host_address_src; - size_t m_cnt; + stream_operation_type m_type; + size_t m_device_address_dst; + size_t m_device_address_src; + void *m_host_address_dst; + const void *m_host_address_src; + size_t m_cnt; - const char *m_symbol; - size_t m_offset; + const char *m_symbol; + size_t m_offset; - bool m_sim_mode; - kernel_info_t *m_kernel; - struct CUevent_st *m_event; + bool m_sim_mode; + kernel_info_t *m_kernel; + struct CUevent_st *m_event; }; struct CUevent_st { - public: - CUevent_st(bool blocking) { - m_uid = ++m_next_event_uid; - m_blocking = blocking; - m_updates = 0; - m_wallclock = 0; - m_gpu_tot_sim_cycle = 0; - m_done = false; - } - void update(double cycle, time_t clk) { - m_updates++; - m_wallclock = clk; - m_gpu_tot_sim_cycle = cycle; - m_done = true; - } - // void set_done() { assert(!m_done); m_done=true; } - int get_uid() const { return m_uid; } - unsigned num_updates() const { return m_updates; } - bool done() const { return m_done; } - time_t clock() const { return m_wallclock; } - - private: - int m_uid; - bool m_blocking; - bool m_done; - int m_updates; - time_t m_wallclock; - double m_gpu_tot_sim_cycle; +public: + CUevent_st( bool blocking ) + { + m_uid = ++m_next_event_uid; + m_blocking = blocking; + m_updates = 0; + m_wallclock = 0; + m_gpu_tot_sim_cycle = 0; + m_done = false; + } + void update( double cycle, time_t clk ) + { + m_updates++; + m_wallclock=clk; + m_gpu_tot_sim_cycle=cycle; + m_done = true; + } + //void set_done() { assert(!m_done); m_done=true; } + int get_uid() const { return m_uid; } + unsigned num_updates() const { return m_updates; } + bool done() const { return m_done; } + time_t clock() const { return m_wallclock; } +private: + int m_uid; + bool m_blocking; + bool m_done; + int m_updates; + time_t m_wallclock; + double m_gpu_tot_sim_cycle; - static int m_next_event_uid; + static int m_next_event_uid; }; struct CUstream_st { - public: - CUstream_st(); - bool empty(); - bool busy(); - void synchronize(); - void push(const stream_operation &op); - void record_next_done(); - stream_operation next(); - void cancel_front(); // front operation fails, cancle the pending status - stream_operation &front() { return m_operations.front(); } - void print(FILE *fp); - unsigned get_uid() const { return m_uid; } +public: + CUstream_st(); + bool empty(); + bool busy(); + void synchronize(); + void push( const stream_operation &op ); + void record_next_done(); + stream_operation next(); + void cancel_front(); //front operation fails, cancle the pending status + stream_operation &front() { return m_operations.front(); } + void print( FILE *fp ); + unsigned get_uid() const { return m_uid; } - private: - unsigned m_uid; - static unsigned sm_next_stream_uid; +private: + unsigned m_uid; + static unsigned sm_next_stream_uid; - std::list m_operations; - bool m_pending; // front operation has started but not yet completed + std::list m_operations; + bool m_pending; // front operation has started but not yet completed - pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream - // operation at one time + pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream operation at one time }; class stream_manager { - public: - stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking); - bool register_finished_kernel(unsigned grid_uid); - bool check_finished_kernel(); - stream_operation front(); - void add_stream(CUstream_st *stream); - void destroy_stream(CUstream_st *stream); - bool concurrent_streams_empty(); - bool empty_protected(); - bool empty(); - void print(FILE *fp); - void push(stream_operation op); - void pushCudaStreamWaitEventToAllStreams(CUevent_st *e, unsigned int flags); - bool operation(bool *sim); - void stop_all_running_kernels(); - unsigned size() { return m_streams.size(); }; - bool is_blocking() { return m_cuda_launch_blocking; }; - - private: - void print_impl(FILE *fp); +public: + stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ); + bool register_finished_kernel(unsigned grid_uid ); + bool check_finished_kernel( ); + stream_operation front(); + void add_stream( CUstream_st *stream ); + void destroy_stream( CUstream_st *stream ); + bool concurrent_streams_empty(); + bool empty_protected(); + bool empty(); + void print( FILE *fp); + void push( stream_operation op ); + void pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ); + bool operation(bool * sim); + void stop_all_running_kernels(); + unsigned size() {return m_streams.size(); }; + bool is_blocking() {return m_cuda_launch_blocking; }; +private: + void print_impl( FILE *fp); - bool m_cuda_launch_blocking; - gpgpu_sim *m_gpu; - std::list m_streams; - std::map m_grid_id_to_stream; - CUstream_st m_stream_zero; - bool m_service_stream_zero; - pthread_mutex_t m_lock; + bool m_cuda_launch_blocking; + gpgpu_sim *m_gpu; + std::list m_streams; + std::map m_grid_id_to_stream; + CUstream_st m_stream_zero; + bool m_service_stream_zero; + pthread_mutex_t m_lock; }; #endif diff --git a/src/tr1_hash_map.h b/src/tr1_hash_map.h index 0672ef8..8c7513e 100644 --- a/src/tr1_hash_map.h +++ b/src/tr1_hash_map.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -27,27 +25,26 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once // detection and fallback for unordered_map in C++0x #ifdef __cplusplus -// detect GCC 4.3 or later and use unordered map (part of C++0x) -// unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its -// enabled. -#if defined(__GNUC__) and not defined(_GLIBCXX_DEBUG) -#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 -#include -#define tr1_hash_map std::unordered_map -#define tr1_hash_map_ismap 0 -#else -#include -#define tr1_hash_map std::map -#define tr1_hash_map_ismap 1 -#endif -#else -#include -#define tr1_hash_map std::map -#define tr1_hash_map_ismap 1 -#endif + // detect GCC 4.3 or later and use unordered map (part of C++0x) + // unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its enabled. + #if defined( __GNUC__ ) and not defined( _GLIBCXX_DEBUG ) + #if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 + #include + #define tr1_hash_map std::unordered_map + #define tr1_hash_map_ismap 0 + #else + #include + #define tr1_hash_map std::map + #define tr1_hash_map_ismap 1 + #endif + #else + #include + #define tr1_hash_map std::map + #define tr1_hash_map_ismap 1 + #endif #endif diff --git a/src/trace.cc b/src/trace.cc index b5c2524..5171e46 100644 --- a/src/trace.cc +++ b/src/trace.cc @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -32,27 +30,27 @@ namespace Trace { + #define TS_TUP_BEGIN(X) const char* trace_streams_str[] = { #define TS_TUP(X) #X -#define TS_TUP_END(X) \ - } \ - ; +#define TS_TUP_END(X) }; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END -bool enabled = false; -int sampling_core = 0; -int sampling_memory_partition = -1; -bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; -const char* config_str; + bool enabled = false; + int sampling_core = 0; + int sampling_memory_partition = -1; + bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; + const char* config_str; -void init() { - for (unsigned i = 0; i < NUM_TRACE_STREAMS; ++i) { - if (strstr(config_str, trace_streams_str[i]) != NULL) { - trace_streams_enabled[i] = true; + void init() + { + for ( unsigned i = 0; i < NUM_TRACE_STREAMS; ++i ) { + if ( strstr( config_str, trace_streams_str[i] ) != NULL ) { + trace_streams_enabled[ i ] = true; + } + } } - } -} -} +} diff --git a/src/trace.h b/src/trace.h index c4e0001..0b96dcf 100644 --- a/src/trace.h +++ b/src/trace.h @@ -7,16 +7,14 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, -// this +// Redistributions in binary form must reproduce the above copyright notice, this // list of conditions and the following disclaimer in the documentation and/or // other materials provided with the distribution. // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE @@ -37,57 +35,52 @@ namespace Trace { #define TS_TUP_BEGIN(X) enum X { #define TS_TUP(X) X -#define TS_TUP_END(X) \ - } \ - ; +#define TS_TUP_END(X) }; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END -extern bool enabled; -extern int sampling_core; -extern int sampling_memory_partition; -extern const char* trace_streams_str[]; -extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; -extern const char* config_str; + extern bool enabled; + extern int sampling_core; + extern int sampling_memory_partition; + extern const char* trace_streams_str[]; + extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; + extern const char* config_str; -void init(); + void init(); + +} // namespace Trace -} // namespace Trace #if TRACING_ON #define SIM_PRINT_STR "GPGPU-Sim Cycle %llu: %s - " #define DTRACE(x) ((Trace::trace_streams_enabled[Trace::x]) && Trace::enabled) -#define DPRINTF(x, ...) \ - do { \ - if (DTRACE(x)) { \ - printf(SIM_PRINT_STR, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ - Trace::trace_streams_str[Trace::x]); \ - printf(__VA_ARGS__); \ - } \ - } while (0) +#define DPRINTF(x, ...) do {\ + if (DTRACE(x)) {\ + printf( SIM_PRINT_STR,\ + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ + Trace::trace_streams_str[Trace::x] );\ + printf(__VA_ARGS__);\ + }\ +} while (0) -#define DPRINTFG(x, ...) \ - do { \ - if (DTRACE(x)) { \ - printf(SIM_PRINT_STR, gpu_sim_cycle + gpu_tot_sim_cycle, \ - Trace::trace_streams_str[Trace::x]); \ - printf(__VA_ARGS__); \ - } \ - } while (0) +#define DPRINTFG(x, ...) do {\ + if (DTRACE(x)) {\ + printf( SIM_PRINT_STR,\ + gpu_sim_cycle + gpu_tot_sim_cycle,\ + Trace::trace_streams_str[Trace::x] );\ + printf(__VA_ARGS__);\ + }\ +} while (0) -#else +#else #define DTRACE(x) (false) -#define DPRINTF(x, ...) \ - do { \ - } while (0) -#define DPRINTFG(x, ...) \ - do { \ - } while (0) +#define DPRINTF(x, ...) do {} while (0) +#define DPRINTFG(x, ...) do {} while (0) -#endif +#endif -#endif +#endif -- cgit v1.3 From a34a4baad45a40840308167307c79371b4024a9d Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 07:54:00 -0400 Subject: Big reformat change using clang-format-6.0 --- src/abstract_hardware_model.cc | 2059 +++---- src/abstract_hardware_model.h | 2300 +++---- src/cuda-sim/cuda-math.h | 483 +- src/cuda-sim/cuda-sim.cc | 4923 +++++++-------- src/cuda-sim/cuda-sim.h | 314 +- src/cuda-sim/cuda_device_printf.cc | 192 +- src/cuda-sim/cuda_device_printf.h | 37 +- src/cuda-sim/cuda_device_runtime.cc | 568 +- src/cuda-sim/cuda_device_runtime.h | 91 +- src/cuda-sim/half.h | 6543 +++++++++++--------- src/cuda-sim/instructions.cc | 10867 ++++++++++++++++++---------------- src/cuda-sim/memory.cc | 363 +- src/cuda-sim/memory.h | 185 +- src/cuda-sim/opcodes.h | 104 +- src/cuda-sim/ptx-stats.cc | 403 +- src/cuda-sim/ptx-stats.h | 81 +- src/cuda-sim/ptx_ir.cc | 2692 +++++---- src/cuda-sim/ptx_ir.h | 2973 +++++----- src/cuda-sim/ptx_loader.cc | 978 +-- src/cuda-sim/ptx_loader.h | 65 +- src/cuda-sim/ptx_parser.cc | 1812 +++--- src/cuda-sim/ptx_parser.h | 329 +- src/cuda-sim/ptx_sim.cc | 1088 ++-- src/cuda-sim/ptx_sim.h | 939 ++- src/debug.cc | 372 +- src/debug.h | 118 +- src/gpgpu-sim/addrdec.cc | 996 ++-- src/gpgpu-sim/addrdec.h | 132 +- src/gpgpu-sim/delayqueue.h | 312 +- src/gpgpu-sim/dram.cc | 1612 +++-- src/gpgpu-sim/dram.h | 417 +- src/gpgpu-sim/dram_sched.cc | 429 +- src/gpgpu-sim/dram_sched.h | 95 +- src/gpgpu-sim/gpu-cache.cc | 3121 +++++----- src/gpgpu-sim/gpu-cache.h | 3179 +++++----- src/gpgpu-sim/gpu-misc.cc | 63 +- src/gpgpu-sim/gpu-misc.h | 44 +- src/gpgpu-sim/gpu-sim.cc | 3324 ++++++----- src/gpgpu-sim/gpu-sim.h | 1192 ++-- src/gpgpu-sim/histogram.cc | 197 +- src/gpgpu-sim/histogram.h | 89 +- src/gpgpu-sim/icnt_wrapper.cc | 280 +- src/gpgpu-sim/icnt_wrapper.h | 74 +- src/gpgpu-sim/l2cache.cc | 1525 ++--- src/gpgpu-sim/l2cache.h | 452 +- src/gpgpu-sim/l2cache_trace.h | 94 +- src/gpgpu-sim/local_interconnect.cc | 581 +- src/gpgpu-sim/local_interconnect.h | 212 +- src/gpgpu-sim/mem_fetch.cc | 200 +- src/gpgpu-sim/mem_fetch.h | 274 +- src/gpgpu-sim/mem_latency_stat.cc | 899 +-- src/gpgpu-sim/mem_latency_stat.h | 182 +- src/gpgpu-sim/power_interface.cc | 243 +- src/gpgpu-sim/power_interface.h | 46 +- src/gpgpu-sim/power_stat.cc | 584 +- src/gpgpu-sim/power_stat.h | 1342 +++-- src/gpgpu-sim/scoreboard.cc | 239 +- src/gpgpu-sim/scoreboard.h | 75 +- src/gpgpu-sim/shader.cc | 7527 +++++++++++------------ src/gpgpu-sim/shader.h | 4000 +++++++------ src/gpgpu-sim/shader_trace.h | 90 +- src/gpgpu-sim/stack.cc | 100 +- src/gpgpu-sim/stack.h | 43 +- src/gpgpu-sim/stat-tool.cc | 1189 ++-- src/gpgpu-sim/stat-tool.h | 493 +- src/gpgpu-sim/stats.h | 76 +- src/gpgpu-sim/traffic_breakdown.cc | 89 +- src/gpgpu-sim/traffic_breakdown.h | 48 +- src/gpgpu-sim/visualizer.cc | 642 +- src/gpgpu-sim/visualizer.h | 38 +- src/gpgpusim_entrypoint.cc | 463 +- src/gpgpusim_entrypoint.h | 97 +- src/option_parser.cc | 921 +-- src/option_parser.h | 77 +- src/statwrapper.cc | 59 +- src/statwrapper.h | 14 +- src/stream_manager.cc | 765 ++- src/stream_manager.h | 450 +- src/tr1_hash_map.h | 70 +- src/trace.cc | 63 +- src/trace.h | 100 +- 81 files changed, 41712 insertions(+), 39085 deletions(-) (limited to 'src/stream_manager.cc') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 07232ee..4bfe3c9 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -7,1203 +7,1210 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "abstract_hardware_model.h" +#include +#include +#include +#include +#include "../libcuda/gpgpu_context.h" +#include "cuda-sim/cuda-sim.h" #include "cuda-sim/memory.h" -#include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx-stats.h" -#include "cuda-sim/cuda-sim.h" +#include "cuda-sim/ptx_ir.h" #include "gpgpu-sim/gpu-sim.h" -#include "option_parser.h" #include "gpgpusim_entrypoint.h" -#include -#include -#include -#include -#include "../libcuda/gpgpu_context.h" +#include "option_parser.h" -void mem_access_t::init(gpgpu_context* ctx) -{ - gpgpu_ctx = ctx; - m_uid=++(gpgpu_ctx->sm_next_access_uid); - m_addr=0; - m_req_size=0; +void mem_access_t::init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = ++(gpgpu_ctx->sm_next_access_uid); + m_addr = 0; + m_req_size = 0; } -void warp_inst_t::issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ) -{ - m_warp_active_mask = mask; - m_warp_issued_mask = mask; - m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); - m_warp_id = warp_id; - m_dynamic_warp_id = dynamic_warp_id; - issue_cycle = cycle; - cycles = initiation_interval; - m_cache_hit=false; - m_empty=false; - m_scheduler_id=sch_id; +void warp_inst_t::issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, + int sch_id) { + m_warp_active_mask = mask; + m_warp_issued_mask = mask; + m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); + m_warp_id = warp_id; + m_dynamic_warp_id = dynamic_warp_id; + issue_cycle = cycle; + cycles = initiation_interval; + m_cache_hit = false; + m_empty = false; + m_scheduler_id = sch_id; } -checkpoint::checkpoint() -{ - - struct stat st = {0}; - - if (stat("checkpoint_files", &st) == -1) { - mkdir("checkpoint_files", 0777); - } +checkpoint::checkpoint() { + struct stat st = {0}; + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); + } } -void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) -{ - - FILE * fp2 = fopen(f1name, "r"); - assert(fp2!=NULL); - char line [ 128 ]; /* or other suitable maximum line size */ - unsigned int offset ; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - unsigned int index; - char * pch; - pch = strtok (line," "); - if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') - { - - pch = strtok (NULL, " "); - - std::stringstream ss; - ss << std::hex << pch; - ss >> index; - - offset=0; - } - else { - unsigned int data; - std::stringstream ss; - ss << std::hex << pch; - ss >> data; - temp_mem->write_only(offset,index, 4,&data); - offset= offset+4; - } - //fputs ( line, stdout ); /* write the line */ - } - fclose ( fp2 ); +void checkpoint::load_global_mem(class memory_space *temp_mem, char *f1name) { + FILE *fp2 = fopen(f1name, "r"); + assert(fp2 != NULL); + char line[128]; /* or other suitable maximum line size */ + unsigned int offset; + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + unsigned int index; + char *pch; + pch = strtok(line, " "); + if (pch[0] == 'g' || pch[0] == 's' || pch[0] == 'l') { + pch = strtok(NULL, " "); + + std::stringstream ss; + ss << std::hex << pch; + ss >> index; + + offset = 0; + } else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset, index, 4, &data); + offset = offset + 4; + } + // fputs ( line, stdout ); /* write the line */ + } + fclose(fp2); } -void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) -{ - - FILE * fp3 = fopen(fname, "w"); - assert(fp3!=NULL); - mem->print(format,fp3); - fclose(fp3); +void checkpoint::store_global_mem(class memory_space *mem, char *fname, + char *format) { + FILE *fp3 = fopen(fname, "w"); + assert(fp3 != NULL); + mem->print(format, fp3); + fclose(fp3); } -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) -{ - assert( dst->empty() ); - warp_inst_t* temp = dst; - dst = src; - src = temp; - src->clear(); +void move_warp(warp_inst_t *&dst, warp_inst_t *&src) { + assert(dst->empty()); + warp_inst_t *temp = dst; + dst = src; + src = temp; + src->clear(); } - -void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) -{ - option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, - &m_ptx_use_cuobjdump, - "Use cuobjdump to extract ptx and sass from binaries", +void gpgpu_functional_sim_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, + &m_ptx_use_cuobjdump, + "Use cuobjdump to extract ptx and sass from binaries", #if (CUDART_VERSION >= 4000) - "1" + "1" #else - "0" + "0" #endif - ); - option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, - &m_experimental_lib_support, - "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", - "0"); - option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, - " checkpointing flag (0 = no checkpoint)", - "0"); - option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, - " checkpointing during execution of which kernel (1- 1st kernel)", - "1"); - option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, - " checkpointing after # of CTA (< less than total CTA)", - "0"); - option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, - " resume flag (0 = no resume)", - "0"); - option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, - " Resume from which kernel (1= 1st kernel)", - "0"); - option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, - " resume from which CTA ", - "0"); - - option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, - &m_ptx_convert_to_ptxplus, - "Convert SASS (native ISA) to ptxplus and run ptxplus", - "0"); - option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, - &m_ptx_force_max_capability, - "Force maximum compute capability", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, - &g_ptx_inst_debug_to_file, - "Dump executed instructions' debug information to file", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, - "Executed instructions' debug output file", - "inst_debug.txt"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, &g_ptx_inst_debug_thread_uid, - "Thread UID for executed instructions' debug output", - "1"); + ); + option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, + &m_experimental_lib_support, + "Try to extract code from cuda libraries [Broken " + "because of unknown cudaGetExportTable]", + "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, + &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", "0"); + option_parser_register( + opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", "1"); + option_parser_register( + opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, + &checkpoint_insn_Y, " resume from which CTA ", "0"); + + option_parser_register( + opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, + "Convert SASS (native ISA) to ptxplus and run ptxplus", "0"); + option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, + &m_ptx_force_max_capability, + "Force maximum compute capability", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, &g_ptx_inst_debug_to_file, + "Dump executed instructions' debug information to file", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, + "Executed instructions' debug output file", "inst_debug.txt"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, + &g_ptx_inst_debug_thread_uid, + "Thread UID for executed instructions' debug output", + "1"); } -void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize(unsigned linesize) -{ - m_texcache_linesize = linesize; +void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize( + unsigned linesize) { + m_texcache_linesize = linesize; } -gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ) - : m_function_model_config(config) -{ - gpgpu_ctx = ctx; - m_global_mem = new memory_space_impl<8192>("global",64*1024); - - m_tex_mem = new memory_space_impl<8192>("tex",64*1024); - m_surf_mem = new memory_space_impl<8192>("surf",64*1024); - - m_dev_malloc=GLOBAL_HEAP_START; - checkpoint_option = m_function_model_config.get_checkpoint_option(); - checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); - checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); - resume_option = m_function_model_config.get_resume_option(); - resume_kernel = m_function_model_config.get_resume_kernel(); - resume_CTA = m_function_model_config.get_resume_CTA(); - checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); - checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); - - // initialize texture mappings to empty - m_NameToTextureInfo.clear(); - m_NameToCudaArray.clear(); - m_TextureRefToName.clear(); - m_NameToAttribute.clear(); - - if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) - ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); - - gpu_sim_cycle=0; - gpu_tot_sim_cycle=0; +gpgpu_t::gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx) + : m_function_model_config(config) { + gpgpu_ctx = ctx; + m_global_mem = new memory_space_impl<8192>("global", 64 * 1024); + + m_tex_mem = new memory_space_impl<8192>("tex", 64 * 1024); + m_surf_mem = new memory_space_impl<8192>("surf", 64 * 1024); + + m_dev_malloc = GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + + // initialize texture mappings to empty + m_NameToTextureInfo.clear(); + m_NameToCudaArray.clear(); + m_TextureRefToName.clear(); + m_NameToAttribute.clear(); + + if (m_function_model_config.get_ptx_inst_debug_to_file() != 0) + ptx_inst_debug_file = + fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); + + gpu_sim_cycle = 0; + gpu_tot_sim_cycle = 0; } -address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_size) -{ - //gives the tag for an address based on a given line size - return address & ~(line_size-1); +address_type line_size_based_tag_func(new_addr_type address, + new_addr_type line_size) { + // gives the tag for an address based on a given line size + return address & ~(line_size - 1); } -const char * mem_access_type_str(enum mem_access_type access_type) -{ - #define MA_TUP_BEGIN(X) static const char* access_type_str[] = { - #define MA_TUP(X) #X - #define MA_TUP_END(X) }; - MEM_ACCESS_TYPE_TUP_DEF - #undef MA_TUP_BEGIN - #undef MA_TUP - #undef MA_TUP_END +const char *mem_access_type_str(enum mem_access_type access_type) { +#define MA_TUP_BEGIN(X) static const char *access_type_str[] = { +#define MA_TUP(X) #X +#define MA_TUP_END(X) \ + } \ + ; + MEM_ACCESS_TYPE_TUP_DEF +#undef MA_TUP_BEGIN +#undef MA_TUP +#undef MA_TUP_END - assert(access_type < NUM_MEM_ACCESS_TYPE); + assert(access_type < NUM_MEM_ACCESS_TYPE); - return access_type_str[access_type]; + return access_type_str[access_type]; } - -void warp_inst_t::clear_active( const active_mask_t &inactive ) { - active_mask_t test = m_warp_active_mask; - test &= inactive; - assert( test == inactive ); // verify threads being disabled were active - m_warp_active_mask &= ~inactive; +void warp_inst_t::clear_active(const active_mask_t &inactive) { + active_mask_t test = m_warp_active_mask; + test &= inactive; + assert(test == inactive); // verify threads being disabled were active + m_warp_active_mask &= ~inactive; } -void warp_inst_t::set_not_active( unsigned lane_id ) { - m_warp_active_mask.reset(lane_id); +void warp_inst_t::set_not_active(unsigned lane_id) { + m_warp_active_mask.reset(lane_id); } -void warp_inst_t::set_active( const active_mask_t &active ) { - m_warp_active_mask = active; - if( m_isatomic ) { - for( unsigned i=0; i < m_config->warp_size; i++ ) { - if( !m_warp_active_mask.test(i) ) { - m_per_scalar_thread[i].callback.function = NULL; - m_per_scalar_thread[i].callback.instruction = NULL; - m_per_scalar_thread[i].callback.thread = NULL; - } +void warp_inst_t::set_active(const active_mask_t &active) { + m_warp_active_mask = active; + if (m_isatomic) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (!m_warp_active_mask.test(i)) { + m_per_scalar_thread[i].callback.function = NULL; + m_per_scalar_thread[i].callback.instruction = NULL; + m_per_scalar_thread[i].callback.thread = NULL; } - } + } + } } void warp_inst_t::do_atomic(bool forceDo) { - do_atomic( m_warp_active_mask,forceDo ); + do_atomic(m_warp_active_mask, forceDo); } - -void warp_inst_t::do_atomic( const active_mask_t& access_mask,bool forceDo ) { - assert( m_isatomic && (!m_empty||forceDo) ); - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ) - cb.function(cb.instruction, cb.thread); - } +void warp_inst_t::do_atomic(const active_mask_t &access_mask, bool forceDo) { + assert(m_isatomic && (!m_empty || forceDo)); + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) cb.function(cb.instruction, cb.thread); } + } } -void warp_inst_t::broadcast_barrier_reduction(const active_mask_t& access_mask) -{ - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ){ - cb.function(cb.instruction, cb.thread); - } - } +void warp_inst_t::broadcast_barrier_reduction( + const active_mask_t &access_mask) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) { + cb.function(cb.instruction, cb.thread); + } } + } } -void warp_inst_t::generate_mem_accesses() -{ - if( empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created ) - return; - if (!((op == LOAD_OP) || (op==TENSOR_CORE_LOAD_OP) || (op == STORE_OP)||(op==TENSOR_CORE_STORE_OP))) - return; - if( m_warp_active_mask.count() == 0 ) - return; // predicated off +void warp_inst_t::generate_mem_accesses() { + if (empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created) return; + if (!((op == LOAD_OP) || (op == TENSOR_CORE_LOAD_OP) || (op == STORE_OP) || + (op == TENSOR_CORE_STORE_OP))) + return; + if (m_warp_active_mask.count() == 0) return; // predicated off - const size_t starting_queue_size = m_accessq.size(); + const size_t starting_queue_size = m_accessq.size(); - assert( is_load() || is_store() ); - assert( m_per_scalar_thread_valid ); // need address information per thread + assert(is_load() || is_store()); + assert(m_per_scalar_thread_valid); // need address information per thread - bool is_write = is_store(); + bool is_write = is_store(); - mem_access_type access_type; - switch (space.get_type()) { + mem_access_type access_type; + switch (space.get_type()) { case const_space: - case param_space_kernel: - access_type = CONST_ACC_R; - break; - case tex_space: - access_type = TEXTURE_ACC_R; - break; - case global_space: - access_type = is_write? GLOBAL_ACC_W: GLOBAL_ACC_R; - break; + case param_space_kernel: + access_type = CONST_ACC_R; + break; + case tex_space: + access_type = TEXTURE_ACC_R; + break; + case global_space: + access_type = is_write ? GLOBAL_ACC_W : GLOBAL_ACC_R; + break; case local_space: - case param_space_local: - access_type = is_write? LOCAL_ACC_W: LOCAL_ACC_R; - break; - case shared_space: break; - case sstarr_space: break; - default: assert(0); break; - } + case param_space_local: + access_type = is_write ? LOCAL_ACC_W : LOCAL_ACC_R; + break; + case shared_space: + break; + case sstarr_space: + break; + default: + assert(0); + break; + } - // Calculate memory accesses generated by this warp - new_addr_type cache_block_size = 0; // in bytes + // Calculate memory accesses generated by this warp + new_addr_type cache_block_size = 0; // in bytes - switch( space.get_type() ) { + switch (space.get_type()) { case shared_space: case sstarr_space: { - unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; - unsigned total_accesses=0; - for( unsigned subwarp=0; subwarp < m_config->mem_warp_parts; subwarp++ ) { - - // data structures used per part warp - std::map > bank_accs; // bank -> word address -> access count - - // step 1: compute accesses to words in banks - for( unsigned thread=subwarp*subwarp_size; thread < (subwarp+1)*subwarp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - //FIXME: deferred allocation of shared memory should not accumulate across kernel launches - //assert( addr < m_config->gpgpu_shmem_size ); - unsigned bank = m_config->shmem_bank_func(addr); - new_addr_type word = line_size_based_tag_func(addr,m_config->WORD_SIZE); - bank_accs[bank][word]++; - } - - if (m_config->shmem_limited_broadcast) { - // step 2: look for and select a broadcast bank/word if one occurs - bool broadcast_detected = false; - new_addr_type broadcast_word=(new_addr_type)-1; - unsigned broadcast_bank=(unsigned)-1; - std::map >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank = b->first; - std::map &access_set = b->second; - std::map::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->second > 1 ) { - // found a broadcast - broadcast_detected=true; - broadcast_bank=bank; - broadcast_word=w->first; - break; - } - } - if( broadcast_detected ) - break; - } - - // step 3: figure out max bank accesses performed, taking account of broadcast case - unsigned max_bank_accesses=0; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank_accesses=0; - std::map &access_set = b->second; - std::map::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) - bank_accesses += w->second; - if( broadcast_detected && broadcast_bank == b->first ) { - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->first == broadcast_word ) { - unsigned n = w->second; - assert(n > 1); // or this wasn't a broadcast - assert(bank_accesses >= (n-1)); - bank_accesses -= (n-1); - break; - } - } - } - if( bank_accesses > max_bank_accesses ) - max_bank_accesses = bank_accesses; - } + unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; + unsigned total_accesses = 0; + for (unsigned subwarp = 0; subwarp < m_config->mem_warp_parts; + subwarp++) { + // data structures used per part warp + std::map > + bank_accs; // bank -> word address -> access count + + // step 1: compute accesses to words in banks + for (unsigned thread = subwarp * subwarp_size; + thread < (subwarp + 1) * subwarp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + // FIXME: deferred allocation of shared memory should not accumulate + // across kernel launches assert( addr < m_config->gpgpu_shmem_size ); + unsigned bank = m_config->shmem_bank_func(addr); + new_addr_type word = + line_size_based_tag_func(addr, m_config->WORD_SIZE); + bank_accs[bank][word]++; + } - // step 4: accumulate - total_accesses+= max_bank_accesses; - } else { - // step 2: look for the bank with the maximum number of access to different words - unsigned max_bank_accesses=0; - std::map >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - max_bank_accesses = std::max(max_bank_accesses, (unsigned)b->second.size()); + if (m_config->shmem_limited_broadcast) { + // step 2: look for and select a broadcast bank/word if one occurs + bool broadcast_detected = false; + new_addr_type broadcast_word = (new_addr_type)-1; + unsigned broadcast_bank = (unsigned)-1; + std::map >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank = b->first; + std::map &access_set = b->second; + std::map::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->second > 1) { + // found a broadcast + broadcast_detected = true; + broadcast_bank = bank; + broadcast_word = w->first; + break; + } + } + if (broadcast_detected) break; + } + + // step 3: figure out max bank accesses performed, taking account of + // broadcast case + unsigned max_bank_accesses = 0; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank_accesses = 0; + std::map &access_set = b->second; + std::map::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) + bank_accesses += w->second; + if (broadcast_detected && broadcast_bank == b->first) { + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->first == broadcast_word) { + unsigned n = w->second; + assert(n > 1); // or this wasn't a broadcast + assert(bank_accesses >= (n - 1)); + bank_accesses -= (n - 1); + break; } - - // step 3: accumulate - total_accesses+= max_bank_accesses; + } } + if (bank_accesses > max_bank_accesses) + max_bank_accesses = bank_accesses; + } + + // step 4: accumulate + total_accesses += max_bank_accesses; + } else { + // step 2: look for the bank with the maximum number of access to + // different words + unsigned max_bank_accesses = 0; + std::map >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + max_bank_accesses = + std::max(max_bank_accesses, (unsigned)b->second.size()); + } + + // step 3: accumulate + total_accesses += max_bank_accesses; } - assert( total_accesses > 0 && total_accesses <= m_config->warp_size ); - cycles = total_accesses; // shared memory conflicts modeled as larger initiation interval - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); - break; + } + assert(total_accesses > 0 && total_accesses <= m_config->warp_size); + cycles = total_accesses; // shared memory conflicts modeled as larger + // initiation interval + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( + pc, total_accesses); + break; } - case tex_space: - cache_block_size = m_config->gpgpu_cache_texl1_linesize; - break; - case const_space: case param_space_kernel: - cache_block_size = m_config->gpgpu_cache_constl1_linesize; - break; + case tex_space: + cache_block_size = m_config->gpgpu_cache_texl1_linesize; + break; + case const_space: + case param_space_kernel: + cache_block_size = m_config->gpgpu_cache_constl1_linesize; + break; - case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch >= 13) { - if(isatomic()) - memory_coalescing_arch_atomic(is_write, access_type); - else - memory_coalescing_arch(is_write, access_type); - } else abort(); + case global_space: + case local_space: + case param_space_local: + if (m_config->gpgpu_coalesce_arch >= 13) { + if (isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else + abort(); - break; + break; default: - abort(); + abort(); + } + + if (cache_block_size) { + assert(m_accessq.empty()); + mem_access_byte_mask_t byte_mask; + std::map + accesses; // block address -> set of thread offsets in warp + std::map::iterator a; + for (unsigned thread = 0; thread < m_config->warp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, cache_block_size); + accesses[block_address].set(thread); + unsigned idx = addr - block_address; + for (unsigned i = 0; i < data_size; i++) byte_mask.set(idx + i); } + for (a = accesses.begin(); a != accesses.end(); ++a) + m_accessq.push_back(mem_access_t( + access_type, a->first, cache_block_size, is_write, a->second, + byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); + } + + if (space.get_type() == global_space) { + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( + pc, m_accessq.size() - starting_queue_size); + } + m_mem_accesses_created = true; +} - if( cache_block_size ) { - assert( m_accessq.empty() ); - mem_access_byte_mask_t byte_mask; - std::map accesses; // block address -> set of thread offsets in warp - std::map::iterator a; - for( unsigned thread=0; thread < m_config->warp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr,cache_block_size); - accesses[block_address].set(thread); - unsigned idx = addr-block_address; - for( unsigned i=0; i < data_size; i++ ) - byte_mask.set(idx+i); +void warp_inst_t::memory_coalescing_arch(bool is_write, + mem_access_type access_type) { + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } + + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map subwarp_transactions; + + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; + + unsigned data_size_coales = data_size; + unsigned num_accesses = 1; + + if (space.get_type() == local_space || + space.get_type() == param_space_local) { + // Local memory accesses >4B were split into 4B chunks + if (data_size >= 4) { + data_size_coales = 4; + num_accesses = data_size / 4; } - for( a=accesses.begin(); a != accesses.end(); ++a ) - m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); - } + // Otherwise keep the same data_size for sub-4B access to local memory + } - if ( space.get_type() == global_space ) { - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); + assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); + + // for(unsigned access=0; accessmem_warp_parts; - bool sector_segment_size = false; - - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } + // step 2: reduce each transaction size, if possible + std::map::iterator t; + for (t = subwarp_transactions.begin(); t != subwarp_transactions.end(); + t++) { + new_addr_type addr = t->first; + const transaction_info &info = t->second; - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, + segment_size); } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map subwarp_transactions; - - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread4B were split into 4B chunks - if(data_size >= 4) { - data_size_coales = 4; - num_accesses = data_size/4; - } - // Otherwise keep the same data_size for sub-4B access to local memory - } - - - assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); - -// for(unsigned access=0; access::iterator t; - for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { - new_addr_type addr = t->first; - const transaction_info &info = t->second; - - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); + } +} +void warp_inst_t::memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type) { + assert(space.get_type() == + global_space); // Atomics allowed only for global memory + + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } + + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; + + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map > + subwarp_transactions; // each block addr maps to a list of transactions + + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; + + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, segment_size); + unsigned chunk = + (addr & 127) / 32; // which 32-byte chunk within in a 128-byte chunk + // does this thread access? + + // can only write to one segment + assert(block_address == + line_size_based_tag_func(addr + data_size - 1, segment_size)); + + // Find a transaction that does not conflict with this thread's accesses + bool new_transaction = true; + std::list::iterator it; + transaction_info *info; + for (it = subwarp_transactions[block_address].begin(); + it != subwarp_transactions[block_address].end(); it++) { + unsigned idx = (addr & 127); + if (not it->test_bytes(idx, idx + data_size - 1)) { + new_transaction = false; + info = &(*it); + break; } + } + if (new_transaction) { + // Need a new transaction + subwarp_transactions[block_address].push_back(transaction_info()); + info = &subwarp_transactions[block_address].back(); + } + assert(info); + + info->chunks.set(chunk); + info->active.set(thread); + unsigned idx = (addr & 127); + for (unsigned i = 0; i < data_size; i++) { + assert(!info->bytes.test(idx + i)); + info->bytes.set(idx + i); + } } -} -void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) -{ - - assert(space.get_type() == global_space); // Atomics allowed only for global memory - - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; - - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } - - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; - - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map > subwarp_transactions; // each block addr maps to a list of transactions - - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread::iterator it; - transaction_info* info; - for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) { - unsigned idx = (addr&127); - if(not it->test_bytes(idx,idx+data_size-1)) { - new_transaction = false; - info = &(*it); - break; - } - } - if(new_transaction) { - // Need a new transaction - subwarp_transactions[block_address].push_back(transaction_info()); - info = &subwarp_transactions[block_address].back(); - } - assert(info); - - info->chunks.set(chunk); - info->active.set(thread); - unsigned idx = (addr&127); - for( unsigned i=0; i < data_size; i++ ) { - assert(!info->bytes.test(idx+i)); - info->bytes.set(idx+i); - } - } - - // step 2: reduce each transaction size, if possible - std::map< new_addr_type, std::list >::iterator t_list; - for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) { - // For each block addr - new_addr_type addr = t_list->first; - const std::list& transaction_list = t_list->second; - - std::list::const_iterator t; - for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { - // For each transaction - const transaction_info &info = *t; - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); - } - } - } + // step 2: reduce each transaction size, if possible + std::map >::iterator t_list; + for (t_list = subwarp_transactions.begin(); + t_list != subwarp_transactions.end(); t_list++) { + // For each block addr + new_addr_type addr = t_list->first; + const std::list &transaction_list = t_list->second; + + std::list::const_iterator t; + for (t = transaction_list.begin(); t != transaction_list.end(); t++) { + // For each transaction + const transaction_info &info = *t; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, + addr, segment_size); + } + } + } } -void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) -{ - assert( (addr & (segment_size-1)) == 0 ); - - const std::bitset<4> &q = info.chunks; - assert( q.count() >= 1 ); - std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) - - unsigned size=segment_size; - if( segment_size == 128 ) { - bool lower_half_used = q[0] || q[1]; - bool upper_half_used = q[2] || q[3]; - if( lower_half_used && !upper_half_used ) { - // only lower 64 bytes used - size = 64; - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else if ( (!lower_half_used) && upper_half_used ) { - // only upper 64 bytes used - addr = addr+64; - size = 64; - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } else { - assert(lower_half_used && upper_half_used); - } - } else if( segment_size == 64 ) { - // need to set halves - if( (addr % 128) == 0 ) { - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else { - assert( (addr % 128) == 64 ); - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } - } - if( size == 64 ) { - bool lower_half_used = h[0]; - bool upper_half_used = h[1]; - if( lower_half_used && !upper_half_used ) { - size = 32; - } else if ( (!lower_half_used) && upper_half_used ) { - addr = addr+32; - size = 32; - } else { - assert(lower_half_used && upper_half_used); - } - } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks,m_config->gpgpu_ctx) ); +void warp_inst_t::memory_coalescing_arch_reduce_and_send( + bool is_write, mem_access_type access_type, const transaction_info &info, + new_addr_type addr, unsigned segment_size) { + assert((addr & (segment_size - 1)) == 0); + + const std::bitset<4> &q = info.chunks; + assert(q.count() >= 1); + std::bitset<2> h; // halves (used to check if 64 byte segment can be + // compressed into a single 32 byte segment) + + unsigned size = segment_size; + if (segment_size == 128) { + bool lower_half_used = q[0] || q[1]; + bool upper_half_used = q[2] || q[3]; + if (lower_half_used && !upper_half_used) { + // only lower 64 bytes used + size = 64; + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else if ((!lower_half_used) && upper_half_used) { + // only upper 64 bytes used + addr = addr + 64; + size = 64; + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } else { + assert(lower_half_used && upper_half_used); + } + } else if (segment_size == 64) { + // need to set halves + if ((addr % 128) == 0) { + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else { + assert((addr % 128) == 64); + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } + } + if (size == 64) { + bool lower_half_used = h[0]; + bool upper_half_used = h[1]; + if (lower_half_used && !upper_half_used) { + size = 32; + } else if ((!lower_half_used) && upper_half_used) { + addr = addr + 32; + size = 32; + } else { + assert(lower_half_used && upper_half_used); + } + } + m_accessq.push_back(mem_access_t(access_type, addr, size, is_write, + info.active, info.bytes, info.chunks, + m_config->gpgpu_ctx)); } -void warp_inst_t::completed( unsigned long long cycle ) const -{ - unsigned long long latency = cycle - issue_cycle; - assert(latency <= cycle); // underflow detection - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency(pc, latency * active_count()); +void warp_inst_t::completed(unsigned long long cycle) const { + unsigned long long latency = cycle - issue_cycle; + assert(latency <= cycle); // underflow detection + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency( + pc, latency * active_count()); } - -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); - - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set=false; +kernel_info_t::kernel_info_t(dim3 gridDim, dim3 blockDim, + class function_info *entry) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); + + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set = false; } -/*A snapshot of the texture mappings needs to be stored in the kernel's info as +/*A snapshot of the texture mappings needs to be stored in the kernel's info as kernels should use the texture bindings seen at the time of launch and textures can be bound/unbound asynchronously with respect to streams. */ -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); - - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set=false; - m_NameToCudaArray = nameToCudaArray; - m_NameToTextureInfo = nameToTextureInfo; +kernel_info_t::kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map nameToCudaArray, + std::map nameToTextureInfo) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); + + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; + + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + + volta_cache_config_set = false; + m_NameToCudaArray = nameToCudaArray; + m_NameToTextureInfo = nameToTextureInfo; } -kernel_info_t::~kernel_info_t() -{ - assert( m_active_threads.empty() ); - destroy_cta_streams(); - delete m_param_mem; +kernel_info_t::~kernel_info_t() { + assert(m_active_threads.empty()); + destroy_cta_streams(); + delete m_param_mem; } -std::string kernel_info_t::name() const -{ - return m_kernel_entry->get_name(); -} +std::string kernel_info_t::name() const { return m_kernel_entry->get_name(); } -//Jin: parent and child kernel management for CDP -void kernel_info_t::set_parent(kernel_info_t * parent, - dim3 parent_ctaid, dim3 parent_tid) { - m_parent_kernel = parent; - m_parent_ctaid = parent_ctaid; - m_parent_tid = parent_tid; - parent->set_child(this); +// Jin: parent and child kernel management for CDP +void kernel_info_t::set_parent(kernel_info_t *parent, dim3 parent_ctaid, + dim3 parent_tid) { + m_parent_kernel = parent; + m_parent_ctaid = parent_ctaid; + m_parent_tid = parent_tid; + parent->set_child(this); } -void kernel_info_t::set_child(kernel_info_t * child) { - m_child_kernels.push_back(child); +void kernel_info_t::set_child(kernel_info_t *child) { + m_child_kernels.push_back(child); } -void kernel_info_t::remove_child(kernel_info_t * child) { - assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) - != m_child_kernels.end()); - m_child_kernels.remove(child); +void kernel_info_t::remove_child(kernel_info_t *child) { + assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) != + m_child_kernels.end()); + m_child_kernels.remove(child); } bool kernel_info_t::is_finished() { - if(done() && children_all_finished()) - return true; + if (done() && children_all_finished()) + return true; else - return false; + return false; } bool kernel_info_t::children_all_finished() { - if(!m_child_kernels.empty()) - return false; - - return true; + if (!m_child_kernels.empty()) return false; + + return true; } void kernel_info_t::notify_parent_finished() { - if(m_parent_kernel) { - m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= ((m_kernel_entry->get_args_aligned_size() + 255)/256*256); - m_parent_kernel->remove_child(this); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(m_parent_kernel->get_uid()); - } + if (m_parent_kernel) { + m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= + ((m_kernel_entry->get_args_aligned_size() + 255) / 256 * 256); + m_parent_kernel->remove_child(this); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager + ->register_finished_kernel(m_parent_kernel->get_uid()); + } } -CUstream_st * kernel_info_t::create_stream_cta(dim3 ctaid) { - assert(get_default_stream_cta(ctaid)); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); - assert(m_cta_streams[ctaid].size() >= 1); //must have default stream - m_cta_streams[ctaid].push_back(stream); +CUstream_st *kernel_info_t::create_stream_cta(dim3 ctaid) { + assert(get_default_stream_cta(ctaid)); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); + assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); + assert(m_cta_streams[ctaid].size() >= 1); // must have default stream + m_cta_streams[ctaid].push_back(stream); - return stream; + return stream; } -CUstream_st * kernel_info_t::get_default_stream_cta(dim3 ctaid) { - if(m_cta_streams.find(ctaid) != m_cta_streams.end()) { - assert(m_cta_streams[ctaid].size() >= 1); //already created, must have default stream - return *(m_cta_streams[ctaid].begin()); - } - else { - m_cta_streams[ctaid] = std::list(); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - m_cta_streams[ctaid].push_back(stream); - return stream; - } +CUstream_st *kernel_info_t::get_default_stream_cta(dim3 ctaid) { + if (m_cta_streams.find(ctaid) != m_cta_streams.end()) { + assert(m_cta_streams[ctaid].size() >= + 1); // already created, must have default stream + return *(m_cta_streams[ctaid].begin()); + } else { + m_cta_streams[ctaid] = std::list(); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream( + stream); + m_cta_streams[ctaid].push_back(stream); + return stream; + } } -bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st* stream) { - if(m_cta_streams.find(ctaid) == m_cta_streams.end()) - return false; +bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st *stream) { + if (m_cta_streams.find(ctaid) == m_cta_streams.end()) return false; - std::list &stream_list = m_cta_streams[ctaid]; - if(std::find(stream_list.begin(), stream_list.end(), stream) - == stream_list.end()) - return false; - else - return true; + std::list &stream_list = m_cta_streams[ctaid]; + if (std::find(stream_list.begin(), stream_list.end(), stream) == + stream_list.end()) + return false; + else + return true; } void kernel_info_t::print_parent_info() { - if(m_parent_kernel) { - printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", - m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), - m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, - m_parent_tid.x, m_parent_tid.y, m_parent_tid.z); - } + if (m_parent_kernel) { + printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", + m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), + m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, m_parent_tid.x, + m_parent_tid.y, m_parent_tid.z); + } } void kernel_info_t::destroy_cta_streams() { - printf("Destroy streams for kernel %d: ", get_uid()); size_t stream_size = 0; - for(auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { - stream_size += s->second.size(); - for(auto ss = s->second.begin(); ss != s->second.end(); ss++) - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream(*ss); - s->second.clear(); - } - printf("size %lu\n", stream_size); - m_cta_streams.clear(); + printf("Destroy streams for kernel %d: ", get_uid()); + size_t stream_size = 0; + for (auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { + stream_size += s->second.size(); + for (auto ss = s->second.begin(); ss != s->second.end(); ss++) + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream( + *ss); + s->second.clear(); + } + printf("size %lu\n", stream_size); + m_cta_streams.clear(); } -simt_stack::simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu) -{ - m_warp_id=wid; - m_warp_size = warpSize; - m_gpu=gpu; - reset(); +simt_stack::simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu) { + m_warp_id = wid; + m_warp_size = warpSize; + m_gpu = gpu; + reset(); } -void simt_stack::reset() -{ - m_stack.clear(); -} +void simt_stack::reset() { m_stack.clear(); } -void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) -{ - reset(); - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = start_pc; - new_stack_entry.m_calldepth = 1; - new_stack_entry.m_active_mask = active_mask; - new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; - m_stack.push_back(new_stack_entry); +void simt_stack::launch(address_type start_pc, const simt_mask_t &active_mask) { + reset(); + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = start_pc; + new_stack_entry.m_calldepth = 1; + new_stack_entry.m_active_mask = active_mask; + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + m_stack.push_back(new_stack_entry); } -void simt_stack::resume( char * fname ) -{ - reset(); - - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - - char line [ 200 ]; /* or other suitable maximum line size */ - - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - simt_stack_entry new_stack_entry; - char * pch; - pch = strtok (line," "); - for (unsigned j=0; j 0); - return m_stack.back().m_active_mask; -} + char line[200]; /* or other suitable maximum line size */ -void simt_stack::get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const -{ - assert(m_stack.size() > 0); - *pc = m_stack.back().m_pc; - *rpc = m_stack.back().m_recvg_pc; + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + simt_stack_entry new_stack_entry; + char *pch; + pch = strtok(line, " "); + for (unsigned j = 0; j < m_warp_size; j++) { + if (pch[0] == '1') + new_stack_entry.m_active_mask.set(j); + else + new_stack_entry.m_active_mask.reset(j); + pch = strtok(NULL, " "); + } + + new_stack_entry.m_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_calldepth = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_recvg_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_branch_div_cycle = atoi(pch); + pch = strtok(NULL, " "); + if (pch[0] == '0') + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + else + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + } + fclose(fp2); } -unsigned simt_stack::get_rp() const -{ - assert(m_stack.size() > 0); - return m_stack.back().m_recvg_pc; +const simt_mask_t &simt_stack::get_active_mask() const { + assert(m_stack.size() > 0); + return m_stack.back().m_active_mask; } -void simt_stack::print (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - if ( k==0 ) { - fprintf(fout, "w%02d %1u ", m_warp_id, k ); - } else { - fprintf(fout, " %1u ", k ); - } - for (unsigned j=0; jgpgpu_ctx->func_sim->ptx_print_insn( stack_entry.m_pc, fout ); - fprintf(fout,"\n"); - } +void simt_stack::get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const { + assert(m_stack.size() > 0); + *pc = m_stack.back().m_pc; + *rpc = m_stack.back().m_recvg_pc; +} +unsigned simt_stack::get_rp() const { + assert(m_stack.size() > 0); + return m_stack.back().m_recvg_pc; } -void simt_stack::print_checkpoint (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - - for (unsigned j=0; jgpgpu_ctx->func_sim->ptx_print_insn(stack_entry.m_pc, fout); + fprintf(fout, "\n"); + } } -void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ) -{ - assert(m_stack.size() > 0); - - assert( next_pc.size() == m_warp_size ); - - simt_mask_t top_active_mask = m_stack.back().m_active_mask; - address_type top_recvg_pc = m_stack.back().m_recvg_pc; - address_type top_pc = m_stack.back().m_pc; // the pc of the instruction just executed - stack_entry_type top_type = m_stack.back().m_type; - assert(top_pc==next_inst_pc); - assert(top_active_mask.any()); - - const address_type null_pc = -1; - bool warp_diverged = false; - address_type new_recvg_pc = null_pc; - unsigned num_divergent_paths=0; - - std::map divergent_paths; - while (top_active_mask.any()) { - - // extract a group of threads with the same next PC among the active threads in the warp - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - for (int i = m_warp_size - 1; i >= 0; i--) { - if ( top_active_mask.test(i) ) { // is this thread active? - if (thread_done.test(i)) { - top_active_mask.reset(i); // remove completed thread from active mask - } else if (tmp_next_pc == null_pc) { - tmp_next_pc = next_pc[i]; - tmp_active_mask.set(i); - top_active_mask.reset(i); - } else if (tmp_next_pc == next_pc[i]) { - tmp_active_mask.set(i); - top_active_mask.reset(i); - } - } - } +void simt_stack::print_checkpoint(FILE *fout) const { + for (unsigned k = 0; k < m_stack.size(); k++) { + simt_stack_entry stack_entry = m_stack[k]; + + for (unsigned j = 0; j < m_warp_size; j++) + fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j) ? '1' : '0')); + fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc, + stack_entry.m_calldepth, stack_entry.m_recvg_pc, + stack_entry.m_branch_div_cycle, stack_entry.m_type); + fprintf(fout, "%d %d\n", m_warp_id, m_warp_size); + } +} - if(tmp_next_pc == null_pc) { - assert(!top_active_mask.any()); // all threads done - continue; +void simt_stack::update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc) { + assert(m_stack.size() > 0); + + assert(next_pc.size() == m_warp_size); + + simt_mask_t top_active_mask = m_stack.back().m_active_mask; + address_type top_recvg_pc = m_stack.back().m_recvg_pc; + address_type top_pc = + m_stack.back().m_pc; // the pc of the instruction just executed + stack_entry_type top_type = m_stack.back().m_type; + assert(top_pc == next_inst_pc); + assert(top_active_mask.any()); + + const address_type null_pc = -1; + bool warp_diverged = false; + address_type new_recvg_pc = null_pc; + unsigned num_divergent_paths = 0; + + std::map divergent_paths; + while (top_active_mask.any()) { + // extract a group of threads with the same next PC among the active threads + // in the warp + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + for (int i = m_warp_size - 1; i >= 0; i--) { + if (top_active_mask.test(i)) { // is this thread active? + if (thread_done.test(i)) { + top_active_mask.reset(i); // remove completed thread from active mask + } else if (tmp_next_pc == null_pc) { + tmp_next_pc = next_pc[i]; + tmp_active_mask.set(i); + top_active_mask.reset(i); + } else if (tmp_next_pc == next_pc[i]) { + tmp_active_mask.set(i); + top_active_mask.reset(i); } - - divergent_paths[tmp_next_pc]=tmp_active_mask; - num_divergent_paths++; + } } + if (tmp_next_pc == null_pc) { + assert(!top_active_mask.any()); // all threads done + continue; + } - address_type not_taken_pc = next_inst_pc+next_inst_size; - assert(num_divergent_paths<=2); - for(unsigned i=0; i:: iterator it=divergent_paths.begin(); - tmp_next_pc=it->first; - tmp_active_mask=divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); - } - - // HANDLE THE SPECIAL CASES FIRST - if (next_inst_op== CALL_OPS){ - // Since call is not a divergent instruction, all threads should have executed a call instruction - assert(num_divergent_paths == 1); - - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = tmp_next_pc; - new_stack_entry.m_active_mask = tmp_active_mask; - new_stack_entry.m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; - new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; - m_stack.push_back(new_stack_entry); - return; - }else if(next_inst_op == RET_OPS && top_type==STACK_ENTRY_TYPE_CALL){ - // pop the CALL Entry - assert(num_divergent_paths == 1); - m_stack.pop_back(); - - assert(m_stack.size() > 0); - m_stack.back().m_pc=tmp_next_pc;// set the PC of the stack top entry to return PC from the call stack; - // Check if the New top of the stack is reconverging - if (tmp_next_pc == m_stack.back().m_recvg_pc && m_stack.back().m_type!=STACK_ENTRY_TYPE_CALL){ - assert(m_stack.back().m_type==STACK_ENTRY_TYPE_NORMAL); - m_stack.pop_back(); - } - return; - } - - // discard the new entry if its PC matches with reconvergence PC - // that automatically reconverges the entry - // If the top stack entry is CALL, dont reconverge. - if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) continue; - - // this new entry is not converging - // if this entry does not include thread from the warp, divergence occurs - if ((num_divergent_paths>1) && !warp_diverged ) { - warp_diverged = true; - // modify the existing top entry into a reconvergence entry in the pdom stack - new_recvg_pc = recvg_pc; - if (new_recvg_pc != top_recvg_pc) { - m_stack.back().m_pc = new_recvg_pc; - m_stack.back().m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; - - m_stack.push_back(simt_stack_entry()); - } - } + divergent_paths[tmp_next_pc] = tmp_active_mask; + num_divergent_paths++; + } + + address_type not_taken_pc = next_inst_pc + next_inst_size; + assert(num_divergent_paths <= 2); + for (unsigned i = 0; i < num_divergent_paths; i++) { + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + tmp_active_mask.reset(); + if (divergent_paths.find(not_taken_pc) != divergent_paths.end()) { + assert(i == 0); + tmp_next_pc = not_taken_pc; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } else { + std::map::iterator it = + divergent_paths.begin(); + tmp_next_pc = it->first; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } - // discard the new entry if its PC matches with reconvergence PC - if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // HANDLE THE SPECIAL CASES FIRST + if (next_inst_op == CALL_OPS) { + // Since call is not a divergent instruction, all threads should have + // executed a call instruction + assert(num_divergent_paths == 1); + + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = tmp_next_pc; + new_stack_entry.m_active_mask = tmp_active_mask; + new_stack_entry.m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + return; + } else if (next_inst_op == RET_OPS && top_type == STACK_ENTRY_TYPE_CALL) { + // pop the CALL Entry + assert(num_divergent_paths == 1); + m_stack.pop_back(); + + assert(m_stack.size() > 0); + m_stack.back().m_pc = tmp_next_pc; // set the PC of the stack top entry + // to return PC from the call stack; + // Check if the New top of the stack is reconverging + if (tmp_next_pc == m_stack.back().m_recvg_pc && + m_stack.back().m_type != STACK_ENTRY_TYPE_CALL) { + assert(m_stack.back().m_type == STACK_ENTRY_TYPE_NORMAL); + m_stack.pop_back(); + } + return; + } - // update the current top of pdom stack - m_stack.back().m_pc = tmp_next_pc; - m_stack.back().m_active_mask = tmp_active_mask; - if (warp_diverged) { - m_stack.back().m_calldepth = 0; - m_stack.back().m_recvg_pc = new_recvg_pc; - } else { - m_stack.back().m_recvg_pc = top_recvg_pc; - } + // discard the new entry if its PC matches with reconvergence PC + // that automatically reconverges the entry + // If the top stack entry is CALL, dont reconverge. + if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) + continue; + + // this new entry is not converging + // if this entry does not include thread from the warp, divergence occurs + if ((num_divergent_paths > 1) && !warp_diverged) { + warp_diverged = true; + // modify the existing top entry into a reconvergence entry in the pdom + // stack + new_recvg_pc = recvg_pc; + if (new_recvg_pc != top_recvg_pc) { + m_stack.back().m_pc = new_recvg_pc; + m_stack.back().m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; m_stack.push_back(simt_stack_entry()); + } } - assert(m_stack.size() > 0); - m_stack.pop_back(); + // discard the new entry if its PC matches with reconvergence PC + if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // update the current top of pdom stack + m_stack.back().m_pc = tmp_next_pc; + m_stack.back().m_active_mask = tmp_active_mask; if (warp_diverged) { - m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + m_stack.back().m_calldepth = 0; + m_stack.back().m_recvg_pc = new_recvg_pc; + } else { + m_stack.back().m_recvg_pc = top_recvg_pc; } + + m_stack.push_back(simt_stack_entry()); + } + assert(m_stack.size() > 0); + m_stack.pop_back(); + + if (warp_diverged) { + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + } } -void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) -{ - for ( unsigned t=0; t < m_warp_size; t++ ) { - if( inst.active(t) ) { - if(warpId==(unsigned (-1))) - warpId = inst.warp_id(); - unsigned tid=m_warp_size*warpId+t; - m_thread[tid]->ptx_exec_inst(inst,t); - - //virtual function - checkExecutionStatusAndUpdate(inst,t,tid); - } - } +void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) { + for (unsigned t = 0; t < m_warp_size; t++) { + if (inst.active(t)) { + if (warpId == (unsigned(-1))) warpId = inst.warp_id(); + unsigned tid = m_warp_size * warpId + t; + m_thread[tid]->ptx_exec_inst(inst, t); + + // virtual function + checkExecutionStatusAndUpdate(inst, t, tid); + } + } } - -bool core_t::ptx_thread_done( unsigned hw_thread_id ) const -{ - return ((m_thread[ hw_thread_id ]==NULL) || m_thread[ hw_thread_id ]->is_done()); + +bool core_t::ptx_thread_done(unsigned hw_thread_id) const { + return ((m_thread[hw_thread_id] == NULL) || + m_thread[hw_thread_id]->is_done()); } - -void core_t::updateSIMTStack(unsigned warpId, warp_inst_t * inst) -{ - simt_mask_t thread_done; - addr_vector_t next_pc; - unsigned wtid = warpId * m_warp_size; - for (unsigned i = 0; i < m_warp_size; i++) { - if( ptx_thread_done(wtid+i) ) { - thread_done.set(i); - next_pc.push_back( (address_type)-1 ); - } else { - if( inst->reconvergence_pc == RECONVERGE_RETURN_PC ) - inst->reconvergence_pc = get_return_pc(m_thread[wtid+i]); - next_pc.push_back( m_thread[wtid+i]->get_pc() ); - } + +void core_t::updateSIMTStack(unsigned warpId, warp_inst_t *inst) { + simt_mask_t thread_done; + addr_vector_t next_pc; + unsigned wtid = warpId * m_warp_size; + for (unsigned i = 0; i < m_warp_size; i++) { + if (ptx_thread_done(wtid + i)) { + thread_done.set(i); + next_pc.push_back((address_type)-1); + } else { + if (inst->reconvergence_pc == RECONVERGE_RETURN_PC) + inst->reconvergence_pc = get_return_pc(m_thread[wtid + i]); + next_pc.push_back(m_thread[wtid + i]->get_pc()); } - m_simt_stack[warpId]->update(thread_done,next_pc,inst->reconvergence_pc, inst->op,inst->isize,inst->pc); + } + m_simt_stack[warpId]->update(thread_done, next_pc, inst->reconvergence_pc, + inst->op, inst->isize, inst->pc); } //! Get the warp to be executed using the data taken form the SIMT stack -warp_inst_t core_t::getExecuteWarp(unsigned warpId) -{ - unsigned pc,rpc; - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - warp_inst_t wi= *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); - wi.set_active(m_simt_stack[warpId]->get_active_mask()); - return wi; +warp_inst_t core_t::getExecuteWarp(unsigned warpId) { + unsigned pc, rpc; + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + warp_inst_t wi = *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); + wi.set_active(m_simt_stack[warpId]->get_active_mask()); + return wi; } -void core_t::deleteSIMTStack() -{ - if ( m_simt_stack ) { - for (unsigned i = 0; i < m_warp_count; ++i) - delete m_simt_stack[i]; - delete[] m_simt_stack; - m_simt_stack = NULL; - } +void core_t::deleteSIMTStack() { + if (m_simt_stack) { + for (unsigned i = 0; i < m_warp_count; ++i) delete m_simt_stack[i]; + delete[] m_simt_stack; + m_simt_stack = NULL; + } } -void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) -{ - m_simt_stack = new simt_stack*[warp_count]; - for (unsigned i = 0; i < warp_count; ++i) - m_simt_stack[i] = new simt_stack(i,warp_size,m_gpu); - m_warp_size = warp_size; - m_warp_count = warp_count; +void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) { + m_simt_stack = new simt_stack *[warp_count]; + for (unsigned i = 0; i < warp_count; ++i) + m_simt_stack[i] = new simt_stack(i, warp_size, m_gpu); + m_warp_size = warp_size; + m_warp_count = warp_count; } -void core_t::get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const -{ - m_simt_stack[warpId]->get_pdom_stack_top_info(pc,rpc); +void core_t::get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const { + m_simt_stack[warpId]->get_pdom_stack_top_info(pc, rpc); } diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 29e4a30..dda4ead 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED #define ABSTRACT_HARDWARE_MODEL_INCLUDED @@ -33,45 +34,41 @@ class gpgpu_sim; class kernel_info_t; class gpgpu_context; - -//Set a hard limit of 32 CTAs per shader [cuda only has 8] +// Set a hard limit of 32 CTAs per shader [cuda only has 8] #define MAX_CTA_PER_SHADER 32 #define MAX_BARRIERS_PER_CTA 16 -//After expanding the vector input and output operands +// After expanding the vector input and output operands #define MAX_INPUT_VALUES 24 #define MAX_OUTPUT_VALUES 8 enum _memory_space_t { - undefined_space=0, - reg_space, - local_space, - shared_space, - sstarr_space, - param_space_unclassified, - param_space_kernel, /* global to all threads in a kernel : read-only */ - param_space_local, /* local to a thread : read-writable */ - const_space, - tex_space, - surf_space, - global_space, - generic_space, - instruction_space + undefined_space = 0, + reg_space, + local_space, + shared_space, + sstarr_space, + param_space_unclassified, + param_space_kernel, /* global to all threads in a kernel : read-only */ + param_space_local, /* local to a thread : read-writable */ + const_space, + tex_space, + surf_space, + global_space, + generic_space, + instruction_space }; - -enum FuncCache -{ +enum FuncCache { FuncCachePreferNone = 0, FuncCachePreferShared = 1, FuncCachePreferL1 = 2 }; - #ifdef __cplusplus -#include #include +#include #include typedef unsigned long long new_addr_type; @@ -79,371 +76,357 @@ typedef unsigned long long cudaTextureObject_t; typedef unsigned address_type; typedef unsigned addr_t; -// the following are operations the timing model can see +// the following are operations the timing model can see enum uarch_op_t { - NO_OP=-1, - ALU_OP=1, - SFU_OP, - TENSOR_CORE_OP, - DP_OP, - SP_OP, - INTP_OP, - ALU_SFU_OP, - LOAD_OP, - TENSOR_CORE_LOAD_OP, - TENSOR_CORE_STORE_OP, - STORE_OP, - BRANCH_OP, - BARRIER_OP, - MEMORY_BARRIER_OP, - CALL_OPS, - RET_OPS + NO_OP = -1, + ALU_OP = 1, + SFU_OP, + TENSOR_CORE_OP, + DP_OP, + SP_OP, + INTP_OP, + ALU_SFU_OP, + LOAD_OP, + TENSOR_CORE_LOAD_OP, + TENSOR_CORE_STORE_OP, + STORE_OP, + BRANCH_OP, + BARRIER_OP, + MEMORY_BARRIER_OP, + CALL_OPS, + RET_OPS }; typedef enum uarch_op_t op_type; - -enum uarch_bar_t { - NOT_BAR=-1, - SYNC=1, - ARRIVE, - RED -}; +enum uarch_bar_t { NOT_BAR = -1, SYNC = 1, ARRIVE, RED }; typedef enum uarch_bar_t barrier_type; -enum uarch_red_t { - NOT_RED=-1, - POPC_RED=1, - AND_RED, - OR_RED -}; +enum uarch_red_t { NOT_RED = -1, POPC_RED = 1, AND_RED, OR_RED }; typedef enum uarch_red_t reduction_type; - -enum uarch_operand_type_t { - UN_OP=-1, - INT_OP, - FP_OP -}; +enum uarch_operand_type_t { UN_OP = -1, INT_OP, FP_OP }; typedef enum uarch_operand_type_t types_of_operands; enum special_operations_t { - OTHER_OP, - INT__OP, - INT_MUL24_OP, - INT_MUL32_OP, - INT_MUL_OP, - INT_DIV_OP, - FP_MUL_OP, - FP_DIV_OP, - FP__OP, - FP_SQRT_OP, - FP_LG_OP, - FP_SIN_OP, - FP_EXP_OP + OTHER_OP, + INT__OP, + INT_MUL24_OP, + INT_MUL32_OP, + INT_MUL_OP, + INT_DIV_OP, + FP_MUL_OP, + FP_DIV_OP, + FP__OP, + FP_SQRT_OP, + FP_LG_OP, + FP_SIN_OP, + FP_EXP_OP }; -typedef enum special_operations_t special_ops; // Required to identify for the power model +typedef enum special_operations_t + special_ops; // Required to identify for the power model enum operation_pipeline_t { - UNKOWN_OP, - SP__OP, - DP__OP, - INTP__OP, - SFU__OP, - TENSOR_CORE__OP, - MEM__OP + UNKOWN_OP, + SP__OP, + DP__OP, + INTP__OP, + SFU__OP, + TENSOR_CORE__OP, + MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; -enum mem_operation_t { - NOT_TEX, - TEX -}; +enum mem_operation_t { NOT_TEX, TEX }; typedef enum mem_operation_t mem_operation; -enum _memory_op_t { - no_memory_op = 0, - memory_load, - memory_store -}; +enum _memory_op_t { no_memory_op = 0, memory_load, memory_store }; -#include -#include -#include #include #include -#include -#include #include +#include +#include +#include +#include +#include #if !defined(__VECTOR_TYPES_H__) #include "vector_types.h" #endif struct dim3comp { - bool operator() (const dim3 & a, const dim3 & b) const - { - if(a.z < b.z) - return true; - else if(a.y < b.y) - return true; - else if (a.x < b.x) - return true; - else - return false; - } + bool operator()(const dim3 &a, const dim3 &b) const { + if (a.z < b.z) + return true; + else if (a.y < b.y) + return true; + else if (a.x < b.x) + return true; + else + return false; + } }; -void increment_x_then_y_then_z( dim3 &i, const dim3 &bound); +void increment_x_then_y_then_z(dim3 &i, const dim3 &bound); -//Jin: child kernel information for CDP +// Jin: child kernel information for CDP #include "stream_manager.h" class stream_manager; struct CUstream_st; -//extern stream_manager * g_stream_manager; -//support for pinned memories added -extern std::map pinned_memory; +// extern stream_manager * g_stream_manager; +// support for pinned memories added +extern std::map pinned_memory; extern std::map pinned_memory_size; class kernel_info_t { -public: -// kernel_info_t() -// { -// m_valid=false; -// m_kernel_entry=NULL; -// m_uid=0; -// m_num_cores_running=0; -// m_param_mem=NULL; -// } - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry); - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map nameToCudaArray, std::map nameToTextureInfo); - ~kernel_info_t(); - - void inc_running() { m_num_cores_running++; } - void dec_running() - { - assert( m_num_cores_running > 0 ); - m_num_cores_running--; - } - bool running() const { return m_num_cores_running>0; } - bool done() const - { - return no_more_ctas_to_run() && !running(); - } - class function_info *entry() { return m_kernel_entry; } - const class function_info *entry() const { return m_kernel_entry; } - - size_t num_blocks() const - { - return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; - } - - size_t threads_per_cta() const - { - return m_block_dim.x * m_block_dim.y * m_block_dim.z; - } - - dim3 get_grid_dim() const { return m_grid_dim; } - dim3 get_cta_dim() const { return m_block_dim; } - - void increment_cta_id() - { - increment_x_then_y_then_z(m_next_cta,m_grid_dim); - m_next_tid.x=0; - m_next_tid.y=0; - m_next_tid.z=0; - } - dim3 get_next_cta_id() const { return m_next_cta; } - unsigned get_next_cta_id_single() const - { - return m_next_cta.x + m_grid_dim.x*m_next_cta.y + m_grid_dim.x*m_grid_dim.y*m_next_cta.z; - } - bool no_more_ctas_to_run() const - { - return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || m_next_cta.z >= m_grid_dim.z ); - } - - void increment_thread_id() { increment_x_then_y_then_z(m_next_tid,m_block_dim); } - dim3 get_next_thread_id_3d() const { return m_next_tid; } - unsigned get_next_thread_id() const - { - return m_next_tid.x + m_block_dim.x*m_next_tid.y + m_block_dim.x*m_block_dim.y*m_next_tid.z; - } - bool more_threads_in_cta() const - { - return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && m_next_tid.x < m_block_dim.x; - } - unsigned get_uid() const { return m_uid; } - std::string name() const; - - std::list &active_threads() { return m_active_threads; } - class memory_space *get_param_memory() { return m_param_mem; } - - - //The following functions access texture bindings present at the kernel's launch - - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - -private: - kernel_info_t( const kernel_info_t & ); // disable copy constructor - void operator=( const kernel_info_t & ); // disable copy operator - - class function_info *m_kernel_entry; - - unsigned m_uid; - - //These maps contain the snapshot of the texture mappings at kernel launch - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - - dim3 m_grid_dim; - dim3 m_block_dim; - dim3 m_next_cta; - dim3 m_next_tid; - - unsigned m_num_cores_running; - - std::list m_active_threads; - class memory_space *m_param_mem; - -public: - //Jin: parent and child kernel management for CDP - void set_parent(kernel_info_t * parent, dim3 parent_ctaid, dim3 parent_tid); - void set_child(kernel_info_t * child); - void remove_child(kernel_info_t * child); - bool is_finished(); - bool children_all_finished(); - void notify_parent_finished(); - CUstream_st * create_stream_cta(dim3 ctaid); - CUstream_st * get_default_stream_cta(dim3 ctaid); - bool cta_has_stream(dim3 ctaid, CUstream_st* stream); - void destroy_cta_streams(); - void print_parent_info(); - kernel_info_t * get_parent() { return m_parent_kernel; } - -private: - kernel_info_t * m_parent_kernel; - dim3 m_parent_ctaid; - dim3 m_parent_tid; - std::list m_child_kernels; //child kernel launched - std::map< dim3, std::list, dim3comp > m_cta_streams; //streams created in each CTA - -//Jin: kernel timing -public: - unsigned long long launch_cycle; - unsigned long long start_cycle; - unsigned long long end_cycle; - unsigned m_launch_latency; - - mutable bool volta_cache_config_set; + public: + // kernel_info_t() + // { + // m_valid=false; + // m_kernel_entry=NULL; + // m_uid=0; + // m_num_cores_running=0; + // m_param_mem=NULL; + // } + kernel_info_t(dim3 gridDim, dim3 blockDim, class function_info *entry); + kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map nameToCudaArray, + std::map nameToTextureInfo); + ~kernel_info_t(); + + void inc_running() { m_num_cores_running++; } + void dec_running() { + assert(m_num_cores_running > 0); + m_num_cores_running--; + } + bool running() const { return m_num_cores_running > 0; } + bool done() const { return no_more_ctas_to_run() && !running(); } + class function_info *entry() { + return m_kernel_entry; + } + const class function_info *entry() const { return m_kernel_entry; } + + size_t num_blocks() const { + return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; + } + + size_t threads_per_cta() const { + return m_block_dim.x * m_block_dim.y * m_block_dim.z; + } + + dim3 get_grid_dim() const { return m_grid_dim; } + dim3 get_cta_dim() const { return m_block_dim; } + + void increment_cta_id() { + increment_x_then_y_then_z(m_next_cta, m_grid_dim); + m_next_tid.x = 0; + m_next_tid.y = 0; + m_next_tid.z = 0; + } + dim3 get_next_cta_id() const { return m_next_cta; } + unsigned get_next_cta_id_single() const { + return m_next_cta.x + m_grid_dim.x * m_next_cta.y + + m_grid_dim.x * m_grid_dim.y * m_next_cta.z; + } + bool no_more_ctas_to_run() const { + return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || + m_next_cta.z >= m_grid_dim.z); + } + + void increment_thread_id() { + increment_x_then_y_then_z(m_next_tid, m_block_dim); + } + dim3 get_next_thread_id_3d() const { return m_next_tid; } + unsigned get_next_thread_id() const { + return m_next_tid.x + m_block_dim.x * m_next_tid.y + + m_block_dim.x * m_block_dim.y * m_next_tid.z; + } + bool more_threads_in_cta() const { + return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && + m_next_tid.x < m_block_dim.x; + } + unsigned get_uid() const { return m_uid; } + std::string name() const; + + std::list &active_threads() { + return m_active_threads; + } + class memory_space *get_param_memory() { + return m_param_mem; + } + + // The following functions access texture bindings present at the kernel's + // launch + + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + + private: + kernel_info_t(const kernel_info_t &); // disable copy constructor + void operator=(const kernel_info_t &); // disable copy operator + + class function_info *m_kernel_entry; + + unsigned m_uid; + + // These maps contain the snapshot of the texture mappings at kernel launch + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + + dim3 m_grid_dim; + dim3 m_block_dim; + dim3 m_next_cta; + dim3 m_next_tid; + + unsigned m_num_cores_running; + + std::list m_active_threads; + class memory_space *m_param_mem; + + public: + // Jin: parent and child kernel management for CDP + void set_parent(kernel_info_t *parent, dim3 parent_ctaid, dim3 parent_tid); + void set_child(kernel_info_t *child); + void remove_child(kernel_info_t *child); + bool is_finished(); + bool children_all_finished(); + void notify_parent_finished(); + CUstream_st *create_stream_cta(dim3 ctaid); + CUstream_st *get_default_stream_cta(dim3 ctaid); + bool cta_has_stream(dim3 ctaid, CUstream_st *stream); + void destroy_cta_streams(); + void print_parent_info(); + kernel_info_t *get_parent() { return m_parent_kernel; } + + private: + kernel_info_t *m_parent_kernel; + dim3 m_parent_ctaid; + dim3 m_parent_tid; + std::list m_child_kernels; // child kernel launched + std::map, dim3comp> + m_cta_streams; // streams created in each CTA + + // Jin: kernel timing + public: + unsigned long long launch_cycle; + unsigned long long start_cycle; + unsigned long long end_cycle; + unsigned m_launch_latency; + + mutable bool volta_cache_config_set; }; class core_config { - public: - core_config(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_valid = false; - num_shmem_bank=16; - shmem_limited_broadcast = false; - gpgpu_shmem_sizeDefault=(unsigned)-1; - gpgpu_shmem_sizePrefL1=(unsigned)-1; - gpgpu_shmem_sizePrefShared=(unsigned)-1; - } - virtual void init() = 0; - - bool m_valid; - unsigned warp_size; - // backward pointer - class gpgpu_context* gpgpu_ctx; - - // off-chip memory request architecture parameters - int gpgpu_coalesce_arch; - - // shared memory bank conflict checking parameters - bool shmem_limited_broadcast; - static const address_type WORD_SIZE=4; - unsigned num_shmem_bank; - unsigned shmem_bank_func(address_type addr) const - { - return ((addr/WORD_SIZE) % num_shmem_bank); - } - unsigned mem_warp_parts; - mutable unsigned gpgpu_shmem_size; - unsigned gpgpu_shmem_sizeDefault; - unsigned gpgpu_shmem_sizePrefL1; - unsigned gpgpu_shmem_sizePrefShared; - unsigned mem_unit_ports; - - // texture and constant cache line sizes (used to determine number of memory accesses) - unsigned gpgpu_cache_texl1_linesize; - unsigned gpgpu_cache_constl1_linesize; - - unsigned gpgpu_max_insn_issue_per_warp; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - - bool adaptive_volta_cache_config; + public: + core_config(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_valid = false; + num_shmem_bank = 16; + shmem_limited_broadcast = false; + gpgpu_shmem_sizeDefault = (unsigned)-1; + gpgpu_shmem_sizePrefL1 = (unsigned)-1; + gpgpu_shmem_sizePrefShared = (unsigned)-1; + } + virtual void init() = 0; + + bool m_valid; + unsigned warp_size; + // backward pointer + class gpgpu_context *gpgpu_ctx; + + // off-chip memory request architecture parameters + int gpgpu_coalesce_arch; + + // shared memory bank conflict checking parameters + bool shmem_limited_broadcast; + static const address_type WORD_SIZE = 4; + unsigned num_shmem_bank; + unsigned shmem_bank_func(address_type addr) const { + return ((addr / WORD_SIZE) % num_shmem_bank); + } + unsigned mem_warp_parts; + mutable unsigned gpgpu_shmem_size; + unsigned gpgpu_shmem_sizeDefault; + unsigned gpgpu_shmem_sizePrefL1; + unsigned gpgpu_shmem_sizePrefShared; + unsigned mem_unit_ports; + + // texture and constant cache line sizes (used to determine number of memory + // accesses) + unsigned gpgpu_cache_texl1_linesize; + unsigned gpgpu_cache_constl1_linesize; + + unsigned gpgpu_max_insn_issue_per_warp; + bool gmem_skip_L1D; // on = global memory access always skip the L1 cache + + bool adaptive_volta_cache_config; }; -// bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper +// bounded stack that implements simt reconvergence using pdom mechanism from +// MICRO'07 paper const unsigned MAX_WARP_SIZE = 32; typedef std::bitset active_mask_t; -#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE +#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE typedef std::bitset simt_mask_t; typedef std::vector addr_vector_t; class simt_stack { -public: - simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu); - - void reset(); - void launch( address_type start_pc, const simt_mask_t &active_mask ); - void update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ); - - const simt_mask_t &get_active_mask() const; - void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; - unsigned get_rp() const; - void print(FILE *fp) const; - void resume(char * fname) ; - void print_checkpoint (FILE *fout) const; - -protected: - unsigned m_warp_id; - unsigned m_warp_size; - - - enum stack_entry_type { - STACK_ENTRY_TYPE_NORMAL = 0, - STACK_ENTRY_TYPE_CALL - }; - - struct simt_stack_entry { - address_type m_pc; - unsigned int m_calldepth; - simt_mask_t m_active_mask; - address_type m_recvg_pc; - unsigned long long m_branch_div_cycle; - stack_entry_type m_type; - simt_stack_entry() : - m_pc(-1), m_calldepth(0), m_active_mask(), m_recvg_pc(-1), m_branch_div_cycle(0), m_type(STACK_ENTRY_TYPE_NORMAL) { }; - }; - - std::deque m_stack; - - class gpgpu_sim * m_gpu; + public: + simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu); + + void reset(); + void launch(address_type start_pc, const simt_mask_t &active_mask); + void update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc); + + const simt_mask_t &get_active_mask() const; + void get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const; + unsigned get_rp() const; + void print(FILE *fp) const; + void resume(char *fname); + void print_checkpoint(FILE *fout) const; + + protected: + unsigned m_warp_id; + unsigned m_warp_size; + + enum stack_entry_type { STACK_ENTRY_TYPE_NORMAL = 0, STACK_ENTRY_TYPE_CALL }; + + struct simt_stack_entry { + address_type m_pc; + unsigned int m_calldepth; + simt_mask_t m_active_mask; + address_type m_recvg_pc; + unsigned long long m_branch_div_cycle; + stack_entry_type m_type; + simt_stack_entry() + : m_pc(-1), + m_calldepth(0), + m_active_mask(), + m_recvg_pc(-1), + m_branch_div_cycle(0), + m_type(STACK_ENTRY_TYPE_NORMAL){}; + }; + + std::deque m_stack; + + class gpgpu_sim *m_gpu; }; // Let's just upgrade to C++11 so we can use constexpr here... -// start allocating from this address (lower values used for allocating globals in .ptx file) +// start allocating from this address (lower values used for allocating globals +// in .ptx file) const unsigned long long GLOBAL_HEAP_START = 0xC0000000; // Volta max shmem size is 96kB const unsigned long long SHARED_MEM_SIZE_MAX = 96 * (1 << 10); @@ -455,873 +438,928 @@ const unsigned MAX_STREAMING_MULTIPROCESSORS = 80; const unsigned MAX_THREAD_PER_SM = 1 << 11; // MAX 64 warps / SM const unsigned MAX_WARP_PER_SM = 1 << 6; -const unsigned long long TOTAL_LOCAL_MEM_PER_SM = MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long TOTAL_SHARED_MEM = MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; -const unsigned long long TOTAL_LOCAL_MEM = MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long SHARED_GENERIC_START = GLOBAL_HEAP_START - TOTAL_SHARED_MEM; -const unsigned long long LOCAL_GENERIC_START = SHARED_GENERIC_START - TOTAL_LOCAL_MEM; -const unsigned long long STATIC_ALLOC_LIMIT = GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); +const unsigned long long TOTAL_LOCAL_MEM_PER_SM = + MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long TOTAL_SHARED_MEM = + MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; +const unsigned long long TOTAL_LOCAL_MEM = + MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long SHARED_GENERIC_START = + GLOBAL_HEAP_START - TOTAL_SHARED_MEM; +const unsigned long long LOCAL_GENERIC_START = + SHARED_GENERIC_START - TOTAL_LOCAL_MEM; +const unsigned long long STATIC_ALLOC_LIMIT = + GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); #if !defined(__CUDA_RUNTIME_API_H__) #include "builtin_types.h" struct cudaArray { - void *devPtr; - int devPtr32; - struct cudaChannelFormatDesc desc; - int width; - int height; - int size; //in bytes - unsigned dimensions; + void *devPtr; + int devPtr32; + struct cudaChannelFormatDesc desc; + int width; + int height; + int size; // in bytes + unsigned dimensions; }; #endif -// Struct that record other attributes in the textureReference declaration +// Struct that record other attributes in the textureReference declaration // - These attributes are passed thru __cudaRegisterTexture() struct textureReferenceAttr { - const struct textureReference *m_texref; - int m_dim; - enum cudaTextureReadMode m_readmode; - int m_ext; - textureReferenceAttr(const struct textureReference *texref, - int dim, - enum cudaTextureReadMode readmode, - int ext) - : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) - { } + const struct textureReference *m_texref; + int m_dim; + enum cudaTextureReadMode m_readmode; + int m_ext; + textureReferenceAttr(const struct textureReference *texref, int dim, + enum cudaTextureReadMode readmode, int ext) + : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) {} }; -class gpgpu_functional_sim_config -{ -public: - void reg_options(class OptionParser * opp); - - void ptx_set_tex_cache_linesize(unsigned linesize); - - unsigned get_forced_max_capability() const { return m_ptx_force_max_capability; } - bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } - bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } - bool experimental_lib_support() const { return m_experimental_lib_support; } - - int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } - const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } - int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } - unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const {return checkpoint_option; } - int get_checkpoint_kernel() const {return checkpoint_kernel; } - int get_checkpoint_CTA() const {return checkpoint_CTA; } - int get_resume_option() const {return resume_option; } - int get_resume_kernel() const {return resume_kernel; } - int get_resume_CTA() const {return resume_CTA; } - int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } -private: - // PTX options - int m_ptx_convert_to_ptxplus; - int m_ptx_use_cuobjdump; - int m_experimental_lib_support; - unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - int g_ptx_inst_debug_to_file; - char* g_ptx_inst_debug_file; - int g_ptx_inst_debug_thread_uid; - - unsigned m_texcache_linesize; +class gpgpu_functional_sim_config { + public: + void reg_options(class OptionParser *opp); + + void ptx_set_tex_cache_linesize(unsigned linesize); + + unsigned get_forced_max_capability() const { + return m_ptx_force_max_capability; + } + bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } + bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } + bool experimental_lib_support() const { return m_experimental_lib_support; } + + int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } + const char *get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } + int get_ptx_inst_debug_thread_uid() const { + return g_ptx_inst_debug_thread_uid; + } + unsigned get_texcache_linesize() const { return m_texcache_linesize; } + int get_checkpoint_option() const { return checkpoint_option; } + int get_checkpoint_kernel() const { return checkpoint_kernel; } + int get_checkpoint_CTA() const { return checkpoint_CTA; } + int get_resume_option() const { return resume_option; } + int get_resume_kernel() const { return resume_kernel; } + int get_resume_CTA() const { return resume_CTA; } + int get_checkpoint_CTA_t() const { return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const { return checkpoint_insn_Y; } + + private: + // PTX options + int m_ptx_convert_to_ptxplus; + int m_ptx_use_cuobjdump; + int m_experimental_lib_support; + unsigned m_ptx_force_max_capability; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; + char *g_ptx_inst_debug_file; + int g_ptx_inst_debug_thread_uid; + + unsigned m_texcache_linesize; }; - class gpgpu_t { -public: - gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ); - // backward pointer - class gpgpu_context* gpgpu_ctx; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - - //Move some cycle core stats here instead of being global - unsigned long long gpu_sim_cycle; - unsigned long long gpu_tot_sim_cycle; - - - void* gpu_malloc( size_t size ); - void* gpu_mallocarray( size_t count ); - void gpu_memset( size_t dst_start_addr, int c, size_t count ); - void memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ); - void memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ); - void memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ); - - class memory_space *get_global_memory() { return m_global_mem; } - class memory_space *get_tex_memory() { return m_tex_mem; } - class memory_space *get_surf_memory() { return m_surf_mem; } - - void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array); - void gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext); - void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); - const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); - - const struct textureReference* get_texref( const std::string &texname ) const - { - std::map >::const_iterator t=m_NameToTextureRef.find(texname); - assert( t != m_NameToTextureRef.end() ); - return *(t->second.begin()); - } - - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } - - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } - - const struct textureReferenceAttr* get_texattr( const std::string &texname ) const - { - std::map::const_iterator t=m_NameToAttribute.find(texname); - assert(t != m_NameToAttribute.end()); - return t->second; - } - - const gpgpu_functional_sim_config &get_config() const { return m_function_model_config; } - FILE* get_ptx_inst_debug_file() { return ptx_inst_debug_file; } - - // These maps return the current texture mappings for the GPU at any given time. - std::map getNameArrayMapping() {return m_NameToCudaArray;} - std::map getNameInfoMapping() {return m_NameToTextureInfo;} - -protected: - const gpgpu_functional_sim_config &m_function_model_config; - FILE* ptx_inst_debug_file; - - class memory_space *m_global_mem; - class memory_space *m_tex_mem; - class memory_space *m_surf_mem; - - unsigned long long m_dev_malloc; - // These maps contain the current texture mappings for the GPU at any given time. - std::map > m_NameToTextureRef; - std::map m_TextureRefToName; - std::map m_NameToCudaArray; - std::map m_NameToTextureInfo; - std::map m_NameToAttribute; + public: + gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx); + // backward pointer + class gpgpu_context *gpgpu_ctx; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + + // Move some cycle core stats here instead of being global + unsigned long long gpu_sim_cycle; + unsigned long long gpu_tot_sim_cycle; + + void *gpu_malloc(size_t size); + void *gpu_mallocarray(size_t count); + void gpu_memset(size_t dst_start_addr, int c, size_t count); + void memcpy_to_gpu(size_t dst_start_addr, const void *src, size_t count); + void memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count); + void memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count); + + class memory_space *get_global_memory() { + return m_global_mem; + } + class memory_space *get_tex_memory() { + return m_tex_mem; + } + class memory_space *get_surf_memory() { + return m_surf_mem; + } + + void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference *texref, + const struct cudaArray *array); + void gpgpu_ptx_sim_bindNameToTexture(const char *name, + const struct textureReference *texref, + int dim, int readmode, int ext); + void gpgpu_ptx_sim_unbindTexture(const struct textureReference *texref); + const char *gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref); + + const struct textureReference *get_texref(const std::string &texname) const { + std::map >::const_iterator t = + m_NameToTextureRef.find(texname); + assert(t != m_NameToTextureRef.end()); + return *(t->second.begin()); + } + + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } + + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } + + const struct textureReferenceAttr *get_texattr( + const std::string &texname) const { + std::map::const_iterator + t = m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); + return t->second; + } + + const gpgpu_functional_sim_config &get_config() const { + return m_function_model_config; + } + FILE *get_ptx_inst_debug_file() { return ptx_inst_debug_file; } + + // These maps return the current texture mappings for the GPU at any given + // time. + std::map getNameArrayMapping() { + return m_NameToCudaArray; + } + std::map getNameInfoMapping() { + return m_NameToTextureInfo; + } + + protected: + const gpgpu_functional_sim_config &m_function_model_config; + FILE *ptx_inst_debug_file; + + class memory_space *m_global_mem; + class memory_space *m_tex_mem; + class memory_space *m_surf_mem; + + unsigned long long m_dev_malloc; + // These maps contain the current texture mappings for the GPU at any given + // time. + std::map > + m_NameToTextureRef; + std::map m_TextureRefToName; + std::map m_NameToCudaArray; + std::map m_NameToTextureInfo; + std::map m_NameToAttribute; }; -struct gpgpu_ptx_sim_info -{ - // Holds properties of the kernel (Kernel's resource use). - // These will be set to zero if a ptxinfo file is not present. - int lmem; - int smem; - int cmem; - int gmem; - int regs; - unsigned maxthreads; - unsigned ptx_version; - unsigned sm_target; +struct gpgpu_ptx_sim_info { + // Holds properties of the kernel (Kernel's resource use). + // These will be set to zero if a ptxinfo file is not present. + int lmem; + int smem; + int cmem; + int gmem; + int regs; + unsigned maxthreads; + unsigned ptx_version; + unsigned sm_target; }; - struct gpgpu_ptx_sim_arg { - gpgpu_ptx_sim_arg() { m_start=NULL; } - gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) - { - m_start=arg; - m_nbytes=size; - m_offset=offset; - } - const void *m_start; - size_t m_nbytes; - size_t m_offset; + gpgpu_ptx_sim_arg() { m_start = NULL; } + gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) { + m_start = arg; + m_nbytes = size; + m_offset = offset; + } + const void *m_start; + size_t m_nbytes; + size_t m_offset; }; typedef std::list gpgpu_ptx_sim_arg_list_t; class memory_space_t { -public: - memory_space_t() { m_type = undefined_space; m_bank=0; } - memory_space_t( const enum _memory_space_t &from ) { m_type = from; m_bank = 0; } - bool operator==( const memory_space_t &x ) const { return (m_bank == x.m_bank) && (m_type == x.m_type); } - bool operator!=( const memory_space_t &x ) const { return !(*this == x); } - bool operator<( const memory_space_t &x ) const - { - if(m_type < x.m_type) - return true; - else if(m_type > x.m_type) - return false; - else if( m_bank < x.m_bank ) - return true; + public: + memory_space_t() { + m_type = undefined_space; + m_bank = 0; + } + memory_space_t(const enum _memory_space_t &from) { + m_type = from; + m_bank = 0; + } + bool operator==(const memory_space_t &x) const { + return (m_bank == x.m_bank) && (m_type == x.m_type); + } + bool operator!=(const memory_space_t &x) const { return !(*this == x); } + bool operator<(const memory_space_t &x) const { + if (m_type < x.m_type) + return true; + else if (m_type > x.m_type) return false; - } - enum _memory_space_t get_type() const { return m_type; } - void set_type( enum _memory_space_t t ) { m_type = t; } - unsigned get_bank() const { return m_bank; } - void set_bank( unsigned b ) { m_bank = b; } - bool is_const() const { return (m_type == const_space) || (m_type == param_space_kernel); } - bool is_local() const { return (m_type == local_space) || (m_type == param_space_local); } - bool is_global() const { return (m_type == global_space); } - -private: - enum _memory_space_t m_type; - unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 manual, sec. 5.1.3) + else if (m_bank < x.m_bank) + return true; + return false; + } + enum _memory_space_t get_type() const { return m_type; } + void set_type(enum _memory_space_t t) { m_type = t; } + unsigned get_bank() const { return m_bank; } + void set_bank(unsigned b) { m_bank = b; } + bool is_const() const { + return (m_type == const_space) || (m_type == param_space_kernel); + } + bool is_local() const { + return (m_type == local_space) || (m_type == param_space_local); + } + bool is_global() const { return (m_type == global_space); } + + private: + enum _memory_space_t m_type; + unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 + // manual, sec. 5.1.3) }; const unsigned MAX_MEMORY_ACCESS_SIZE = 128; typedef std::bitset mem_access_byte_mask_t; -const unsigned SECTOR_CHUNCK_SIZE = 4; //four sectors -const unsigned SECTOR_SIZE = 32 ; //sector is 32 bytes width +const unsigned SECTOR_CHUNCK_SIZE = 4; // four sectors +const unsigned SECTOR_SIZE = 32; // sector is 32 bytes width typedef std::bitset mem_access_sector_mask_t; #define NO_PARTIAL_WRITE (mem_access_byte_mask_t()) -#define MEM_ACCESS_TYPE_TUP_DEF \ -MA_TUP_BEGIN( mem_access_type ) \ - MA_TUP( GLOBAL_ACC_R ), \ - MA_TUP( LOCAL_ACC_R ), \ - MA_TUP( CONST_ACC_R ), \ - MA_TUP( TEXTURE_ACC_R ), \ - MA_TUP( GLOBAL_ACC_W ), \ - MA_TUP( LOCAL_ACC_W ), \ - MA_TUP( L1_WRBK_ACC ), \ - MA_TUP( L2_WRBK_ACC ), \ - MA_TUP( INST_ACC_R ), \ - MA_TUP( L1_WR_ALLOC_R ), \ - MA_TUP( L2_WR_ALLOC_R ), \ - MA_TUP( NUM_MEM_ACCESS_TYPE ) \ -MA_TUP_END( mem_access_type ) +#define MEM_ACCESS_TYPE_TUP_DEF \ + MA_TUP_BEGIN(mem_access_type) \ + MA_TUP(GLOBAL_ACC_R), MA_TUP(LOCAL_ACC_R), MA_TUP(CONST_ACC_R), \ + MA_TUP(TEXTURE_ACC_R), MA_TUP(GLOBAL_ACC_W), MA_TUP(LOCAL_ACC_W), \ + MA_TUP(L1_WRBK_ACC), MA_TUP(L2_WRBK_ACC), MA_TUP(INST_ACC_R), \ + MA_TUP(L1_WR_ALLOC_R), MA_TUP(L2_WR_ALLOC_R), \ + MA_TUP(NUM_MEM_ACCESS_TYPE) MA_TUP_END(mem_access_type) #define MA_TUP_BEGIN(X) enum X { #define MA_TUP(X) X -#define MA_TUP_END(X) }; +#define MA_TUP_END(X) \ + } \ + ; MEM_ACCESS_TYPE_TUP_DEF #undef MA_TUP_BEGIN #undef MA_TUP #undef MA_TUP_END -const char * mem_access_type_str(enum mem_access_type access_type); +const char *mem_access_type_str(enum mem_access_type access_type); enum cache_operator_type { - CACHE_UNDEFINED, - - // loads - CACHE_ALL, // .ca - CACHE_LAST_USE, // .lu - CACHE_VOLATILE, // .cv - CACHE_L1, // .nc - - // loads and stores - CACHE_STREAMING, // .cs - CACHE_GLOBAL, // .cg - - // stores - CACHE_WRITE_BACK, // .wb - CACHE_WRITE_THROUGH // .wt + CACHE_UNDEFINED, + + // loads + CACHE_ALL, // .ca + CACHE_LAST_USE, // .lu + CACHE_VOLATILE, // .cv + CACHE_L1, // .nc + + // loads and stores + CACHE_STREAMING, // .cs + CACHE_GLOBAL, // .cg + + // stores + CACHE_WRITE_BACK, // .wb + CACHE_WRITE_THROUGH // .wt }; class mem_access_t { -public: - mem_access_t(gpgpu_context* ctx) { init(ctx); } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - gpgpu_context* ctx) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - const active_mask_t &active_mask, - const mem_access_byte_mask_t &byte_mask, - const mem_access_sector_mask_t §or_mask, - gpgpu_context* ctx) - : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - - new_addr_type get_addr() const { return m_addr; } - void set_addr(new_addr_type addr) {m_addr=addr;} - unsigned get_size() const { return m_req_size; } - const active_mask_t &get_warp_mask() const { return m_warp_mask; } - bool is_write() const { return m_write; } - enum mem_access_type get_type() const { return m_type; } - mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } - mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } - - void print(FILE *fp) const - { - fprintf(fp,"addr=0x%llx, %s, size=%u, ", m_addr, m_write?"store":"load ", m_req_size ); - switch(m_type) { - case GLOBAL_ACC_R: fprintf(fp,"GLOBAL_R"); break; - case LOCAL_ACC_R: fprintf(fp,"LOCAL_R "); break; - case CONST_ACC_R: fprintf(fp,"CONST "); break; - case TEXTURE_ACC_R: fprintf(fp,"TEXTURE "); break; - case GLOBAL_ACC_W: fprintf(fp,"GLOBAL_W"); break; - case LOCAL_ACC_W: fprintf(fp,"LOCAL_W "); break; - case L2_WRBK_ACC: fprintf(fp,"L2_WRBK "); break; - case INST_ACC_R: fprintf(fp,"INST "); break; - case L1_WRBK_ACC: fprintf(fp,"L1_WRBK "); break; - default: fprintf(fp,"unknown "); break; - } - } - - gpgpu_context* gpgpu_ctx; -private: - void init(gpgpu_context* ctx); - - unsigned m_uid; - new_addr_type m_addr; // request address - bool m_write; - unsigned m_req_size; // bytes - mem_access_type m_type; - active_mask_t m_warp_mask; - mem_access_byte_mask_t m_byte_mask; - mem_access_sector_mask_t m_sector_mask; + public: + mem_access_t(gpgpu_context *ctx) { init(ctx); } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, gpgpu_context *ctx) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, const active_mask_t &active_mask, + const mem_access_byte_mask_t &byte_mask, + const mem_access_sector_mask_t §or_mask, gpgpu_context *ctx) + : m_warp_mask(active_mask), + m_byte_mask(byte_mask), + m_sector_mask(sector_mask) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + + new_addr_type get_addr() const { return m_addr; } + void set_addr(new_addr_type addr) { m_addr = addr; } + unsigned get_size() const { return m_req_size; } + const active_mask_t &get_warp_mask() const { return m_warp_mask; } + bool is_write() const { return m_write; } + enum mem_access_type get_type() const { return m_type; } + mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } + mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } + + void print(FILE *fp) const { + fprintf(fp, "addr=0x%llx, %s, size=%u, ", m_addr, + m_write ? "store" : "load ", m_req_size); + switch (m_type) { + case GLOBAL_ACC_R: + fprintf(fp, "GLOBAL_R"); + break; + case LOCAL_ACC_R: + fprintf(fp, "LOCAL_R "); + break; + case CONST_ACC_R: + fprintf(fp, "CONST "); + break; + case TEXTURE_ACC_R: + fprintf(fp, "TEXTURE "); + break; + case GLOBAL_ACC_W: + fprintf(fp, "GLOBAL_W"); + break; + case LOCAL_ACC_W: + fprintf(fp, "LOCAL_W "); + break; + case L2_WRBK_ACC: + fprintf(fp, "L2_WRBK "); + break; + case INST_ACC_R: + fprintf(fp, "INST "); + break; + case L1_WRBK_ACC: + fprintf(fp, "L1_WRBK "); + break; + default: + fprintf(fp, "unknown "); + break; + } + } + + gpgpu_context *gpgpu_ctx; + + private: + void init(gpgpu_context *ctx); + + unsigned m_uid; + new_addr_type m_addr; // request address + bool m_write; + unsigned m_req_size; // bytes + mem_access_type m_type; + active_mask_t m_warp_mask; + mem_access_byte_mask_t m_byte_mask; + mem_access_sector_mask_t m_sector_mask; }; class mem_fetch; class mem_fetch_interface { -public: - virtual bool full( unsigned size, bool write ) const = 0; - virtual void push( mem_fetch *mf ) = 0; + public: + virtual bool full(unsigned size, bool write) const = 0; + virtual void push(mem_fetch *mf) = 0; }; class mem_fetch_allocator { -public: - virtual mem_fetch *alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const = 0; - virtual mem_fetch *alloc( const class warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle ) const = 0; + public: + virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type, + unsigned size, bool wr, + unsigned long long cycle) const = 0; + virtual mem_fetch *alloc(const class warp_inst_t &inst, + const mem_access_t &access, + unsigned long long cycle) const = 0; }; -// the maximum number of destination, source, or address uarch operands in a instruction -#define MAX_REG_OPERANDS 32 +// the maximum number of destination, source, or address uarch operands in a +// instruction +#define MAX_REG_OPERANDS 32 struct dram_callback_t { - dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } - void (*function)(const class inst_t*, class ptx_thread_info*); - - const class inst_t* instruction; - class ptx_thread_info *thread; + dram_callback_t() { + function = NULL; + instruction = NULL; + thread = NULL; + } + void (*function)(const class inst_t *, class ptx_thread_info *); + + const class inst_t *instruction; + class ptx_thread_info *thread; }; class inst_t { -public: - inst_t() - { - m_decoded=false; - pc=(address_type)-1; - reconvergence_pc=(address_type)-1; - op=NO_OP; - bar_type=NOT_BAR; - red_type=NOT_RED; - bar_id=(unsigned)-1; - bar_count=(unsigned)-1; - oprnd_type=UN_OP; - sp_op=OTHER_OP; - op_pipe=UNKOWN_OP; - mem_op=NOT_TEX; - num_operands=0; - num_regs=0; - memset(out, 0, sizeof(unsigned)); - memset(in, 0, sizeof(unsigned)); - is_vectorin=0; - is_vectorout=0; - space = memory_space_t(); - cache_op = CACHE_UNDEFINED; - latency = 1; - initiation_interval = 1; - for( unsigned i=0; i < MAX_REG_OPERANDS; i++ ) { - arch_reg.src[i] = -1; - arch_reg.dst[i] = -1; - } - isize=0; - } - bool valid() const { return m_decoded; } - virtual void print_insn( FILE *fp ) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); + public: + inst_t() { + m_decoded = false; + pc = (address_type)-1; + reconvergence_pc = (address_type)-1; + op = NO_OP; + bar_type = NOT_BAR; + red_type = NOT_RED; + bar_id = (unsigned)-1; + bar_count = (unsigned)-1; + oprnd_type = UN_OP; + sp_op = OTHER_OP; + op_pipe = UNKOWN_OP; + mem_op = NOT_TEX; + num_operands = 0; + num_regs = 0; + memset(out, 0, sizeof(unsigned)); + memset(in, 0, sizeof(unsigned)); + is_vectorin = 0; + is_vectorout = 0; + space = memory_space_t(); + cache_op = CACHE_UNDEFINED; + latency = 1; + initiation_interval = 1; + for (unsigned i = 0; i < MAX_REG_OPERANDS; i++) { + arch_reg.src[i] = -1; + arch_reg.dst[i] = -1; } - bool is_load() const { return (op == LOAD_OP ||op==TENSOR_CORE_LOAD_OP || memory_op == memory_load); } - bool is_store() const { return (op == STORE_OP ||op==TENSOR_CORE_STORE_OP || memory_op == memory_store); } - unsigned get_num_operands() const {return num_operands;} - unsigned get_num_regs() const {return num_regs;} - void set_num_regs(unsigned num) {num_regs=num;} - void set_num_operands(unsigned num) {num_operands=num;} - void set_bar_id(unsigned id) {bar_id=id;} - void set_bar_count(unsigned count) {bar_count=count;} - - address_type pc; // program counter address of instruction - unsigned isize; // size of instruction in bytes - op_type op; // opcode (uarch visible) - - barrier_type bar_type; - reduction_type red_type; - unsigned bar_id; - unsigned bar_count; - - types_of_operands oprnd_type; // code (uarch visible) identify if the operation is an interger or a floating point - special_ops sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... - operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of the operation (SP, SFU or MEM) - mem_operation mem_op; // code (uarch visible) identify memory type - _memory_op_t memory_op; // memory_op used by ptxplus - unsigned num_operands; - unsigned num_regs; // count vector operand as one register operand - - address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address - - unsigned out[8]; - unsigned outcount; - unsigned in[24]; - unsigned incount; - unsigned char is_vectorin; - unsigned char is_vectorout; - int pred; // predicate register number - int ar1, ar2; - // register number for bank conflict evaluation - struct { - int dst[MAX_REG_OPERANDS]; - int src[MAX_REG_OPERANDS]; - } arch_reg; - //int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation - unsigned latency; // operation latency - unsigned initiation_interval; - - unsigned data_size; // what is the size of the word being operated on? - memory_space_t space; - cache_operator_type cache_op; - -protected: - bool m_decoded; - virtual void pre_decode() {} + isize = 0; + } + bool valid() const { return m_decoded; } + virtual void print_insn(FILE *fp) const { + fprintf(fp, " [inst @ pc=0x%04x] ", pc); + } + bool is_load() const { + return (op == LOAD_OP || op == TENSOR_CORE_LOAD_OP || + memory_op == memory_load); + } + bool is_store() const { + return (op == STORE_OP || op == TENSOR_CORE_STORE_OP || + memory_op == memory_store); + } + unsigned get_num_operands() const { return num_operands; } + unsigned get_num_regs() const { return num_regs; } + void set_num_regs(unsigned num) { num_regs = num; } + void set_num_operands(unsigned num) { num_operands = num; } + void set_bar_id(unsigned id) { bar_id = id; } + void set_bar_count(unsigned count) { bar_count = count; } + + address_type pc; // program counter address of instruction + unsigned isize; // size of instruction in bytes + op_type op; // opcode (uarch visible) + + barrier_type bar_type; + reduction_type red_type; + unsigned bar_id; + unsigned bar_count; + + types_of_operands oprnd_type; // code (uarch visible) identify if the + // operation is an interger or a floating point + special_ops + sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... + operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of + // the operation (SP, SFU or MEM) + mem_operation mem_op; // code (uarch visible) identify memory type + _memory_op_t memory_op; // memory_op used by ptxplus + unsigned num_operands; + unsigned num_regs; // count vector operand as one register operand + + address_type reconvergence_pc; // -1 => not a branch, -2 => use function + // return address + + unsigned out[8]; + unsigned outcount; + unsigned in[24]; + unsigned incount; + unsigned char is_vectorin; + unsigned char is_vectorout; + int pred; // predicate register number + int ar1, ar2; + // register number for bank conflict evaluation + struct { + int dst[MAX_REG_OPERANDS]; + int src[MAX_REG_OPERANDS]; + } arch_reg; + // int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict + // evaluation + unsigned latency; // operation latency + unsigned initiation_interval; + + unsigned data_size; // what is the size of the word being operated on? + memory_space_t space; + cache_operator_type cache_op; + + protected: + bool m_decoded; + virtual void pre_decode() {} }; -enum divergence_support_t { - POST_DOMINATOR = 1, - NUM_SIMD_MODEL -}; +enum divergence_support_t { POST_DOMINATOR = 1, NUM_SIMD_MODEL }; const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; -class warp_inst_t: public inst_t { -public: - // constructors - warp_inst_t() - { - m_uid=0; - m_empty=true; - m_config=NULL; - } - warp_inst_t( const core_config *config ) - { - m_uid=0; - assert(config->warp_size<=MAX_WARP_SIZE); - m_config=config; - m_empty=true; - m_isatomic=false; - m_per_scalar_thread_valid=false; - m_mem_accesses_created=false; - m_cache_hit=false; - m_is_printf=false; - m_is_cdp = 0; +class warp_inst_t : public inst_t { + public: + // constructors + warp_inst_t() { + m_uid = 0; + m_empty = true; + m_config = NULL; + } + warp_inst_t(const core_config *config) { + m_uid = 0; + assert(config->warp_size <= MAX_WARP_SIZE); + m_config = config; + m_empty = true; + m_isatomic = false; + m_per_scalar_thread_valid = false; + m_mem_accesses_created = false; + m_cache_hit = false; + m_is_printf = false; + m_is_cdp = 0; + } + virtual ~warp_inst_t() {} + + // modifiers + void broadcast_barrier_reduction(const active_mask_t &access_mask); + void do_atomic(bool forceDo = false); + void do_atomic(const active_mask_t &access_mask, bool forceDo = false); + void clear() { m_empty = true; } + + void issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, int sch_id); + + const active_mask_t &get_active_mask() const { return m_warp_active_mask; } + void completed(unsigned long long cycle) + const; // stat collection: called when the instruction is completed + + void set_addr(unsigned n, new_addr_type addr) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - virtual ~warp_inst_t(){ + m_per_scalar_thread[n].memreqaddr[0] = addr; + } + void set_addr(unsigned n, new_addr_type *addr, unsigned num_addrs) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - - // modifiers - void broadcast_barrier_reduction( const active_mask_t& access_mask); - void do_atomic(bool forceDo=false); - void do_atomic( const active_mask_t& access_mask, bool forceDo=false ); - void clear() - { - m_empty=true; + assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); + for (unsigned i = 0; i < num_addrs; i++) + m_per_scalar_thread[n].memreqaddr[i] = addr[i]; + } + void print_m_accessq() { + if (accessq_empty()) + return; + else { + printf("Printing mem access generated\n"); + std::list::iterator it; + for (it = m_accessq.begin(); it != m_accessq.end(); ++it) { + printf("MEM_TXN_GEN:%s:%llx, Size:%d \n", + mem_access_type_str(it->get_type()), it->get_addr(), + it->get_size()); + } } - - void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ); - - const active_mask_t & get_active_mask() const - { - return m_warp_active_mask; + } + struct transaction_info { + std::bitset<4> chunks; // bitmask: 32-byte chunks accessed + mem_access_byte_mask_t bytes; + active_mask_t active; // threads in this transaction + + bool test_bytes(unsigned start_bit, unsigned end_bit) { + for (unsigned i = start_bit; i <= end_bit; i++) + if (bytes.test(i)) return true; + return false; } - void completed( unsigned long long cycle ) const; // stat collection: called when the instruction is completed - - void set_addr( unsigned n, new_addr_type addr ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - m_per_scalar_thread[n].memreqaddr[0] = addr; + }; + + void generate_mem_accesses(); + void memory_coalescing_arch(bool is_write, mem_access_type access_type); + void memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type); + void memory_coalescing_arch_reduce_and_send(bool is_write, + mem_access_type access_type, + const transaction_info &info, + new_addr_type addr, + unsigned segment_size); + + void add_callback(unsigned lane_id, + void (*function)(const class inst_t *, + class ptx_thread_info *), + const inst_t *inst, class ptx_thread_info *thread, + bool atomic) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; + if (atomic) m_isatomic = true; } - void set_addr( unsigned n, new_addr_type* addr, unsigned num_addrs ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); - for(unsigned i=0; iwarp_size - 1; i >= 0; i--) + fprintf(fp, "%c", ((m_warp_active_mask[i]) ? '1' : '0')); + } + bool active(unsigned thread) const { return m_warp_active_mask.test(thread); } + unsigned active_count() const { return m_warp_active_mask.count(); } + unsigned issued_count() const { + assert(m_empty == false); + return m_warp_issued_mask.count(); + } // for instruction counting + bool empty() const { return m_empty; } + unsigned warp_id() const { + assert(!m_empty); + return m_warp_id; + } + unsigned warp_id_func() const // to be used in functional simulations only + { + return m_warp_id; + } + unsigned dynamic_warp_id() const { + assert(!m_empty); + return m_dynamic_warp_id; + } + bool has_callback(unsigned n) const { + return m_warp_active_mask[n] && m_per_scalar_thread_valid && + (m_per_scalar_thread[n].callback.function != NULL); + } + new_addr_type get_addr(unsigned n) const { + assert(m_per_scalar_thread_valid); + return m_per_scalar_thread[n].memreqaddr[0]; + } + + bool isatomic() const { return m_isatomic; } + + unsigned warp_size() const { return m_config->warp_size; } + + bool accessq_empty() const { return m_accessq.empty(); } + unsigned accessq_count() const { return m_accessq.size(); } + const mem_access_t &accessq_back() { return m_accessq.back(); } + void accessq_pop_back() { m_accessq.pop_back(); } + + bool dispatch_delay() { + if (cycles > 0) cycles--; + return cycles > 0; + } + + bool has_dispatch_delay() { return cycles > 0; } + + void print(FILE *fout) const; + unsigned get_uid() const { return m_uid; } + unsigned get_schd_id() const { return m_scheduler_id; } + + protected: + unsigned m_uid; + bool m_empty; + bool m_cache_hit; + unsigned long long issue_cycle; + unsigned cycles; // used for implementing initiation interval delay + bool m_isatomic; + bool m_is_printf; + unsigned m_warp_id; + unsigned m_dynamic_warp_id; + const core_config *m_config; + active_mask_t m_warp_active_mask; // dynamic active mask for timing model + // (after predication) + active_mask_t + m_warp_issued_mask; // active mask at issue (prior to predication test) + // -- for instruction counting + + struct per_thread_info { + per_thread_info() { + for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) + memreqaddr[i] = 0; } - void print_m_accessq(){ - - if(accessq_empty()) - return; - else{ - printf("Printing mem access generated\n"); - std::list::iterator it; - for (it = m_accessq.begin(); it != m_accessq.end(); ++it){ - printf("MEM_TXN_GEN:%s:%llx, Size:%d \n",mem_access_type_str(it->get_type()), it->get_addr(),it->get_size()); - } - } - } - struct transaction_info { - std::bitset<4> chunks; // bitmask: 32-byte chunks accessed - mem_access_byte_mask_t bytes; - active_mask_t active; // threads in this transaction - - bool test_bytes(unsigned start_bit, unsigned end_bit) { - for( unsigned i=start_bit; i<=end_bit; i++ ) - if(bytes.test(i)) - return true; - return false; - } - }; - - void generate_mem_accesses(); - void memory_coalescing_arch( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); - - void add_callback( unsigned lane_id, - void (*function)(const class inst_t*, class ptx_thread_info*), - const inst_t *inst, - class ptx_thread_info *thread, - bool atomic) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - if(atomic) m_isatomic=true; - } - m_per_scalar_thread[lane_id].callback.function = function; - m_per_scalar_thread[lane_id].callback.instruction = inst; - m_per_scalar_thread[lane_id].callback.thread = thread; - } - void set_active( const active_mask_t &active ); + dram_callback_t callback; + new_addr_type + memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; // effective address, + // upto 8 different + // requests (to support + // 32B access in 8 chunks + // of 4B each) + }; + bool m_per_scalar_thread_valid; + std::vector m_per_scalar_thread; + bool m_mem_accesses_created; + std::list m_accessq; + + unsigned m_scheduler_id; // the scheduler that issues this inst + + // Jin: cdp support + public: + int m_is_cdp; +}; + +void move_warp(warp_inst_t *&dst, warp_inst_t *&src); - void clear_active( const active_mask_t &inactive ); - void set_not_active( unsigned lane_id ); +size_t get_kernel_code_size(class function_info *entry); +class checkpoint { + public: + checkpoint(); + ~checkpoint() { printf("clasfsfss destructed\n"); } - // accessors - virtual void print_insn(FILE *fp) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); - for (int i=(int)m_config->warp_size-1; i>=0; i--) - fprintf(fp, "%c", ((m_warp_active_mask[i])?'1':'0') ); + void load_global_mem(class memory_space *temp_mem, char *f1name); + void store_global_mem(class memory_space *mem, char *fname, char *format); + unsigned radnom; +}; +/* + * This abstract class used as a base for functional and performance and + * simulation, it has basic functional simulation data structures and + * procedures. + */ +class core_t { + public: + core_t(gpgpu_sim *gpu, kernel_info_t *kernel, unsigned warp_size, + unsigned threads_per_shader) + : m_gpu(gpu), + m_kernel(kernel), + m_simt_stack(NULL), + m_thread(NULL), + m_warp_size(warp_size) { + m_warp_count = threads_per_shader / m_warp_size; + // Handle the case where the number of threads is not a + // multiple of the warp size + if (threads_per_shader % m_warp_size != 0) { + m_warp_count += 1; } - bool active( unsigned thread ) const { return m_warp_active_mask.test(thread); } - unsigned active_count() const { return m_warp_active_mask.count(); } - unsigned issued_count() const { assert(m_empty == false); return m_warp_issued_mask.count(); } // for instruction counting - bool empty() const { return m_empty; } - unsigned warp_id() const - { - assert( !m_empty ); - return m_warp_id; + assert(m_warp_count * m_warp_size > 0); + m_thread = (ptx_thread_info **)calloc(m_warp_count * m_warp_size, + sizeof(ptx_thread_info *)); + initilizeSIMTStack(m_warp_count, m_warp_size); + + for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) { + for (unsigned j = 0; j < MAX_BARRIERS_PER_CTA; j++) { + reduction_storage[i][j] = 0; + } } - unsigned warp_id_func() const // to be used in functional simulations only - { - return m_warp_id; + } + virtual ~core_t() { free(m_thread); } + virtual void warp_exit(unsigned warp_id) = 0; + virtual bool warp_waiting_at_barrier(unsigned warp_id) const = 0; + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) = 0; + class gpgpu_sim *get_gpu() { + return m_gpu; + } + void execute_warp_inst_t(warp_inst_t &inst, unsigned warpId = (unsigned)-1); + bool ptx_thread_done(unsigned hw_thread_id) const; + void updateSIMTStack(unsigned warpId, warp_inst_t *inst); + void initilizeSIMTStack(unsigned warp_count, unsigned warps_size); + void deleteSIMTStack(); + warp_inst_t getExecuteWarp(unsigned warpId); + void get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const; + kernel_info_t *get_kernel_info() { return m_kernel; } + class ptx_thread_info **get_thread_info() { + return m_thread; + } + unsigned get_warp_size() const { return m_warp_size; } + void and_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] &= value; + } + void or_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] |= value; + } + void popc_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] += value; + } + unsigned get_reduction_value(unsigned ctaid, unsigned barid) { + return reduction_storage[ctaid][barid]; + } + + protected: + class gpgpu_sim *m_gpu; + kernel_info_t *m_kernel; + simt_stack **m_simt_stack; // pdom based reconvergence context for each warp + class ptx_thread_info **m_thread; + unsigned m_warp_size; + unsigned m_warp_count; + unsigned reduction_storage[MAX_CTA_PER_SHADER][MAX_BARRIERS_PER_CTA]; +}; + +// register that can hold multiple instructions. +class register_set { + public: + register_set(unsigned num, const char *name) { + for (unsigned i = 0; i < num; i++) { + regs.push_back(new warp_inst_t()); } - unsigned dynamic_warp_id() const - { - assert( !m_empty ); - return m_dynamic_warp_id; + m_name = name; + } + bool has_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return true; + } } - bool has_callback( unsigned n ) const - { - return m_warp_active_mask[n] && m_per_scalar_thread_valid && - (m_per_scalar_thread[n].callback.function!=NULL); + return false; + } + bool has_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return has_free(); + + assert(reg_id < regs.size()); + return regs[reg_id]->empty(); + } + bool has_ready() { + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + return true; + } } - new_addr_type get_addr( unsigned n ) const - { - assert( m_per_scalar_thread_valid ); - return m_per_scalar_thread[n].memreqaddr[0]; + return false; + } + + void move_in(warp_inst_t *&src) { + warp_inst_t **free = get_free(); + move_warp(*free, src); + } + // void copy_in( warp_inst_t* src ){ + // src->copy_contents_to(*get_free()); + //} + void move_out_to(warp_inst_t *&dest) { + warp_inst_t **ready = get_ready(); + move_warp(dest, *ready); + } + + warp_inst_t **get_ready() { + warp_inst_t **ready; + ready = NULL; + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + if (ready and (*ready)->get_uid() < regs[i]->get_uid()) { + // ready is oldest + } else { + ready = ®s[i]; + } + } } - - bool isatomic() const { return m_isatomic; } - - unsigned warp_size() const { return m_config->warp_size; } - - bool accessq_empty() const { return m_accessq.empty(); } - unsigned accessq_count() const { return m_accessq.size(); } - const mem_access_t &accessq_back() { return m_accessq.back(); } - void accessq_pop_back() { m_accessq.pop_back(); } - - bool dispatch_delay() - { - if( cycles > 0 ) - cycles--; - return cycles > 0; + return ready; + } + + void print(FILE *fp) const { + fprintf(fp, "%s : @%p\n", m_name, this); + for (unsigned i = 0; i < regs.size(); i++) { + fprintf(fp, " "); + regs[i]->print(fp); + fprintf(fp, "\n"); } + } - bool has_dispatch_delay(){ - return cycles > 0; + warp_inst_t **get_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return ®s[i]; + } } - - void print( FILE *fout ) const; - unsigned get_uid() const { return m_uid; } - unsigned get_schd_id() const { return m_scheduler_id; } - -protected: - - unsigned m_uid; - bool m_empty; - bool m_cache_hit; - unsigned long long issue_cycle; - unsigned cycles; // used for implementing initiation interval delay - bool m_isatomic; - bool m_is_printf; - unsigned m_warp_id; - unsigned m_dynamic_warp_id; - const core_config *m_config; - active_mask_t m_warp_active_mask; // dynamic active mask for timing model (after predication) - active_mask_t m_warp_issued_mask; // active mask at issue (prior to predication test) -- for instruction counting - - struct per_thread_info { - per_thread_info() { - for(unsigned i=0; i m_per_scalar_thread; - bool m_mem_accesses_created; - std::list m_accessq; - - unsigned m_scheduler_id; //the scheduler that issues this inst - - //Jin: cdp support -public: - int m_is_cdp; - -}; - -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ); - -size_t get_kernel_code_size( class function_info *entry ); -class checkpoint -{ -public: - - checkpoint(); - ~checkpoint(){ - printf("clasfsfss destructed\n"); + assert(0 && "No free registers found"); + return NULL; + } + + warp_inst_t **get_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return get_free(); + + assert(reg_id < regs.size()); + if (regs[reg_id]->empty()) { + return ®s[reg_id]; } + assert(0 && "No free register found"); + return NULL; + } - void load_global_mem(class memory_space *temp_mem, char * f1name); - void store_global_mem(class memory_space *mem, char * fname , char * format); - unsigned radnom; + unsigned get_size() { return regs.size(); } - -}; -/* - * This abstract class used as a base for functional and performance and simulation, it has basic functional simulation - * data structures and procedures. - */ -class core_t { - public: - core_t( gpgpu_sim *gpu, - kernel_info_t *kernel, - unsigned warp_size, - unsigned threads_per_shader ) - : m_gpu( gpu ), - m_kernel( kernel ), - m_simt_stack( NULL ), - m_thread( NULL ), - m_warp_size( warp_size ) - { - m_warp_count = threads_per_shader/m_warp_size; - // Handle the case where the number of threads is not a - // multiple of the warp size - if ( threads_per_shader % m_warp_size != 0 ) { - m_warp_count += 1; - } - assert( m_warp_count * m_warp_size > 0 ); - m_thread = ( ptx_thread_info** ) - calloc( m_warp_count * m_warp_size, - sizeof( ptx_thread_info* ) ); - initilizeSIMTStack(m_warp_count,m_warp_size); - - for(unsigned i=0; iempty() ) { - return true; - } - } - return false; - } - bool has_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return has_free(); - - assert(reg_id < regs.size()); - return regs[reg_id]->empty(); - } - bool has_ready(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - return true; - } - } - return false; - } - - void move_in( warp_inst_t *&src ){ - warp_inst_t** free = get_free(); - move_warp(*free, src); - } - //void copy_in( warp_inst_t* src ){ - // src->copy_contents_to(*get_free()); - //} - void move_out_to( warp_inst_t *&dest ){ - warp_inst_t **ready=get_ready(); - move_warp(dest, *ready); - } - - warp_inst_t** get_ready(){ - warp_inst_t** ready; - ready = NULL; - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - if( ready and (*ready)->get_uid() < regs[i]->get_uid() ) { - // ready is oldest - } else { - ready = ®s[i]; - } - } - } - return ready; - } - - void print(FILE* fp) const{ - fprintf(fp, "%s : @%p\n", m_name, this); - for( unsigned i = 0; i < regs.size(); i++ ) { - fprintf(fp, " "); - regs[i]->print(fp); - fprintf(fp, "\n"); - } - } - - warp_inst_t ** get_free(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( regs[i]->empty() ) { - return ®s[i]; - } - } - assert(0 && "No free registers found"); - return NULL; - } - - warp_inst_t ** get_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return get_free(); - - assert(reg_id < regs.size()); - if( regs[reg_id]->empty() ) { - return ®s[reg_id]; - } - assert(0 && "No free register found"); - return NULL; - } - - unsigned get_size(){ - return regs.size(); - } - -private: - std::vector regs; - const char* m_name; + private: + std::vector regs; + const char *m_name; }; -#endif // #ifdef __cplusplus +#endif // #ifdef __cplusplus -#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED +#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 9a5468c..67539d4 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,61 +10,60 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ - #ifndef CUDA_MATH #define CUDA_MATH @@ -74,32 +73,31 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN - struct int4 { - int x, y, z, w; - }; - struct uint4 { - unsigned int x, y, z, w; - }; - struct float4 { - float x, y, z, w; - }; - struct float2 { - float x, y; - }; - +struct int4 { + int x, y, z, w; +}; +struct uint4 { + unsigned int x, y, z, w; +}; +struct float4 { + float x, y, z, w; +}; +struct float2 { + float x, y; +}; // DEVICE_BUILTIN - typedef struct int4 int4; - typedef struct uint4 uint4; - typedef struct float4 float4; - typedef struct float2 float2; +typedef struct int4 int4; +typedef struct uint4 uint4; +typedef struct float4 float4; +typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include @@ -108,38 +106,36 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -147,205 +143,211 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include #include - // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; -} - -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -int __internal_float2int(float a, enum cudaRoundMode mode) -{ - return float2int(a, mode); -} - -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - unsigned int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) -{ - return float2uint(a, mode); -} - -// intrinsic for division -float fdividef(float a, float b) -{ - return (a / b); -} - -float __internal_accurate_fdividef(float a, float b) -{ - return fdividef(a, b); -} + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; +} + +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +int __internal_float2int(float a, enum cudaRoundMode mode) { + return float2int(a, mode); +} + +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + unsigned int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { + return float2uint(a, mode); +} + +// intrinsic for division +float fdividef(float a, float b) { return (a / b); } + +float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) -{ - float b; - if (std::isnan(a)) b = 0.0f; - else if (a >= 1.0f) b = 1.0f; - else if (a <= 0.0f) b = 0.0f; - else b = a; - return b; -} - -// intrinsic for power -float __powf(float a, float b) -{ - return powf(a, b); -} +float __saturatef(float a) { + float b; + if (std::isnan(a)) + b = 0.0f; + else if (a >= 1.0f) + b = 1.0f; + else if (a <= 0.0f) + b = 0.0f; + else + b = a; + return b; +} + +// intrinsic for power +float __powf(float a, float b) { return powf(a, b); } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) -{ - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) { + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -355,14 +357,11 @@ int __signbitd(double d) #endif -} +} // namespace cuda_math // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) -{ - return (std::isnan(a)); -} -#endif +int isnanf(float a) { return (std::isnan(a)); } +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 7a130ea..39e2b7e 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,283 +8,340 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda-sim.h" #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "ptx_sim.h" +typedef void *yyscan_t; #include -#include -#include "opcodes.h" -#include "../statwrapper.h" -#include #include +#include +#include +#include "../../libcuda/gpgpu_context.h" #include "../abstract_hardware_model.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "../gpgpusim_entrypoint.h" +#include "../statwrapper.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "decuda_pred_table/decuda_pred_table.h" #include "memory.h" +#include "opcodes.h" #include "ptx-stats.h" +#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" -#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "../gpgpusim_entrypoint.h" -#include "decuda_pred_table/decuda_pred_table.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" int g_debug_execution = 0; // Output debug information to file options - -void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, - "Opcode latencies for integers " - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, - "Opcode latencies for single precision floating points " - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, - "Opcode latencies for double precision floating points " - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers " - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, - "Opcode initiation intervals for single precision floating points " - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, - "Opcode initiation intervals for double precision floating points " - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency " + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, + &opcode_latency_fp, + "Opcode latencies for single precision floating " + "points " + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, + &opcode_latency_dp, + "Opcode latencies for double precision floating " + "points " + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, + &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, + &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register( + opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers " + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, + &opcode_initiation_fp, + "Opcode initiation intervals for single precision " + "floating points " + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, + &opcode_initiation_dp, + "Opcode initiation intervals for double precision " + "floating points " + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, + &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, + &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency " - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); -} - -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) -{ - std::string texname(name); - if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ - m_NameToTextureRef[texname] = std::set(); - }else{ - const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); - assert(tr!=NULL); - //asserts that all texrefs in set have same fields - assert(tr->normalized==texref->normalized&& - tr->filterMode==texref->filterMode&& - tr->addressMode[0]==texref->addressMode[0]&& - tr->addressMode[1]==texref->addressMode[1]&& - tr->addressMode[2]==texref->addressMode[2]&& - tr->channelDesc.x==texref->channelDesc.x&& - tr->channelDesc.y==texref->channelDesc.y&& - tr->channelDesc.z==texref->channelDesc.z&& - tr->channelDesc.w==texref->channelDesc.w&& - tr->channelDesc.f==texref->channelDesc.f - ); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; -} - -const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) -{ - std::map::const_iterator t=m_TextureRefToName.find(texref); - assert( t != m_TextureRefToName.end() ); - return t->second.c_str(); -} - -unsigned int intLOGB2( unsigned int v ) { - unsigned int shift; - unsigned int r; - - r = 0; - - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; - - return r; -} - -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) -{ - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - - std::map::const_iterator t=m_NameToCudaArray.find(texname); - //check that there's nothing there first - if(t != m_NameToCudaArray.end()){ - printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits/8; - unsigned int Tx, Ty; - int r; - - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); - //first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: Tx = 4; break; - case 32: Tx = 8; break; - case 64: Tx = 8; break; - case 128: Tx = 16; break; - case 256: Tx = 16; break; - default: - printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); +} + +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( + const char *name, const struct textureReference *texref, int dim, + int readmode, int ext) { + std::string texname(name); + if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { + m_NameToTextureRef[texname] = std::set(); + } else { + const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); + assert(tr != NULL); + // asserts that all texrefs in set have same fields + assert(tr->normalized == texref->normalized && + tr->filterMode == texref->filterMode && + tr->addressMode[0] == texref->addressMode[0] && + tr->addressMode[1] == texref->addressMode[1] && + tr->addressMode[2] == texref->addressMode[2] && + tr->channelDesc.x == texref->channelDesc.x && + tr->channelDesc.y == texref->channelDesc.y && + tr->channelDesc.z == texref->channelDesc.z && + tr->channelDesc.w == texref->channelDesc.w && + tr->channelDesc.f == texref->channelDesc.f); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr( + texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; +} + +const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref) { + std::map::const_iterator t = + m_TextureRefToName.find(texref); + assert(t != m_TextureRefToName.end()); + return t->second.c_str(); +} + +unsigned int intLOGB2(unsigned int v) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; + + return r; +} + +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( + const struct textureReference *texref, const struct cudaArray *array) { + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + // check that there's nothing there first + if (t != m_NameToCudaArray.end()) { + printf( + "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " + "was previously bound.\nImplicitly unbinding texref associated to %s " + "first\n", + texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = + array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits / 8; + unsigned int Tx, Ty; + int r; + + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", + m_function_model_config.get_texcache_linesize()); + // first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: + Tx = 4; + break; + case 32: + Tx = 8; + break; + case 64: + Tx = 8; + break; + case 128: + Tx = 16; + break; + case 256: + Tx = 16; + break; + default: + printf( + "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", + m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - //modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - //by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); - - printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); - printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; -} - -void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) -{ - //assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); + } + r = texel_size >> 2; + // modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + // by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); + + printf( + "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", + Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", + texel_size, intLOGB2(texel_size)); + printf( + "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", + array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo *texInfo = + (struct textureInfo *)malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; +} + +void gpgpu_t::gpgpu_ptx_sim_unbindTexture( + const struct textureReference *texref) { + // assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() -{ - if( m_assembled ) { - return; - } - - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); - m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; - - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); - fflush(stdout); - std::list::iterator i; - - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) - // start function on an aligned address - for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - PC += PC%MAX_INST_SIZE; - m_start_PC = PC; - - addr_t n=0; // offset in m_instr_mem - //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. - //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); - for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if ( pI->is_label() ) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert( pI->inst_size() <= MAX_INST_SIZE ); - for( unsigned i=1; i < pI->inst_size(); i++ ) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - m_instr_mem[n+i]=NULL; - } - n += pI->inst_size(); - PC += pI->inst_size(); +void function_info::ptx_assemble() { + if (m_assembled) { + return; + } + + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); + m_instr_mem = new ptx_instruction *[m_instr_mem_size]; + + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", + m_name.c_str()); + fflush(stdout); + std::list::iterator i; + + addr_t PC = + gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address + // (across functions) + // start function on an aligned address + for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + PC += PC % MAX_INST_SIZE; + m_start_PC = PC; + + addr_t n = 0; // offset in m_instr_mem + // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? + // reserve is cumulative. s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + + // MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); + for (i = m_instructions.begin(); i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert(pI->inst_size() <= MAX_INST_SIZE); + for (unsigned i = 1; i < pI->inst_size(); i++) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + m_instr_mem[n + i] = NULL; } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; - for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); //get operand, e.g. target name - if ( labels.find(target.name()) == labels.end() ) { - printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", - pI->source_file(),pI->source_line(), target.name().c_str() ); - abort(); - } - unsigned index = labels[ target.name() ]; //determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address( target.get_symbol(), PC ); - target.set_type(label_t); + n += pI->inst_size(); + PC += pI->inst_size(); + } + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; + for (unsigned ii = 0; ii < n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || + pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); // get operand, e.g. target name + if (labels.find(target.name()) == labels.end()) { + printf( + "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " + "appear in assembly code.", + pI->source_file(), pI->source_line(), target.name().c_str()); + abort(); } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + unsigned index = labels[target.name()]; // determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address(target.get_symbol(), PC); + target.set_type(label_t); + } + } + m_n = n; + printf(" done.\n"); + fflush(stdout); - //disable pdom analysis here and do it at runtime + // disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -323,2247 +380,2445 @@ void function_info::ptx_assemble() #endif } -addr_t shared_to_generic( unsigned smid, addr_t addr ) -{ - assert( addr < SHARED_MEM_SIZE_MAX ); - return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; -} - -addr_t global_to_generic( addr_t addr ) -{ - return addr; -} - -bool isspace_shared( unsigned smid, addr_t addr ) -{ - addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -bool isspace_global( addr_t addr ) -{ - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); -} - -memory_space_t whichspace( addr_t addr ) -{ - if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { - return global_space; - } else if( addr >= SHARED_GENERIC_START ) { - return shared_space; - } else { - return local_space; - } -} - -addr_t generic_to_shared( unsigned smid, addr_t addr ) -{ - assert(isspace_shared(smid,addr)); - return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); -} - -addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; -} - -bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(isspace_local(smid,hwtid,addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); -} - -addr_t generic_to_global( addr_t addr ) -{ - return addr; -} - - -void* gpgpu_t::gpu_malloc( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - -void* gpgpu_t::gpu_mallocarray( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - - -void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - char *src_data = (char*)src; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char*)dst; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->read(src_start_addr+n,1,dst_data+n); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, - (unsigned long long) src, (unsigned long long) dst ); - fflush(stdout); - } - for (unsigned n=0; n < count; n ++ ) { - unsigned char tmp; - m_global_mem->read(src+n,1,&tmp); - m_global_mem->write(dst+n,1, &tmp,NULL,NULL); - } - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", - count, (unsigned char) c, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} +addr_t shared_to_generic(unsigned smid, addr_t addr) { + assert(addr < SHARED_MEM_SIZE_MAX); + return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; +} + +addr_t global_to_generic(addr_t addr) { return addr; } -void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - fprintf(fp,"", pc ); - return; - } - function_info *finfo = f->second; - assert( finfo ); - finfo->print_insn(pc,fp); -} - -std::string cuda_sim::ptx_get_insn_str( address_type pc ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - #define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE,"", pc ); - return std::string(buff); - } - function_info *finfo = f->second; - assert( finfo ); - return finfo->get_insn_str(pc); -} - -void ptx_instruction::set_fp_or_int_archop(){ - oprnd_type=UN_OP; - if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ - // do nothing - }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ - if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - - }else{ - if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - } -} -void ptx_instruction::set_mul_div_or_other_archop(){ - sp_op=OTHER_OP; - if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ - if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - switch(get_opcode()){ - case MUL_OP: - case MAD_OP: - sp_op=FP_MUL_OP; - break; - case DIV_OP: - sp_op=FP_DIV_OP; - break; - case LG2_OP: - sp_op=FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op=FP_SQRT_OP; - break; - case RCP_OP: - sp_op=FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op=FP_SIN_OP; - break; - case EX2_OP: - sp_op=FP_EXP_OP; - break; - default: - if((op==ALU_OP)||(op==TENSOR_CORE_OP)) - sp_op=FP__OP; - break; - - } - }else { - switch(get_opcode()){ - case MUL24_OP: - case MAD24_OP: - sp_op=INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) - sp_op=INT_MUL32_OP; - else - sp_op=INT_MUL_OP; - break; - case DIV_OP: - sp_op=INT_DIV_OP; - break; - default: - if((op==ALU_OP)) - sp_op=INT__OP; - break; - } - } - } - -} - - - -void ptx_instruction::set_bar_type() -{ - if(m_opcode==BAR_OP) { - switch(m_barrier_op){ - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch(m_atomic_spec){ - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } - else if(m_opcode==SST_OP) { - bar_type = SYNC; - } -} - - -void ptx_instruction::set_opcode_and_latency() -{ - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0],&int_latency[1],&int_latency[2], - &int_latency[3],&int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0],&fp_latency[1],&fp_latency[2], - &fp_latency[3],&fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0],&dp_latency[1],&dp_latency[2], - &dp_latency[3],&dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", - &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", - &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0],&int_init[1],&int_init[2], - &int_init[3],&int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0],&fp_init[1],&fp_init[2], - &fp_init[3],&fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0],&dp_init[1],&dp_init[2], - &dp_init[3],&dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", - &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", - &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - - if(!m_operands.empty()){ - std::vector::iterator it; - for(it=++m_operands.begin();it!=m_operands.end();it++){ - num_operands++; - if((it->is_reg() || it->is_vector())){ - num_regs++; - } - } - } - op = ALU_OP; - mem_op= NOT_TEX; - initiation_interval = latency = 1; - switch( m_opcode ) { - case MOV_OP: - assert( !(has_memory_read() && has_memory_write()) ); - if ( has_memory_read() ) op = LOAD_OP; - if ( has_memory_write() ) op = STORE_OP; - break; - case LD_OP: op = LOAD_OP; break; - case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; - case LDU_OP: op = LOAD_OP; break; - case ST_OP: op = STORE_OP; break; - case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; - case BRA_OP: op = BRANCH_OP; break; - case BREAKADDR_OP: op = BRANCH_OP; break; - case TEX_OP: op = LOAD_OP; mem_op=TEX; break; - case ATOM_OP: op = LOAD_OP; break; - case BAR_OP: op = BARRIER_OP; break; - case SST_OP: op = BARRIER_OP; break; - case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; - case CALL_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case CALLP_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case RET_OP: case RETP_OP: op = RET_OPS;break; - case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: - //ADD,SUB latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: case MIN_OP: - //MAX,MIN latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - //MUL latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: case MADC_OP: case MADP_OP: - //MAD latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; +bool isspace_shared(unsigned smid, addr_t addr) { + addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; + if ((addr >= end) || (addr < start)) return false; + return true; +} + +bool isspace_global(addr_t addr) { + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +} + +memory_space_t whichspace(addr_t addr) { + if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { + return global_space; + } else if (addr >= SHARED_GENERIC_START) { + return shared_space; + } else { + return local_space; + } +} + +addr_t generic_to_shared(unsigned smid, addr_t addr) { + assert(isspace_shared(smid, addr)); + return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); +} + +addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +} + +bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); + if ((addr >= end) || (addr < start)) return false; + return true; +} + +addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { + assert(isspace_local(smid, hwtid, addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid)); +} + +addr_t generic_to_global(addr_t addr) { return addr; } + +void *gpgpu_t::gpu_malloc(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void *gpgpu_t::gpu_mallocarray(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, + size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", + count, (unsigned long long)src, (unsigned long long)dst_start_addr); + fflush(stdout); + } + char *src_data = (char *)src; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", + count, (unsigned long long)src_start_addr, (unsigned long long)dst); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char *)dst; + for (unsigned n = 0; n < count; n++) + m_global_mem->read(src_start_addr + n, 1, dst_data + n); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", + count, (unsigned long long)src, (unsigned long long)dst); + fflush(stdout); + } + for (unsigned n = 0; n < count; n++) { + unsigned char tmp; + m_global_mem->read(src + n, 1, &tmp); + m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); + } + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " + "0x%Lx... ", + count, (unsigned char)c, (unsigned long long)dst_start_addr); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { + fprintf(fp, "", pc); + return; + } + function_info *finfo = f->second; + assert(finfo); + finfo->print_insn(pc, fp); +} + +std::string cuda_sim::ptx_get_insn_str(address_type pc) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { +#define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); + return std::string(buff); + } + function_info *finfo = f->second; + assert(finfo); + return finfo->get_insn_str(pc); +} + +void ptx_instruction::set_fp_or_int_archop() { + oprnd_type = UN_OP; + if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || + (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || + (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || + (m_opcode == CALL_OP)) { + // do nothing + } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || + m_opcode == SLCT_OP)) { + if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || + get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + + } else { + if (get_type() == F16_TYPE || get_type() == F32_TYPE || + get_type() == F64_TYPE || get_type() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + } +} +void ptx_instruction::set_mul_div_or_other_archop() { + sp_op = OTHER_OP; + if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && + (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && + (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && + (m_opcode != CALL_OP)) { + if (get_type() == F32_TYPE || get_type() == F64_TYPE || + get_type() == FF64_TYPE) { + switch (get_opcode()) { + case MUL_OP: + case MAD_OP: + sp_op = FP_MUL_OP; + break; + case DIV_OP: + sp_op = FP_DIV_OP; + break; + case LG2_OP: + sp_op = FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op = FP_SQRT_OP; + break; + case RCP_OP: + sp_op = FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op = FP_SIN_OP; + break; + case EX2_OP: + sp_op = FP_EXP_OP; + break; + default: + if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; + break; + } + } else { + switch (get_opcode()) { + case MUL24_OP: + case MAD24_OP: + sp_op = INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if (get_type() == U32_TYPE || get_type() == S32_TYPE || + get_type() == B32_TYPE) + sp_op = INT_MUL32_OP; + else + sp_op = INT_MUL_OP; + break; + case DIV_OP: + sp_op = INT_DIV_OP; + break; + default: + if ((op == ALU_OP)) sp_op = INT__OP; + break; + } + } + } +} + +void ptx_instruction::set_bar_type() { + if (m_opcode == BAR_OP) { + switch (m_barrier_op) { + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch (m_atomic_spec) { + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } else if (m_opcode == SST_OP) { + bar_type = SYNC; + } +} + +void ptx_instruction::set_opcode_and_latency() { + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], + &int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], + &fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], + &dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); + + if (!m_operands.empty()) { + std::vector::iterator it; + for (it = ++m_operands.begin(); it != m_operands.end(); it++) { + num_operands++; + if ((it->is_reg() || it->is_vector())) { + num_regs++; + } + } + } + op = ALU_OP; + mem_op = NOT_TEX; + initiation_interval = latency = 1; + switch (m_opcode) { + case MOV_OP: + assert(!(has_memory_read() && has_memory_write())); + if (has_memory_read()) op = LOAD_OP; + if (has_memory_write()) op = STORE_OP; + break; + case LD_OP: + op = LOAD_OP; + break; + case MMA_LD_OP: + op = TENSOR_CORE_LOAD_OP; + break; + case LDU_OP: + op = LOAD_OP; + break; + case ST_OP: + op = STORE_OP; + break; + case MMA_ST_OP: + op = TENSOR_CORE_STORE_OP; + break; + case BRA_OP: + op = BRANCH_OP; + break; + case BREAKADDR_OP: + op = BRANCH_OP; + break; + case TEX_OP: + op = LOAD_OP; + mem_op = TEX; + break; + case ATOM_OP: + op = LOAD_OP; + break; + case BAR_OP: + op = BARRIER_OP; + break; + case SST_OP: + op = BARRIER_OP; + break; + case MEMBAR_OP: + op = MEMORY_BARRIER_OP; + break; + case CALL_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case CALLP_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case RET_OP: + case RETP_OP: + op = RET_OPS; + break; + case ADD_OP: + case ADDP_OP: + case ADDC_OP: + case SUB_OP: + case SUBC_OP: + // ADD,SUB latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: + case MIN_OP: + // MAX,MIN latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + // MUL latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: + case MADC_OP: + case MADP_OP: + // MAD latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only op = SFU_OP; + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op=TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); - -} - -void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const -{ - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t&)*pI; - assert( inst.valid() ); -} - -static unsigned datatype2size( unsigned data_type ) -{ - unsigned data_size; - switch ( data_type ) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; break; - case BB128_TYPE: - data_size = 16; break; - default: assert(0); break; - } - return data_size; + case SQRT_OP: + case SIN_OP: + case COS_OP: + case EX2_OP: + case LG2_OP: + case RSQRT_OP: + case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; + op = SFU_OP; + break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op = TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); } -void ptx_instruction::pre_decode() -{ - pc = m_PC; - isize = m_inst_size; - for(unsigned i=0; iget_instruction(pc); + inst = (const inst_t &)*pI; + assert(inst.valid()); +} - switch ( get_opcode() ) { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +static unsigned datatype2size(unsigned data_type) { + unsigned data_size; + switch (data_type) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; + break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; + break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; + break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; + break; + case BB128_TYPE: + data_size = 16; + break; + default: + assert(0); + break; + } + return data_size; +} + +void ptx_instruction::pre_decode() { + pc = m_PC; + isize = m_inst_size; + for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { + out[i] = 0; + } + for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { + in[i] = 0; + } + incount = 0; + outcount = 0; + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if (has_memory_read() || has_memory_write()) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } + + bool has_dst = false; + + switch (get_opcode()) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF - default: - printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); + default: + printf("Execution error: Invalid opcode (0x%x)\n", get_opcode()); break; - } + } - switch( m_cache_option ) { - case CA_OPTION: cache_op = CACHE_ALL; break; - case NC_OPTION: cache_op = CACHE_L1; break; - case CG_OPTION: cache_op = CACHE_GLOBAL; break; - case CS_OPTION: cache_op = CACHE_STREAMING; break; - case LU_OPTION: cache_op = CACHE_LAST_USE; break; - case CV_OPTION: cache_op = CACHE_VOLATILE; break; - case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; - case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; - default: - //if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) - cache_op = CACHE_ALL; - //else if( m_opcode == ST_OP ) - else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) - cache_op = CACHE_WRITE_BACK; - else if( m_opcode == ATOM_OP ) - cache_op = CACHE_GLOBAL; + switch (m_cache_option) { + case CA_OPTION: + cache_op = CACHE_ALL; break; - } - - set_opcode_and_latency(); - set_bar_type(); - // Get register operands - int n=0,m=0; - ptx_instruction::const_iterator opr=op_iter_begin(); - for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands - const operand_info &o = *opr; - if ( has_dst && n==0 ) { - // Do not set the null register "_" as an architectural register - if ( o.is_reg() && !o.is_non_arch_reg() ) { - out[0] = o.reg_num(); - arch_reg.dst[0] = o.arch_reg_num(); - } else if ( o.is_vector() ) { - is_vectorin = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) out[0] = o.reg1_num(); - if( num_elem >= 2 ) out[1] = o.reg2_num(); - if( num_elem >= 3 ) out[2] = o.reg3_num(); - if( num_elem >= 4 ) out[3] = o.reg4_num(); - if( num_elem >= 5 ) out[4] = o.reg5_num(); - if( num_elem >= 6 ) out[5] = o.reg6_num(); - if( num_elem >= 7 ) out[6] = o.reg7_num(); - if( num_elem >= 8 ) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if ( o.is_reg() && !o.is_non_arch_reg() ) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch ( m ) { - case 0: in[0] = reg_num; break; - case 1: in[1] = reg_num; break; - case 2: in[2] = reg_num; break; - default: break; - } - m++; - } else if ( o.is_vector() ) { - //assert(m == 0); //only support 1 vector operand (for textures) right now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[m+0] = o.reg1_num(); - if( num_elem >= 2 ) in[m+1] = o.reg2_num(); - if( num_elem >= 3 ) in[m+2] = o.reg3_num(); - if( num_elem >= 4 ) in[m+3] = o.reg4_num(); - if( num_elem >= 5 ) in[m+4] = o.reg5_num(); - if( num_elem >= 6 ) in[m+5] = o.reg6_num(); - if( num_elem >= 7 ) in[m+6] = o.reg7_num(); - if( num_elem >= 8 ) in[m+7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m+i] = o.arch_reg_num(i); - m+=num_elem; - } + case NC_OPTION: + cache_op = CACHE_L1; + break; + case CG_OPTION: + cache_op = CACHE_GLOBAL; + break; + case CS_OPTION: + cache_op = CACHE_STREAMING; + break; + case LU_OPTION: + cache_op = CACHE_LAST_USE; + break; + case CV_OPTION: + cache_op = CACHE_VOLATILE; + break; + case WB_OPTION: + cache_op = CACHE_WRITE_BACK; + break; + case WT_OPTION: + cache_op = CACHE_WRITE_THROUGH; + break; + default: + // if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if (m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP) + cache_op = CACHE_ALL; + // else if( m_opcode == ST_OP ) + else if (m_opcode == MMA_ST_OP || m_opcode == ST_OP) + cache_op = CACHE_WRITE_BACK; + else if (m_opcode == ATOM_OP) + cache_op = CACHE_GLOBAL; + break; + } + + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n = 0, m = 0; + ptx_instruction::const_iterator opr = op_iter_begin(); + for (; opr != op_iter_end(); opr++, n++) { // process operands + const operand_info &o = *opr; + if (has_dst && n == 0) { + // Do not set the null register "_" as an architectural register + if (o.is_reg() && !o.is_non_arch_reg()) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if (o.is_vector()) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) out[0] = o.reg1_num(); + if (num_elem >= 2) out[1] = o.reg2_num(); + if (num_elem >= 3) out[2] = o.reg3_num(); + if (num_elem >= 4) out[3] = o.reg4_num(); + if (num_elem >= 5) out[4] = o.reg5_num(); + if (num_elem >= 6) out[5] = o.reg6_num(); + if (num_elem >= 7) out[6] = o.reg7_num(); + if (num_elem >= 8) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } - } - - //Setting number of input and output operands which is required for scoreboard check - for(int i=0;i0) - outcount++; - - for(int i=0;i0) - incount++; - - // Get predicate - if(has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } - - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if( has_memory_read() || has_memory_write() ) { - ptx_instruction::const_iterator op=op_iter_begin(); - for ( ; op != op_iter_end(); op++, n++ ) { //process operands - const operand_info &o = *op; - - if(o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert( !o.is_non_arch_reg() ); - - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if(o.is_memory_operand2()) { - - // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) - if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register as well - } - // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) - else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output register as well - } - } - else if(o.is_immediate_address()){ - - } - // Regular PTX operand - else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } - - } + } else { + if (o.is_reg() && !o.is_non_arch_reg()) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch (m) { + case 0: + in[0] = reg_num; + break; + case 1: + in[1] = reg_num; + break; + case 2: + in[2] = reg_num; + break; + default: + break; + } + m++; + } else if (o.is_vector()) { + // assert(m == 0); //only support 1 vector operand (for textures) right + // now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) in[m + 0] = o.reg1_num(); + if (num_elem >= 2) in[m + 1] = o.reg2_num(); + if (num_elem >= 3) in[m + 2] = o.reg3_num(); + if (num_elem >= 4) in[m + 3] = o.reg4_num(); + if (num_elem >= 5) in[m + 4] = o.reg5_num(); + if (num_elem >= 6) in[m + 5] = o.reg6_num(); + if (num_elem >= 7) in[m + 6] = o.reg7_num(); + if (num_elem >= 8) in[m + 7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m + i] = o.arch_reg_num(i); + m += num_elem; } - } - - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - - m_decoded=true; -} - -void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) -{ - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); - int ntokens = sscanf(name.c_str(),buffer,&parsed_index); - if( ntokens == 1 ) { - assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); - } else { - assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } -} - -void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) -{ - const void *data = args->m_start; - - bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory - - std::map::iterator i=m_ptx_kernel_param_info.find(argn); - if( i != m_ptx_kernel_param_info.end() ) { - if (i->second.is_ptr_shared()) { - assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned) args->m_offset); + } + } + + // Setting number of input and output operands which is required for + // scoreboard check + for (int i = 0; i < MAX_OUTPUT_VALUES; i++) + if (out[i] > 0) outcount++; + + for (int i = 0; i < MAX_INPUT_VALUES; i++) + if (in[i] > 0) incount++; + + // Get predicate + if (has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } + + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if (has_memory_read() || has_memory_write()) { + ptx_instruction::const_iterator op = op_iter_begin(); + for (; op != op_iter_end(); op++, n++) { // process operands + const operand_info &o = *op; + + if (o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert(!o.is_non_arch_reg()); + + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if (o.is_memory_operand2()) { + // memory operand with one address register (ex. g[$r1+0x4] or + // s[$r2+=0x4]) + if (o.get_double_operand_type() == 0 || + o.get_double_operand_type() == 3) { + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register + // as well + } + // memory operand with two address register (ex. s[$r1+$r1] or + // g[$r1+=$r2]) + else if (o.get_double_operand_type() == 1 || + o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output + // register as well + } + } else if (o.is_immediate_address()) { + } + // Regular PTX operand + else if (o.get_symbol() + ->type() + ->get_key() + .is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } } - } else { - scratchpad_memory_param = true; - } - - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. - - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); - - symbol *p = m_symtab->lookup(buffer); - if( p == NULL ) { - printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); - abort(); + } + } + + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + + m_decoded = true; +} + +void function_info::add_param_name_type_size(unsigned index, std::string name, + int type, size_t size, bool ptr, + memory_space_t space) { + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); + int ntokens = sscanf(name.c_str(), buffer, &parsed_index); + if (ntokens == 1) { + assert(m_ptx_kernel_param_info.find(parsed_index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[parsed_index] = + param_info(name, type, size, ptr, space); + } else { + assert(m_ptx_kernel_param_info.find(index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } +} + +void function_info::add_param_data(unsigned argn, + struct gpgpu_ptx_sim_arg *args) { + const void *data = args->m_start; + + bool scratchpad_memory_param = + false; // Is this parameter in CUDA shared memory or OpenCL local memory + + std::map::iterator i = + m_ptx_kernel_param_info.find(argn); + if (i != m_ptx_kernel_param_info.end()) { + if (i->second.is_ptr_shared()) { + assert( + args->m_start == NULL && + "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned)args->m_offset); + } + } else { + scratchpad_memory_param = true; + } + + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); + + symbol *p = m_symtab->lookup(buffer); + if (p == NULL) { + printf( + "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " + "bind buffer\n", + buffer); + abort(); + } + if (data) + p->set_address((addr_t) * (size_t *)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if (i != m_ptx_kernel_param_info.end()) { + is_ptr_shared = i->second.is_ptr_shared(); } - if( data ) - p->set_address((addr_t)*(size_t*)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if( i != m_ptx_kernel_param_info.end() ) { - is_ptr_shared = i->second.is_ptr_shared(); - } - - if( !is_ptr_shared and !p->is_shared() ) { - printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); - abort(); - } - unsigned num_bits = 8*args->m_nbytes; - printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), - m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits/8 ); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; - p->set_address( addr+addr_pad ); - m_symtab->alloc_shared( num_bits/8 + addr_pad ); + + if (!is_ptr_shared and !p->is_shared()) { + printf( + "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " + "shared memory\n"); + abort(); } - } + unsigned num_bits = 8 * args->m_nbytes; + printf( + "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " + "0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits / 8); + fflush(stdout); + assert((num_bits % 8) == 0); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = + num_bits + ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) + : 0; + p->set_address(addr + addr_pad); + m_symtab->alloc_shared(num_bits / 8 + addr_pad); + } + } } unsigned function_info::get_args_aligned_size() { - - if(m_args_aligned_size >= 0) - return m_args_aligned_size; - - unsigned param_address = 0; - unsigned int total_size = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } - - m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word - - return m_args_aligned_size; - -} - - -void function_info::finalize( memory_space *param_mem ) -{ - unsigned param_address = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if( size != p.get_size() / 8) { - printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", - size, p.get_size()/8); - size = (size<(p.get_size()/8))?size:(p.get_size()/8); - } - // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over - //Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned - - const size_t word_size = 4; - //param_address = (param_address + size - 1) / size * size; //aligned with size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); + if (m_args_aligned_size >= 0) return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } + + m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word + + return m_args_aligned_size; +} + +void function_info::finalize(memory_space *param_mem) { + unsigned param_address = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if (size != p.get_size() / 8) { + printf( + "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " + "formal size = %zu (using smaller of two)\n", + size, p.get_size() / 8); + size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); + } + // copy the parameter over word-by-word so that parameter that crosses a + // memory page can be copied over + // Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * + align_amount; // aligned + + const size_t word_size = 4; + // param_address = (param_address + size - 1) / size * size; //aligned with + // size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast(param_value.pdata) + + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } +} + +void function_info::param_to_shared(memory_space *shared_mem, + symbol_table *symtab) { + // TODO: call this only for PTXPlus with GT200 models + // extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) + return; + + // copies parameters into simulated shared memory + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype, size, tmp); + + // Write to shared memory - offset + 0x10 + shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); + } +} + +void function_info::list_param(FILE *fout) const { + for (std::map::const_iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); +} + +void function_info::ptx_jit_config( + std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { + static unsigned long long counter = 0; + std::vector > param_data; + std::vector offsets; + std::vector paramIsPointer; + + char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path != NULL); + char *wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path != NULL); + std::string command = + std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/params.config" + + std::to_string(counter)); + + // initialize paramList + char buff[1024]; + std::string filename_c(filename + "_c"); + snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), + filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2 > pos1 && pos1 > 0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while (tok != NULL) { + std::string param(tok); + if (param.find("<") != std::string::npos) { + assert(param.find(">") == std::string::npos); + assert(param.find("*") == std::string::npos); + tmp = param; + } else { + if (tmp.length() > 0) { + tmp = ""; + assert(param.find(">") != std::string::npos); + assert(param.find("<") == std::string::npos); + assert(param.find("*") == std::string::npos); + } + printf("%s\n", param.c_str()); + if (param.find("*") != std::string::npos) { + paramIsPointer.push_back(true); + } else { + paramIsPointer.push_back(false); } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } -} - -void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) -{ - // TODO: call this only for PTXPlus with GT200 models - //extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; - - // copies parameters into simulated shared memory - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype,size,tmp); - - // Write to shared memory - offset + 0x10 - shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); - } -} - - -void function_info::list_param( FILE *fout ) const -{ - for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); -} - -void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) -{ - static unsigned long long counter = 0; - std::vector< std::pair > param_data; - std::vector offsets; - std::vector paramIsPointer; - - char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path!=NULL); - char * wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path!=NULL); - std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); - - //initialize paramList - char buff[1024]; - std::string filename_c(filename+"_c"); - snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2>pos1&&pos1>0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while(tok!=NULL){ - std::string param(tok); - if(param.find("<")!=std::string::npos){ - assert(param.find(">")==std::string::npos); - assert(param.find("*")==std::string::npos); - tmp = param; - } else { - if (tmp.length()>0){ - tmp = ""; - assert(param.find(">")!=std::string::npos); - assert(param.find("<")==std::string::npos); - assert(param.find("*")==std::string::npos); - } - printf("%s\n", param.c_str()); - if(param.find("*")!=std::string::npos){ - paramIsPointer.push_back(true); - }else{ - paramIsPointer.push_back(false); - } - } - tok = strtok(NULL, ","); } - - - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - - if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ - //is pointer - assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; - if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ - array_size = mallocPtr_Size[param_pointer]; - }else{ - for( std::map::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { - if(param_pointer>j->first&¶m_pointerfirst + j->second){ - array_size = j->first + j->second - param_pointer; - break; - } - } - assert(array_size>0&&"pointer was not previously malloc'd"); - } - - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - unsigned char* array_val = (unsigned char*) malloc(array_size); - gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); - param_data.push_back(std::pair(array_size,array_val)); - paramIsPointer.push_back(true); - }else{ - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - param_data.push_back(std::pair(param_value.size,val)); - paramIsPointer.push_back(false); + tok = strtok(NULL, ","); + } + + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); + + if (paramIsPointer[i->first] && + (*(unsigned long long *)param_value.pdata != 0)) { + // is pointer + assert(param_value.size == sizeof(void *) && + "MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = + *(unsigned long long *)param_value.pdata; + if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { + array_size = mallocPtr_Size[param_pointer]; + } else { + for (std::map::iterator j = + mallocPtr_Size.begin(); + j != mallocPtr_Size.end(); j++) { + if (param_pointer > j->first && + param_pointer < j->first + j->second) { + array_size = j->first + j->second - param_pointer; + break; + } } - } + assert(array_size > 0 && "pointer was not previously malloc'd"); + } - FILE *fout = fopen (filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { - if (paramIsPointer[index]){ - fprintf(fout, "*"); - } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; jfirst; j++){ - fprintf(fout, " %u", i->second[j]); - } - fprintf(fout, " : %u", offsets[index]); - free (i->second); - fprintf(fout, "\n"); - index++; + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + unsigned char *array_val = (unsigned char *)malloc(array_size); + gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, + (void *)array_val); + param_data.push_back( + std::pair(array_size, array_val)); + paramIsPointer.push_back(true); + } else { + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + param_data.push_back( + std::pair(param_value.size, val)); + paramIsPointer.push_back(false); } - fflush(fout); - fclose(fout); - - //ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); - snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); + } + + FILE *fout = fopen(filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for (std::vector >::const_iterator i = + param_data.begin(); + i != param_data.end(); i++) { + if (paramIsPointer[index]) { + fprintf(fout, "*"); } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j < i->first; j++) { + fprintf(fout, " %u", i->second[j]); } - fin = fopen(ptx_source, "r"); - assert(fin!=NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout!=NULL); - for (unsigned i = 0; isecond); + fprintf(fout, "\n"); + index++; + } + fflush(fout); + fclose(fout); + + // ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/ptx.config" + + std::to_string(counter)); + snprintf(buff, 1024, + "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", + get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, + "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " + "\"NR>={}&&NR<={}+2\" %s > %s", + ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin != NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout != NULL); + for (unsigned i = 0; i < line_number; i++) { + assert(fgets(buff, 1024, fin) != NULL); + assert(!feof(fin)); + } + fprintf(fout, "\n\n"); + do { + fprintf(fout, "%s", buff); + assert(fgets(buff, 1024, fin) != NULL); + if (feof(fin)) { + break; } - fprintf(fout, "\n\n"); - do{ - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if(feof(fin)){ - break; - } - } while(strstr(buff, "entry")==NULL); + } while (strstr(buff, "entry") == NULL); - fclose(fin); - fflush(fout); - fclose(fout); - counter++; + fclose(fin); + fflush(fout); + fclose(fout); + counter++; } -template -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) -{ - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { - return false; - } - if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { - return false; - } +template +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ((g_debug_thread_uid != 0) && + (thd_uid != (unsigned)g_debug_thread_uid)) { + return false; + } + if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { + return false; + } - return true; - } - - return false; -} - -void cuda_sim::init_inst_classification_stat() -{ - static std::set init; - if( init.find(g_ptx_kernel_count) != init.end() ) - return; - init.insert(g_ptx_kernel_count); - - #define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] =""; - if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); - assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! - g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); - if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); - g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); -} - -static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &src1 = pI->src1(); //the name of the texture - std::string texname = src1.name(); - - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t& k = thread->get_kernel(); - const struct textureInfo* texInfo = k.get_texinfo(texname); - - unsigned data_size = texInfo->texel_size; - return data_size; -} - -int tensorcore_op(int inst_opcode){ - - if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) - return 1; - else - return 0; -} -void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) -{ - - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert( pc == inst.pc ); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); - - set_npc( pc + pI->inst_size() ); - - - try { - - clearRPC(); - m_last_set_operand_value.u64 = 0; - - if(is_done()) - { - printf("attempted to execute instruction on a thread that is already done.\n"); + return true; + } + + return false; +} + +void cuda_sim::init_inst_classification_stat() { + static std::set init; + if (init.find(g_ptx_kernel_count) != init.end()) return; + init.insert(g_ptx_kernel_count); + +#define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] = ""; + if (!g_inst_classification_stat) + g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", + g_ptx_kernel_count); + assert(g_ptx_kernel_count < + MAX_CLASS_KER); // a static limit on number of kernels increase it if + // it fails! + g_inst_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 20); + if (!g_inst_op_classification_stat) + g_inst_op_classification_stat = + (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", + g_ptx_kernel_count); + g_inst_op_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 100); +} + +static unsigned get_tex_datasize(const ptx_instruction *pI, + ptx_thread_info *thread) { + const operand_info &src1 = pI->src1(); // the name of the texture + std::string texname = src1.name(); + + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t &k = thread->get_kernel(); + const struct textureInfo *texInfo = k.get_texinfo(texname); + + unsigned data_size = texInfo->texel_size; + return data_size; +} + +int tensorcore_op(int inst_opcode) { + if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || + (inst_opcode == MMA_ST_OP)) + return 1; + else + return 0; +} +void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert(pc == + inst.pc); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + + set_npc(pc + pI->inst_size()); + + try { + clearRPC(); + m_last_set_operand_value.u64 = 0; + + if (is_done()) { + printf( + "attempted to execute instruction on a thread that is already " + "done.\n"); assert(0); - } - - if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) - || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { - - clear_modifiedregs(); - enable_debug_trace(); + } + + if (g_debug_execution >= 6 || + m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || + (get_uid() == + (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { + clear_modifiedregs(); + enable_debug_trace(); } - } - - - if( pI->has_pred() ) { + } + + if (pI->has_pred()) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if(pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag + if (pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ + pI->get_pred_neg(); // ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode=pI->get_opcode(); - - if( skip ) { + } + int inst_opcode = pI->get_opcode(); + + if (skip) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if( pI->get_opcode() == VOTE_OP ) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t*)pJ) = inst; // copy active mask information - pI = pJ; + if (pI->get_opcode() == VOTE_OP) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t *)pJ) = inst; // copy active mask information + pI = pJ; } - - if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ - if(inst.active_count()!=MAX_WARP_SIZE) - { - printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); - assert(0); - } + + if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || + inst_opcode == MMA_ST_OP))) { + if (inst.active_count() != MAX_WARP_SIZE) { + printf( + "Tensor Core operation are warp synchronous operation. All the " + "threads needs to be active."); + assert(0); + } } - - //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation - if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ - switch ( inst_opcode ) { - #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; - #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; - #include "opcodes.def" - #undef OP_DEF - #undef OP_W_DEF - default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; - } + + // Tensorcore is warp synchronous operation. So these instructions needs + // to be executed only once. To make the simulation faster removing the + // redundant tensorcore operation + if (!tensorcore_op(inst_opcode) || + ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { + switch (inst_opcode) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, this); \ + op_classification = CLASSIFICATION; \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, get_core(), inst); \ + op_classification = CLASSIFICATION; \ + break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: + printf("Execution error: Invalid opcode (0x%x)\n", + pI->get_opcode()); + break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if(pI->is_exit()) - exit_impl(pI,this); - } - - - - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - - // Output instruction information to file and stdout - if( config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { - fprintf(m_gpu->get_ptx_inst_debug_file(), - "[thd=%u] : (%s:%u - %s)\n", - get_uid(), - pI->source_file(), pI->source_line(), pI->get_source() ); - //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + if (pI->is_exit()) exit_impl(pI, this); + } + + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { + fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", + get_uid(), pI->source_file(), pI->source_line(), + pI->get_source()); + // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory + // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), + pc)) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, - get_uid(), - pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, - get_icount(), - pc, pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64 ); + printf( + "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " + "[pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), + ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, + pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ( (pI->has_memory_read() || pI->has_memory_write()) ) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ((pI->has_memory_read() || pI->has_memory_write())) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); - } + } + } + + if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, + false /*not atomic*/); + } - if ( pI->get_opcode() == ATOM_OP ) { + if (pI->get_opcode() == ATOM_OP) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr() ); - assert( inst.space == last_space() ); - insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr()); + assert(inst.space == last_space()); + insn_data_size = get_tex_datasize( + pI, + this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if( config.get_ptx_inst_debug_to_file()!=0 && - (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { + // Output register information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( g_debug_execution >= 6 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) - dump_modifiedregs(stdout); - } - if ( g_debug_execution >= 10 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - //not using it with functional simulation mode - if(!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); - - if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { + if (g_debug_execution >= 6) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), + pc)) + dump_modifiedregs(stdout); + } + if (g_debug_execution >= 10) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), + pc)) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + // not using it with functional simulation mode + if (!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type=0; - switch ( pI->get_space().get_type() ) { - case global_space: space_type = 10; break; - case local_space: space_type = 11; break; - case tex_space: space_type = 12; break; - case surf_space: space_type = 13; break; - case param_space_kernel: - case param_space_local: - space_type = 14; break; - case shared_space: space_type = 15; break; - case const_space: space_type = 16; break; - default: - space_type = 0 ; - break; + unsigned space_type = 0; + switch (pI->get_space().get_type()) { + case global_space: + space_type = 10; + break; + case local_space: + space_type = 11; + break; + case tex_space: + space_type = 12; + break; + case surf_space: + space_type = 13; + break; + case param_space_kernel: + case param_space_local: + space_type = 14; + break; + case shared_space: + space_type = 15; + break; + case const_space: + space_type = 16; + break; + default: + space_type = 0; + break; } - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); - if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); - } - if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + op_classification); + if (space_type) + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)space_type); + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)pI->get_opcode()); + } + if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); + DPRINTF(LIVENESS, + "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, + ctaid.z, tid.x, tid.y, tid.z); fflush(stdout); - } - - // "Return values" - if(!skip) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert( inst.memory_op == insn_memory_op ); - } - } - - } catch ( int x ) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); - abort(); - } - -} - -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) -{ - gpgpu_param_num_shaders = num_shaders; -} - -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) -{ - return kernel->get_kernel_info(); -} - -const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) -{ - return pc_to_instruction(pc); -} - -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool isInFunctionalSimulationMode) -{ - std::list &active_threads = kernel.active_threads(); - - static std::map shared_memory_lookup; - static std::map sstarr_memory_lookup; - static std::map ptx_cta_lookup; - static std::map ptx_warp_lookup; - static std::map > local_memory_lookup; - - if ( *thread_info != NULL ) { - ptx_thread_info *thd = *thread_info; - assert( thd->is_done() ); - if ( g_debug_execution==-1 ) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", - ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } - - if ( !active_threads.empty() ) { - assert( active_threads.size() <= threads_left ); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); - return 1; - } - - if ( kernel.no_more_ctas_to_run() ) { - return 0; //finished! - } - - if ( threads_left < kernel.threads_per_cta() ) { - return 0; - } - - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } - - //initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; - - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 - assert( max_cta_per_sm > 0 ); - - //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - - if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - char buf[512]; - snprintf(buf,512,"shared_%u", sid); - shared_mem = new memory_space_impl<16*1024>(buf,4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf,512,"sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16*1024>(buf,4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } - - std::map &local_mem_lookup = local_memory_lookup[sid]; - while( kernel.more_threads_in_cta() ) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + } - memory_space *local_mem = NULL; - std::map::iterator l = local_mem_lookup.find(new_tid); - if ( l != local_mem_lookup.end() ) { - local_mem = l->second; - } else { - char buf[512]; - snprintf(buf,512,"local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf,32); - local_mem_lookup[new_tid] = local_mem; + // "Return values" + if (!skip) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert(inst.memory_op == insn_memory_op); } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if( kernel.entry()->get_ptx_version().extensions() ) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem,st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); - fflush(stdout); - } - active_threads.push_back(thd); - } - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); - fflush(stdout); - } - - kernel.increment_cta_id(); + } - assert( active_threads.size() <= threads_left ); - *thread_info = active_threads.front(); - (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); - active_threads.pop_front(); - return 1; + } catch (int x) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, + pI->source_file(), pI->source_line()); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); + abort(); + } } -size_t get_kernel_code_size( class function_info *entry ) -{ - return entry->get_function_size(); +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { + gpgpu_param_num_shaders = num_shaders; } +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const function_info *kernel) { + return kernel->get_kernel_info(); +} -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ) -{ - kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); - unsigned argcount=args.size(); - unsigned argn=1; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - entry->add_param_data(argcount-argn,&(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - - return result; +const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { + return pc_to_instruction(pc); } -#include "../../version" -#include "detailed_version" +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, bool isInFunctionalSimulationMode) { + std::list &active_threads = kernel.active_threads(); -void print_splash() -{ - static int splash_printed=0; - if ( !splash_printed ) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); - splash_printed=1; - } -} + static std::map shared_memory_lookup; + static std::map sstarr_memory_lookup; + static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; + static std::map > + local_memory_lookup; -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); - g_const_name_lookup[hostVar] = deviceName; -} + if (*thread_info != NULL) { + ptx_thread_info *thd = *thread_info; + assert(thd->is_done()); + if (g_debug_execution == -1) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf( + "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) uid=%u\n", + ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } + + if (!active_threads.empty()) { + assert(active_threads.size() <= threads_left); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + return 1; + } + + if (kernel.no_more_ctas_to_run()) { + return 0; // finished! + } + + if (threads_left < kernel.threads_per_cta()) { + return 0; + } + + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } + + // initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; + + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 + assert(max_cta_per_sm > 0); + + // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = + hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; + + if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + char buf[512]; + snprintf(buf, 512, "shared_%u", sid); + shared_mem = new memory_space_impl<16 * 1024>(buf, 4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf, 512, "sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } + + std::map &local_mem_lookup = + local_memory_lookup[sid]; + while (kernel.more_threads_in_cta()) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); - g_global_name_lookup[hostVar] = deviceName; + memory_space *local_mem = NULL; + std::map::iterator l = + local_mem_lookup.find(new_tid); + if (l != local_mem_lookup.end()) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf, 512, "local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf, 32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if (kernel.entry()->get_ptx_version().extensions()) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem, st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if (g_debug_execution == -1) { + printf( + "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, + (unsigned long long)thd); + fflush(stdout); + } + active_threads.push_back(thd); + } + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + fflush(stdout); + } + + kernel.increment_cta_id(); + + assert(active_threads.size() <= threads_left); + *thread_info = active_threads.front(); + (*thread_info) + ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + active_threads.pop_front(); + return 1; +} + +size_t get_kernel_code_size(class function_info *entry) { + return entry->get_function_size(); +} + +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( + class function_info *entry, gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { + kernel_info_t *result = + new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), + gpu->getNameInfoMapping()); + unsigned argcount = args.size(); + unsigned argn = 1; + for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); + a++) { + entry->add_param_data(argcount - argn, &(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; } -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) -{ - printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; - - std::map::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { - if ( found_sym ) { - printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar ); - abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if( g_globals.find(hostVar) != g_globals.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if( g_constants.find(hostVar) != g_constants.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } +#include "../../version" +#include "detailed_version" - if ( !found_sym ) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); +void print_splash() { + static int splash_printed = 0; + if (!splash_printed) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", + g_gpgpusim_version_string, g_gpgpusim_build_string); + splash_printed = 1; + } +} + +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", + deviceName, size); + g_const_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", + deviceName); + g_global_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu) { + printf( + "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", + hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; + + std::map::iterator c = + gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map::iterator g = + gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { + if (found_sym) { + printf( + "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " + "both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar); abort(); - } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); - const char *mem_name = NULL; - memory_space *mem = NULL; - - std::map::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); - symbol_table *symtab = st->second; - - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if (g_globals.find(hostVar) != g_globals.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if (g_constants.find(hostVar) != g_constants.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } + + if (!found_sym) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", + (unsigned long long)hostVar); + abort(); + } else + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " + "hostVar=0x%Lx\n", + (unsigned long long)hostVar); + const char *mem_name = NULL; + memory_space *mem = NULL; + + std::map::iterator st = + gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); - for ( unsigned n=0; n < count; n++ ) { - if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); - else mem->read(dst+n,1,((char*)src)+n); - } - fflush(stdout); + } + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " + "%s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); + for (unsigned n = 0; n < count; n++) { + if (to) + mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); + else + mem->read(dst + n, 1, ((char *)src) + n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() -{ - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; - - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if ( mode ) - sscanf(mode,"%u", &g_ptx_sim_mode); - printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); - printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if ( dbg_inter && strlen(dbg_inter) ) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); - fflush(stdout); - sscanf(dbg_level,"%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if ( dbg_thread && strlen(dbg_thread) ) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); - fflush(stdout); - sscanf(dbg_thread,"%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if ( dbg_pc && strlen(dbg_pc) ) { - printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); - fflush(stdout); - sscanf(dbg_pc,"%d", &g_debug_pc); - } +void cuda_sim::read_sim_environment_variables() { + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; + + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); + printf( + "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " + "environment variable:\n", + g_ptx_sim_mode); + printf( + " 1=functional simulation only, 0=detailed performance " + "simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if (dbg_inter && strlen(dbg_inter)) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); + fflush(stdout); + sscanf(dbg_level, "%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if (dbg_thread && strlen(dbg_thread)) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", + dbg_thread); + fflush(stdout); + sscanf(dbg_thread, "%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if (dbg_pc && strlen(dbg_pc)) { + printf( + "GPGPU-Sim PTX: printing debug information for instruction with PC = " + "%s\n", + dbg_pc); + fflush(stdout); + sscanf(dbg_pc, "%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if( blocking && !strcmp(blocking,"1") ) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf( + "GPGPU-Sim PTX: overriding embedded ptx with ptx file " + "(PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if (blocking && !strcmp(blocking, "1")) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if ( g_debug_execution >= 40 ) { - ptx_debug = 1; - } -} - -#define MAX(a,b) (((a)>(b))?(a):(b)) - -unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) -{ - - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; - - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); - printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); - //Limit by CTA - unsigned int result_cta = max_cta_per_core; - - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); - - printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf (" threads"); - if (result == result_shmem) printf (" shmem"); - if (result == result_regs) printf (" regs"); - if (result == result_cta) printf (" cta_limit"); - printf ("\n"); - - return result; + if (g_debug_execution >= 40) { + ptx_debug = 1; + } +} + +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, + unsigned threads_per_cta, unsigned int warp_size, + unsigned int n_thread_per_shader, + unsigned int gpgpu_shmem_size, + unsigned int gpgpu_shader_registers, + unsigned int max_cta_per_core) { + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; + + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / + (padded_cta_size * ((kernel_info->regs + 3) & ~3)); + printf("padded cta size is %d and %d and %d", padded_cta_size, + kernel_info->regs, ((kernel_info->regs + 3) & ~3)); + // Limit by CTA + unsigned int result_cta = max_cta_per_core; + + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf(" threads"); + if (result == result_shmem) printf(" shmem"); + if (result == result_regs) printf(" regs"); + if (result == result_cta) printf(" cta_limit"); + printf("\n"); + + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t parameter -which holds the data for the CUDA kernel to be executed +This function simulates the CUDA code functionally, it takes a kernel_info_t +parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) -{ - printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); - - //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here - //extern gpgpu_sim *g_the_gpu; - //before we execute, we should do PDOM analysis for functional simulation scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); +void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, + bool openCL) { + printf( + "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", + kernel.name().c_str()); + + // using a shader core object for book keeping, it is not needed but as most + // function built for performance simulation need it we use it here + // extern gpgpu_sim *g_the_gpu; + // before we execute, we should do PDOM analysis for functional simulation + // scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = + ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + kernel.name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + kernel.name().c_str()); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + + unsigned max_cta_tot = max_cta( + kernel_info, kernel.threads_per_cta(), + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->n_thread_per_shader, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shmem_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shader_registers, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->max_cta_per_core); + printf("Max CTA : %d\n", max_cta_tot); + + int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched = 0; + + // we excute the kernel one CTA (Block) at the time, as synchronization + // functions work block wise + while (!kernel.no_more_ctas_to_run()) { + unsigned temp = kernel.get_next_cta_id_single(); + + if (cp_op == 0 || + (cp_op == 1 && cta_launched < cp_cta_resume && + kernel.get_uid() == cp_kernel) || + kernel.get_uid() < cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); + cta.execute(cp_count, temp); + +#if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); +#endif } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); + kernel.increment_cta_id(); } - - unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); - printf("Max CTA : %d\n",max_cta_tot); - - int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; - - //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise - while(!kernel.no_more_ctas_to_run()){ - unsigned temp=kernel.get_next_cta_id_single(); - - - if(cp_op==0 || (cp_op==1 && cta_launchedthe_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size - ); - cta.execute(cp_count,temp); - - #if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); - #endif - } - else - { - kernel.increment_cta_id(); - } cta_launched++; + } + + if (cp_op == 1) { + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", + kernel.get_uid()); + g_checkpoint->store_global_mem( + gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, + (char *)"%08x"); + } + + // registering this kernel as done + + // openCL kernel simulation calls don't register the kernel so we don't + // register its exit + if (!openCL) { + // extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( + kernel.get_uid()); + } + + //******PRINTING******* + printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", + g_ptx_sim_num_insn); + if (gpgpu_ptx_instruction_classification) { + StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); + } + + // time_t variables used to calculate the total simulation time + // the start time of simulation is hold by the global variable + // g_simulation_starttime g_simulation_starttime is initilized by + // gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = + MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + + // calculating and printing simulation time in terms of days, hours, minutes + // and seconds + days = elapsed_time / (3600 * 24); + hrs = elapsed_time / 3600 - 24 * days; + minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); + sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); + + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, + (unsigned)elapsed_time); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(g_ptx_sim_num_insn / elapsed_time)); + fflush(stdout); +} + +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { + int ctaLiveThreads = 0; + symbol_table *symtab = m_kernel->entry()->get_symtab(); + + for (int i = 0; i < m_warp_count; i++) { + m_warpAtBarrier[i] = false; + m_liveThreadCount[i] = 0; + } + for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; + + // get threads for a cta + for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { + ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, + m_kernel->threads_per_cta() - i, + m_kernel->threads_per_cta(), this, 0, i / m_warp_size, + (gpgpu_t *)m_gpu, true); + assert(m_thread[i] != NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) + m_thread[i]->resume_reg_thread(fname, symtab); + ctaLiveThreads++; + } + + for (int k = 0; k < m_warp_count; k++) createWarp(k); +} + +void functionalCoreSim::createWarp(unsigned warpId) { + simt_mask_t initialMask; + unsigned liveThreadsCount = 0; + initialMask.set(); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + if (m_thread[i] == NULL) + initialMask.reset(i - warpId * m_warp_size); + else + liveThreadsCount++; + } + + assert(m_thread[warpId * m_warp_size] != NULL); + m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), + initialMask); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); + + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { + unsigned pc, rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } + } + m_liveThreadCount[warpId] = liveThreadsCount; +} + +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { + m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count = 0; + while (true) { + bool someOneLive = false; + bool allAtBarrier = true; + for (unsigned i = 0; i < m_warp_count; i++) { + executeWarp(i, allAtBarrier, someOneLive); + count++; } - - - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); - } - - - - - //registering this kernel as done - - //openCL kernel simulation calls don't register the kernel so we don't register its exit - if(!openCL) { - //extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); - } - - //******PRINTING******* - printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); - if ( gpgpu_ptx_instruction_classification ) { - StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); - } - - //time_t variables used to calculate the total simulation time - //the start time of simulation is hold by the global variable g_simulation_starttime - //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - - - //calculating and printing simulation time in terms of days, hours, minutes and seconds - days = elapsed_time/(3600*24); - hrs = elapsed_time/3600 - 24*days; - minutes = elapsed_time/60 - 60*(hrs + 24*days); - sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); - fflush(stdout); -} - -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) -{ - int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); - - for(int i=0; i< m_warp_count; i++){ - m_warpAtBarrier[i]=false; - m_liveThreadCount[i]=0; + if (inst_count > 0 && count > inst_count && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { + someOneLive = false; + break; } - for(int i=0; i< m_warp_count*m_warp_size;i++) - m_thread[i]=NULL; - - //get threads for a cta - for(unsigned i=0; ithreads_per_cta();i++) { - ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); - assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); - ctaLiveThreads++; + if (!someOneLive) break; + if (allAtBarrier) { + for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; } - - for(int k=0;klaunch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; iset_npc(pc); - m_thread[i]->update_pc(); - } - - } - m_liveThreadCount[warpId]= liveThreadsCount; -} - -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) - { - m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count=0; - while(true){ - bool someOneLive= false; - bool allAtBarrier = true; - for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } - if(!someOneLive) break; - if(allAtBarrier){ - for(unsigned i=0;iget_next_cta_id_single(); + if (m_gpu->checkpoint_option == 1 && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t)) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, + (char *)"%08x"); + for (int i = 0; i < 32 * m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, + ctaid - 1); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", + i, ctaid - 1); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, + (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); } - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - ptx_reg_t regval; - regval.u64= 123; - - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;iprint_checkpoint(fp); - fclose(fp); - } - } - + for (int i = 0; i < m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, + ctaid - 1); + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); + } + } } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) -{ - if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ - warp_inst_t inst =getExecuteWarp(i); - execute_warp_inst_t(inst,i); - if(inst.isatomic()) inst.do_atomic(true); - if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; - updateSIMTStack( i, &inst ); - } - if(m_liveThreadCount[i]>0) someOneLive=true; - if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, + bool &someOneLive) { + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { + warp_inst_t inst = getExecuteWarp(i); + execute_warp_inst_t(inst, i); + if (inst.isatomic()) inst.do_atomic(true); + if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) + m_warpAtBarrier[i] = true; + updateSIMTStack(i, &inst); + } + if (m_liveThreadCount[i] > 0) someOneLive = true; + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) -{ - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is + // the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map get_duplicate(); +extern std::map get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map g_duplicate; +static std::map g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() -{ - return g_ptxinfo_kname; -} +const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } -void print_ptxinfo() -{ - if(! get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if(get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - } +void print_ptxinfo() { + if (!get_ptxinfo_kname()) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if (get_ptxinfo_kname()) { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + } } - -struct gpgpu_ptx_sim_info get_ptxinfo() -{ - return g_ptxinfo; +struct gpgpu_ptx_sim_info get_ptxinfo() { + return g_ptxinfo; } -std::map get_duplicate() -{ - return g_duplicate; -} +std::map get_duplicate() { return g_duplicate; } -void ptxinfo_linenum( unsigned linenum ) -{ - g_duplicate[linenum] = g_last_dup_type; +void ptxinfo_linenum(unsigned linenum) { + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_dup_type( const char *dup_type ) -{ - g_last_dup_type = dup_type; -} +void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } -void ptxinfo_function(const char *fname ) -{ - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_function(const char *fname) { + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_regs( unsigned nregs ) -{ - g_ptxinfo.regs=nregs; -} +void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } -void ptxinfo_lmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.lmem=declared+system; +void ptxinfo_lmem(unsigned declared, unsigned system) { + g_ptxinfo.lmem = declared + system; } -void ptxinfo_gmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.gmem=declared+system; +void ptxinfo_gmem(unsigned declared, unsigned system) { + g_ptxinfo.gmem = declared + system; } -void ptxinfo_smem( unsigned declared, unsigned system ) -{ - g_ptxinfo.smem=declared+system; +void ptxinfo_smem(unsigned declared, unsigned system) { + g_ptxinfo.smem = declared + system; } -void ptxinfo_cmem( unsigned nbytes, unsigned bank ) -{ - g_ptxinfo.cmem+=nbytes; -} +void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } -void clear_ptxinfo() -{ - free(g_ptxinfo_kname); - g_ptxinfo_kname=NULL; - g_ptxinfo.regs=0; - g_ptxinfo.lmem=0; - g_ptxinfo.smem=0; - g_ptxinfo.cmem=0; - g_ptxinfo.gmem=0; - g_ptxinfo.ptx_version=0; - g_ptxinfo.sm_target=0; +void clear_ptxinfo() { + free(g_ptxinfo_kname); + g_ptxinfo_kname = NULL; + g_ptxinfo.regs = 0; + g_ptxinfo.lmem = 0; + g_ptxinfo.smem = 0; + g_ptxinfo.cmem = 0; + g_ptxinfo.gmem = 0; + g_ptxinfo.ptx_version = 0; + g_ptxinfo.sm_target = 0; } +void ptxinfo_opencl_addinfo(std::map &kernels) { + if (!g_ptxinfo_kname) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } -void ptxinfo_opencl_addinfo( std::map &kernels ) -{ - - if(! g_ptxinfo_kname) { - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } - - if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map::iterator k=kernels.find(g_ptxinfo_kname); - if( k==kernels.end() ) { - printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); - abort(); - } else { - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - function_info *finfo = k->second; - assert(finfo!=NULL); - finfo->set_kernel_info( g_ptxinfo ); - } - clear_ptxinfo(); -} - -struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) -{ - rec_pts tmp; - std::map::iterator r=g_rpts.find(finfo); - - if( r==g_rpts.end() ) { - int num_recon = finfo->get_num_reconvergence_pairs(); - - gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); - for (int i=0;iprint_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ " ); - if( kernel_recon_points[i].target_inst ) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); - - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; -} - -address_type get_return_pc( void *thd ) -{ - // function call return - ptx_thread_info *the_thread = (ptx_thread_info*)thd; - assert( the_thread != NULL ); - return the_thread->get_return_PC(); -} - -address_type cuda_sim::get_converge_point( address_type pc ) -{ - // the branch could encode the reconvergence point and/or a bit that indicates the - // reconvergence point is the return PC on the call stack in the case the branch has - // no immediate postdominator in the function (i.e., due to multiple return points). - - std::map::iterator f=g_pc_to_finfo.find(pc); - assert( f != g_pc_to_finfo.end() ); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); - - int i=0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; - } + if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map::iterator k = + kernels.find(g_ptxinfo_kname); + if (k == kernels.end()) { + printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", + g_ptxinfo_kname); + abort(); + } else { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + function_info *finfo = k->second; + assert(finfo != NULL); + finfo->set_kernel_info(g_ptxinfo); + } + clear_ptxinfo(); +} + +struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { + rec_pts tmp; + std::map::iterator r = g_rpts.find(finfo); + + if (r == g_rpts.end()) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = + (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", + finfo->get_name().c_str()); + for (int i = 0; i < num_recon; i++) { + printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); + kernel_recon_points[i].source_inst->print_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ "); + if (kernel_recon_points[i].target_inst) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", + finfo->get_name().c_str()); + + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; +} + +address_type get_return_pc(void *thd) { + // function call return + ptx_thread_info *the_thread = (ptx_thread_info *)thd; + assert(the_thread != NULL); + return the_thread->get_return_PC(); +} + +address_type cuda_sim::get_converge_point(address_type pc) { + // the branch could encode the reconvergence point and/or a bit that indicates + // the reconvergence point is the return PC on the call stack in the case the + // branch has no immediate postdominator in the function (i.e., due to + // multiple return points). + + std::map::iterator f = g_pc_to_finfo.find(pc); + assert(f != g_pc_to_finfo.end()); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); + + int i = 0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; } - } - return NO_BRANCH_DIVERGENCE; + } + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit( unsigned warp_id ) -{ - for(int i=0;im_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; - } +void functionalCoreSim::warp_exit(unsigned warp_id) { + for (int i = 0; i < m_warp_count * m_warp_size; i++) { + if (m_thread[i] != NULL) { + m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; } + } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 1be3d19..21e1ca0 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,34 +7,35 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED -#include "../abstract_hardware_model.h" -#include"../gpgpu-sim/shader.h" #include #include -#include #include -#include"ptx_sim.h" +#include +#include "../abstract_hardware_model.h" +#include "../gpgpu-sim/shader.h" +#include "ptx_sim.h" class gpgpu_context; class memory_space; @@ -44,69 +45,65 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( std::map &kernels ); -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - class ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - class core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( + std::map &kernels); +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + class ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, class core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures + * and procedures in core_t */ -class functionalCoreSim: public core_t -{ -public: - functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) - : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) - { - m_warpAtBarrier = new bool [m_warp_count]; - m_liveThreadCount = new unsigned [m_warp_count]; - } - virtual ~functionalCoreSim(){ - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit( unsigned warp_id ); - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const - { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); - } - -private: - void executeWarp(unsigned, bool &, bool &); - //initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) - { - if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ - m_liveThreadCount[tid/m_warp_size]--; - } +class functionalCoreSim : public core_t { + public: + functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) + : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { + m_warpAtBarrier = new bool[m_warp_count]; + m_liveThreadCount = new unsigned[m_warp_count]; + } + virtual ~functionalCoreSim() { + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit(unsigned warp_id); + virtual bool warp_waiting_at_barrier(unsigned warp_id) const { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); + } + + private: + void executeWarp(unsigned, bool &, bool &); + // initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) { + if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { + m_liveThreadCount[tid / m_warp_size]--; } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - //each warp live thread count and barrier indicator - unsigned * m_liveThreadCount; - bool* m_warpAtBarrier; + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + // each warp live thread count and barrier indicator + unsigned *m_liveThreadCount; + bool *m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc( void *thd ); +address_type get_return_pc(void *thd); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -114,89 +111,98 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; - class cuda_sim { - public: - cuda_sim( gpgpu_context* ctx ) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat= NULL; - g_assemble_code_next_pc=0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count=0; - g_ptx_thread_info_uid_next=1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - //global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map g_global_name_lookup; // indexed by hostVar - std::map g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map g_rpts; - bool g_cuda_launch_blocking; - void ** g_inst_classification_stat; - void ** g_inst_op_classification_stat; - std::set g_globals; - std::set g_constants; - std::map g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set g_ptx_cta_info_sm_idx_used; - ptx_reg_t* ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context* gpgpu_ctx; - //global functions - void ptx_opcocde_latency_options (option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); - int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); - void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points( function_info *finfo ); - address_type get_converge_point( address_type pc ); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); - void ptx_print_insn( address_type pc, FILE *fp ); - std::string ptx_get_insn_str( address_type pc ); - template bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim(gpgpu_context *ctx) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = + -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat = NULL; + g_assemble_code_next_pc = 0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count = 0; + g_ptx_thread_info_uid_next = 1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + // global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map + g_global_name_lookup; // indexed by hostVar + std::map + g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no + // notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map g_rpts; + bool g_cuda_launch_blocking; + void **g_inst_classification_stat; + void **g_inst_op_classification_stat; + std::set g_globals; + std::set g_constants; + std::map g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set g_ptx_cta_info_sm_idx_used; + ptx_reg_t *ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // global functions + void ptx_opcocde_latency_options(option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); + int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size); + void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, + size_t size); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points(function_info *finfo); + address_type get_converge_point(address_type pc); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu); + void ptx_print_insn(address_type pc, FILE *fp); + std::string ptx_get_insn_str(address_type pc); + template + bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 9ac0727..1da6b85 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,108 +7,112 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr,const char *arg_list) -{ - FILE *fp = stdout; - unsigned i=0,j=0; - unsigned arg_offset=0; - char buf[64]; - bool in_fmt=false; - while( fmtstr[i] ) { - char c = fmtstr[i++]; - if( !in_fmt ) { - if( c != '%' ) { - fprintf(fp,"%c",c); - } else { - in_fmt=true; - buf[0] = c; - j=1; - } +void my_cuda_printf(const char *fmtstr, const char *arg_list) { + FILE *fp = stdout; + unsigned i = 0, j = 0; + unsigned arg_offset = 0; + char buf[64]; + bool in_fmt = false; + while (fmtstr[i]) { + char c = fmtstr[i++]; + if (!in_fmt) { + if (c != '%') { + fprintf(fp, "%c", c); } else { - if(!( c == 'u' || c == 'f' || c == 'd' )) { - printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j+1] = 0; - void* ptr = (void*)&arg_list[arg_offset]; - //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if( c == 'u' || c == 'd' ) { - fprintf(fp,buf,*((unsigned long long*)ptr)); - } else if( c == 'f' ) { - double tmp = *((double*)ptr); - fprintf(fp,buf,tmp); - } - arg_offset++; - in_fmt=false; + in_fmt = true; + buf[0] = c; + j = 1; } - } + } else { + if (!(c == 'u' || c == 'f' || c == 'd')) { + printf( + "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " + "%%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j + 1] = 0; + void *ptr = (void *)&arg_list[arg_offset]; + // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if (c == 'u' || c == 'd') { + fprintf(fp, buf, *((unsigned long long *)ptr)); + } else if (c == 'f') { + double tmp = *((double *)ptr); + fprintf(fp, buf, tmp); + } + arg_offset++; + in_fmt = false; + } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) -{ - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size<1024*sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr,size,buffer); - addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem=NULL; - memory_space_t space = generic_space; - decode_space(space,thread,actual_param_op,mem,addr); // figure out which space - if( arg == 0 ) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr+len,1,&b); - len++; - } while(b); - fmtstr = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,fmtstr+i); - //mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,arg_list+i); - //mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr,arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size < 1024 * sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr, size, buffer); + addr_t addr = + (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem = NULL; + memory_space_t space = generic_space; + decode_space(space, thread, actual_param_op, mem, + addr); // figure out which space + if (arg == 0) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr + len, 1, &b); + len++; + } while (b); + fmtstr = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); + // mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); + // mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr, arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 4e9baaa..c4ee75d 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,27 +7,30 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); +void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, + class ptx_thread_info* thread, + const class function_info* target_func); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 4baced5..4a99c1c 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,297 +1,327 @@ -//Jin: cuda_device_runtime.cc -//Defines CUDA device runtime APIs for CDP support - +// Jin: cuda_device_runtime.cc +// Defines CUDA device runtime APIs for CDP support #include #include - - #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include #include +#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "../stream_manager.h" #include "../gpgpusim_entrypoint.h" +#include "../stream_manager.h" +#include "cuda-sim.h" #include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -#define DEV_RUNTIME_REPORT(a) \ - if( g_debug_execution ) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } - - - -//Handling device runtime api: -//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) -{ - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 4 ); - - function_info * child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); - } - else if(arg == 1) { //dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); - } - else if(arg == 2) { //dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); - } - else if(arg == 3) { //unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, & shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); - } +#define DEV_RUNTIME_REPORT(a) \ + if (g_debug_execution) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } + +// Handling device runtime api: +// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 +// blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 4); + + function_info *child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " + << child_kernel_entry->get_name()); + } else if (arg == 1) { // dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " + << grid_dim.z << ")"); + } else if (arg == 2) { // dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " + << block_dim.z << ")"); + } else if (arg == 3) { // unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); } - - //get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); - if(g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; - - //store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; - - //copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); - + } + + // get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " + << child_kernel_arg_size + << ", parameter buffer allocated at " << param_buffer); + if (g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; + + // store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, + child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == + g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + + // copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, + NULL); } -//Handling device runtime api: -//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - kernel_info_t * device_grid = NULL; - function_info * device_kernel_entry = NULL; - void * parameter_buffer; - struct CUstream_st * child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//paramter buffer for child kernel (in global memory) - //get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); - - //get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - //device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); - - //PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() >0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } - - //copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; - - //create child kernel_info_t and index it with parameter_buffer address - gpgpu_t* gpu=thread->get_gpu(); - device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t & parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << - thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << - "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << - ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); //kernel param - size_t param_start_address = 0; - //copy in word - for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); - } - } - else if(arg == 1) { //cudaStream for the child kernel - - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); - - kernel_info_t & parent_kernel = thread->get_kernel(); - if(child_stream == 0) { //default stream on device for current CTA - child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); - } - else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to stream " << child_stream->get_uid() << ": " << child_stream); - } - - device_launch_op.stream = child_stream; - } - +// Handling device runtime api: +// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + kernel_info_t *device_grid = NULL; + function_info *device_kernel_entry = NULL; + void *parameter_buffer; + struct CUstream_st *child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // paramter buffer for child kernel (in global memory) + // get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " + << parameter_buffer); + + // get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != + g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + // device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " + << device_kernel_entry->get_name()); + + // PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + device_kernel_entry->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + device_kernel_entry->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() > 0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } + + // copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = + device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; + + // create child kernel_info_t and index it with parameter_buffer address + gpgpu_t *gpu = thread->get_gpu(); + device_grid = new kernel_info_t( + config.grid_dim, config.block_dim, device_kernel_entry, + gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t &parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT( + "child kernel launched by " + << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " + << thread->get_ctaid().y << ", " << thread->get_ctaid().z + << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y + << ", " << thread->get_tid().z << ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), + thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); // kernel + // param + size_t param_start_address = 0; + // copy in word + for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read( + (size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, + NULL, NULL); + } + } else if (arg == 1) { // cudaStream for the child kernel + + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + + kernel_info_t &parent_kernel = thread->get_kernel(); + if (child_stream == 0) { // default stream on device for current CTA + child_stream = + parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() + << " to default stream of the cta " + << child_stream->get_uid() << ": " << child_stream); + } else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() << " to stream " + << child_stream->get_uid() << ": " << child_stream); + } + + device_launch_op.stream = child_stream; } - - - //launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - -//Handling device runtime api: -//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) -//flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - - //pStream should be non-zero address in local memory - pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } - else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); - } +// Handling device runtime api: +// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int +// flags) flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + + // pStream should be non-zero address in local memory + pStream_addr = generic_to_local( + thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } else if (arg == + 1) { // unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); } - - //create stream and write back to param0 - CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // create stream and write back to param0 + CUstream_st *stream = + thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, + NULL); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - void cuda_device_runtime::launch_one_device_kernel() { - if(!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); - - stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + if (!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); + + stream_operation stream_op = stream_operation( + op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while(!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while (!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 7f7a0ca..1d661b2 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,67 +1,66 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -//Jin: cuda_device_runtime.h -//Defines CUDA device runtime APIs for CDP support +// Jin: cuda_device_runtime.h +// Defines CUDA device runtime APIs for CDP support class device_launch_config_t { + public: + device_launch_config_t() {} -public: - device_launch_config_t() {} - - device_launch_config_t(dim3 _grid_dim, - dim3 _block_dim, - unsigned int _shared_mem, - function_info * _entry): - grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} - - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info * entry; + device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, + unsigned int _shared_mem, function_info* _entry) + : grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info* entry; }; class device_launch_operation_t { + public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) + : grid(_grid), stream(_stream) {} -public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t *_grid, - CUstream_st * _stream) : - grid(_grid), stream(_stream) {} - - kernel_info_t * grid; //a new child grid - - CUstream_st * stream; + kernel_info_t* grid; // a new child grid + CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime( gpgpu_context* ctx ) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map g_cuda_device_launch_param_map; - std::list g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime(gpgpu_context* ctx) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map g_cuda_device_launch_param_map; + std::list g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index 9f74bb7..54bedc2 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,17 +2,23 @@ // // Copyright (c) 2012-2017 Christian Rau // -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, -// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: // -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // Version 1.12.0 @@ -23,180 +29,182 @@ #define HALF_HALF_HPP /// Combined gcc version number. -#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__) - -//check C++11 language features -#if defined(__clang__) //clang - #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif -/*#elif defined(__INTEL_COMPILER) //Intel C++ - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif*/ -#elif defined(__GNUC__) //gcc - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #endif -#elif defined(_MSC_VER) //Visual C++ - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #define HALF_POP_WARNINGS 1 - #pragma warning(push) - #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned -#endif - -//check C++11 library features +#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// check C++11 language features +#if defined(__clang__) // clang +#if __has_feature(cxx_static_assert) && \ + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if __has_feature(cxx_user_literals) && \ + !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \ + !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +/*#elif defined(__INTEL_COMPILER) + //Intel C++ #if __INTEL_COMPILER >= 1100 && + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? #define + HALF_ENABLE_CPP11_STATIC_ASSERT 1 #endif #if __INTEL_COMPILER >= 1300 && + !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? #define + HALF_ENABLE_CPP11_CONSTEXPR 1 #endif #if __INTEL_COMPILER >= 1300 && + !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? #define + HALF_ENABLE_CPP11_NOEXCEPT 1 #endif #if __INTEL_COMPILER >= 1100 && + !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? #define + HALF_ENABLE_CPP11_LONG_LONG 1 #endif*/ +#elif defined(__GNUC__) // gcc +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#endif +#elif defined(_MSC_VER) // Visual C++ +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#define HALF_POP_WARNINGS 1 +#pragma warning(push) +#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if, + // negative unsigned +#endif + +// check C++11 library features #include -#if defined(_LIBCPP_VERSION) //libc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif -#elif defined(__GLIBCXX__) //libstdc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifdef __clang__ - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #else - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #endif -#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++ - #if _CPPLIB_VER >= 520 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #if _CPPLIB_VER >= 610 - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #endif +#if defined(_LIBCPP_VERSION) // libc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#elif defined(__GLIBCXX__) // libstdc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifdef __clang__ +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#else +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#endif +#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ +#if _CPPLIB_VER >= 520 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#if _CPPLIB_VER >= 610 +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#endif #endif #undef HALF_GNUC_VERSION -//support constexpr +// support constexpr #if HALF_ENABLE_CPP11_CONSTEXPR - #define HALF_CONSTEXPR constexpr - #define HALF_CONSTEXPR_CONST constexpr +#define HALF_CONSTEXPR constexpr +#define HALF_CONSTEXPR_CONST constexpr #else - #define HALF_CONSTEXPR - #define HALF_CONSTEXPR_CONST const +#define HALF_CONSTEXPR +#define HALF_CONSTEXPR_CONST const #endif -//support noexcept +// support noexcept #if HALF_ENABLE_CPP11_NOEXCEPT - #define HALF_NOEXCEPT noexcept - #define HALF_NOTHROW noexcept +#define HALF_NOEXCEPT noexcept +#define HALF_NOTHROW noexcept #else - #define HALF_NOEXCEPT - #define HALF_NOTHROW throw() +#define HALF_NOEXCEPT +#define HALF_NOTHROW throw() #endif #include -#include -#include #include #include #include +#include +#include #if HALF_ENABLE_CPP11_TYPE_TRAITS - #include +#include #endif #if HALF_ENABLE_CPP11_CSTDINT - #include +#include #endif #if HALF_ENABLE_CPP11_HASH - #include +#include #endif - /// Default rounding mode. -/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as -/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one -/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`: +/// This specifies the rounding mode used for all conversions between +/// [half](\ref half_float::half)s and `float`s as well as for the half_cast() +/// if not specifying a rounding mode explicitly. It can be redefined (before +/// including half.hpp) to one of the standard rounding modes using their +/// respective constants or the equivalent values of `std::float_round_style`: /// /// `std::float_round_style` | value | rounding /// ---------------------------------|-------|------------------------- @@ -206,2862 +214,3509 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity /// `std::round_toward_neg_infinity` | 3 | toward negative infinity /// -/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows -/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits::round_style` -/// to synchronize the rounding mode with that of the underlying single-precision implementation. +/// By default this is set to `-1` (`std::round_indeterminate`), which uses +/// truncation (round toward zero, but with overflows set to infinity) and is +/// the fastest rounding mode possible. It can even be set to +/// `std::numeric_limits::round_style` to synchronize the rounding mode +/// with that of the underlying single-precision implementation. #ifndef HALF_ROUND_STYLE - #define HALF_ROUND_STYLE -1 // = std::round_indeterminate +#define HALF_ROUND_STYLE -1 // = std::round_indeterminate #endif /// Tie-breaking behaviour for round to nearest. -/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is -/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and -/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant +/// This specifies if ties in round to nearest should be resolved by rounding to +/// the nearest even value. By default this is defined to `0` resulting in the +/// faster but slightly more biased behaviour of rounding away from zero in +/// half-way cases (and thus equal to the round() function), but can be +/// redefined to `1` (before including half.hpp) if more IEEE-conformant /// behaviour is needed. #ifndef HALF_ROUND_TIES_TO_EVEN - #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero +#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero #endif /// Value signaling overflow. -/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an -/// operation, in particular it just evaluates to positive infinity. -#define HUGE_VALH std::numeric_limits::infinity() +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to +/// a positive value signaling the overflow of an operation, in particular it +/// just evaluates to positive infinity. +#define HUGE_VALH std::numeric_limits::infinity() /// Fast half-precision fma function. -/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate -/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all +/// This symbol is only defined if the fma() function generally executes as fast +/// as, or faster than, a separate half-precision multiplication followed by an +/// addition. Due to the internal single-precision implementation of all /// arithmetic operations, this is in fact always the case. -#define FP_FAST_FMAH 1 +#define FP_FAST_FMAH 1 #ifndef FP_ILOGB0 - #define FP_ILOGB0 INT_MIN +#define FP_ILOGB0 INT_MIN #endif #ifndef FP_ILOGBNAN - #define FP_ILOGBNAN INT_MAX +#define FP_ILOGBNAN INT_MAX #endif #ifndef FP_SUBNORMAL - #define FP_SUBNORMAL 0 +#define FP_SUBNORMAL 0 #endif #ifndef FP_ZERO - #define FP_ZERO 1 +#define FP_ZERO 1 #endif #ifndef FP_NAN - #define FP_NAN 2 +#define FP_NAN 2 #endif #ifndef FP_INFINITE - #define FP_INFINITE 3 +#define FP_INFINITE 3 #endif #ifndef FP_NORMAL - #define FP_NORMAL 4 +#define FP_NORMAL 4 #endif - /// Main namespace for half precision functionality. /// This namespace contains all the functionality provided by the library. -namespace half_float -{ - class half; +namespace half_float { +class half; #if HALF_ENABLE_CPP11_USER_LITERALS - /// Library-defined half-precision literals. - /// Import this namespace to enable half-precision floating point literals: - /// ~~~~{.cpp} - /// using namespace half_float::literal; - /// half_float::half = 4.2_h; - /// ~~~~ - namespace literal - { - half operator"" _h(long double); - } -#endif - - /// \internal - /// \brief Implementation details. - namespace detail - { - #if HALF_ENABLE_CPP11_TYPE_TRAITS - /// Conditional type. - template struct conditional : std::conditional {}; - - /// Helper for tag dispatching. - template struct bool_type : std::integral_constant {}; - using std::true_type; - using std::false_type; - - /// Type traits for floating point types. - template struct is_float : std::is_floating_point {}; - #else - /// Conditional type. - template struct conditional { typedef T type; }; - template struct conditional { typedef F type; }; - - /// Helper for tag dispatching. - template struct bool_type {}; - typedef bool_type true_type; - typedef bool_type false_type; - - /// Type traits for floating point types. - template struct is_float : false_type {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - #endif - - /// Type traits for floating point bits. - template struct bits { typedef unsigned char type; }; - template struct bits : bits {}; - template struct bits : bits {}; - template struct bits : bits {}; - - #if HALF_ENABLE_CPP11_CSTDINT - /// Unsigned integer of (at least) 16 bits width. - typedef std::uint_least16_t uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits { typedef std::uint_least32_t type; }; - - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef std::uint_least64_t type; }; - #else - /// Unsigned integer of (at least) 16 bits width. - typedef unsigned short uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; - #else - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef unsigned long type; }; - #endif - #endif - - /// Tag type for binary construction. - struct binary_t {}; - - /// Tag for binary construction. - HALF_CONSTEXPR_CONST binary_t binary = binary_t(); - - /// Temporary half-precision expression. - /// This class represents a half-precision expression which just stores a single-precision value internally. - struct expr - { - /// Conversion constructor. - /// \param f single-precision value to convert - explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } - - private: - /// Internal expression value stored in single-precision. - float value_; - }; - - /// SFINAE helper for generic half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member equivalent to \a T. - /// \tparam T type to return - template struct enable {}; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - - /// Return type for specialized generic 2-argument half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member denoting the appropriate return type. - /// \tparam T first argument type - /// \tparam U first argument type - template struct result : enable {}; - template<> struct result { typedef half type; }; - - /// \name Classification helpers - /// \{ - - /// Check for infinity. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if infinity - /// \retval false else - template bool builtin_isinf(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isinf(arg); - #elif defined(_MSC_VER) - return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); - #else - return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); - #endif - } - - /// Check for NaN. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if not a number - /// \retval false else - template bool builtin_isnan(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isnan(arg); - #elif defined(_MSC_VER) - return ::_isnan(static_cast(arg)) != 0; - #else - return arg != arg; - #endif - } - - /// Check sign. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if signbit set - /// \retval false else - template bool builtin_signbit(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::signbit(arg); - #else - return arg < T() || (arg == T() && T(1)/arg < T()); - #endif - } - - /// \} - /// \name Conversion - /// \{ - - /// Convert IEEE single-precision to half-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value single-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(float value, true_type) - { - typedef bits::type uint32; - uint32 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(float)); -/* uint16 hbits = (bits>>16) & 0x8000; - bits &= 0x7FFFFFFF; - int exp = bits >> 23; - if(exp == 255) - return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0x7FFFFF)!=0)); - if(exp > 142) - { - if(R == std::round_toward_infinity) - return hbits | 0x7C00 - (hbits>>15); - if(R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits>>15); - return hbits | 0x7BFF + (R!=std::round_toward_zero); - } - int g, s; - if(exp > 112) - { - g = (bits>>12) & 1; - s = (bits&0xFFF) != 0; - hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF); - } - else if(exp > 101) - { - int i = 125 - exp; - bits = (bits&0x7FFFFF) | 0x800000; - g = (bits>>i) & 1; - s = (bits&((1L<> (i+1); - } - else - { - g = 0; - s = bits != 0; - } - if(R == std::round_to_nearest) - #if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s|hbits); - #else - hbits += g; - #endif - else if(R == std::round_toward_infinity) - hbits += ~(hbits>>15) & (s|g); - else if(R == std::round_toward_neg_infinity) - hbits += (hbits>>15) & (g|s); -*/ static const uint16 base_table[512] = { - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, - 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, - 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, - 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, - 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 }; - static const unsigned char shift_table[512] = { - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; - uint16 hbits = base_table[bits>>23] + static_cast((bits&0x7FFFFF)>>shift_table[bits>>23]); - if(R == std::round_to_nearest) - hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00) - #if HALF_ROUND_TIES_TO_EVEN - & (((((static_cast(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits) - #endif - ; - else if(R == std::round_toward_zero) - hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23]; - else if(R == std::round_toward_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)& - ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511)); - else if(R == std::round_toward_neg_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)& - ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255)); - return hbits; - } - - /// Convert IEEE double-precision to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value double-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(double value, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint64 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(double)); - uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; - uint16 hbits = (hi>>16) & 0x8000; - hi &= 0x7FFFFFFF; - int exp = hi >> 20; - if(exp == 2047) - return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0xFFFFFFFFFFFFF)!=0)); - if(exp > 1038) - { - if(R == std::round_toward_infinity) - return hbits | 0x7C00 - (hbits>>15); - if(R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits>>15); - return hbits | 0x7BFF + (R!=std::round_toward_zero); - } - int g, s = lo != 0; - if(exp > 1008) - { - g = (hi>>9) & 1; - s |= (hi&0x1FF) != 0; - hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF); - } - else if(exp > 997) - { - int i = 1018 - exp; - hi = (hi&0xFFFFF) | 0x100000; - g = (hi>>i) & 1; - s |= (hi&((1L<> (i+1); - } - else - { - g = 0; - s |= hi != 0; - } - if(R == std::round_to_nearest) - #if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s|hbits); - #else - hbits += g; - #endif - else if(R == std::round_toward_infinity) - hbits += ~(hbits>>15) & (s|g); - else if(R == std::round_toward_neg_infinity) - hbits += (hbits>>15) & (g|s); - return hbits; - } - - /// Convert non-IEEE floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half_impl(T value, ...) - { - uint16 hbits = static_cast(builtin_signbit(value)) << 15; - if(value == T()) - return hbits; - if(builtin_isnan(value)) - return hbits | 0x7FFF; - if(builtin_isinf(value)) - return hbits | 0x7C00; - int exp; - std::frexp(value, &exp); - if(exp > 16) - { - if(R == std::round_toward_infinity) - return hbits | (0x7C00 - (hbits>>15)); - else if(R == std::round_toward_neg_infinity) - return hbits | (0x7BFF + (hbits>>15)); - return hbits | (0x7BFF + (R!=std::round_toward_zero)); - } - if(exp < -13) - value = std::ldexp(value, 24); - else - { - value = std::ldexp(value, 11-exp); - hbits |= ((exp+13)<<10); - } - T ival, frac = std::modf(value, &ival); - hbits += static_cast(std::abs(static_cast(ival))); - if(R == std::round_to_nearest) - { - frac = std::abs(frac); - #if HALF_ROUND_TIES_TO_EVEN - hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits); - #else - hbits += frac >= T(0.5); - #endif - } - else if(R == std::round_toward_infinity) - hbits += frac > T(); - else if(R == std::round_toward_neg_infinity) - hbits += frac < T(); - return hbits; - } - - /// Convert floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half(T value) - { - return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam S `true` if value negative, `false` else - /// \tparam T type to convert (builtin integer type) - /// \param value non-negative integral value - /// \return binary representation of half-precision value - template uint16 int2half_impl(T value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "int to half conversion only supports builtin integer types"); - #endif - if(S) - value = -value; - uint16 bits = S << 15; - if(value > 0xFFFF) - { - if(R == std::round_toward_infinity) - bits |= 0x7C00 - S; - else if(R == std::round_toward_neg_infinity) - bits |= 0x7BFF + S; - else - bits |= 0x7BFF + (R!=std::round_toward_zero); - } - else if(value) - { - unsigned int m = value, exp = 24; - for(; m<0x400; m<<=1,--exp) ; - for(; m>0x7FF; m>>=1,++exp) ; - bits |= (exp<<10) + m; - if(exp > 24) - { - if(R == std::round_to_nearest) - bits += (value>>(exp-25)) & 1 - #if HALF_ROUND_TIES_TO_EVEN - & (((((1<<(exp-25))-1)&value)!=0)|bits) - #endif - ; - else if(R == std::round_toward_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & !S; - else if(R == std::round_toward_neg_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & S; - } - } - return bits; - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert (builtin integer type) - /// \param value integral value - /// \return binary representation of half-precision value - template uint16 int2half(T value) - { - return (value<0) ? int2half_impl(value) : int2half_impl(value); - } - - /// Convert half-precision to IEEE single-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \param value binary representation of half-precision value - /// \return single-precision value - inline float half2float_impl(uint16 value, float, true_type) - { - typedef bits::type uint32; -/* uint32 bits = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - bits |= 0x38000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,bits-=0x800000) ; - bits += static_cast(abs) << 13; - } -*/ static const uint32 mantissa_table[2048] = { - 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, - 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, - 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, - 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, - 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, - 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, - 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, - 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, - 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, - 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, - 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, - 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, - 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, - 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, - 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, - 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, - 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, - 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, - 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, - 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, - 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, - 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, - 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, - 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, - 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, - 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, - 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, - 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, - 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, - 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, - 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, - 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, - 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, - 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, - 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, - 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, - 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, - 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, - 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, - 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, - 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, - 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, - 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, - 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, - 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, - 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, - 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, - 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, - 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, - 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, - 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, - 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, - 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, - 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, - 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, - 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, - 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, - 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, - 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, - 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, - 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, - 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, - 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, - 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, - 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, - 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, - 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, - 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, - 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, - 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, - 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, - 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, - 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, - 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, - 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, - 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, - 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, - 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, - 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, - 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, - 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, - 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, - 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, - 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, - 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, - 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, - 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, - 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, - 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, - 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, - 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, - 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, - 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, - 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, - 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, - 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, - 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, - 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, - 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, - 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, - 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, - 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, - 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, - 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, - 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, - 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, - 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, - 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, - 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, - 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, - 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, - 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, - 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, - 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, - 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, - 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, - 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, - 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, - 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, - 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, - 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, - 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, - 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, - 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, - 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, - 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, - 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, - 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; - static const uint32 exponent_table[64] = { - 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, - 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, - 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, - 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; - static const unsigned short offset_table[64] = { - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; - uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; -// return *reinterpret_cast(&bits); //violating strict aliasing! - float out; - std::memcpy(&out, &bits, sizeof(float)); - return out; - } - - /// Convert half-precision to IEEE double-precision. - /// \param value binary representation of half-precision value - /// \return double-precision value - inline double half2float_impl(uint16 value, double, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint32 hi = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - hi |= 0x3F000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,hi-=0x100000) ; - hi += static_cast(abs) << 10; - } - uint64 bits = static_cast(hi) << 32; -// return *reinterpret_cast(&bits); //violating strict aliasing! - double out; - std::memcpy(&out, &bits, sizeof(double)); - return out; - } - - /// Convert half-precision to non-IEEE floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float_impl(uint16 value, T, ...) - { - T out; - int abs = value & 0x7FFF; - if(abs > 0x7C00) - out = std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); - else if(abs == 0x7C00) - out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); - else if(abs > 0x3FF) - out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); - else - out = std::ldexp(static_cast(abs), -24); - return (value&0x8000) ? -out : out; - } - - /// Convert half-precision to floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float(uint16 value) - { - return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_impl(uint16 value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "half to int conversion only supports builtin integer types"); - #endif - unsigned int e = value & 0x7FFF; - if(e >= 0x7C00) - return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); - if(e < 0x3800) - { - if(R == std::round_toward_infinity) - return T(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - return -T(value>0x8000); - return T(); - } - unsigned int m = (value&0x3FF) | 0x400; - e >>= 10; - if(e < 25) - { - if(R == std::round_to_nearest) - m += (1<<(24-e)) - (~(m>>(25-e))&E); - else if(R == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(25-e))-1U); - else if(R == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(25-e))-1U); - m >>= 25 - e; - } - else - m <<= e - 25; - return (value&0x8000) ? -static_cast(m) : static_cast(m); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int(uint16 value) { return half2int_impl(value); } - - /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero. - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_up(uint16 value) { return half2int_impl(value); } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half_impl(uint16 value) - { - unsigned int e = value & 0x7FFF; - uint16 result = value; - if(e < 0x3C00) - { - result &= 0x8000; - if(R == std::round_to_nearest) - result |= 0x3C00U & -(e>=(0x3800+E)); - else if(R == std::round_toward_infinity) - result |= 0x3C00U & -(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - result |= 0x3C00U & -(value>0x8000); - } - else if(e < 0x6400) - { - e = 25 - (e>>10); - unsigned int mask = (1<>e)&E); - else if(R == std::round_toward_infinity) - result += mask & ((value>>15)-1); - else if(R == std::round_toward_neg_infinity) - result += mask & -(value>>15); - result &= ~mask; - } - return result; - } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half(uint16 value) { return round_half_impl(value); } - - /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero. - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - inline uint16 round_half_up(uint16 value) { return round_half_impl(value); } - /// \} - - struct functions; - template struct unary_specialized; - template struct binary_specialized; - template struct half_caster; - } - - /// Half-precision floating point type. - /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and - /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and - /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations - /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to - /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic - /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type). - /// - /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and - /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which - /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the - /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of - /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most - /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit - /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if - /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on - /// nearly any reasonable platform. - /// - /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable - /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. - class half - { - friend struct detail::functions; - friend struct detail::unary_specialized; - friend struct detail::binary_specialized; - template friend struct detail::half_caster; - friend class std::numeric_limits; - #if HALF_ENABLE_CPP11_HASH - friend struct std::hash; - #endif - #if HALF_ENABLE_CPP11_USER_LITERALS - friend half literal::operator"" _h(long double); - #endif - - public: - /// Default constructor. - /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics - /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. - HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} - - /// Copy constructor. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - half(detail::expr rhs) : data_(detail::float2half(static_cast(rhs))) {} - - /// Conversion constructor. - /// \param rhs float to convert - explicit half(float rhs) : data_(detail::float2half(rhs)) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - operator float() const { return detail::half2float(data_); } - - /// Assignment operator. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - /// \return reference to this half - half& operator=(detail::expr rhs) { return *this = static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to add - /// \return reference to this half - template typename detail::enable::type operator+=(T rhs) { return *this += static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to subtract - /// \return reference to this half - template typename detail::enable::type operator-=(T rhs) { return *this -= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to multiply with - /// \return reference to this half - template typename detail::enable::type operator*=(T rhs) { return *this *= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to divide by - /// \return reference to this half - template typename detail::enable::type operator/=(T rhs) { return *this /= static_cast(rhs); } - - /// Assignment operator. - /// \param rhs single-precision value to copy from - /// \return reference to this half - half& operator=(float rhs) { data_ = detail::float2half(rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to add - /// \return reference to this half - half& operator+=(float rhs) { data_ = detail::float2half(detail::half2float(data_)+rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to subtract - /// \return reference to this half - half& operator-=(float rhs) { data_ = detail::float2half(detail::half2float(data_)-rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to multiply with - /// \return reference to this half - half& operator*=(float rhs) { data_ = detail::float2half(detail::half2float(data_)*rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to divide by - /// \return reference to this half - half& operator/=(float rhs) { data_ = detail::float2half(detail::half2float(data_)/rhs); return *this; } - - /// Prefix increment. - /// \return incremented half value - half& operator++() { return *this += 1.0f; } - - /// Prefix decrement. - /// \return decremented half value - half& operator--() { return *this -= 1.0f; } - - /// Postfix increment. - /// \return non-incremented half value - half operator++(int) { half out(*this); ++*this; return out; } - - /// Postfix decrement. - /// \return non-decremented half value - half operator--(int) { half out(*this); --*this; return out; } - - private: - /// Rounding mode to use - static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); - - /// Constructor. - /// \param bits binary representation to set half to - HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {} - - /// Internal binary representation - detail::uint16 data_; - }; +/// Library-defined half-precision literals. +/// Import this namespace to enable half-precision floating point literals: +/// ~~~~{.cpp} +/// using namespace half_float::literal; +/// half_float::half = 4.2_h; +/// ~~~~ +namespace literal { +half operator"" _h(long double); +} +#endif + +/// \internal +/// \brief Implementation details. +namespace detail { +#if HALF_ENABLE_CPP11_TYPE_TRAITS +/// Conditional type. +template +struct conditional : std::conditional {}; + +/// Helper for tag dispatching. +template +struct bool_type : std::integral_constant {}; +using std::false_type; +using std::true_type; + +/// Type traits for floating point types. +template +struct is_float : std::is_floating_point {}; +#else +/// Conditional type. +template +struct conditional { + typedef T type; +}; +template +struct conditional { + typedef F type; +}; + +/// Helper for tag dispatching. +template +struct bool_type {}; +typedef bool_type true_type; +typedef bool_type false_type; + +/// Type traits for floating point types. +template +struct is_float : false_type {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +#endif + +/// Type traits for floating point bits. +template +struct bits { + typedef unsigned char type; +}; +template +struct bits : bits {}; +template +struct bits : bits {}; +template +struct bits : bits {}; + +#if HALF_ENABLE_CPP11_CSTDINT +/// Unsigned integer of (at least) 16 bits width. +typedef std::uint_least16_t uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits { + typedef std::uint_least32_t type; +}; + +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef std::uint_least64_t type; +}; +#else +/// Unsigned integer of (at least) 16 bits width. +typedef unsigned short uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits + : conditional::digits >= 32, unsigned int, + unsigned long> {}; -#if HALF_ENABLE_CPP11_USER_LITERALS - namespace literal - { - /// Half literal. - /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due - /// to rather involved conversions. - /// \param value literal value - /// \return half with given value (if representable) - inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half(value)); } - } -#endif - - namespace detail - { - /// Wrapper implementing unspecialized half-precision functions. - struct functions - { - /// Addition implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision sum stored in single-precision - static expr plus(float x, float y) { return expr(x+y); } - - /// Subtraction implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision difference stored in single-precision - static expr minus(float x, float y) { return expr(x-y); } - - /// Multiplication implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision product stored in single-precision - static expr multiplies(float x, float y) { return expr(x*y); } - - /// Division implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision quotient stored in single-precision - static expr divides(float x, float y) { return expr(x/y); } - - /// Output implementation. - /// \param out stream to write to - /// \param arg value to write - /// \return reference to stream - template static std::basic_ostream& write(std::basic_ostream &out, float arg) { return out << arg; } - - /// Input implementation. - /// \param in stream to read from - /// \param arg half to read into - /// \return reference to stream - template static std::basic_istream& read(std::basic_istream &in, half &arg) - { - float f; - if(in >> f) - arg = f; - return in; - } - - /// Modulo implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr remainder(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remainder(x, y)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return expr(builtin_signbit(x) ? -0.0f : 0.0f); - ax = std::fmod(ax, ay+ay); - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - if(ax >= y2) - ax -= ay; - } - return expr(builtin_signbit(x) ? -ax : ax); - #endif - } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \param quo address to store quotient bits at - /// \return Half-precision division remainder stored in single-precision - static expr remquo(float x, float y, int *quo) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remquo(x, y, quo)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - bool sign = builtin_signbit(x), qsign = static_cast(sign^builtin_signbit(y)); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); - ax = std::fmod(ax, 8.0f*ay); - int cquo = 0; - if(ax >= 4.0f * ay) - { - ax -= 4.0f * ay; - cquo += 4; - } - if(ax >= 2.0f * ay) - { - ax -= 2.0f * ay; - cquo += 2; - } - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - ++cquo; - if(ax >= y2) - { - ax -= ay; - ++cquo; - } - } - return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); - #endif - } - - /// Positive difference implementation. - /// \param x first operand - /// \param y second operand - /// \return Positive difference stored in single-precision - static expr fdim(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fdim(x, y)); - #else - return expr((x<=y) ? 0.0f : (x-y)); - #endif - } - - /// Fused multiply-add implementation. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return \a x * \a y + \a z stored in single-precision - static expr fma(float x, float y, float z) - { - #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) - return expr(std::fma(x, y, z)); - #else - return expr(x*y+z); - #endif - } - - /// Get NaN. - /// \return Half-precision quiet NaN - static half nanh() { return half(binary, 0x7FFF); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp(float arg) { return expr(std::exp(arg)); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr expm1(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::expm1(arg)); - #else - return expr(static_cast(std::exp(static_cast(arg))-1.0)); - #endif - } - - /// Binary exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::exp2(arg)); - #else - return expr(static_cast(std::exp(arg*0.69314718055994530941723212145818))); - #endif - } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log(float arg) { return expr(std::log(arg)); } - - /// Common logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log10(float arg) { return expr(std::log10(arg)); } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log1p(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log1p(arg)); - #else - return expr(static_cast(std::log(1.0+arg))); - #endif - } - - /// Binary logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log2(arg)); - #else - return expr(static_cast(std::log(static_cast(arg))*1.4426950408889634073599246810019)); - #endif - } - - /// Square root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sqrt(float arg) { return expr(std::sqrt(arg)); } - - /// Cubic root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cbrt(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::cbrt(arg)); - #else - if(builtin_isnan(arg) || builtin_isinf(arg)) - return expr(arg); - return expr(builtin_signbit(arg) ? -static_cast(std::pow(-static_cast(arg), 1.0/3.0)) : - static_cast(std::pow(static_cast(arg), 1.0/3.0))); - #endif - } - - /// Hypotenuse implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr hypot(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::hypot(x, y)); - #else - return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits::infinity() : - static_cast(std::sqrt(static_cast(x)*x+static_cast(y)*y))); - #endif - } - - /// Power implementation. - /// \param base value to exponentiate - /// \param exp power to expontiate to - /// \return function value stored in single-preicision - static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } - - /// Sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sin(float arg) { return expr(std::sin(arg)); } - - /// Cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cos(float arg) { return expr(std::cos(arg)); } - - /// Tan implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tan(float arg) { return expr(std::tan(arg)); } - - /// Arc sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asin(float arg) { return expr(std::asin(arg)); } - - /// Arc cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acos(float arg) { return expr(std::acos(arg)); } - - /// Arc tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atan(float arg) { return expr(std::atan(arg)); } - - /// Arc tangent implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } - - /// Hyperbolic sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sinh(float arg) { return expr(std::sinh(arg)); } - - /// Hyperbolic cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cosh(float arg) { return expr(std::cosh(arg)); } - - /// Hyperbolic tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tanh(float arg) { return expr(std::tanh(arg)); } - - /// Hyperbolic area sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asinh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::asinh(arg)); - #else - return expr((arg==-std::numeric_limits::infinity()) ? arg : static_cast(std::log(arg+std::sqrt(arg*arg+1.0)))); - #endif - } - - /// Hyperbolic area cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acosh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::acosh(arg)); - #else - return expr((arg<-1.0f) ? std::numeric_limits::quiet_NaN() : static_cast(std::log(arg+std::sqrt(arg*arg-1.0)))); - #endif - } - - /// Hyperbolic area tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atanh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::atanh(arg)); - #else - return expr(static_cast(0.5*std::log((1.0+arg)/(1.0-arg)))); - #endif - } - - /// Error function implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erf(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erf(arg)); - #else - return expr(static_cast(erf(static_cast(arg)))); - #endif - } - - /// Complementary implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erfc(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erfc(arg)); - #else - return expr(static_cast(1.0-erf(static_cast(arg)))); - #endif - } - - /// Gamma logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr lgamma(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::lgamma(arg)); - #else - if(builtin_isinf(arg)) - return expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::infinity()); - return expr(static_cast(1.1447298858494001741434273513531- - std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg))); - } - return expr(static_cast(lgamma(static_cast(arg)))); - #endif - } - - /// Gamma implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tgamma(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::tgamma(arg)); - #else - if(arg == 0.0f) - return builtin_signbit(arg) ? expr(-std::numeric_limits::infinity()) : expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::quiet_NaN()); - double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg))); - return expr(static_cast((std::fmod(i, 2.0f)==0.0f) ? -value : value)); - } - if(builtin_isinf(arg)) - return expr(arg); - return expr(static_cast(std::exp(lgamma(static_cast(arg))))); - #endif - } - - /// Floor implementation. - /// \param arg value to round - /// \return rounded value - static half floor(half arg) { return half(binary, round_half(arg.data_)); } - - /// Ceiling implementation. - /// \param arg value to round - /// \return rounded value - static half ceil(half arg) { return half(binary, round_half(arg.data_)); } - - /// Truncation implementation. - /// \param arg value to round - /// \return rounded value - static half trunc(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half round(half arg) { return half(binary, round_half_up(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half rint(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lrint(half arg) { return detail::half2int(arg.data_); } - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llrint(half arg) { return detail::half2int(arg.data_); } - #endif - - /// Decompression implementation. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return normalized significant - static half frexp(half arg, int *exp) - { - int m = arg.data_ & 0x7FFF, e = -14; - if(m >= 0x7C00 || !m) - return *exp = 0, arg; - for(; m<0x400; m<<=1,--e) ; - return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF)); - } - - /// Decompression implementation. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part - static half modf(half arg, half *iptr) - { - unsigned int e = arg.data_ & 0x7FFF; - if(e >= 0x6400) - return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00))); - if(e < 0x3C00) - return iptr->data_ = arg.data_ & 0x8000, arg; - e >>= 10; - unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask; - iptr->data_ = arg.data_ & ~mask; - if(!m) - return half(binary, arg.data_&0x8000); - for(; m<0x400; m<<=1,--e) ; - return half(binary, static_cast((arg.data_&0x8000)|(e<<10)|(m&0x3FF))); - } - - /// Scaling implementation. - /// \param arg number to scale - /// \param exp power of two to scale by - /// \return scaled number - static half scalbln(half arg, long exp) - { - unsigned int m = arg.data_ & 0x7FFF; - if(m >= 0x7C00 || !m) - return arg; - for(; m<0x400; m<<=1,--exp) ; - exp += m >> 10; - uint16 value = arg.data_ & 0x8000; - if(exp > 30) - { - if(half::round_style == std::round_toward_zero) - value |= 0x7BFF; - else if(half::round_style == std::round_toward_infinity) - value |= 0x7C00 - (value>>15); - else if(half::round_style == std::round_toward_neg_infinity) - value |= 0x7BFF + (value>>15); - else - value |= 0x7C00; - } - else if(exp > 0) - value |= (exp<<10) | (m&0x3FF); - else if(exp > -11) - { - m = (m&0x3FF) | 0x400; - if(half::round_style == std::round_to_nearest) - { - m += 1 << -exp; - #if HALF_ROUND_TIES_TO_EVEN - m -= (m>>(1-exp)) & 1; - #endif - } - else if(half::round_style == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(1-exp))-1U); - else if(half::round_style == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(1-exp))-1U); - value |= m >> (1-exp); - } - else if(half::round_style == std::round_toward_infinity) - value -= (value>>15) - 1; - else if(half::round_style == std::round_toward_neg_infinity) - value += value >> 15; - return half(binary, value); - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static int ilogb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return FP_ILOGB0; - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - return exp; - } - if(abs > 0x7C00) - return FP_ILOGBNAN; - return INT_MAX; - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static half logb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(binary, 0xFC00); - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - uint16 bits = (exp<0) << 15; - if(exp) - { - unsigned int m = std::abs(exp) << 6, e = 18; - for(; m<0x400; m<<=1,--e) ; - bits |= (e<<10) + m; - } - return half(binary, bits); - } - if(abs > 0x7C00) - return arg; - return half(binary, 0x7C00); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nextafter(half from, half to) - { - uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; - if(fabs > 0x7C00) - return from; - if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs)) - return to; - if(!fabs) - return half(binary, (to.data_&0x8000)+1); - bool lt = ((fabs==from.data_) ? static_cast(fabs) : -static_cast(fabs)) < - ((tabs==to.data_) ? static_cast(tabs) : -static_cast(tabs)); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lt))<<1)-1); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nexttoward(half from, long double to) - { - if(isnan(from)) - return from; - long double lfrom = static_cast(from); - if(builtin_isnan(to) || lfrom == to) - return half(static_cast(to)); - if(!(from.data_&0x7FFF)) - return half(binary, (static_cast(builtin_signbit(to))<<15)+1); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lfrom0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO; - } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if finite number - /// \retval false else - static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if not a number - /// \retval false else - static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if normal number - /// \retval false else - static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } - - /// Sign bit implementation. - /// \param arg value to check - /// \retval true if signed - /// \retval false if unsigned - static bool signbit(half arg) { return (arg.data_&0x8000) != 0; } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x > \a y - /// \retval false else - static bool isgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x >= \a y - /// \retval false else - static bool isgreaterequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x < \a y - /// \retval false else - static bool isless(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x <= \a y - /// \retval false else - static bool islessequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if either \a x > \a y nor \a x < \a y - /// \retval false else - static bool islessgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00 || yabs > 0x7C00) - return false; - int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs; - return a < b || a > b; - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operand unordered - /// \retval false else - static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } - - private: - static double erf(double arg) - { - if(builtin_isinf(arg)) - return (arg<0.0) ? -1.0 : 1.0; - double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2))); - return builtin_signbit(arg) ? -value : value; - } - - static double lgamma(double arg) - { - double v = 1.0; - for(; arg<8.0; ++arg) v *= arg; - double w = 1.0 / (arg*arg); - return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+ - -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+ - -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+ - -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg + - 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg); - } - }; - - /// Wrapper for unary half-precision functions needing specialization for individual argument types. - /// \tparam T argument type - template struct unary_specialized - { - /// Negation implementation. - /// \param arg value to negate - /// \return negated value - static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); } - - /// Absolute value implementation. - /// \param arg function argument - /// \return absolute value - static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); } - }; - template<> struct unary_specialized - { - static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } - static expr fabs(float arg) { return expr(std::fabs(arg)); } - }; - - /// Wrapper for binary half-precision functions needing specialization for individual argument types. - /// \tparam T first argument type - /// \tparam U first argument type - template struct binary_specialized - { - /// Minimum implementation. - /// \param x first operand - /// \param y second operand - /// \return minimum value - static expr fmin(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmin(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::min(x, y)); - #endif - } - - /// Maximum implementation. - /// \param x first operand - /// \param y second operand - /// \return maximum value - static expr fmax(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmax(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::max(x, y)); - #endif - } - }; - template<> struct binary_specialized - { - static half fmin(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - static half fmax(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - }; - - /// Helper class for half casts. - /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member - /// function and a corresponding `type` member denoting its return type. - /// \tparam T destination type - /// \tparam U source type - /// \tparam R rounding mode to use - template struct half_caster {}; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); - #endif - - static half cast(U arg) { return cast_impl(arg, is_float()); }; - - private: - static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } - static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(half arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(half arg, true_type) { return half2float(arg.data_); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(expr arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(float arg, true_type) { return static_cast(arg); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - static half cast(half arg) { return arg; } - }; - template struct half_caster : half_caster {}; - - /// \name Comparison operators - /// \{ - - /// Comparison for equality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - template typename enable::type operator==(T x, U y) { return functions::isequal(x, y); } - - /// Comparison for inequality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - template typename enable::type operator!=(T x, U y) { return functions::isnotequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else - template typename enable::type operator<(T x, U y) { return functions::isless(x, y); } - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else - template typename enable::type operator>(T x, U y) { return functions::isgreater(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else - template typename enable::type operator<=(T x, U y) { return functions::islessequal(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else - template typename enable::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); } - - /// \} - /// \name Arithmetic operators - /// \{ - - /// Add halfs. - /// \param x left operand - /// \param y right operand - /// \return sum of half expressions - template typename enable::type operator+(T x, U y) { return functions::plus(x, y); } - - /// Subtract halfs. - /// \param x left operand - /// \param y right operand - /// \return difference of half expressions - template typename enable::type operator-(T x, U y) { return functions::minus(x, y); } - - /// Multiply halfs. - /// \param x left operand - /// \param y right operand - /// \return product of half expressions - template typename enable::type operator*(T x, U y) { return functions::multiplies(x, y); } - - /// Divide halfs. - /// \param x left operand - /// \param y right operand - /// \return quotient of half expressions - template typename enable::type operator/(T x, U y) { return functions::divides(x, y); } - - /// Identity. - /// \param arg operand - /// \return uncahnged operand - template HALF_CONSTEXPR typename enable::type operator+(T arg) { return arg; } - - /// Negation. - /// \param arg operand - /// \return negated operand - template HALF_CONSTEXPR typename enable::type operator-(T arg) { return unary_specialized::negate(arg); } - - /// \} - /// \name Input and output - /// \{ - - /// Output operator. - /// \param out output stream to write into - /// \param arg half expression to write - /// \return reference to output stream - template typename enable&,T>::type - operator<<(std::basic_ostream &out, T arg) { return functions::write(out, arg); } - - /// Input operator. - /// \param in input stream to read from - /// \param arg half to read into - /// \return reference to input stream - template std::basic_istream& - operator>>(std::basic_istream &in, half &arg) { return functions::read(in, arg); } - - /// \} - /// \name Basic mathematical operations - /// \{ - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type abs(T arg) { return unary_specialized::fabs(arg); } - inline half abs(half arg) { return unary_specialized::fabs(arg); } - inline expr abs(expr arg) { return unary_specialized::fabs(arg); } - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type fabs(T arg) { return unary_specialized::fabs(arg); } - inline half fabs(half arg) { return unary_specialized::fabs(arg); } - inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type fmod(T x, U y) { return functions::fmod(x, y); } - inline expr fmod(half x, half y) { return functions::fmod(x, y); } - inline expr fmod(half x, expr y) { return functions::fmod(x, y); } - inline expr fmod(expr x, half y) { return functions::fmod(x, y); } - inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type remainder(T x, U y) { return functions::remainder(x, y); } - inline expr remainder(half x, half y) { return functions::remainder(x, y); } - inline expr remainder(half x, expr y) { return functions::remainder(x, y); } - inline expr remainder(expr x, half y) { return functions::remainder(x, y); } - inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \param quo address to store some bits of quotient at - /// \return remainder of floating point division. -// template typename enable::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); } - - /// Fused multiply add. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return ( \a x * \a y ) + \a z rounded as one operation. -// template typename enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } - - /// Maximum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return maximum of operands -// template typename result::type fmax(T x, U y) { return binary_specialized::fmax(x, y); } - inline half fmax(half x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(half x, expr y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, expr y) { return binary_specialized::fmax(x, y); } - - /// Minimum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return minimum of operands -// template typename result::type fmin(T x, U y) { return binary_specialized::fmin(x, y); } - inline half fmin(half x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(half x, expr y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, expr y) { return binary_specialized::fmin(x, y); } - - /// Positive difference. - /// \param x first operand - /// \param y second operand - /// \return \a x - \a y or 0 if difference negative -// template typename enable::type fdim(T x, U y) { return functions::fdim(x, y); } - inline expr fdim(half x, half y) { return functions::fdim(x, y); } - inline expr fdim(half x, expr y) { return functions::fdim(x, y); } - inline expr fdim(expr x, half y) { return functions::fdim(x, y); } - inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } - - /// Get NaN value. - /// \return quiet NaN - inline half nanh(const char*) { return functions::nanh(); } - - /// \} - /// \name Exponential functions - /// \{ - - /// Exponential function. - /// \param arg function argument - /// \return e raised to \a arg -// template typename enable::type exp(T arg) { return functions::exp(arg); } - inline expr exp(half arg) { return functions::exp(arg); } - inline expr exp(expr arg) { return functions::exp(arg); } - - /// Exponential minus one. - /// \param arg function argument - /// \return e raised to \a arg subtracted by 1 -// template typename enable::type expm1(T arg) { return functions::expm1(arg); } - inline expr expm1(half arg) { return functions::expm1(arg); } - inline expr expm1(expr arg) { return functions::expm1(arg); } - - /// Binary exponential. - /// \param arg function argument - /// \return 2 raised to \a arg -// template typename enable::type exp2(T arg) { return functions::exp2(arg); } - inline expr exp2(half arg) { return functions::exp2(arg); } - inline expr exp2(expr arg) { return functions::exp2(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base e -// template typename enable::type log(T arg) { return functions::log(arg); } - inline expr log(half arg) { return functions::log(arg); } - inline expr log(expr arg) { return functions::log(arg); } - - /// Common logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 10 -// template typename enable::type log10(T arg) { return functions::log10(arg); } - inline expr log10(half arg) { return functions::log10(arg); } - inline expr log10(expr arg) { return functions::log10(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg plus 1 to base e -// template typename enable::type log1p(T arg) { return functions::log1p(arg); } - inline expr log1p(half arg) { return functions::log1p(arg); } - inline expr log1p(expr arg) { return functions::log1p(arg); } - - /// Binary logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 2 -// template typename enable::type log2(T arg) { return functions::log2(arg); } - inline expr log2(half arg) { return functions::log2(arg); } - inline expr log2(expr arg) { return functions::log2(arg); } - - /// \} - /// \name Power functions - /// \{ - - /// Square root. - /// \param arg function argument - /// \return square root of \a arg -// template typename enable::type sqrt(T arg) { return functions::sqrt(arg); } - inline expr sqrt(half arg) { return functions::sqrt(arg); } - inline expr sqrt(expr arg) { return functions::sqrt(arg); } - - /// Cubic root. - /// \param arg function argument - /// \return cubic root of \a arg -// template typename enable::type cbrt(T arg) { return functions::cbrt(arg); } - inline expr cbrt(half arg) { return functions::cbrt(arg); } - inline expr cbrt(expr arg) { return functions::cbrt(arg); } - - /// Hypotenuse function. - /// \param x first argument - /// \param y second argument - /// \return square root of sum of squares without internal over- or underflows -// template typename enable::type hypot(T x, U y) { return functions::hypot(x, y); } - inline expr hypot(half x, half y) { return functions::hypot(x, y); } - inline expr hypot(half x, expr y) { return functions::hypot(x, y); } - inline expr hypot(expr x, half y) { return functions::hypot(x, y); } - inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } - - /// Power function. - /// \param base first argument - /// \param exp second argument - /// \return \a base raised to \a exp -// template typename enable::type pow(T base, U exp) { return functions::pow(base, exp); } - inline expr pow(half base, half exp) { return functions::pow(base, exp); } - inline expr pow(half base, expr exp) { return functions::pow(base, exp); } - inline expr pow(expr base, half exp) { return functions::pow(base, exp); } - inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } - - /// \} - /// \name Trigonometric functions - /// \{ - - /// Sine function. - /// \param arg function argument - /// \return sine value of \a arg -// template typename enable::type sin(T arg) { return functions::sin(arg); } - inline expr sin(half arg) { return functions::sin(arg); } - inline expr sin(expr arg) { return functions::sin(arg); } - - /// Cosine function. - /// \param arg function argument - /// \return cosine value of \a arg -// template typename enable::type cos(T arg) { return functions::cos(arg); } - inline expr cos(half arg) { return functions::cos(arg); } - inline expr cos(expr arg) { return functions::cos(arg); } - - /// Tangent function. - /// \param arg function argument - /// \return tangent value of \a arg -// template typename enable::type tan(T arg) { return functions::tan(arg); } - inline expr tan(half arg) { return functions::tan(arg); } - inline expr tan(expr arg) { return functions::tan(arg); } - - /// Arc sine. - /// \param arg function argument - /// \return arc sine value of \a arg -// template typename enable::type asin(T arg) { return functions::asin(arg); } - inline expr asin(half arg) { return functions::asin(arg); } - inline expr asin(expr arg) { return functions::asin(arg); } - - /// Arc cosine function. - /// \param arg function argument - /// \return arc cosine value of \a arg -// template typename enable::type acos(T arg) { return functions::acos(arg); } - inline expr acos(half arg) { return functions::acos(arg); } - inline expr acos(expr arg) { return functions::acos(arg); } - - /// Arc tangent function. - /// \param arg function argument - /// \return arc tangent value of \a arg -// template typename enable::type atan(T arg) { return functions::atan(arg); } - inline expr atan(half arg) { return functions::atan(arg); } - inline expr atan(expr arg) { return functions::atan(arg); } - - /// Arc tangent function. - /// \param x first argument - /// \param y second argument - /// \return arc tangent value -// template typename enable::type atan2(T x, U y) { return functions::atan2(x, y); } - inline expr atan2(half x, half y) { return functions::atan2(x, y); } - inline expr atan2(half x, expr y) { return functions::atan2(x, y); } - inline expr atan2(expr x, half y) { return functions::atan2(x, y); } - inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } - - /// \} - /// \name Hyperbolic functions - /// \{ - - /// Hyperbolic sine. - /// \param arg function argument - /// \return hyperbolic sine value of \a arg -// template typename enable::type sinh(T arg) { return functions::sinh(arg); } - inline expr sinh(half arg) { return functions::sinh(arg); } - inline expr sinh(expr arg) { return functions::sinh(arg); } - - /// Hyperbolic cosine. - /// \param arg function argument - /// \return hyperbolic cosine value of \a arg -// template typename enable::type cosh(T arg) { return functions::cosh(arg); } - inline expr cosh(half arg) { return functions::cosh(arg); } - inline expr cosh(expr arg) { return functions::cosh(arg); } - - /// Hyperbolic tangent. - /// \param arg function argument - /// \return hyperbolic tangent value of \a arg -// template typename enable::type tanh(T arg) { return functions::tanh(arg); } - inline expr tanh(half arg) { return functions::tanh(arg); } - inline expr tanh(expr arg) { return functions::tanh(arg); } - - /// Hyperbolic area sine. - /// \param arg function argument - /// \return area sine value of \a arg -// template typename enable::type asinh(T arg) { return functions::asinh(arg); } - inline expr asinh(half arg) { return functions::asinh(arg); } - inline expr asinh(expr arg) { return functions::asinh(arg); } - - /// Hyperbolic area cosine. - /// \param arg function argument - /// \return area cosine value of \a arg -// template typename enable::type acosh(T arg) { return functions::acosh(arg); } - inline expr acosh(half arg) { return functions::acosh(arg); } - inline expr acosh(expr arg) { return functions::acosh(arg); } - - /// Hyperbolic area tangent. - /// \param arg function argument - /// \return area tangent value of \a arg -// template typename enable::type atanh(T arg) { return functions::atanh(arg); } - inline expr atanh(half arg) { return functions::atanh(arg); } - inline expr atanh(expr arg) { return functions::atanh(arg); } - - /// \} - /// \name Error and gamma functions - /// \{ - - /// Error function. - /// \param arg function argument - /// \return error function value of \a arg -// template typename enable::type erf(T arg) { return functions::erf(arg); } - inline expr erf(half arg) { return functions::erf(arg); } - inline expr erf(expr arg) { return functions::erf(arg); } - - /// Complementary error function. - /// \param arg function argument - /// \return 1 minus error function value of \a arg -// template typename enable::type erfc(T arg) { return functions::erfc(arg); } - inline expr erfc(half arg) { return functions::erfc(arg); } - inline expr erfc(expr arg) { return functions::erfc(arg); } - - /// Natural logarithm of gamma function. - /// \param arg function argument - /// \return natural logarith of gamma function for \a arg -// template typename enable::type lgamma(T arg) { return functions::lgamma(arg); } - inline expr lgamma(half arg) { return functions::lgamma(arg); } - inline expr lgamma(expr arg) { return functions::lgamma(arg); } - - /// Gamma function. - /// \param arg function argument - /// \return gamma function value of \a arg -// template typename enable::type tgamma(T arg) { return functions::tgamma(arg); } - inline expr tgamma(half arg) { return functions::tgamma(arg); } - inline expr tgamma(expr arg) { return functions::tgamma(arg); } - - /// \} - /// \name Rounding - /// \{ - - /// Nearest integer not less than half value. - /// \param arg half to round - /// \return nearest integer not less than \a arg -// template typename enable::type ceil(T arg) { return functions::ceil(arg); } - inline half ceil(half arg) { return functions::ceil(arg); } - inline half ceil(expr arg) { return functions::ceil(arg); } - - /// Nearest integer not greater than half value. - /// \param arg half to round - /// \return nearest integer not greater than \a arg -// template typename enable::type floor(T arg) { return functions::floor(arg); } - inline half floor(half arg) { return functions::floor(arg); } - inline half floor(expr arg) { return functions::floor(arg); } - - /// Nearest integer not greater in magnitude than half value. - /// \param arg half to round - /// \return nearest integer not greater in magnitude than \a arg -// template typename enable::type trunc(T arg) { return functions::trunc(arg); } - inline half trunc(half arg) { return functions::trunc(arg); } - inline half trunc(expr arg) { return functions::trunc(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type round(T arg) { return functions::round(arg); } - inline half round(half arg) { return functions::round(arg); } - inline half round(expr arg) { return functions::round(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type lround(T arg) { return functions::lround(arg); } - inline long lround(half arg) { return functions::lround(arg); } - inline long lround(expr arg) { return functions::lround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type nearbyint(T arg) { return functions::nearbyint(arg); } - inline half nearbyint(half arg) { return functions::rint(arg); } - inline half nearbyint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type rint(T arg) { return functions::rint(arg); } - inline half rint(half arg) { return functions::rint(arg); } - inline half rint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type lrint(T arg) { return functions::lrint(arg); } - inline long lrint(half arg) { return functions::lrint(arg); } - inline long lrint(expr arg) { return functions::lrint(arg); } - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type llround(T arg) { return functions::llround(arg); } - inline long long llround(half arg) { return functions::llround(arg); } - inline long long llround(expr arg) { return functions::llround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type llrint(T arg) { return functions::llrint(arg); } - inline long long llrint(half arg) { return functions::llrint(arg); } - inline long long llrint(expr arg) { return functions::llrint(arg); } - #endif - - /// \} - /// \name Floating point manipulation - /// \{ - - /// Decompress floating point number. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return significant in range [0.5, 1) -// template typename enable::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Extract integer and fractional parts. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part -// template typename enable::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent - /// \retval FP_ILOGB0 for zero - /// \retval FP_ILOGBNAN for NaN - /// \retval MAX_INT for infinity -// template typename enable::type ilogb(T arg) { return functions::ilogb(arg); } - inline int ilogb(half arg) { return functions::ilogb(arg); } - inline int ilogb(expr arg) { return functions::ilogb(arg); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent -// template typename enable::type logb(T arg) { return functions::logb(arg); } - inline half logb(half arg) { return functions::logb(arg); } - inline half logb(expr arg) { return functions::logb(arg); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nextafter(T from, U to) { return functions::nextafter(from, to); } - inline half nextafter(half from, half to) { return functions::nextafter(from, to); } - inline half nextafter(half from, expr to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, half to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); } - - /// Take sign. - /// \param x value to change sign for - /// \param y value to take sign from - /// \return value equal to \a x in magnitude and to \a y in sign -// template typename enable::type copysign(T x, U y) { return functions::copysign(x, y); } - inline half copysign(half x, half y) { return functions::copysign(x, y); } - inline half copysign(half x, expr y) { return functions::copysign(x, y); } - inline half copysign(expr x, half y) { return functions::copysign(x, y); } - inline half copysign(expr x, expr y) { return functions::copysign(x, y); } - - /// \} - /// \name Floating point classification - /// \{ - - - /// Classify floating point value. - /// \param arg number to classify - /// \retval FP_ZERO for positive and negative zero - /// \retval FP_SUBNORMAL for subnormal numbers - /// \retval FP_INFINITY for positive and negative infinity - /// \retval FP_NAN for NaNs - /// \retval FP_NORMAL for all other (normal) values -// template typename enable::type fpclassify(T arg) { return functions::fpclassify(arg); } - inline int fpclassify(half arg) { return functions::fpclassify(arg); } - inline int fpclassify(expr arg) { return functions::fpclassify(arg); } - - /// Check if finite number. - /// \param arg number to check - /// \retval true if neither infinity nor NaN - /// \retval false else -// template typename enable::type isfinite(T arg) { return functions::isfinite(arg); } - inline bool isfinite(half arg) { return functions::isfinite(arg); } - inline bool isfinite(expr arg) { return functions::isfinite(arg); } - - /// Check for infinity. - /// \param arg number to check - /// \retval true for positive or negative infinity - /// \retval false else -// template typename enable::type isinf(T arg) { return functions::isinf(arg); } - inline bool isinf(half arg) { return functions::isinf(arg); } - inline bool isinf(expr arg) { return functions::isinf(arg); } - - /// Check for NaN. - /// \param arg number to check - /// \retval true for NaNs - /// \retval false else -// template typename enable::type isnan(T arg) { return functions::isnan(arg); } - inline bool isnan(half arg) { return functions::isnan(arg); } - inline bool isnan(expr arg) { return functions::isnan(arg); } - - /// Check if normal number. - /// \param arg number to check - /// \retval true if normal number - /// \retval false if either subnormal, zero, infinity or NaN -// template typename enable::type isnormal(T arg) { return functions::isnormal(arg); } - inline bool isnormal(half arg) { return functions::isnormal(arg); } - inline bool isnormal(expr arg) { return functions::isnormal(arg); } - - /// Check sign. - /// \param arg number to check - /// \retval true for negative number - /// \retval false for positive number -// template typename enable::type signbit(T arg) { return functions::signbit(arg); } - inline bool signbit(half arg) { return functions::signbit(arg); } - inline bool signbit(expr arg) { return functions::signbit(arg); } - - /// \} - /// \name Comparison - /// \{ - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else -// template typename enable::type isgreater(T x, U y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else -// template typename enable::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else -// template typename enable::type isless(T x, U y) { return functions::isless(x, y); } - inline bool isless(half x, half y) { return functions::isless(x, y); } - inline bool isless(half x, expr y) { return functions::isless(x, y); } - inline bool isless(expr x, half y) { return functions::isless(x, y); } - inline bool isless(expr x, expr y) { return functions::isless(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else -// template typename enable::type islessequal(T x, U y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } - - /// Comarison for less or greater. - /// \param x first operand - /// \param y second operand - /// \retval true if either less or greater - /// \retval false else -// template typename enable::type islessgreater(T x, U y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); } - - /// Check if unordered. - /// \param x first operand - /// \param y second operand - /// \retval true if unordered (one or two NaN operands) - /// \retval false else -// template typename enable::type isunordered(T x, U y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } - - /// \name Casting - /// \{ - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// It uses the default rounding mode. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam R rounding mode to use. - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - /// \} - } - - using detail::operator==; - using detail::operator!=; - using detail::operator<; - using detail::operator>; - using detail::operator<=; - using detail::operator>=; - using detail::operator+; - using detail::operator-; - using detail::operator*; - using detail::operator/; - using detail::operator<<; - using detail::operator>>; - - using detail::abs; - using detail::fabs; - using detail::fmod; - using detail::remainder; - using detail::remquo; - using detail::fma; - using detail::fmax; - using detail::fmin; - using detail::fdim; - using detail::nanh; - using detail::exp; - using detail::expm1; - using detail::exp2; - using detail::log; - using detail::log10; - using detail::log1p; - using detail::log2; - using detail::sqrt; - using detail::cbrt; - using detail::hypot; - using detail::pow; - using detail::sin; - using detail::cos; - using detail::tan; - using detail::asin; - using detail::acos; - using detail::atan; - using detail::atan2; - using detail::sinh; - using detail::cosh; - using detail::tanh; - using detail::asinh; - using detail::acosh; - using detail::atanh; - using detail::erf; - using detail::erfc; - using detail::lgamma; - using detail::tgamma; - using detail::ceil; - using detail::floor; - using detail::trunc; - using detail::round; - using detail::lround; - using detail::nearbyint; - using detail::rint; - using detail::lrint; #if HALF_ENABLE_CPP11_LONG_LONG - using detail::llround; - using detail::llrint; -#endif - using detail::frexp; - using detail::ldexp; - using detail::modf; - using detail::scalbn; - using detail::scalbln; - using detail::ilogb; - using detail::logb; - using detail::nextafter; - using detail::nexttoward; - using detail::copysign; - using detail::fpclassify; - using detail::isfinite; - using detail::isinf; - using detail::isnan; - using detail::isnormal; - using detail::signbit; - using detail::isgreater; - using detail::isgreaterequal; - using detail::isless; - using detail::islessequal; - using detail::islessgreater; - using detail::isunordered; - - using detail::half_cast; +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits + : conditional::digits >= 64, + unsigned long, unsigned long long> {}; +#else +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef unsigned long type; +}; +#endif +#endif + +/// Tag type for binary construction. +struct binary_t {}; + +/// Tag for binary construction. +HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + +/// Temporary half-precision expression. +/// This class represents a half-precision expression which just stores a +/// single-precision value internally. +struct expr { + /// Conversion constructor. + /// \param f single-precision value to convert + explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } + + private: + /// Internal expression value stored in single-precision. + float value_; +}; + +/// SFINAE helper for generic half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding `type` member equivalent to \a T. +/// \tparam T type to return +template +struct enable {}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; + +/// Return type for specialized generic 2-argument half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding `type` member denoting the +/// appropriate return type. \tparam T first argument type \tparam U first +/// argument type +template +struct result : enable {}; +template <> +struct result { + typedef half type; +}; + +/// \name Classification helpers +/// \{ + +/// Check for infinity. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if infinity +/// \retval false else +template +bool builtin_isinf(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); +#elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && + !::_isnan(static_cast(arg)); +#else + return arg == std::numeric_limits::infinity() || + arg == -std::numeric_limits::infinity(); +#endif } +/// Check for NaN. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if not a number +/// \retval false else +template +bool builtin_isnan(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); +#elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; +#else + return arg != arg; +#endif +} -/// Extensions to the C++ standard library. -namespace std -{ - /// Numeric limits for half-precision floats. - /// Because of the underlying single-precision implementation of many operations, it inherits some properties from - /// `std::numeric_limits`. - template<> class numeric_limits : public numeric_limits - { - public: - /// Supports signed values. - static HALF_CONSTEXPR_CONST bool is_signed = true; +/// Check sign. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if signbit set +/// \retval false else +template +bool builtin_signbit(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); +#else + return arg < T() || (arg == T() && T(1) / arg < T()); +#endif +} + +/// \} +/// \name Conversion +/// \{ + +/// Convert IEEE single-precision to half-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \tparam R +/// rounding mode to use, `std::round_indeterminate` for fastest rounding \param +/// value single-precision value \return binary representation of half-precision +/// value +template +uint16 float2half_impl(float value, true_type) { + typedef bits::type uint32; + uint32 bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(float)); + /* uint16 hbits = (bits>>16) & 0x8000; + bits &= 0x7FFFFFFF; + int exp = bits >> 23; + if(exp == 255) + return hbits | 0x7C00 | + (0x3FF&-static_cast((bits&0x7FFFFF)!=0)); if(exp > 142) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + + (R!=std::round_toward_zero); + } + int g, s; + if(exp > 112) + { + g = (bits>>12) & 1; + s = (bits&0xFFF) != 0; + hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF); + } + else if(exp > 101) + { + int i = 125 - exp; + bits = (bits&0x7FFFFF) | 0x800000; + g = (bits>>i) & 1; + s = (bits&((1L<> (i+1); + } + else + { + g = 0; + s = bits != 0; + } + if(R == std::round_to_nearest) + #if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s|hbits); + #else + hbits += g; + #endif + else if(R == std::round_toward_infinity) + hbits += ~(hbits>>15) & (s|g); + else if(R == std::round_toward_neg_infinity) + hbits += (hbits>>15) & (g|s); + */ + static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, + 0x0020, 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, + 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, + 0x3800, 0x3C00, 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, + 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, + 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, 0x8200, + 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, + 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, 0xC000, 0xC400, 0xC800, + 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, + 0xF000, 0xF400, 0xF800, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00}; + static const unsigned char shift_table[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, + 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, + 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 13}; + uint16 hbits = + base_table[bits >> 23] + + static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); + if (R == std::round_to_nearest) + hbits += + (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) | + (((bits >> 23) & 0xFF) == 102)) & + ((hbits & 0x7C00) != 0x7C00) +#if HALF_ROUND_TIES_TO_EVEN + & (((((static_cast(1) << (shift_table[bits >> 23] - 1)) - 1) & + bits) != 0) | + hbits) +#endif + ; + else if (R == std::round_toward_zero) + hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23]; + else if (R == std::round_toward_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 102) & ((bits >> 23) != 0))) & + (hbits < 0x7C00)) - + ((hbits == 0xFC00) & ((bits >> 23) != 511)); + else if (R == std::round_toward_neg_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 358) & ((bits >> 23) != 256))) & + (hbits < 0xFC00) & (hbits >> 15)) - + ((hbits == 0x7C00) & ((bits >> 23) != 255)); + return hbits; +} + +/// Convert IEEE double-precision to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \param value double-precision value \return binary representation +/// of half-precision value +template +uint16 float2half_impl(double value, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint64 bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(double)); + uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; + uint16 hbits = (hi >> 16) & 0x8000; + hi &= 0x7FFFFFFF; + int exp = hi >> 20; + if (exp == 2047) + return hbits | 0x7C00 | + (0x3FF & -static_cast((bits & 0xFFFFFFFFFFFFF) != 0)); + if (exp > 1038) { + if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15); + if (R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits >> 15); + return hbits | 0x7BFF + (R != std::round_toward_zero); + } + int g, s = lo != 0; + if (exp > 1008) { + g = (hi >> 9) & 1; + s |= (hi & 0x1FF) != 0; + hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF); + } else if (exp > 997) { + int i = 1018 - exp; + hi = (hi & 0xFFFFF) | 0x100000; + g = (hi >> i) & 1; + s |= (hi & ((1L << i) - 1)) != 0; + hbits |= hi >> (i + 1); + } else { + g = 0; + s |= hi != 0; + } + if (R == std::round_to_nearest) +#if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s | hbits); +#else + hbits += g; +#endif + else if (R == std::round_toward_infinity) + hbits += ~(hbits >> 15) & (s | g); + else if (R == std::round_toward_neg_infinity) + hbits += (hbits >> 15) & (g | s); + return hbits; +} + +/// Convert non-IEEE floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T source type (builtin floating point type) \param value +/// floating point value \return binary representation of half-precision value +template +uint16 float2half_impl(T value, ...) { + uint16 hbits = static_cast(builtin_signbit(value)) << 15; + if (value == T()) return hbits; + if (builtin_isnan(value)) return hbits | 0x7FFF; + if (builtin_isinf(value)) return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if (exp > 16) { + if (R == std::round_toward_infinity) + return hbits | (0x7C00 - (hbits >> 15)); + else if (R == std::round_toward_neg_infinity) + return hbits | (0x7BFF + (hbits >> 15)); + return hbits | (0x7BFF + (R != std::round_toward_zero)); + } + if (exp < -13) + value = std::ldexp(value, 24); + else { + value = std::ldexp(value, 11 - exp); + hbits |= ((exp + 13) << 10); + } + T ival, frac = std::modf(value, &ival); + hbits += static_cast(std::abs(static_cast(ival))); + if (R == std::round_to_nearest) { + frac = std::abs(frac); +#if HALF_ROUND_TIES_TO_EVEN + hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits); +#else + hbits += frac >= T(0.5); +#endif + } else if (R == std::round_toward_infinity) + hbits += frac > T(); + else if (R == std::round_toward_neg_infinity) + hbits += frac < T(); + return hbits; +} + +/// Convert floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T source type (builtin floating point type) \param value +/// floating point value \return binary representation of half-precision value +template +uint16 float2half(T value) { + return float2half_impl( + value, bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} + +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam S `true` if value negative, `false` else \tparam T type to +/// convert (builtin integer type) \param value non-negative integral value +/// \return binary representation of half-precision value +template +uint16 int2half_impl(T value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "int to half conversion only supports builtin integer types"); +#endif + if (S) value = -value; + uint16 bits = S << 15; + if (value > 0xFFFF) { + if (R == std::round_toward_infinity) + bits |= 0x7C00 - S; + else if (R == std::round_toward_neg_infinity) + bits |= 0x7BFF + S; + else + bits |= 0x7BFF + (R != std::round_toward_zero); + } else if (value) { + unsigned int m = value, exp = 24; + for (; m < 0x400; m <<= 1, --exp) + ; + for (; m > 0x7FF; m >>= 1, ++exp) + ; + bits |= (exp << 10) + m; + if (exp > 24) { + if (R == std::round_to_nearest) + bits += (value >> (exp - 25)) & 1 +#if HALF_ROUND_TIES_TO_EVEN + & (((((1 << (exp - 25)) - 1) & value) != 0) | bits) +#endif + ; + else if (R == std::round_toward_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S; + else if (R == std::round_toward_neg_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S; + } + } + return bits; +} + +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T type to convert (builtin integer type) \param value +/// integral value \return binary representation of half-precision value +template +uint16 int2half(T value) { + return (value < 0) ? int2half_impl(value) + : int2half_impl(value); +} + +/// Convert half-precision to IEEE single-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \param +/// value binary representation of half-precision value \return single-precision +/// value +inline float half2float_impl(uint16 value, float, true_type) { + typedef bits::type uint32; + /* uint32 bits = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + bits |= 0x38000000 << + static_cast(abs>=0x7C00); for(; abs<0x400; + abs<<=1,bits-=0x800000) ; bits += static_cast(abs) << 13; + } + */ + static const uint32 mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, + 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, + 0x35400000, 0x35500000, 0x35600000, 0x35700000, 0x35800000, 0x35880000, + 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, + 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, + 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, 0x360C0000, + 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, + 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, + 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, + 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000, 0x36820000, + 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, + 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, + 0x369C0000, 0x369E0000, 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, + 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, + 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, + 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, + 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000, + 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, + 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, + 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, 0x37020000, 0x37030000, + 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, + 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, + 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, + 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, 0x37200000, 0x37210000, + 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, + 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, + 0x372E0000, 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, + 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, + 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, + 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, + 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000, + 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, + 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, + 0x375E0000, 0x375F0000, 0x37600000, 0x37610000, 0x37620000, 0x37630000, + 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, + 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, + 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, + 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, 0x37800000, 0x37808000, + 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, + 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, + 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, 0x37898000, + 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, + 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, + 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, + 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, 0x37988000, + 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, + 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, + 0x379F0000, 0x379F8000, 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, + 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, + 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, + 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, + 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, + 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, + 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, + 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, + 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, + 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, + 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, + 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, 0x37C80000, 0x37C88000, + 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, + 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, + 0x37CF0000, 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, + 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, + 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, + 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, + 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000, + 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, + 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, + 0x37E70000, 0x37E78000, 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, + 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, + 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, + 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, + 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, 0x37F80000, 0x37F88000, + 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, + 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, + 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, 0x3800C000, + 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, + 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, + 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, + 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, 0x38084000, + 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, + 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, + 0x380B8000, 0x380BC000, 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, + 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, + 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, + 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, + 0x38130000, 0x38134000, 0x38138000, 0x3813C000, 0x38140000, 0x38144000, + 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, + 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, + 0x38178000, 0x3817C000, 0x38180000, 0x38184000, 0x38188000, 0x3818C000, + 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, + 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, + 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, + 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, 0x38200000, 0x38204000, + 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, + 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, + 0x38238000, 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000, + 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, + 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, + 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, + 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000, + 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, + 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, + 0x382F8000, 0x382FC000, 0x38300000, 0x38304000, 0x38308000, 0x3830C000, + 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, + 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, + 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, + 0x38370000, 0x38374000, 0x38378000, 0x3837C000, 0x38380000, 0x38384000, + 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, + 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, + 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, + 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, + 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, + 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, + 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000, 0x38444000, + 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, + 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, + 0x38478000, 0x3847C000, 0x38480000, 0x38484000, 0x38488000, 0x3848C000, + 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, + 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, + 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, + 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, 0x38500000, 0x38504000, + 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, + 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, + 0x38538000, 0x3853C000, 0x38540000, 0x38544000, 0x38548000, 0x3854C000, + 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, + 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, + 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, + 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, 0x385C0000, 0x385C4000, + 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, + 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, + 0x385F8000, 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000, + 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, + 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, + 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, + 0x38670000, 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000, + 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, + 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, + 0x386B8000, 0x386BC000, 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, + 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, + 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, + 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, + 0x38730000, 0x38734000, 0x38738000, 0x3873C000, 0x38740000, 0x38744000, + 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, + 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, + 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000, 0x3878C000, + 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, + 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, + 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, + 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000, 0x38002000, + 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, + 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, + 0x3801C000, 0x3801E000, 0x38020000, 0x38022000, 0x38024000, 0x38026000, + 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, + 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, + 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, + 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, 0x38060000, 0x38062000, + 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, + 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, + 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, 0x38084000, 0x38086000, + 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, + 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, + 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, + 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, 0x380C0000, 0x380C2000, + 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, + 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, + 0x380DC000, 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, + 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, + 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, + 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, + 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000, + 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, + 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, + 0x3813C000, 0x3813E000, 0x38140000, 0x38142000, 0x38144000, 0x38146000, + 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, + 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, + 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, + 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, 0x38180000, 0x38182000, + 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, + 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, + 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, + 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, + 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, + 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, + 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, 0x381E2000, + 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, + 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, + 0x381FC000, 0x381FE000, 0x38200000, 0x38202000, 0x38204000, 0x38206000, + 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, + 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, + 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, + 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, + 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, + 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, + 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, 0x38264000, 0x38266000, + 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, + 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, + 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, + 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, 0x382A0000, 0x382A2000, + 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, + 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, + 0x382BC000, 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, + 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, + 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, + 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, + 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000, + 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, + 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, + 0x3831C000, 0x3831E000, 0x38320000, 0x38322000, 0x38324000, 0x38326000, + 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, + 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, + 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, + 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, 0x38360000, 0x38362000, + 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, + 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, + 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, 0x38386000, + 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, + 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, + 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, + 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, 0x383C2000, + 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, + 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, + 0x383DC000, 0x383DE000, 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, + 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, + 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, + 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, + 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, + 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, + 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, + 0x3843C000, 0x3843E000, 0x38440000, 0x38442000, 0x38444000, 0x38446000, + 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, + 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, + 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, + 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, 0x38480000, 0x38482000, + 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, + 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, + 0x3849C000, 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, + 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, + 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, + 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, + 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000, + 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, + 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, + 0x384FC000, 0x384FE000, 0x38500000, 0x38502000, 0x38504000, 0x38506000, + 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, + 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, + 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, + 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, 0x38540000, 0x38542000, + 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, + 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, + 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, 0x38566000, + 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, + 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, + 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, + 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000, 0x385A2000, + 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, + 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, + 0x385BC000, 0x385BE000, 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, + 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, + 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, + 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, + 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, 0x38600000, 0x38602000, + 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, + 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, + 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, 0x38624000, 0x38626000, + 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, + 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, + 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, + 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, 0x38660000, 0x38662000, + 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, + 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, + 0x3867C000, 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, + 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, + 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, + 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, + 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000, + 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, + 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, + 0x386DC000, 0x386DE000, 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, + 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, + 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, + 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, + 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, 0x38720000, 0x38722000, + 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, + 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, + 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000, 0x38746000, + 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, + 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, + 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, + 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, 0x38782000, + 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, + 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, + 0x3879C000, 0x3879E000, 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, + 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, + 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, + 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, + 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, + 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, + 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, + 0x387FC000, 0x387FE000}; + static const uint32 exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, + 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, + 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000, + 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, + 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, + 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000, + 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, + 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, + 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, + 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000}; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; + uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] + + exponent_table[value >> 10]; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + float out; + std::memcpy(&out, &bits, sizeof(float)); + return out; +} + +/// Convert half-precision to IEEE double-precision. +/// \param value binary representation of half-precision value +/// \return double-precision value +inline double half2float_impl(uint16 value, double, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint32 hi = static_cast(value & 0x8000) << 16; + int abs = value & 0x7FFF; + if (abs) { + hi |= 0x3F000000 << static_cast(abs >= 0x7C00); + for (; abs < 0x400; abs <<= 1, hi -= 0x100000) + ; + hi += static_cast(abs) << 10; + } + uint64 bits = static_cast(hi) << 32; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + double out; + std::memcpy(&out, &bits, sizeof(double)); + return out; +} + +/// Convert half-precision to non-IEEE floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float_impl(uint16 value, T, ...) { + T out; + int abs = value & 0x7FFF; + if (abs > 0x7C00) + out = std::numeric_limits::has_quiet_NaN + ? std::numeric_limits::quiet_NaN() + : T(); + else if (abs == 0x7C00) + out = std::numeric_limits::has_infinity + ? std::numeric_limits::infinity() + : std::numeric_limits::max(); + else if (abs > 0x3FF) + out = std::ldexp(static_cast((abs & 0x3FF) | 0x400), (abs >> 10) - 25); + else + out = std::ldexp(static_cast(abs), -24); + return (value & 0x8000) ? -out : out; +} + +/// Convert half-precision to floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float(uint16 value) { + return half2float_impl(value, T(), + bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} + +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam E `true` for round to even, `false` for round away from +/// zero \tparam T type to convert to (buitlin integer type with at least 16 +/// bits precision, excluding any implicit sign bits) \param value binary +/// representation of half-precision value \return integral value +template +T half2int_impl(uint16 value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "half to int conversion only supports builtin integer types"); +#endif + unsigned int e = value & 0x7FFF; + if (e >= 0x7C00) + return (value & 0x8000) ? std::numeric_limits::min() + : std::numeric_limits::max(); + if (e < 0x3800) { + if (R == std::round_toward_infinity) + return T(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + return -T(value > 0x8000); + return T(); + } + unsigned int m = (value & 0x3FF) | 0x400; + e >>= 10; + if (e < 25) { + if (R == std::round_to_nearest) + m += (1 << (24 - e)) - (~(m >> (25 - e)) & E); + else if (R == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U); + else if (R == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (25 - e)) - 1U); + m >>= 25 - e; + } else + m <<= e - 25; + return (value & 0x8000) ? -static_cast(m) : static_cast(m); +} + +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T type to convert to (buitlin integer type with at least 16 +/// bits precision, excluding any implicit sign bits) \param value binary +/// representation of half-precision value \return integral value +template +T half2int(uint16 value) { + return half2int_impl(value); +} + +/// Convert half-precision floating point to integer using +/// round-to-nearest-away-from-zero. \tparam T type to convert to (buitlin +/// integer type with at least 16 bits precision, excluding any implicit sign +/// bits) \param value binary representation of half-precision value \return +/// integral value +template +T half2int_up(uint16 value) { + return half2int_impl(value); +} + +/// Round half-precision number to nearest integer value. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam E `true` for round to even, `false` for round away from +/// zero \param value binary representation of half-precision value \return +/// half-precision bits for nearest integral value +template +uint16 round_half_impl(uint16 value) { + unsigned int e = value & 0x7FFF; + uint16 result = value; + if (e < 0x3C00) { + result &= 0x8000; + if (R == std::round_to_nearest) + result |= 0x3C00U & -(e >= (0x3800 + E)); + else if (R == std::round_toward_infinity) + result |= 0x3C00U & -(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + result |= 0x3C00U & -(value > 0x8000); + } else if (e < 0x6400) { + e = 25 - (e >> 10); + unsigned int mask = (1 << e) - 1; + if (R == std::round_to_nearest) + result += (1 << (e - 1)) - (~(result >> e) & E); + else if (R == std::round_toward_infinity) + result += mask & ((value >> 15) - 1); + else if (R == std::round_toward_neg_infinity) + result += mask & -(value >> 15); + result &= ~mask; + } + return result; +} + +/// Round half-precision number to nearest integer value. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \param value binary representation of half-precision value \return +/// half-precision bits for nearest integral value +template +uint16 round_half(uint16 value) { + return round_half_impl(value); +} + +/// Round half-precision number to nearest integer value using +/// round-to-nearest-away-from-zero. \param value binary representation of +/// half-precision value \return half-precision bits for nearest integral value +inline uint16 round_half_up(uint16 value) { + return round_half_impl(value); +} +/// \} + +struct functions; +template +struct unary_specialized; +template +struct binary_specialized; +template +struct half_caster; +} // namespace detail + +/// Half-precision floating point type. +/// This class implements an IEEE-conformant half-precision floating point type +/// with the usual arithmetic operators and conversions. It is implicitly +/// convertible to single-precision floating point, which makes artihmetic +/// expressions and functions with mixed-type operands to be of the most precise +/// operand type. Additionally all arithmetic operations (and many mathematical +/// functions) are carried out in single-precision internally. All conversions +/// from single- to half-precision are done using the library's default rounding +/// mode, but temporary results inside chained arithmetic expressions are kept +/// in single-precision as long as possible (while of course still maintaining a +/// strong half-precision type). +/// +/// According to the C++98/03 definition, the half type is not a POD type. But +/// according to C++11's less strict and extended definitions it is both a +/// standard layout type and a trivially copyable type (even if not a POD type), +/// which means it can be standard-conformantly copied using raw binary copies. +/// But in this context some more words about the actual size of the type. +/// Although the half is representing an IEEE 16-bit type, it does not +/// neccessarily have to be of exactly 16-bits size. But on any reasonable +/// implementation the actual binary representation of this type will most +/// probably not ivolve any additional "magic" or padding beyond the simple +/// binary representation of the underlying 16-bit IEEE number, even if not +/// strictly guaranteed by the standard. But even then it only has an actual +/// size of 16 bits if your C++ implementation supports an unsigned integer type +/// of exactly 16 bits width. But this should be the case on nearly any +/// reasonable platform. +/// +/// So if your C++ implementation is not totally exotic or imposes special +/// alignment requirements, it is a reasonable assumption that the data of a +/// half is just comprised of the 2 bytes of the underlying IEEE representation. +class half { + friend struct detail::functions; + friend struct detail::unary_specialized; + friend struct detail::binary_specialized; + template + friend struct detail::half_caster; + friend class std::numeric_limits; +#if HALF_ENABLE_CPP11_HASH + friend struct std::hash; +#endif +#if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator"" _h(long double); +#endif - /// Is not exact. - static HALF_CONSTEXPR_CONST bool is_exact = false; + public: + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin + /// types' default-initialization semantics and may be less efficient than no + /// initialization, it is needed to provide proper value-initialization + /// semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Copy constructor. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + half(detail::expr rhs) + : data_(detail::float2half(static_cast(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + explicit half(float rhs) : data_(detail::float2half(rhs)) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + /// \return reference to this half + half &operator=(detail::expr rhs) { return *this = static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + template + typename detail::enable::type operator+=(T rhs) { + return *this += static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + template + typename detail::enable::type operator-=(T rhs) { + return *this -= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + template + typename detail::enable::type operator*=(T rhs) { + return *this *= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + template + typename detail::enable::type operator/=(T rhs) { + return *this /= static_cast(rhs); + } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + half &operator=(float rhs) { + data_ = detail::float2half(rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + half &operator+=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) + rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + half &operator-=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) - rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + half &operator*=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) * rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + half &operator/=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) / rhs); + return *this; + } + + /// Prefix increment. + /// \return incremented half value + half &operator++() { return *this += 1.0f; } + + /// Prefix decrement. + /// \return decremented half value + half &operator--() { return *this -= 1.0f; } + + /// Postfix increment. + /// \return non-incremented half value + half operator++(int) { + half out(*this); + ++*this; + return out; + } + + /// Postfix decrement. + /// \return non-decremented half value + half operator--(int) { + half out(*this); + --*this; + return out; + } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = + (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT + : data_(bits) {} + + /// Internal binary representation + detail::uint16 data_; +}; - /// Doesn't provide modulo arithmetic. - static HALF_CONSTEXPR_CONST bool is_modulo = false; +#if HALF_ENABLE_CPP11_USER_LITERALS +namespace literal { +/// Half literal. +/// While this returns an actual half-precision value, half literals can +/// unfortunately not be constant expressions due to rather involved +/// conversions. \param value literal value \return half with given value (if +/// representable) +inline half operator"" _h(long double value) { + return half(detail::binary, detail::float2half(value)); +} +} // namespace literal +#endif - /// IEEE conformant. - static HALF_CONSTEXPR_CONST bool is_iec559 = true; +namespace detail { +/// Wrapper implementing unspecialized half-precision functions. +struct functions { + /// Addition implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision sum stored in single-precision + static expr plus(float x, float y) { return expr(x + y); } + + /// Subtraction implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision difference stored in single-precision + static expr minus(float x, float y) { return expr(x - y); } + + /// Multiplication implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision product stored in single-precision + static expr multiplies(float x, float y) { return expr(x * y); } + + /// Division implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision quotient stored in single-precision + static expr divides(float x, float y) { return expr(x / y); } + + /// Output implementation. + /// \param out stream to write to + /// \param arg value to write + /// \return reference to stream + template + static std::basic_ostream &write( + std::basic_ostream &out, float arg) { + return out << arg; + } + + /// Input implementation. + /// \param in stream to read from + /// \param arg half to read into + /// \return reference to stream + template + static std::basic_istream &read( + std::basic_istream &in, half &arg) { + float f; + if (in >> f) arg = f; + return in; + } + + /// Modulo implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr remainder(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remainder(x, y)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f); + ax = std::fmod(ax, ay + ay); + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + if (ax >= y2) ax -= ay; + } + return expr(builtin_signbit(x) ? -ax : ax); +#endif + } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \param quo address to store quotient bits at + /// \return Half-precision division remainder stored in single-precision + static expr remquo(float x, float y, int *quo) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remquo(x, y, quo)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + bool sign = builtin_signbit(x), + qsign = static_cast(sign ^ builtin_signbit(y)); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); + ax = std::fmod(ax, 8.0f * ay); + int cquo = 0; + if (ax >= 4.0f * ay) { + ax -= 4.0f * ay; + cquo += 4; + } + if (ax >= 2.0f * ay) { + ax -= 2.0f * ay; + cquo += 2; + } + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + ++cquo; + if (ax >= y2) { + ax -= ay; + ++cquo; + } + } + return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); +#endif + } + + /// Positive difference implementation. + /// \param x first operand + /// \param y second operand + /// \return Positive difference stored in single-precision + static expr fdim(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fdim(x, y)); +#else + return expr((x <= y) ? 0.0f : (x - y)); +#endif + } + + /// Fused multiply-add implementation. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return \a x * \a y + \a z stored in single-precision + static expr fma(float x, float y, float z) { +#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) + return expr(std::fma(x, y, z)); +#else + return expr(x * y + z); +#endif + } + + /// Get NaN. + /// \return Half-precision quiet NaN + static half nanh() { return half(binary, 0x7FFF); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp(float arg) { return expr(std::exp(arg)); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr expm1(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::expm1(arg)); +#else + return expr(static_cast(std::exp(static_cast(arg)) - 1.0)); +#endif + } + + /// Binary exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::exp2(arg)); +#else + return expr( + static_cast(std::exp(arg * 0.69314718055994530941723212145818))); +#endif + } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log(float arg) { return expr(std::log(arg)); } + + /// Common logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log10(float arg) { return expr(std::log10(arg)); } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log1p(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log1p(arg)); +#else + return expr(static_cast(std::log(1.0 + arg))); +#endif + } + + /// Binary logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log2(arg)); +#else + return expr(static_cast(std::log(static_cast(arg)) * + 1.4426950408889634073599246810019)); +#endif + } + + /// Square root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sqrt(float arg) { return expr(std::sqrt(arg)); } + + /// Cubic root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cbrt(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::cbrt(arg)); +#else + if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg); + return expr(builtin_signbit(arg) + ? -static_cast( + std::pow(-static_cast(arg), 1.0 / 3.0)) + : static_cast( + std::pow(static_cast(arg), 1.0 / 3.0))); +#endif + } + + /// Hypotenuse implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr hypot(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::hypot(x, y)); +#else + return expr( + (builtin_isinf(x) || builtin_isinf(y)) + ? std::numeric_limits::infinity() + : static_cast(std::sqrt(static_cast(x) * x + + static_cast(y) * y))); +#endif + } + + /// Power implementation. + /// \param base value to exponentiate + /// \param exp power to expontiate to + /// \return function value stored in single-preicision + static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } + + /// Sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sin(float arg) { return expr(std::sin(arg)); } + + /// Cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cos(float arg) { return expr(std::cos(arg)); } + + /// Tan implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tan(float arg) { return expr(std::tan(arg)); } + + /// Arc sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asin(float arg) { return expr(std::asin(arg)); } + + /// Arc cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acos(float arg) { return expr(std::acos(arg)); } + + /// Arc tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atan(float arg) { return expr(std::atan(arg)); } + + /// Arc tangent implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } + + /// Hyperbolic sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sinh(float arg) { return expr(std::sinh(arg)); } + + /// Hyperbolic cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cosh(float arg) { return expr(std::cosh(arg)); } + + /// Hyperbolic tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tanh(float arg) { return expr(std::tanh(arg)); } + + /// Hyperbolic area sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asinh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::asinh(arg)); +#else + return expr( + (arg == -std::numeric_limits::infinity()) + ? arg + : static_cast(std::log(arg + std::sqrt(arg * arg + 1.0)))); +#endif + } + + /// Hyperbolic area cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acosh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::acosh(arg)); +#else + return expr((arg < -1.0f) ? std::numeric_limits::quiet_NaN() + : static_cast(std::log( + arg + std::sqrt(arg * arg - 1.0)))); +#endif + } + + /// Hyperbolic area tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atanh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::atanh(arg)); +#else + return expr(static_cast(0.5 * std::log((1.0 + arg) / (1.0 - arg)))); +#endif + } + + /// Error function implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erf(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erf(arg)); +#else + return expr(static_cast(erf(static_cast(arg)))); +#endif + } + + /// Complementary implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erfc(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erfc(arg)); +#else + return expr(static_cast(1.0 - erf(static_cast(arg)))); +#endif + } + + /// Gamma logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr lgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::lgamma(arg)); +#else + if (builtin_isinf(arg)) return expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::infinity()); + return expr(static_cast( + 1.1447298858494001741434273513531 - + std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) - + lgamma(1.0 - arg))); + } + return expr(static_cast(lgamma(static_cast(arg)))); +#endif + } + + /// Gamma implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::tgamma(arg)); +#else + if (arg == 0.0f) + return builtin_signbit(arg) + ? expr(-std::numeric_limits::infinity()) + : expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::quiet_NaN()); + double value = 3.1415926535897932384626433832795 / + (std::sin(3.1415926535897932384626433832795 * f) * + std::exp(lgamma(1.0 - arg))); + return expr( + static_cast((std::fmod(i, 2.0f) == 0.0f) ? -value : value)); + } + if (builtin_isinf(arg)) return expr(arg); + return expr(static_cast(std::exp(lgamma(static_cast(arg))))); +#endif + } + + /// Floor implementation. + /// \param arg value to round + /// \return rounded value + static half floor(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Ceiling implementation. + /// \param arg value to round + /// \return rounded value + static half ceil(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Truncation implementation. + /// \param arg value to round + /// \return rounded value + static half trunc(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half round(half arg) { return half(binary, round_half_up(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half rint(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lrint(half arg) { + return detail::half2int(arg.data_); + } - /// Supports infinity. - static HALF_CONSTEXPR_CONST bool has_infinity = true; +#if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llround(half arg) { + return detail::half2int_up(arg.data_); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llrint(half arg) { + return detail::half2int(arg.data_); + } +#endif - /// Supports quiet NaNs. - static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + /// Decompression implementation. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return normalized significant + static half frexp(half arg, int *exp) { + int m = arg.data_ & 0x7FFF, e = -14; + if (m >= 0x7C00 || !m) return *exp = 0, arg; + for (; m < 0x400; m <<= 1, --e) + ; + return *exp = e + (m >> 10), + half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF)); + } + + /// Decompression implementation. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + static half modf(half arg, half *iptr) { + unsigned int e = arg.data_ & 0x7FFF; + if (e >= 0x6400) + return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00))); + if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg; + e >>= 10; + unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if (!m) return half(binary, arg.data_ & 0x8000); + for (; m < 0x400; m <<= 1, --e) + ; + return half(binary, static_cast((arg.data_ & 0x8000) | (e << 10) | + (m & 0x3FF))); + } + + /// Scaling implementation. + /// \param arg number to scale + /// \param exp power of two to scale by + /// \return scaled number + static half scalbln(half arg, long exp) { + unsigned int m = arg.data_ & 0x7FFF; + if (m >= 0x7C00 || !m) return arg; + for (; m < 0x400; m <<= 1, --exp) + ; + exp += m >> 10; + uint16 value = arg.data_ & 0x8000; + if (exp > 30) { + if (half::round_style == std::round_toward_zero) + value |= 0x7BFF; + else if (half::round_style == std::round_toward_infinity) + value |= 0x7C00 - (value >> 15); + else if (half::round_style == std::round_toward_neg_infinity) + value |= 0x7BFF + (value >> 15); + else + value |= 0x7C00; + } else if (exp > 0) + value |= (exp << 10) | (m & 0x3FF); + else if (exp > -11) { + m = (m & 0x3FF) | 0x400; + if (half::round_style == std::round_to_nearest) { + m += 1 << -exp; +#if HALF_ROUND_TIES_TO_EVEN + m -= (m >> (1 - exp)) & 1; +#endif + } else if (half::round_style == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U); + else if (half::round_style == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (1 - exp)) - 1U); + value |= m >> (1 - exp); + } else if (half::round_style == std::round_toward_infinity) + value -= (value >> 15) - 1; + else if (half::round_style == std::round_toward_neg_infinity) + value += value >> 15; + return half(binary, value); + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static int ilogb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return FP_ILOGB0; + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + return exp; + } + if (abs > 0x7C00) return FP_ILOGBNAN; + return INT_MAX; + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static half logb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return half(binary, 0xFC00); + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + uint16 bits = (exp < 0) << 15; + if (exp) { + unsigned int m = std::abs(exp) << 6, e = 18; + for (; m < 0x400; m <<= 1, --e) + ; + bits |= (e << 10) + m; + } + return half(binary, bits); + } + if (abs > 0x7C00) return arg; + return half(binary, 0x7C00); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nextafter(half from, half to) { + uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if (fabs > 0x7C00) return from; + if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to; + if (!fabs) return half(binary, (to.data_ & 0x8000) + 1); + bool lt = + ((fabs == from.data_) ? static_cast(fabs) + : -static_cast(fabs)) < + ((tabs == to.data_) ? static_cast(tabs) : -static_cast(tabs)); + return half(binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lt)) << 1) - + 1); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nexttoward(half from, long double to) { + if (isnan(from)) return from; + long double lfrom = static_cast(from); + if (builtin_isnan(to) || lfrom == to) return half(static_cast(to)); + if (!(from.data_ & 0x7FFF)) + return half(binary, + (static_cast(builtin_signbit(to)) << 15) + 1); + return half( + binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lfrom < to)) << 1) - + 1); + } + + /// Sign implementation + /// \param x first operand + /// \param y second operand + /// \return composed value + static half copysign(half x, half y) { + return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000)); + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static int fpclassify(half arg) { + unsigned int abs = arg.data_ & 0x7FFF; + return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00) + ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE) + : FP_NORMAL) + : FP_SUBNORMAL) + : FP_ZERO; + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if finite number + /// \retval false else + static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if not a number + /// \retval false else + static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if normal number + /// \retval false else + static bool isnormal(half arg) { + return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00); + } + + /// Sign bit implementation. + /// \param arg value to check + /// \retval true if signed + /// \retval false if unsigned + static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + static bool isequal(half x, half y) { + return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + static bool isnotequal(half x, half y) { + return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x > \a y + /// \retval false else + static bool isgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x >= \a y + /// \retval false else + static bool isgreaterequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) >= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x < \a y + /// \retval false else + static bool isless(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x <= \a y + /// \retval false else + static bool islessequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) <= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if either \a x > \a y nor \a x < \a y + /// \retval false else + static bool islessgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00 || yabs > 0x7C00) return false; + int a = (xabs == x.data_) ? xabs : -xabs, + b = (yabs == y.data_) ? yabs : -yabs; + return a < b || a > b; + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operand unordered + /// \retval false else + static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + private: + static double erf(double arg) { + if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0; + double x2 = arg * arg, ax2 = 0.147 * x2, + value = std::sqrt( + 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) / + (1.0 + ax2))); + return builtin_signbit(arg) ? -value : value; + } + + static double lgamma(double arg) { + double v = 1.0; + for (; arg < 8.0; ++arg) v *= arg; + double w = 1.0 / (arg * arg); + return (((((((-0.02955065359477124183006535947712 * w + + 0.00641025641025641025641025641026) * + w + + -0.00191752691752691752691752691753) * + w + + 8.4175084175084175084175084175084e-4) * + w + + -5.952380952380952380952380952381e-4) * + w + + 7.9365079365079365079365079365079e-4) * + w + + -0.00277777777777777777777777777778) * + w + + 0.08333333333333333333333333333333) / + arg + + 0.91893853320467274178032973640562 - std::log(v) - arg + + (arg - 0.5) * std::log(arg); + } +}; + +/// Wrapper for unary half-precision functions needing specialization for +/// individual argument types. \tparam T argument type +template +struct unary_specialized { + /// Negation implementation. + /// \param arg value to negate + /// \return negated value + static HALF_CONSTEXPR half negate(half arg) { + return half(binary, arg.data_ ^ 0x8000); + } + + /// Absolute value implementation. + /// \param arg function argument + /// \return absolute value + static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); } +}; +template <> +struct unary_specialized { + static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } + static expr fabs(float arg) { return expr(std::fabs(arg)); } +}; + +/// Wrapper for binary half-precision functions needing specialization for +/// individual argument types. \tparam T first argument type \tparam U first +/// argument type +template +struct binary_specialized { + /// Minimum implementation. + /// \param x first operand + /// \param y second operand + /// \return minimum value + static expr fmin(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmin(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::min(x, y)); +#endif + } + + /// Maximum implementation. + /// \param x first operand + /// \param y second operand + /// \return maximum value + static expr fmax(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmax(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::max(x, y)); +#endif + } +}; +template <> +struct binary_specialized { + static half fmin(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } + static half fmax(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } +}; + +/// Helper class for half casts. +/// This class template has to be specialized for all valid cast argument to +/// define an appropriate static `cast` member function and a corresponding +/// `type` member denoting its return type. \tparam T destination type \tparam U +/// source type \tparam R rounding mode to use +template +struct half_caster {}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast from non-arithmetic type unsupported"); +#endif - /// Supports subnormal values. - static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { + return half(binary, float2half(arg)); + } + static half cast_impl(U arg, false_type) { + return half(binary, int2half(arg)); + } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - /// Rounding mode. - /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying - /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding - /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the - /// single-precision rounding mode. - static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits::round_style== - half_float::half::round_style) ? half_float::half::round_style : round_indeterminate; + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - /// Significant digits. - static HALF_CONSTEXPR_CONST int digits = 11; + static T cast(expr arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(float arg, true_type) { return static_cast(arg); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { + static half cast(half arg) { return arg; } +}; +template +struct half_caster : half_caster {}; + +/// \name Comparison operators +/// \{ + +/// Comparison for equality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands equal +/// \retval false else +template +typename enable::type operator==(T x, U y) { + return functions::isequal(x, y); +} - /// Significant decimal digits. - static HALF_CONSTEXPR_CONST int digits10 = 3; +/// Comparison for inequality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands not equal +/// \retval false else +template +typename enable::type operator!=(T x, U y) { + return functions::isnotequal(x, y); +} - /// Required decimal digits to represent all possible values. - static HALF_CONSTEXPR_CONST int max_digits10 = 5; +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +template +typename enable::type operator<(T x, U y) { + return functions::isless(x, y); +} - /// Number base. - static HALF_CONSTEXPR_CONST int radix = 2; +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +template +typename enable::type operator>(T x, U y) { + return functions::isgreater(x, y); +} - /// One more than smallest exponent. - static HALF_CONSTEXPR_CONST int min_exponent = -13; +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +template +typename enable::type operator<=(T x, U y) { + return functions::islessequal(x, y); +} - /// Smallest normalized representable power of 10. - static HALF_CONSTEXPR_CONST int min_exponent10 = -4; +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +template +typename enable::type operator>=(T x, U y) { + return functions::isgreaterequal(x, y); +} - /// One more than largest exponent - static HALF_CONSTEXPR_CONST int max_exponent = 16; +/// \} +/// \name Arithmetic operators +/// \{ + +/// Add halfs. +/// \param x left operand +/// \param y right operand +/// \return sum of half expressions +template +typename enable::type operator+(T x, U y) { + return functions::plus(x, y); +} - /// Largest finitely representable power of 10. - static HALF_CONSTEXPR_CONST int max_exponent10 = 4; +/// Subtract halfs. +/// \param x left operand +/// \param y right operand +/// \return difference of half expressions +template +typename enable::type operator-(T x, U y) { + return functions::minus(x, y); +} - /// Smallest positive normal value. - static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } +/// Multiply halfs. +/// \param x left operand +/// \param y right operand +/// \return product of half expressions +template +typename enable::type operator*(T x, U y) { + return functions::multiplies(x, y); +} - /// Smallest finite value. - static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } +/// Divide halfs. +/// \param x left operand +/// \param y right operand +/// \return quotient of half expressions +template +typename enable::type operator/(T x, U y) { + return functions::divides(x, y); +} - /// Largest finite value. - static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } +/// Identity. +/// \param arg operand +/// \return uncahnged operand +template +HALF_CONSTEXPR typename enable::type operator+(T arg) { + return arg; +} - /// Difference between one and next representable value. - static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } +/// Negation. +/// \param arg operand +/// \return negated operand +template +HALF_CONSTEXPR typename enable::type operator-(T arg) { + return unary_specialized::negate(arg); +} - /// Maximum rounding error. - static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW - { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } +/// \} +/// \name Input and output +/// \{ + +/// Output operator. +/// \param out output stream to write into +/// \param arg half expression to write +/// \return reference to output stream +template +typename enable &, T>::type operator<<( + std::basic_ostream &out, T arg) { + return functions::write(out, arg); +} - /// Positive infinity. - static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } +/// Input operator. +/// \param in input stream to read from +/// \param arg half to read into +/// \return reference to input stream +template +std::basic_istream &operator>>( + std::basic_istream &in, half &arg) { + return functions::read(in, arg); +} - /// Quiet NaN. - static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } +/// \} +/// \name Basic mathematical operations +/// \{ + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type abs(T arg) { +//return unary_specialized::fabs(arg); } +inline half abs(half arg) { return unary_specialized::fabs(arg); } +inline expr abs(expr arg) { return unary_specialized::fabs(arg); } + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type fabs(T arg) { +//return unary_specialized::fabs(arg); } +inline half fabs(half arg) { return unary_specialized::fabs(arg); } +inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//fmod(T x, U y) { return functions::fmod(x, y); } +inline expr fmod(half x, half y) { return functions::fmod(x, y); } +inline expr fmod(half x, expr y) { return functions::fmod(x, y); } +inline expr fmod(expr x, half y) { return functions::fmod(x, y); } +inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//remainder(T x, U y) { return functions::remainder(x, y); } +inline expr remainder(half x, half y) { return functions::remainder(x, y); } +inline expr remainder(half x, expr y) { return functions::remainder(x, y); } +inline expr remainder(expr x, half y) { return functions::remainder(x, y); } +inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \param quo address to store some bits of quotient at +/// \return remainder of floating point division. +// template typename enable::type +//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } +inline expr remquo(half x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(half x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} - /// Signalling NaN. - static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } +/// Fused multiply add. +/// \param x first operand +/// \param y second operand +/// \param z third operand +/// \return ( \a x * \a y ) + \a z rounded as one operation. +// template typename +//enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); +//} +inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } + +/// Maximum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return maximum of operands +// template typename result::type fmax(T +//x, U y) { return binary_specialized::fmax(x, y); } +inline half fmax(half x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(half x, expr y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, expr y) { + return binary_specialized::fmax(x, y); +} - /// Smallest positive subnormal value. - static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } - }; +/// Minimum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return minimum of operands +// template typename result::type fmin(T +//x, U y) { return binary_specialized::fmin(x, y); } +inline half fmin(half x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(half x, expr y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, expr y) { + return binary_specialized::fmin(x, y); +} -#if HALF_ENABLE_CPP11_HASH - /// Hash function for half-precision floats. - /// This is only defined if C++11 `std::hash` is supported and enabled. - template<> struct hash //: unary_function - { - /// Type of function argument. - typedef half_float::half argument_type; - - /// Function return type. - typedef size_t result_type; - - /// Compute hash function. - /// \param arg half to hash - /// \return hash value - result_type operator()(argument_type arg) const - { return hash()(static_cast(arg.data_)&-(arg.data_!=0x8000)); } - }; +/// Positive difference. +/// \param x first operand +/// \param y second operand +/// \return \a x - \a y or 0 if difference negative +// template typename enable::type +//fdim(T x, U y) { return functions::fdim(x, y); } +inline expr fdim(half x, half y) { return functions::fdim(x, y); } +inline expr fdim(half x, expr y) { return functions::fdim(x, y); } +inline expr fdim(expr x, half y) { return functions::fdim(x, y); } +inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } + +/// Get NaN value. +/// \return quiet NaN +inline half nanh(const char *) { return functions::nanh(); } + +/// \} +/// \name Exponential functions +/// \{ + +/// Exponential function. +/// \param arg function argument +/// \return e raised to \a arg +// template typename enable::type exp(T arg) { +//return functions::exp(arg); } +inline expr exp(half arg) { return functions::exp(arg); } +inline expr exp(expr arg) { return functions::exp(arg); } + +/// Exponential minus one. +/// \param arg function argument +/// \return e raised to \a arg subtracted by 1 +// template typename enable::type expm1(T arg) { +//return functions::expm1(arg); } +inline expr expm1(half arg) { return functions::expm1(arg); } +inline expr expm1(expr arg) { return functions::expm1(arg); } + +/// Binary exponential. +/// \param arg function argument +/// \return 2 raised to \a arg +// template typename enable::type exp2(T arg) { +//return functions::exp2(arg); } +inline expr exp2(half arg) { return functions::exp2(arg); } +inline expr exp2(expr arg) { return functions::exp2(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base e +// template typename enable::type log(T arg) { +//return functions::log(arg); } +inline expr log(half arg) { return functions::log(arg); } +inline expr log(expr arg) { return functions::log(arg); } + +/// Common logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 10 +// template typename enable::type log10(T arg) { +//return functions::log10(arg); } +inline expr log10(half arg) { return functions::log10(arg); } +inline expr log10(expr arg) { return functions::log10(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg plus 1 to base e +// template typename enable::type log1p(T arg) { +//return functions::log1p(arg); } +inline expr log1p(half arg) { return functions::log1p(arg); } +inline expr log1p(expr arg) { return functions::log1p(arg); } + +/// Binary logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 2 +// template typename enable::type log2(T arg) { +//return functions::log2(arg); } +inline expr log2(half arg) { return functions::log2(arg); } +inline expr log2(expr arg) { return functions::log2(arg); } + +/// \} +/// \name Power functions +/// \{ + +/// Square root. +/// \param arg function argument +/// \return square root of \a arg +// template typename enable::type sqrt(T arg) { +//return functions::sqrt(arg); } +inline expr sqrt(half arg) { return functions::sqrt(arg); } +inline expr sqrt(expr arg) { return functions::sqrt(arg); } + +/// Cubic root. +/// \param arg function argument +/// \return cubic root of \a arg +// template typename enable::type cbrt(T arg) { +//return functions::cbrt(arg); } +inline expr cbrt(half arg) { return functions::cbrt(arg); } +inline expr cbrt(expr arg) { return functions::cbrt(arg); } + +/// Hypotenuse function. +/// \param x first argument +/// \param y second argument +/// \return square root of sum of squares without internal over- or underflows +// template typename enable::type +//hypot(T x, U y) { return functions::hypot(x, y); } +inline expr hypot(half x, half y) { return functions::hypot(x, y); } +inline expr hypot(half x, expr y) { return functions::hypot(x, y); } +inline expr hypot(expr x, half y) { return functions::hypot(x, y); } +inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } + +/// Power function. +/// \param base first argument +/// \param exp second argument +/// \return \a base raised to \a exp +// template typename enable::type +//pow(T base, U exp) { return functions::pow(base, exp); } +inline expr pow(half base, half exp) { return functions::pow(base, exp); } +inline expr pow(half base, expr exp) { return functions::pow(base, exp); } +inline expr pow(expr base, half exp) { return functions::pow(base, exp); } +inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } + +/// \} +/// \name Trigonometric functions +/// \{ + +/// Sine function. +/// \param arg function argument +/// \return sine value of \a arg +// template typename enable::type sin(T arg) { +//return functions::sin(arg); } +inline expr sin(half arg) { return functions::sin(arg); } +inline expr sin(expr arg) { return functions::sin(arg); } + +/// Cosine function. +/// \param arg function argument +/// \return cosine value of \a arg +// template typename enable::type cos(T arg) { +//return functions::cos(arg); } +inline expr cos(half arg) { return functions::cos(arg); } +inline expr cos(expr arg) { return functions::cos(arg); } + +/// Tangent function. +/// \param arg function argument +/// \return tangent value of \a arg +// template typename enable::type tan(T arg) { +//return functions::tan(arg); } +inline expr tan(half arg) { return functions::tan(arg); } +inline expr tan(expr arg) { return functions::tan(arg); } + +/// Arc sine. +/// \param arg function argument +/// \return arc sine value of \a arg +// template typename enable::type asin(T arg) { +//return functions::asin(arg); } +inline expr asin(half arg) { return functions::asin(arg); } +inline expr asin(expr arg) { return functions::asin(arg); } + +/// Arc cosine function. +/// \param arg function argument +/// \return arc cosine value of \a arg +// template typename enable::type acos(T arg) { +//return functions::acos(arg); } +inline expr acos(half arg) { return functions::acos(arg); } +inline expr acos(expr arg) { return functions::acos(arg); } + +/// Arc tangent function. +/// \param arg function argument +/// \return arc tangent value of \a arg +// template typename enable::type atan(T arg) { +//return functions::atan(arg); } +inline expr atan(half arg) { return functions::atan(arg); } +inline expr atan(expr arg) { return functions::atan(arg); } + +/// Arc tangent function. +/// \param x first argument +/// \param y second argument +/// \return arc tangent value +// template typename enable::type +//atan2(T x, U y) { return functions::atan2(x, y); } +inline expr atan2(half x, half y) { return functions::atan2(x, y); } +inline expr atan2(half x, expr y) { return functions::atan2(x, y); } +inline expr atan2(expr x, half y) { return functions::atan2(x, y); } +inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } + +/// \} +/// \name Hyperbolic functions +/// \{ + +/// Hyperbolic sine. +/// \param arg function argument +/// \return hyperbolic sine value of \a arg +// template typename enable::type sinh(T arg) { +//return functions::sinh(arg); } +inline expr sinh(half arg) { return functions::sinh(arg); } +inline expr sinh(expr arg) { return functions::sinh(arg); } + +/// Hyperbolic cosine. +/// \param arg function argument +/// \return hyperbolic cosine value of \a arg +// template typename enable::type cosh(T arg) { +//return functions::cosh(arg); } +inline expr cosh(half arg) { return functions::cosh(arg); } +inline expr cosh(expr arg) { return functions::cosh(arg); } + +/// Hyperbolic tangent. +/// \param arg function argument +/// \return hyperbolic tangent value of \a arg +// template typename enable::type tanh(T arg) { +//return functions::tanh(arg); } +inline expr tanh(half arg) { return functions::tanh(arg); } +inline expr tanh(expr arg) { return functions::tanh(arg); } + +/// Hyperbolic area sine. +/// \param arg function argument +/// \return area sine value of \a arg +// template typename enable::type asinh(T arg) { +//return functions::asinh(arg); } +inline expr asinh(half arg) { return functions::asinh(arg); } +inline expr asinh(expr arg) { return functions::asinh(arg); } + +/// Hyperbolic area cosine. +/// \param arg function argument +/// \return area cosine value of \a arg +// template typename enable::type acosh(T arg) { +//return functions::acosh(arg); } +inline expr acosh(half arg) { return functions::acosh(arg); } +inline expr acosh(expr arg) { return functions::acosh(arg); } + +/// Hyperbolic area tangent. +/// \param arg function argument +/// \return area tangent value of \a arg +// template typename enable::type atanh(T arg) { +//return functions::atanh(arg); } +inline expr atanh(half arg) { return functions::atanh(arg); } +inline expr atanh(expr arg) { return functions::atanh(arg); } + +/// \} +/// \name Error and gamma functions +/// \{ + +/// Error function. +/// \param arg function argument +/// \return error function value of \a arg +// template typename enable::type erf(T arg) { +//return functions::erf(arg); } +inline expr erf(half arg) { return functions::erf(arg); } +inline expr erf(expr arg) { return functions::erf(arg); } + +/// Complementary error function. +/// \param arg function argument +/// \return 1 minus error function value of \a arg +// template typename enable::type erfc(T arg) { +//return functions::erfc(arg); } +inline expr erfc(half arg) { return functions::erfc(arg); } +inline expr erfc(expr arg) { return functions::erfc(arg); } + +/// Natural logarithm of gamma function. +/// \param arg function argument +/// \return natural logarith of gamma function for \a arg +// template typename enable::type lgamma(T arg) { +//return functions::lgamma(arg); } +inline expr lgamma(half arg) { return functions::lgamma(arg); } +inline expr lgamma(expr arg) { return functions::lgamma(arg); } + +/// Gamma function. +/// \param arg function argument +/// \return gamma function value of \a arg +// template typename enable::type tgamma(T arg) { +//return functions::tgamma(arg); } +inline expr tgamma(half arg) { return functions::tgamma(arg); } +inline expr tgamma(expr arg) { return functions::tgamma(arg); } + +/// \} +/// \name Rounding +/// \{ + +/// Nearest integer not less than half value. +/// \param arg half to round +/// \return nearest integer not less than \a arg +// template typename enable::type ceil(T arg) { +//return functions::ceil(arg); } +inline half ceil(half arg) { return functions::ceil(arg); } +inline half ceil(expr arg) { return functions::ceil(arg); } + +/// Nearest integer not greater than half value. +/// \param arg half to round +/// \return nearest integer not greater than \a arg +// template typename enable::type floor(T arg) { +//return functions::floor(arg); } +inline half floor(half arg) { return functions::floor(arg); } +inline half floor(expr arg) { return functions::floor(arg); } + +/// Nearest integer not greater in magnitude than half value. +/// \param arg half to round +/// \return nearest integer not greater in magnitude than \a arg +// template typename enable::type trunc(T arg) { +//return functions::trunc(arg); } +inline half trunc(half arg) { return functions::trunc(arg); } +inline half trunc(expr arg) { return functions::trunc(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type round(T arg) { +//return functions::round(arg); } +inline half round(half arg) { return functions::round(arg); } +inline half round(expr arg) { return functions::round(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type lround(T arg) { +//return functions::lround(arg); } +inline long lround(half arg) { return functions::lround(arg); } +inline long lround(expr arg) { return functions::lround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type nearbyint(T +//arg) { return functions::nearbyint(arg); } +inline half nearbyint(half arg) { return functions::rint(arg); } +inline half nearbyint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type rint(T arg) { +//return functions::rint(arg); } +inline half rint(half arg) { return functions::rint(arg); } +inline half rint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type lrint(T arg) { +//return functions::lrint(arg); } +inline long lrint(half arg) { return functions::lrint(arg); } +inline long lrint(expr arg) { return functions::lrint(arg); } +#if HALF_ENABLE_CPP11_LONG_LONG +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type llround(T +//arg) { return functions::llround(arg); } +inline long long llround(half arg) { return functions::llround(arg); } +inline long long llround(expr arg) { return functions::llround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type llrint(T +//arg) { return functions::llrint(arg); } +inline long long llrint(half arg) { return functions::llrint(arg); } +inline long long llrint(expr arg) { return functions::llrint(arg); } #endif + +/// \} +/// \name Floating point manipulation +/// \{ + +/// Decompress floating point number. +/// \param arg number to decompress +/// \param exp address to store exponent at +/// \return significant in range [0.5, 1) +// template typename enable::type frexp(T arg, +//int *exp) { return functions::frexp(arg, exp); } +inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } +inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type ldexp(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Extract integer and fractional parts. +/// \param arg number to decompress +/// \param iptr address to store integer part at +/// \return fractional part +// template typename enable::type modf(T arg, +//half *iptr) { return functions::modf(arg, iptr); } +inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } +inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbn(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbln(T arg, +//long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +/// \retval FP_ILOGB0 for zero +/// \retval FP_ILOGBNAN for NaN +/// \retval MAX_INT for infinity +// template typename enable::type ilogb(T arg) { +//return functions::ilogb(arg); } +inline int ilogb(half arg) { return functions::ilogb(arg); } +inline int ilogb(expr arg) { return functions::ilogb(arg); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +// template typename enable::type logb(T arg) { +//return functions::logb(arg); } +inline half logb(half arg) { return functions::logb(arg); } +inline half logb(expr arg) { return functions::logb(arg); } + +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type +//nextafter(T from, U to) { return functions::nextafter(from, to); } +inline half nextafter(half from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(half from, expr to) { + return functions::nextafter(from, to); } +inline half nextafter(expr from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(expr from, expr to) { + return functions::nextafter(from, to); +} + +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type nexttoward(T +//from, long double to) { return functions::nexttoward(from, to); } +inline half nexttoward(half from, long double to) { + return functions::nexttoward(from, to); +} +inline half nexttoward(expr from, long double to) { + return functions::nexttoward(from, to); +} + +/// Take sign. +/// \param x value to change sign for +/// \param y value to take sign from +/// \return value equal to \a x in magnitude and to \a y in sign +// template typename enable::type +//copysign(T x, U y) { return functions::copysign(x, y); } +inline half copysign(half x, half y) { return functions::copysign(x, y); } +inline half copysign(half x, expr y) { return functions::copysign(x, y); } +inline half copysign(expr x, half y) { return functions::copysign(x, y); } +inline half copysign(expr x, expr y) { return functions::copysign(x, y); } + +/// \} +/// \name Floating point classification +/// \{ + +/// Classify floating point value. +/// \param arg number to classify +/// \retval FP_ZERO for positive and negative zero +/// \retval FP_SUBNORMAL for subnormal numbers +/// \retval FP_INFINITY for positive and negative infinity +/// \retval FP_NAN for NaNs +/// \retval FP_NORMAL for all other (normal) values +// template typename enable::type fpclassify(T +//arg) { return functions::fpclassify(arg); } +inline int fpclassify(half arg) { return functions::fpclassify(arg); } +inline int fpclassify(expr arg) { return functions::fpclassify(arg); } + +/// Check if finite number. +/// \param arg number to check +/// \retval true if neither infinity nor NaN +/// \retval false else +// template typename enable::type isfinite(T arg) +//{ return functions::isfinite(arg); } +inline bool isfinite(half arg) { return functions::isfinite(arg); } +inline bool isfinite(expr arg) { return functions::isfinite(arg); } + +/// Check for infinity. +/// \param arg number to check +/// \retval true for positive or negative infinity +/// \retval false else +// template typename enable::type isinf(T arg) { +//return functions::isinf(arg); } +inline bool isinf(half arg) { return functions::isinf(arg); } +inline bool isinf(expr arg) { return functions::isinf(arg); } + +/// Check for NaN. +/// \param arg number to check +/// \retval true for NaNs +/// \retval false else +// template typename enable::type isnan(T arg) { +//return functions::isnan(arg); } +inline bool isnan(half arg) { return functions::isnan(arg); } +inline bool isnan(expr arg) { return functions::isnan(arg); } + +/// Check if normal number. +/// \param arg number to check +/// \retval true if normal number +/// \retval false if either subnormal, zero, infinity or NaN +// template typename enable::type isnormal(T arg) +//{ return functions::isnormal(arg); } +inline bool isnormal(half arg) { return functions::isnormal(arg); } +inline bool isnormal(expr arg) { return functions::isnormal(arg); } + +/// Check sign. +/// \param arg number to check +/// \retval true for negative number +/// \retval false for positive number +// template typename enable::type signbit(T arg) +//{ return functions::signbit(arg); } +inline bool signbit(half arg) { return functions::signbit(arg); } +inline bool signbit(expr arg) { return functions::signbit(arg); } + +/// \} +/// \name Comparison +/// \{ + +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +// template typename enable::type +//isgreater(T x, U y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } + +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +// template typename enable::type +//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } +inline bool isgreaterequal(half x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(half x, expr y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, expr y) { + return functions::isgreaterequal(x, y); +} + +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +// template typename enable::type +//isless(T x, U y) { return functions::isless(x, y); } +inline bool isless(half x, half y) { return functions::isless(x, y); } +inline bool isless(half x, expr y) { return functions::isless(x, y); } +inline bool isless(expr x, half y) { return functions::isless(x, y); } +inline bool isless(expr x, expr y) { return functions::isless(x, y); } + +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +// template typename enable::type +//islessequal(T x, U y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } + +/// Comarison for less or greater. +/// \param x first operand +/// \param y second operand +/// \retval true if either less or greater +/// \retval false else +// template typename enable::type +//islessgreater(T x, U y) { return functions::islessgreater(x, y); } +inline bool islessgreater(half x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(half x, expr y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, expr y) { + return functions::islessgreater(x, y); +} + +/// Check if unordered. +/// \param x first operand +/// \param y second operand +/// \retval true if unordered (one or two NaN operands) +/// \retval false else +// template typename enable::type +//isunordered(T x, U y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } + +/// \name Casting +/// \{ + +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted directly using the given rounding mode, +/// without any roundtrip over `float` that a `static_cast` would otherwise do. +/// It uses the default rounding mode. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types not being a built-in +/// arithmetic type (apart from [half](\ref half_float::half), of course) +/// results in a compiler error and casting between [half](\ref +/// half_float::half)s is just a no-op. \tparam T destination type (half or +/// built-in arithmetic type) \tparam U source type (half or built-in arithmetic +/// type) \param arg value to cast \return \a arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} + +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted directly using the given rounding mode, +/// without any roundtrip over `float` that a `static_cast` would otherwise do. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types not being a built-in +/// arithmetic type (apart from [half](\ref half_float::half), of course) +/// results in a compiler error and casting between [half](\ref +/// half_float::half)s is just a no-op. \tparam T destination type (half or +/// built-in arithmetic type) \tparam R rounding mode to use. \tparam U source +/// type (half or built-in arithmetic type) \param arg value to cast \return \a +/// arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} +/// \} +} // namespace detail + +using detail::operator==; +using detail::operator!=; +using detail::operator<; +using detail::operator>; +using detail::operator<=; +using detail::operator>=; +using detail::operator+; +using detail::operator-; +using detail::operator*; +using detail::operator/; +using detail::operator<<; +using detail::operator>>; + +using detail::abs; +using detail::acos; +using detail::acosh; +using detail::asin; +using detail::asinh; +using detail::atan; +using detail::atan2; +using detail::atanh; +using detail::cbrt; +using detail::ceil; +using detail::cos; +using detail::cosh; +using detail::erf; +using detail::erfc; +using detail::exp; +using detail::exp2; +using detail::expm1; +using detail::fabs; +using detail::fdim; +using detail::floor; +using detail::fma; +using detail::fmax; +using detail::fmin; +using detail::fmod; +using detail::hypot; +using detail::lgamma; +using detail::log; +using detail::log10; +using detail::log1p; +using detail::log2; +using detail::lrint; +using detail::lround; +using detail::nanh; +using detail::nearbyint; +using detail::pow; +using detail::remainder; +using detail::remquo; +using detail::rint; +using detail::round; +using detail::sin; +using detail::sinh; +using detail::sqrt; +using detail::tan; +using detail::tanh; +using detail::tgamma; +using detail::trunc; +#if HALF_ENABLE_CPP11_LONG_LONG +using detail::llrint; +using detail::llround; +#endif +using detail::copysign; +using detail::fpclassify; +using detail::frexp; +using detail::ilogb; +using detail::isfinite; +using detail::isgreater; +using detail::isgreaterequal; +using detail::isinf; +using detail::isless; +using detail::islessequal; +using detail::islessgreater; +using detail::isnan; +using detail::isnormal; +using detail::isunordered; +using detail::ldexp; +using detail::logb; +using detail::modf; +using detail::nextafter; +using detail::nexttoward; +using detail::scalbln; +using detail::scalbn; +using detail::signbit; + +using detail::half_cast; +} // namespace half_float + +/// Extensions to the C++ standard library. +namespace std { +/// Numeric limits for half-precision floats. +/// Because of the underlying single-precision implementation of many +/// operations, it inherits some properties from `std::numeric_limits`. +template <> +class numeric_limits : public numeric_limits { + public: + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Rounding mode. + /// Due to the mix of internal single-precision computations (using the + /// rounding mode of the underlying single-precision implementation) with the + /// rounding mode of the single-to-half conversions, the actual rounding mode + /// might be `std::round_indeterminate` if the default half-precision rounding + /// mode doesn't match the single-precision rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = + (std::numeric_limits::round_style == half_float::half::round_style) + ? half_float::half::round_style + : round_indeterminate; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0400); + } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0xFBFF); + } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7BFF); + } + + /// Difference between one and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x1400); + } + + /// Maximum rounding error. + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW { + return half_float::half( + half_float::detail::binary, + (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00); + } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7C00); + } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7FFF); + } + + /// Signalling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7DFF); + } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0001); + } +}; +#if HALF_ENABLE_CPP11_HASH +/// Hash function for half-precision floats. +/// This is only defined if C++11 `std::hash` is supported and enabled. +template <> +struct hash //: unary_function +{ + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { + return hash()(static_cast(arg.data_) & + -(arg.data_ != 0x8000)); + } +}; +#endif +} // namespace std #undef HALF_CONSTEXPR #undef HALF_CONSTEXPR_CONST #undef HALF_NOEXCEPT #undef HALF_NOTHROW #ifdef HALF_POP_WARNINGS - #pragma warning(pop) - #undef HALF_POP_WARNINGS +#pragma warning(pop) +#undef HALF_POP_WARNINGS #endif #endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 014e588..0d56558 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,51 +8,51 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include "instructions.h" #include "half.h" #include "half.hpp" -#include "instructions.h" -#include "ptx_ir.h" #include "opcodes.h" +#include "ptx_ir.h" #include "ptx_sim.h" -typedef void * yyscan_t; +typedef void *yyscan_t; class ptx_recognizer; -#include "ptx.tab.h" -#include +#include +#include #include +#include +#include +#include #include -#include -#include "cuda-math.h" +#include +#include +#include #include "../abstract_hardware_model.h" -#include "ptx_loader.h" -#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include -#include -#include -#include -#include -#include -#include +#include "cuda-math.h" +#include "cuda_device_printf.h" +#include "ptx.tab.h" +#include "ptx_loader.h" -//Jin: include device runtime for CDP +// Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include @@ -60,5629 +60,6420 @@ class ptx_recognizer; using half_float::half; - - const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -//Using profiled information::check the TensorCoreMatrixArrangement.xls for details -unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ - - unsigned offset; - unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; - unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; - unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; - unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; - unsigned thread_group = thread/4; - unsigned in_tg_index = thread%4; - - switch(wmma_type){ - case LOAD_A: - if(wmma_layout==ROW) - offset=load_a_row[thread_group]+16*in_tg_index; - else - offset=load_a_col[thread_group]+16*in_tg_index; - break; - - - case LOAD_B: - if(wmma_layout==ROW) - offset=load_b_row[thread_group]+16*in_tg_index; - else - offset=load_b_col[thread_group]+16*in_tg_index; - break; - - case LOAD_C: - case STORE_D: - if(type==F16_TYPE){ - if(wmma_layout==ROW) - offset=load_c_half_row[thread_group]+16*in_tg_index; - else - offset=load_c_half_col[thread_group]+in_tg_index; - } - else{ - if(wmma_layout==ROW) - offset=load_c_float_row[thread_group]; - else - offset=load_c_float_col[thread_group]; - - switch(in_tg_index){ - case 0: - break; - case 1: - if(wmma_layout==ROW) - offset+=16; - else - offset+=1; - break; - case 2: - if(wmma_layout==ROW) - offset+=2; - else - offset+=32; - break; - case 3: - if(wmma_layout==ROW) - offset+=18; - else - offset+=33; - break; - default: - abort(); - } - } - break; - - default: - abort(); - - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -int acc_float_offset(int index,int wmma_layout,int stride){ - - int c_row_offset[]={0,1,32,33,4,5,36,37}; - int c_col_offset[]={0,16,2,18,64,80,66,82}; - int offset; - - - if(wmma_layout==ROW) - offset=c_row_offset[index]; - else if(wmma_layout==COL) - offset=c_col_offset[index]; - else{ - printf("wrong layout"); - abort(); - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -void inst_not_implemented( const ptx_instruction * pI ) ; -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); - -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); - -void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) -{ - assert( reg != NULL ); - if( reg->name() == "_" ) return; - assert( !m_regs.empty() ); - assert( reg->uid() > 0 ); - m_regs.back()[ reg ] = value; - if (m_enable_debug_trace ) - m_debug_trace_regs_modified.back()[ reg ] = value; - m_last_set_operand_value = value; -} - -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); +// Using profiled information::check the TensorCoreMatrixArrangement.xls for +// details +unsigned thread_group_offset(int thread, unsigned wmma_type, + unsigned wmma_layout, unsigned type, int stride) { + unsigned offset; + unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned thread_group = thread / 4; + unsigned in_tg_index = thread % 4; + + switch (wmma_type) { + case LOAD_A: + if (wmma_layout == ROW) + offset = load_a_row[thread_group] + 16 * in_tg_index; + else + offset = load_a_col[thread_group] + 16 * in_tg_index; + break; - int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); - + case LOAD_B: + if (wmma_layout == ROW) + offset = load_b_row[thread_group] + 16 * in_tg_index; + else + offset = load_b_col[thread_group] + 16 * in_tg_index; + break; + + case LOAD_C: + case STORE_D: + if (type == F16_TYPE) { + if (wmma_layout == ROW) + offset = load_c_half_row[thread_group] + 16 * in_tg_index; + else + offset = load_c_half_col[thread_group] + in_tg_index; + } else { + if (wmma_layout == ROW) + offset = load_c_float_row[thread_group]; + else + offset = load_c_float_col[thread_group]; + + switch (in_tg_index) { + case 0: + break; + case 1: + if (wmma_layout == ROW) + offset += 16; + else + offset += 1; + break; + case 2: + if (wmma_layout == ROW) + offset += 2; + else + offset += 32; + break; + case 3: + if (wmma_layout == ROW) + offset += 18; + else + offset += 33; + break; + default: + abort(); } - //m_regs.pop_back(); + } + break; + + default: + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +int acc_float_offset(int index, int wmma_layout, int stride) { + int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; + int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; + int offset; + + if (wmma_layout == ROW) + offset = c_row_offset[index]; + else if (wmma_layout == COL) + offset = c_col_offset[index]; + else { + printf("wrong layout"); + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +void inst_not_implemented(const ptx_instruction *pI); +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread); + +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); + +void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { + assert(reg != NULL); + if (reg->name() == "_") return; + assert(!m_regs.empty()); + assert(reg->uid() > 0); + m_regs.back()[reg] = value; + if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; + m_last_set_operand_value = value; +} + +void ptx_thread_info::print_reg_thread(char *fname) { + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + + int size = m_regs.size(); + + if (size > 0) { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) { + const std::string &name = it->first->name(); + const std::string &dec = it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), + size); + } + // m_regs.pop_back(); } fclose(fp); +} +void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { + FILE *fp2 = fopen(fname, "r"); + assert(fp2 != NULL); + // m_regs.push_back( reg_map_t() ); + char line[200]; + while (fgets(line, sizeof line, fp2) != NULL) { + symbol *reg; + char *pch; + pch = strtok(line, " "); + char *name = pch; + reg = symtab->lookup(name); + ptx_reg_t data; + pch = strtok(NULL, " "); + data = atoi(pch); + pch = strtok(NULL, " "); + pch = strtok(NULL, " "); + m_regs.back()[reg] = data; } + fclose(fp2); +} -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - pch = strtok (NULL," "); - m_regs.back()[reg] = data; - } - fclose ( fp2 ); -} - - -ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) -{ - static bool unfound_register_warned = false; - assert( reg != NULL ); - assert( !m_regs.empty() ); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert( reg->type()->get_key().is_reg() ); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway - std::string file_loc = get_location(); - if( !unfound_register_warned ) { - printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid ); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); - } - if (m_enable_debug_trace ) - m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; - return regs_iter->second; -} - -ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) -{ - ptx_reg_t result, tmp; - - - if(op.get_double_operand_type() == 0) { - if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { - if ( op.is_reg() ) { - result = get_reg( op.get_symbol() ); - } else if ( op.is_builtin()) { - result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); - } else if(op.is_immediate_address()){ - result.u64 = op.get_addr_offset(); - } else if ( op.is_memory_operand() ) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); - - if ( info.is_reg() ) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if ( info.is_param_kernel() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_param_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_global() ) { - assert( op.get_addr_offset() == 0 ); - result.u64 = sym->get_address(); - } else if ( info.is_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_const() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); - abort(); - } +ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { + static bool unfound_register_warned = false; + assert(reg != NULL); + assert(!m_regs.empty()); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert(reg->type()->get_key().is_reg()); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the + // user anyway + std::string file_loc = get_location(); + if (!unfound_register_warned) { + printf( + "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " + "(cuid:%u). Setting to 0X00000000. This is okay if you are " + "simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace) + m_debug_trace_regs_read.back()[reg] = regs_iter->second; + return regs_iter->second; +} - } else if ( op.is_literal() ) { - result = op.get_literal_value(); - } else if ( op.is_label() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_const() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_global() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_local() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if ( op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); - }else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if ( info2.is_param_kernel() ) { - result.u64 = sym2->get_address()+ op.get_addr_offset(); - } - else{ - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); - assert(0); - } - } - - if(op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - result.u64 = (result.u64>>16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; - result.u128.low = get_reg( op.vec_symbol(1) ).u32; - result.u128.high = get_reg( op.vec_symbol(2) ).u32; - result.u128.highest = get_reg( op.vec_symbol(3) ).u32; +ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, + operand_info dstInfo, + unsigned opType, + ptx_thread_info *thread, + int derefFlag) { + ptx_reg_t result, tmp; + + if (op.get_double_operand_type() == 0) { + if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && + (opType != FF64_TYPE)) || + (op.get_addr_space() != undefined_space)) { + if (op.is_reg()) { + result = get_reg(op.get_symbol()); + } else if (op.is_builtin()) { + result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); + } else if (op.is_immediate_address()) { + result.u64 = op.get_addr_offset(); + } else if (op.is_memory_operand()) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if (info.is_reg()) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if (info.is_param_kernel()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_param_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_global()) { + assert(op.get_addr_offset() == 0); + result.u64 = sym->get_address(); + } else if (info.is_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_const()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " + "operand type for %s\n", + name); + abort(); + } + + } else if (op.is_literal()) { + result = op.get_literal_value(); + } else if (op.is_label()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_const()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_global()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_local()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_function_address()) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if (op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { - // bb64 or ff64 - result.bits.ls = get_reg( op.vec_symbol(0) ).u32; - result.bits.ms = get_reg( op.vec_symbol(1) ).u32; + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if (info2.is_param_kernel()) { + result.u64 = sym2->get_address() + op.get_addr_offset(); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " + "type for %s\n", + name); + assert(0); + } } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; - secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0),firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(),firstHalf); - } - - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size=0; - int t=0; - finalResult.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if((op.get_addr_space() == global_space)&&(derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == const_space)&&(derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == local_space)&&(derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else { - finalResult = result; - } - - if((op.get_operand_neg() == true)&&(derefFlag)) { - switch( opType ) { + + if (op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + result.u64 = (result.u64 >> 16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg(op.vec_symbol(0)).u32; + result.u128.low = get_reg(op.vec_symbol(1)).u32; + result.u128.high = get_reg(op.vec_symbol(2)).u32; + result.u128.highest = get_reg(op.vec_symbol(3)).u32; + } else { + // bb64 or ff64 + result.bits.ls = get_reg(op.vec_symbol(0)).u32; + result.bits.ms = get_reg(op.vec_symbol(1)).u32; + } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is + // s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0), firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(), firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size = 0; + int t = 0; + finalResult.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if ((op.get_addr_space() == global_space) && (derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == const_space) && (derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read((result.u32 + op.get_const_mem_offset()), size / 8, + &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == local_space) && (derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else { + finalResult = result; + } + + if ((op.get_operand_neg() == true) && (derefFlag)) { + switch (opType) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } + assert(0); + } + } + + return finalResult; +} - } - - return finalResult; - -} - -unsigned get_operand_nbits( const operand_info &op ) -{ - if ( op.is_reg() ) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch( t.scalar_type() ) { - case PRED_TYPE: - return 1; - case B8_TYPE: case S8_TYPE: case U8_TYPE: - return 8; - case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: - return 16; - case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: - return 32; - case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: - return 64; +unsigned get_operand_nbits(const operand_info &op) { + if (op.is_reg()) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch (t.scalar_type()) { + case PRED_TYPE: + return 1; + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + return 8; + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + case B16_TYPE: + return 16; + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + case B32_TYPE: + return 32; + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf( + "ERROR: Need to implement get_operand_nbits() for currently " + "unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) -{ - assert( op.is_vector() ); - assert( num_elements <= 8 ); +void ptx_thread_info::get_vector_operand_values(const operand_info &op, + ptx_reg_t *ptx_regs, + unsigned num_elements) { + assert(op.is_vector()); + assert(num_elements <= 8); + + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if (strcmp(sym->name().c_str(), "_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert(reg_iter != m_regs.back().end()); + ptx_regs[idx] = reg_iter->second; + } + } +} - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if( strcmp(sym->name().c_str(),"_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert( reg_iter != m_regs.back().end() ); - ptx_regs[idx] = reg_iter->second; - } - } +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { + if (!dst.is_reg()) return; + unsigned dst_size = get_operand_nbits(dst); + if (src_size >= dst_size) return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size - 1); + if ((mask & data.u64) == 0) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size - src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) -{ - if( !dst.is_reg() ) - return; - unsigned dst_size = get_operand_nbits( dst ); - if( src_size >= dst_size ) - return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size-1); - if( (mask & data.u64) == 0 ) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size-src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI, int overflow, + int carry) { + thread->set_operand_value(dst, data, type, thread, pI); + + if (dst.get_double_operand_type() == -2) { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01) << 3); + predValue.u64 |= ((carry & 0x01) << 2); + + set_reg(sym, predValue); + } else if (dst.get_double_operand_type() == 0) { + // intentionally do nothing + } else { + printf("Unexpected double destination\n"); + assert(0); + } } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) -{ - thread->set_operand_value( dst, data, type, thread, pI ); +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI) { + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; + + type_info_key::type_decode(type, size, t); + + /*complete this section for other cases*/ + if (dst.get_addr_space() == undefined_space) { + ptx_reg_t setValue; + setValue.u64 = data.u64; + + // Double destination in set instruction ($p0|$p1) - second is negation of + // first + if (dst.get_double_operand_type() == -1) { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); + + if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || + (type == FF64_TYPE)) { + setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; + } else { + setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; + } - if (dst.get_double_operand_type() == -2) - { - ptx_reg_t predValue; - - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01)<<3); - predValue.u64 |= ((carry & 0x01)<<2); - - set_reg(sym,predValue); - } - else if (dst.get_double_operand_type() == 0) - { - //intentionally do nothing + set_reg(name1, setValue); + set_reg(name2, setValue2); } - else - { - printf("Unexpected double destination\n"); - assert(0); + + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second + // register operand receives data, first predicate operand is set as + // $p0=($r4!=0) Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2) || + (dst.get_double_operand_type() == -3)) { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; + + switch (type) { + case S8_TYPE: + if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; + break; + case S16_TYPE: + if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; + break; + case S32_TYPE: + if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; + break; + case S64_TYPE: + if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if (setValue.u8 == 0) predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if (setValue.u16 == 0) predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if (setValue.u32 == 0) predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if (setValue.u64 == 0) predValue.u64 |= 1; + break; + case F16_TYPE: + if (setValue.f16 == 0) predValue.u64 |= 1; + break; + case F32_TYPE: + if (setValue.f32 == 0) predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if (setValue.f64 == 0) predValue.u64 |= 1; + break; + default: + assert(0); + break; + } + + if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || + (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || + (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || + (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { + if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; + } + if (type == F32_TYPE) { + if (setValue.f32 < 0) predValue.u64 |= 1 << 1; + } + + if (dst.get_operand_lohi() == 1) { + setValue.u64 = + ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + + set_reg(predName, predValue); + set_reg(regName, setValue); + } else if (type == BB128_TYPE) { + // b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; + + const symbol *name1, *name2, *name3, *name4 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + set_reg(name3, setValue3); + set_reg(name4, setValue4); + } else if (type == BB64_TYPE || type == FF64_TYPE) { + // ptxplus version of storing 64 bit values to registers stores to two + // adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; + + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; + + const symbol *name1, *name2 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + } else { + if (dst.get_operand_lohi() == 1) { + setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = + ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(), setValue); } + } -} + // global memory - g[4], g[$r0] + else if (dst.get_addr_space() == global_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) -{ - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } - type_info_key::type_decode(type,size,t); + // shared memory - s[4], s[$r0] + else if (dst.get_addr_space() == shared_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); - /*complete this section for other cases*/ - if(dst.get_addr_space() == undefined_space) - { - ptx_reg_t setValue; - setValue.u64 = data.u64; + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } - // Double destination in set instruction ($p0|$p1) - second is negation of first - if (dst.get_double_operand_type() == -1) - { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); + // local memory - l0[4], l0[$r0] + else if (dst.get_addr_space() == local_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type, size, t); - if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { - setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; - } else { - setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; - } + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + + else { + printf("Destination stores to unknown location."); + assert(0); + } +} - set_reg(name1,setValue); - set_reg(name2,setValue2); +void ptx_thread_info::set_vector_operand_values(const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } } + } + } - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) - { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; - - switch ( type ) { - case S8_TYPE: - if((setValue.s8 & 0x7F) == 0) - predValue.u64 |= 1; - break; - case S16_TYPE: - if((setValue.s16 & 0x7FFF) == 0) - predValue.u64 |= 1; - break; - case S32_TYPE: - if((setValue.s32 & 0x7FFFFFFF) == 0) - predValue.u64 |= 1; - break; - case S64_TYPE: - if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) - predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if(setValue.u8 == 0) - predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if(setValue.u16 == 0) - predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if(setValue.u32 == 0) - predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if(setValue.u64 == 0) - predValue.u64 |= 1; - break; - case F16_TYPE: - if(setValue.f16 == 0) - predValue.u64 |= 1; - break; - case F32_TYPE: - if(setValue.f32 == 0) - predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if(setValue.f64 == 0) - predValue.u64 |= 1; - break; - default: assert(0); break; - } + m_last_set_operand_value = data1; +} +void ptx_thread_info::set_wmma_vector_operand_values( + const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, + const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } else { + printf("error:set_wmma_vector_operands"); + } + m_last_set_operand_value = data8; +} - if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| - (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| - (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { - if((setValue.u32 & (1<<(size-1))) != 0) - predValue.u64 |= 1<<1; - } - if ( type==F32_TYPE ) { - if(setValue.f32 < 0) - predValue.u64 |= 1<<1; - } +#define my_abs(a) (((a) < 0) ? (-a) : (a)) - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } +#define MY_MAX_I(a, b) (a > b) ? a : b +#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - set_reg(predName,predValue); - set_reg(regName,setValue); - } - else if (type == BB128_TYPE) - { - //b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; - - const symbol *name1, *name2, *name3, *name4 = NULL; - - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); - - set_reg(name1,setValue); - set_reg(name2,setValue2); - set_reg(name3,setValue3); - set_reg(name4,setValue4); - } - else if (type == BB64_TYPE || type == FF64_TYPE) - { - //ptxplus version of storing 64 bit values to registers stores to two adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; +#define MY_MIN_I(a, b) (a < b) ? a : b +#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; +#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 +#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 - const symbol *name1, *name2 = NULL; +#define MY_CAS_I(a, b, c) (a == b) ? c : a - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); +#define MY_EXCH(a, b) b - set_reg(name1,setValue); - set_reg(name2,setValue2); - } - else - { - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } - set_reg(dst.get_symbol(),setValue); - } - } - - // global memory - g[4], g[$r0] - else if(dst.get_addr_space() == global_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } - - // shared memory - s[4], s[$r0] - else if(dst.get_addr_space() == shared_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } - - // local memory - l0[4], l0[$r0] - else if(dst.get_addr_space() == local_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } - - else - { - printf("Destination stores to unknown location."); - assert(0); - } - - -} - -void ptx_thread_info::set_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } - - m_last_set_operand_value = data1; -} -void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4, - const ptx_reg_t &data5, - const ptx_reg_t &data6, - const ptx_reg_t &data7, - const ptx_reg_t &data8 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } - else{ - printf("error:set_wmma_vector_operands"); - } - - m_last_set_operand_value = data8; -} - -#define my_abs(a) (((a)<0)?(-a):(a)) - -#define MY_MAX_I(a,b) (a > b) ? a : b -#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - -#define MY_MIN_I(a,b) (a < b) ? a : b -#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - -#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 -#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 - -#define MY_CAS_I(a,b,c) (a == b) ? c : a - -#define MY_EXCH(a,b) b - -void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S16_TYPE: d.s16 = my_abs(a.s16); break; - case S32_TYPE: d.s32 = my_abs(a.s32); break; - case S64_TYPE: d.s64 = my_abs(a.s64); break; - case U16_TYPE: d.s16 = my_abs(a.u16); break; - case U32_TYPE: d.s32 = my_abs(a.u32); break; - case U64_TYPE: d.s64 = my_abs(a.u64); break; - case F32_TYPE: d.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; - default: +void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case S16_TYPE: + d.s16 = my_abs(a.s16); + break; + case S32_TYPE: + d.s32 = my_abs(a.s32); + break; + case S64_TYPE: + d.s64 = my_abs(a.s64); + break; + case U16_TYPE: + d.s16 = my_abs(a.u16); + break; + case U32_TYPE: + d.s32 = my_abs(a.u32); + break; + case U64_TYPE: + d.s64 = my_abs(a.u64); + break; + case F32_TYPE: + d.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = my_abs(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - //src3_data.pred&0x4 is the carry flag - switch ( i_type ) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; - break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; - break; - case S64_TYPE: + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + // src3_data.pred&0x4 is the carry flag + switch (i_type) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; + break; + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100)>>8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000)>>16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000)>>32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); - - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); -} - -void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - switch ( i_type ) { - case S8_TYPE: + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); +} + +void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000)>>32; + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; - case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; - case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; - default: + switch (i_type) { + case B16_TYPE: + src2_data.u16 = ~src2_data.u16; + break; + case B32_TYPE: + src2_data.u32 = ~src2_data.u32; + break; + case B64_TYPE: + src2_data.u64 = ~src2_data.u64; + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - data.u64 = src1_data.u64 & src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void bar_callback( const inst_t* inst, ptx_thread_info* thread) -{ - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid,barid); - const ptx_instruction *pI = dynamic_cast(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst,value, U32_TYPE, thread, pI); -} - -void atom_callback( const inst_t* inst, ptx_thread_info* thread) -{ - const ptx_instruction *pI = dynamic_cast(inst); - - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); - - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result - - bool data_ready = false; - - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b - - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()){ - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - //This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } - - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if( whichspace(effective_address) == global_space ) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid,effective_address); - space = shared_space; - } else { - abort(); - } - } - assert( space == global_space || space == shared_space ); - - memory_space *mem = NULL; - if(space == global_space) - mem = thread->get_global_memory(); - else if(space == shared_space) - mem = thread->m_shared_mem; - else - abort(); - - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address,size/8,&data.s64); - if (dst.get_symbol()->type()){ - thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' - } - - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); - - switch ( m_atomic_spec ) { - // AND - case ATOMIC_AND: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); - assert(0); - break; - } + assert(0); + break; + } - break; - } - // OR - case ATOMIC_OR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); - assert(0); - break; - } + data.u64 = src1_data.u64 & src2_data.u64; - break; - } - // XOR - case ATOMIC_XOR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); - assert(0); - break; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - break; - } - // CAS - case ATOMIC_CAS: - { - - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); - assert(0); - break; - } +void bar_callback(const inst_t *inst, ptx_thread_info *thread) { + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid, barid); + const ptx_instruction *pI = dynamic_cast(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst, value, U32_TYPE, thread, pI); +} - break; - } - // EXCH - case ATOMIC_EXCH: - { - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); - assert(0); - break; - } +void atom_callback(const inst_t *inst, ptx_thread_info *thread) { + const ptx_instruction *pI = dynamic_cast(inst); + + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); + + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result + + bool data_ready = false; + + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()) { + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + // This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } + + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if (whichspace(effective_address) == global_space) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } + assert(space == global_space || space == shared_space); + + memory_space *mem = NULL; + if (space == global_space) + mem = thread->get_global_memory(); + else if (space == shared_space) + mem = thread->m_shared_mem; + else + abort(); + + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address, size / 8, &data.s64); + if (dst.get_symbol()->type()) { + thread->set_operand_value(dst, data, to_type, thread, + pI); // Write value into register 'd' + } - break; + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); + + switch (m_atomic_spec) { + // AND + case ATOMIC_AND: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.AND " + "only accepts b32\n", + to_type); + assert(0); + break; } - // ADD - case ATOMIC_ADD: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); - assert(0); - break; - } + break; + } + // OR + case ATOMIC_OR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.OR " + "only accepts b32\n", + to_type); + assert(0); + break; + } - break; + break; + } + // XOR + case ATOMIC_XOR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.XOR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // INC - case ATOMIC_INC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // CAS + case ATOMIC_CAS: { + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.CAS " + "only accepts b32 and b64\n", + to_type); + assert(0); + break; + } + + break; + } + // EXCH + case ATOMIC_EXCH: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.EXCH " + "only accepts b32\n", + to_type); + assert(0); + break; + } + + break; + } + // ADD + case ATOMIC_ADD: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.ADD only " + "accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } + + break; + } + // INC + case ATOMIC_INC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.INC only " + "accepts u32 and s32\n"); + assert(0); + break; } - // DEC - case ATOMIC_DEC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // DEC + case ATOMIC_DEC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.DEC only " + "accepts u32 and s32\n"); + assert(0); + break; } + + break; + } // MIN - case ATOMIC_MIN: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; + case ATOMIC_MIN: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MIN only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MAX + case ATOMIC_MAX: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MAX only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEFAULT + default: { + assert(0); + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if (data_ready) { + mem->write(effective_address, size / 8, &op_result.s64, thread, pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } +} + +// atom_impl will now result in a callback being called in mem_ctrl_pop +// (gpu-sim.c) +void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) + + // obtain memory space of the operation + memory_space_t space = pI->get_space(); + + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address + // calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; + + addr_t effective_address_final; + + // handle generic memory space by converting it to global + if (space == undefined_space) { + if (whichspace(effective_address) == global_space) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert(space == global_space || space == shared_space); + effective_address_final = effective_address; + } + + // Check state space + assert(space == global_space || space == shared_space); + + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; +} + +void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { + ptx_instruction *pI = const_cast(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); + + switch (bar_op) { + case SYNC_OPTION: { + if (pI->get_num_operands() > 1) { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + } else { + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: { + if (pI->get_num_operands() > 3) { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32 = !(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); - assert(0); + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); break; - } - - break; - } - // MAX - case ATOMIC_MAX: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; + default: + abort(); break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; + } + } else { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32 = !(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * + thread->get_ntid().z); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); - assert(0); + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); break; - } - - break; + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + default: + abort(); + break; + } } - // DEFAULT - default: - { - assert(0); - break; + break; + } + default: + abort(); + break; + } + + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; +} + +void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) { + case U32_TYPE: { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } else { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; } - } - - // Write operation result into memory - // (i.e. copy src1_data to dst) - if ( data_ready ) { - mem->write(effective_address,size/8,&op_result.s64,thread,pI); - } else { - printf("Execution error: data_ready not set\n"); - assert(0); - } -} - -// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) -void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) - - // obtain memory space of the operation - memory_space_t space = pI->get_space(); - - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; - - addr_t effective_address_final; - - // handle generic memory space by converting it to global - if ( space == undefined_space ) { - if( whichspace(effective_address) == global_space ) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid,effective_address); - space = shared_space; + break; + } + case S64_TYPE: { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; } else { - abort(); + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; } - } else { - assert( space == global_space || space == shared_space ); - effective_address_final = effective_address; - } - - // Check state space - assert( space == global_space || space == shared_space ); - - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; -} - -void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) -{ - ptx_instruction * pI = const_cast(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); - - switch(bar_op){ - case SYNC_OPTION: - { - if(pI->get_num_operands()>1){ - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - }else{ - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - } - break; - } - case ARRIVE_OPTION: - { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: - { - if(pI->get_num_operands()>3){ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32=!(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - default: - abort(); - break; - } - }else{ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32=!(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - default: - abort(); - break; - } - } - break; - } - default: - abort(); - break; - } - - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; -} - -void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) - { - case U32_TYPE: - { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: - { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: - { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } - else - { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; - } - break; - } - case S64_TYPE: - { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; - } - else - { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; - } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - int i,max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - - switch ( i_type ) { - case B32_TYPE: +void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + int i, max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + + switch (i_type) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data=src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for(i=0;iset_operand_value(dst, data, i_type, thread, pI); + } + data = src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for (i = 0; i < len && pos + i < max; i++) { + data.u32 = (~((0x00000001) << (pos + i))) & data.u32; + data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated +void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert( + pI->has_pred() == + false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); +void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + unsigned msb; + switch (i_type) { + case B32_TYPE: + msb = 31; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); + } + break; + default: + assert(0); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - unsigned msb; - switch(i_type){ - case B32_TYPE: - msb = 31; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u32 & (1 << i))) - data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u64 & (1 << i))) - data.u64 |= 1 << (msb - i); +unsigned trunc(unsigned num, unsigned precision) { + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise + // it is 0 + // int round_up = (num & (1 << (latest_one-precision))) >> + // (latest_one-precision); unsigned shifted_output = num >> + // (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + // if (shifted_output == (pow(2,precision)-1)) round_up = 0; + // num = shifted_output + round_up; + num >>= (latest_one - precision + 1); + } + return num; +} +void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, + int stride, int &row, int &col, int &assg_offset) { + int offset; + int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; + int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; + int c_tg_inside_row_offset[] = {0, 1, 0, 1}; + int c_tg_inside_col_offset[] = {0, 0, 2, 2}; + int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; + int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; + + offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); + + if (wmma_type == LOAD_A) { + if (wmma_layout == ROW) { + offset += index + 8 * ((thread % 16) / 8); + } else { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } else { + offset += index + 8 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_C) { + if (type == F16_TYPE) { + row = c_row_offset[thread / 4] + thread % 4; + col = c_col_offset[thread / 4] + index; + } else { + row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + + c_inside_row_offset[index]; + col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + + c_inside_col_offset[index]; + } + assg_offset = index; + } + + if (wmma_type == LOAD_A || wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + row = offset / 16; + col = offset % 16; + } else { + col = offset / 16; + row = offset % 16; + } + } +} + +void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + int i, j, k, thrd; + int row, col, offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; + + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid; + const operand_info &dst = pI->operand_lookup(0); + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + float temp; + half temp2; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:", thrd); + for (int operand_num = 1; operand_num <= 3; operand_num++) { + const operand_info &src_a = pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values(src_a, v, nelem); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for (k = 0; k < nelem; k++) { + printf("%llx ", v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + if (k % 2 == 1) + hex_val = (v[k / 2].s64 & 0xffff); + else + hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); + nw_v[k].f16 = *((half *)&hex_val); + } + } + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + temp = nw_v[k].f16; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ", temp); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } else { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + for (k = 0; k < 8; k++) { + printf("%.2f ", v[k].f32); + } + printf("\n"); + } + } + switch (operand_num) { + case 1: // operand 1 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_a[row][col] = nw_v[offset]; + } + break; + case 2: // operand 2 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_b[row][col] = nw_v[offset]; + } + break; + case 3: // operand 3 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + if (type2 != F16_TYPE) { + matrix_c[row][col] = v[offset]; + } else { + matrix_c[row][col] = nw_v[offset]; } - break; - default: assert(0); + } + break; + default: + printf("Invalid Operand Index\n"); + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_A\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_a[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_b[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type2 == F16_TYPE) { + temp = matrix_c[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + matrix_d[i][j].f16 = 0; + } + } + + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + for (k = 0; k < 16; k++) { + matrix_d[i][j].f16 = + matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; + } + if ((type == F16_TYPE) && (type2 == F16_TYPE)) + matrix_d[i][j].f16 += matrix_c[i][j].f16; + else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { + temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; + temp = temp2; + matrix_d[i][j].f32 = temp; + } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f16 = half(temp); + } else { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f32 = temp; + } + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_D\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type == F16_TYPE) { + temp = matrix_d[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + int row_t[8]; + int col_t[8]; + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); + } + thread = core->get_thread_info()[tid + thrd]; + + if (type == F32_TYPE) { + thread->set_wmma_vector_operand_values( + dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], + matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], + matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], + matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + temp = matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ", temp); + } + printf("\n"); + + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | + ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); + nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | + ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); + nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | + ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); + nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | + ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); + thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, + nw_data4); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, + nw_data3.s64, nw_data4.s64); + + } else { + printf("wmma:mma:wrong type\n"); + abort(); } - thread->set_operand_value(dst,data, i_type, thread, pI); + } } -void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 - //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); - //unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - //if (shifted_output == (pow(2,precision)-1)) round_up = 0; - //num = shifted_output + round_up; - num >>= (latest_one-precision+1); - } - return num; -} -void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ - int offset; - int c_row_offset[]={0,8,0,8,4,12,4,12}; - int c_col_offset[]={0,0,8,8,0,0,8,8}; - int c_tg_inside_row_offset[]={0,1,0,1}; - int c_tg_inside_col_offset[]={0,0,2,2}; - int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; - int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; - - offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); - - if(wmma_type==LOAD_A){ - if(wmma_layout==ROW){ - offset+=index+8*((thread%16)/8); - } - else{ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if(wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - else{ - offset+=index+8*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if( wmma_type==LOAD_C){ - if(type==F16_TYPE){ - row=c_row_offset[thread/4]+thread%4; - col=c_col_offset[thread/4]+index; - } - else{ - row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; - col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; - } - assg_offset=index; - } - - if(wmma_type==LOAD_A||wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - row=offset/16; - col=offset%16; - } - else{ - col=offset/16; - row=offset%16; - } - } -} - -void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - int i,j,k,thrd; - int row,col,offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; - - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid ; - const operand_info &dst = pI->operand_lookup(0); - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - float temp; - half temp2; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:",thrd); - for(int operand_num=1;operand_num<=3;operand_num++){ - const operand_info &src_a= pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for(k = 0; k < nelem; k++){ - printf("%llx ",v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; - - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - if(k%2==1) - hex_val=(v[k/2].s64&0xffff); - else - hex_val=((v[k/2].s64&0xffff0000)>>16); - nw_v[k].f16 =*((half *)&hex_val); - } - } - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - temp=nw_v[k].f16; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ",temp); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - else{ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - for(k=0;k<8;k++){ - printf("%.2f ",v[k].f32); - } - printf("\n"); - } - } - switch(operand_num) { - case 1 ://operand 1 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_a[row][col]=nw_v[offset]; - } - break; - case 2 ://operand 2 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_b[row][col]=nw_v[offset]; - } - break; - case 3 ://operand 3 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - if(type2!=F16_TYPE){ - matrix_c[row][col]=v[offset]; - } - else { - matrix_c[row][col]=nw_v[offset]; - } - } - break; - default : - printf("Invalid Operand Index\n" ); - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_A\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_a[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_b[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type2==F16_TYPE){ - temp=matrix_c[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - matrix_d[i][j].f16=0; - } - } - - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - for(k=0;k<16;k++){ - matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; - } - if((type==F16_TYPE)&&(type2==F16_TYPE)) - matrix_d[i][j].f16+=matrix_c[i][j].f16; - else if((type==F32_TYPE)&&(type2==F16_TYPE)){ - temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; - temp=temp2; - matrix_d[i][j].f32=temp; - } - else if((type==F16_TYPE)&&(type2==F32_TYPE)){ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f16=half(temp); - } - else{ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f32=temp; - } - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_D\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type==F16_TYPE){ - temp=matrix_d[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - int row_t[8]; - int col_t[8]; - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); - } - thread = core->get_thread_info()[tid+thrd]; - - - if(type==F32_TYPE){ - thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - { - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - temp=matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ",temp); - } - printf("\n"); - - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); - nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); - nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); - nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); - thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); - - } - else{ - printf("wmma:mma:wrong type\n"); - abort(); - } - } -} - -void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static unsigned call_uid_next = 1; - - const operand_info &target = pI->func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (target_func->get_function_size() >0) - target_func->do_pdom(); - target_func->set_pdom(); - } - - // check that number of args and return match function requirements - if( pI->has_return() ^ target_func->has_return() ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); - - if( n_operands != (n_return+1+n_args) ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" - " call instruction and function declaration\n"); - abort(); - } - - // handle intrinsic functions - std::string fname = target_func->get_name(); - if( fname == "vprintf" ) { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } +void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + target_func->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + target_func->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() > 0) target_func->do_pdom(); + target_func->set_pdom(); + } + + // check that number of args and return match function requirements + if (pI->has_return() ^ target_func->has_return()) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of return values " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); + + if (n_operands != (n_return + 1 + n_args)) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + // handle intrinsic functions + std::string fname = target_func->get_name(); + if (fname == "vprintf") { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } #if (CUDART_VERSION >= 5000) - //Jin: handle device runtime apis for CDP - else if(fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); - return; - } - else if(fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); - return; - } - else if(fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); - return; - } + // Jin: handle device runtime apis for CDP + else if (fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( + pI, thread, target_func); + return; + } else if (fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( + pI, thread, target_func); + return; + } else if (fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( + pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if( target_func->has_return() ) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if (target_func->has_return()) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -//Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - - static unsigned call_uid_next = 1; +// Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch ( i_type ) { - case B32_TYPE: + switch (i_type) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64&mask) == 0) ) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64 & mask) == 0)) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst,d, B32_TYPE, thread, pI); + thread->set_operand_value(dst, d, B32_TYPE, thread, pI); } -void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; - case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; - case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; - case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; - default: + switch (i_type) { + case PRED_TYPE: + d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; + break; + case B16_TYPE: + d.u16 = (a.u16 == 0) ? 1 : 0; + break; + case B32_TYPE: + d.u32 = (a.u32 == 0) ? 1 : 0; + break; + case B64_TYPE: + d.u64 = (a.u64 == 0) ? 1 : 0; + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - switch ( to_width ) { - case 8: x.mask_and(0,0xFF); break; - case 16: x.mask_and(0,0xFFFF); break; - case 32: x.mask_and(0,0xFFFFFFFF); break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( from_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 -ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( to_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - return chop(x,0,from_width,0,rounding_mode,saturation_mode); -} - -int saturatei(int a, int max, int min) -{ - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -unsigned int saturatei(unsigned int a, unsigned int max) -{ - if (a > max) a = max; - return a; -} - -ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - half mytemp; - half_float::half tmp_h; - //assert( from_width == 32); - - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: mode = cudaRoundZero; break; - case RNI_OPTION: mode = cudaRoundNearest; break; - case RMI_OPTION: mode = cudaRoundMinInf; break; - case RPI_OPTION: mode = cudaRoundPosInf; break; - default: break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) - tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1<::round_style>(x.f32);//mytemp; - break; - case 32: - y.f32=float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: assert(0); break; - } - } - return y; -} - -double saturated2i (double a, double max, double min) { - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - assert( from_width == 64); - - double tmp; - switch (rounding_mode) { - case RZI_OPTION: tmp = trunc(x.f64); break; - case RNI_OPTION: tmp = nearbyint(x.f64); break; - case RMI_OPTION: tmp = floor(x.f64); break; - case RPI_OPTION: tmp = ceil(x.f64); break; - default: tmp = x.f64; break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); - switch ( to_width ) { - case 8: y.s8 = (char)tmp; break; - case 16: y.s16 = (short)tmp; break; - case 32: y.s32 = (int)tmp; break; - case 64: y.s64 = (long long)tmp; break; - default: assert(0); break; - } - } else if ( to_sign == 0 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); - switch ( to_width ) { - case 8: y.u8 = (unsigned char)tmp; break; - case 16: y.u16 = (unsigned short)tmp; break; - case 32: y.u32 = (unsigned int)tmp; break; - case 64: y.u64 = (unsigned long long)tmp; break; - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: assert(0); break; - } - } - return y; -} - -ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = sext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; - case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; - case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; - case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; - default: break; - } - break; - case 64: y.f64 = y.s32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; - case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; - case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; - case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; - default: break; - } - break; - case 64: y.f64 = y.s64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = zext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; - case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; - case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; - case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; - default: break; - } - break; - case 64: y.f64 = y.u32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - default: break; - } - break; - case 64: y.f64 = y.u64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - if (from_width == 16){ - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float(val); - }else{ - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: - #if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); - #else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); - #endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } - #if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) - #else - if (cuda_math::__cuda___isnanf(y.f32)) - #endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } - - return y; -} - -ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); -#else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); -#endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: - y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; -} - -ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode ) = { - { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, - { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} -}; + assert(0); + break; + } -void ptx_round(ptx_reg_t& data, int rounding_mode, int type) -{ - if (rounding_mode == RN_OPTION) { - return; - } - switch ( rounding_mode ) { - case RZI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) data.f64 = ceil(data.f64); //negative - else data.f64 = floor(data.f64); //positive - break; - default: assert(0); break; + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + switch (to_width) { + case 8: + x.mask_and(0, 0xFF); + break; + case 16: + x.mask_and(0, 0xFFFF); + break; + case 32: + x.mask_and(0, 0xFFFFFFFF); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (from_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +// sign extend depending on the destination register size - hack to get +// SobelFilter working in CUDA 4.2 +ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (to_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); +} + +int saturatei(int a, int max, int min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +unsigned int saturatei(unsigned int a, unsigned int max) { + if (a > max) a = max; + return a; +} + +ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + half mytemp; + half_float::half tmp_h; + // assert( from_width == 32); + + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: + mode = cudaRoundZero; + break; + case RNI_OPTION: + mode = cudaRoundNearest; + break; + case RMI_OPTION: + mode = cudaRoundMinInf; + break; + case RPI_OPTION: + mode = cudaRoundPosInf; + break; + default: + break; + } + + ptx_reg_t y; + if (to_sign == 1) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); + } + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + unsigned int tmp = cuda_math::float2uint(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1); + } + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + y.f16 = half_float::half_cast::round_style>( + x.f32); // mytemp; + break; + case 32: + y.f32 = float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: + assert(0); + break; + } + } + return y; +} + +double saturated2i(double a, double max, double min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + assert(from_width == 64); + + double tmp; + switch (rounding_mode) { + case RZI_OPTION: + tmp = trunc(x.f64); + break; + case RNI_OPTION: + tmp = nearbyint(x.f64); + break; + case RMI_OPTION: + tmp = floor(x.f64); + break; + case RPI_OPTION: + tmp = ceil(x.f64); + break; + default: + tmp = x.f64; + break; + } + + ptx_reg_t y; + if (to_sign == 1) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: + assert(0); + break; + } + } + return y; +} + +ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__int2float_rz(y.s32); + break; + case RN_OPTION: + y.f32 = cuda_math::__int2float_rn(y.s32); + break; + case RM_OPTION: + y.f32 = cuda_math::__int2float_rd(y.s32); + break; + case RP_OPTION: + y.f32 = cuda_math::__int2float_ru(y.s32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ll2float_rz(y.s64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ll2float_rn(y.s64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ll2float_rd(y.s64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ll2float_ru(y.s64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__uint2float_rz(y.u32); + break; + case RN_OPTION: + y.f32 = cuda_math::__uint2float_rn(y.u32); + break; + case RM_OPTION: + y.f32 = cuda_math::__uint2float_rd(y.u32); + break; + case RP_OPTION: + y.f32 = cuda_math::__uint2float_ru(y.u32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + if (from_width == 16) { + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float(val); + } else { + switch (rounding_mode) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); +#else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); +#endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } +#if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) +#else + if (cuda_math::__cuda___isnanf(y.f32)) +#endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } + + return y; +} + +ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + switch (rounding_mode) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f64 = nearbyint(x.f64); +#else + y.f64 = cuda_math::__internal_nearbyintf(x.f64); +#endif + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: + y.f64 = x.f64; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; +} + +ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, + unsigned to_width, int to_sign, int rounding_mode, + int saturation_mode) = { + {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, + {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; + +void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { + if (rounding_mode == RN_OPTION) { + return; + } + switch (rounding_mode) { + case RZI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = truncf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) + data.f64 = ceil(data.f64); // negative + else + data.f64 = floor(data.f64); // positive + break; + default: + assert(0); + break; } break; - case RNI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE:// assert(0); break; + case RNI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + data.f16 = nearbyintf(data.f16); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); #endif - break; - case F32_TYPE: + break; + case F32_TYPE: #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + data.f32 = nearbyintf(data.f32); #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); #endif - break; - case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; - default: assert(0); break; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = round(data.f64); + break; + default: + assert(0); + break; } break; - case RMI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; - default: assert(0); break; + case RMI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = floorf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = floor(data.f64); + break; + default: + assert(0); + break; } break; - case RPI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; - case F32_TYPE: data.f32 = ceilf(data.f32); break; - case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; - default: assert(0); break; + case RPI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = ceilf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = ceilf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = ceil(data.f64); + break; + default: + assert(0); + break; } break; - default: break; - } + default: + break; + } - if (type == F32_TYPE) { + if (type == F32_TYPE) { #if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) + if (isnanf(data.f32)) #else - if (cuda_math::__cuda___isnanf(data.f32)) + if (cuda_math::__cuda___isnanf(data.f32)) #endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE)||(type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE) || (type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) -{ - if (!saturation_mode) { - return; - } - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: //assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; //negative - if (data.f16 < 0.0f) data.f16 = 0.0f; //positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; //negative - if (data.f32 < 0.0f) data.f32 = 0.0f; //positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; //negative - if (data.f64 < 0.0f) data.f64 = 0.0f; //positive - break; - default: assert(0); break; - } - -} - -void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); - -// if ( to_type == F16_TYPE || from_type == F16_TYPE ) -// abort(); - - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - - if(pI->is_neg()){ - - switch( from_type ) { +void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { + if (!saturation_mode) { + return; + } + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; // negative + if (data.f16 < 0.0f) data.f16 = 0.0f; // positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; // negative + if (data.f32 < 0.0f) data.f32 = 0.0f; // positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; // negative + if (data.f64 < 0.0f) data.f64 = 0.0f; // positive + break; + default: + assert(0); + break; + } +} + +void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + + // if ( to_type == F16_TYPE || from_type == F16_TYPE ) + // abort(); + + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = + type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + + if (pI->is_neg()) { + switch (from_type) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - data.s8 = -data.s8; - break; + data.s8 = -data.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - data.s16 = -data.s16; - break; + data.s16 = -data.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - data.s32 = -data.s32; - break; + data.s32 = -data.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - data.s64 = -data.s64; - break; + data.s64 = -data.s64; + break; case F16_TYPE: - data.f16 = -data.f16; - break; + data.f16 = -data.f16; + break; case F32_TYPE: - data.f32 = -data.f32; - break; + data.f32 = -data.f32; + break; case F64_TYPE: case FF64_TYPE: - data.f64 = -data.f64; - break; + data.f64 = -data.f64; + break; default: - assert(0); + assert(0); + } + } + + if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( + data, from_width, to_width, to_sign, rounding_mode, saturation_mode); + data = result; + } + + thread->set_operand_value(dst, data, to_type, thread, pI); +} + +void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); + + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (to_non_generic) { + switch (space.get_type()) { + case shared_space: + to_addr_hw = generic_to_shared(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); + break; + case global_space: + to_addr_hw = generic_to_global(from_addr_hw); + break; + default: + abort(); + } + } else { + switch (space.get_type()) { + case shared_space: + to_addr_hw = shared_to_generic(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + + thread->get_local_mem_stack_pointer(); + break; // add stack ptr here so that it can be passed as a pointer at + // function call + case global_space: + to_addr_hw = global_to_generic(from_addr_hw); + break; + default: + abort(); + } + } + + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(), to_addr); +} + +void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8; + break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; + break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; + break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 / src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 / src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 / src2_data.f64; + break; + default: + assert(0); + break; + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + printf("DP4A instruction not implemented yet"); + assert(0); +} + +void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + data.f32 = cuda_math::__powf(2.0, src1_data.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); +} + +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry = false); + +void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread); +} + +void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a; + bool t = false; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + switch (space.get_type()) { + case shared_space: + t = isspace_shared(smid, addr); + case local_space: + t = isspace_local(smid, hwtid, addr); + case global_space: + t = isspace_global(addr); + default: + abort(); + } + + ptx_reg_t p; + p.pred = t ? 1 : 0; + + thread->set_reg(dst.get_symbol(), p); +} + +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr) { + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (space == param_space_unclassified) { + // need to op to determine whether it refers to a kernel param or local + // param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if (ti.is_param_kernel()) + space = param_space_kernel; + else if (ti.is_param_local()) { + space = param_space_local; + } + // mov r1, param-label + else if (ti.is_reg()) { + space = param_space_kernel; + } else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", + s->name().c_str()); + abort(); + } + } + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; + addr += thread->get_local_mem_stack_pointer(); + break; + case tex_space: + mem = thread->get_tex_memory(); + break; + case surf_space: + mem = thread->get_surf_memory(); + break; + case param_space_kernel: + mem = thread->get_param_memory(); + break; + case shared_space: + mem = thread->m_shared_mem; + break; + case sstarr_space: + mem = thread->m_sstarr_mem; + break; + case const_space: + mem = thread->get_global_memory(); + break; + case generic_space: + if (thread->get_ptx_version().ver() >= 2.0) { + // convert generic address to memory space address + space = whichspace(addr); + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + addr = generic_to_global(addr); + break; + case local_space: + mem = thread->m_local_mem; + addr = generic_to_local(smid, hwtid, addr); + break; + case shared_space: + mem = thread->m_shared_mem; + addr = generic_to_shared(smid, addr); + break; + default: + abort(); + } + } else { + abort(); } + break; + case param_space_unclassified: + case undefined_space: + default: + abort(); + } +} + +void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + data.u64 = 0; + type_info_key::type_decode(type, size, t); + if (!vector_spec) { + mem->read(addr, size / 8, &data.s64); + if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); + thread->set_operand_value(dst, data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr, size / 8, &data1.s64); + mem->read(addr + size / 8, size / 8, &data2.s64); + if (vector_spec != V2_TYPE) { // either V3 or V4 + mem->read(addr + 2 * size / 8, size / 8, &data3.s64); + if (vector_spec != V3_TYPE) { // v4 + mem->read(addr + 3 * size / 8, size / 8, &data4.s64); + thread->set_vector_operand_values(dst, data1, data2, data3, data4); + } else // v3 + thread->set_vector_operand_values(dst, data1, data2, data3, data3); + } else // v2 + thread->set_vector_operand_values(dst, data1, data2, data2, data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} - } +void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); +} +void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); +} +void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a = pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t *v = new ptx_reg_t[8]; + thread->get_vector_operand_values(src_a, v, nelem); + stride = src2_data.u32; - if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); - data = result; - } + memory_space_t space = pI->get_space(); - thread->set_operand_value(dst, data, to_type, thread, pI ); -} + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; -void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + decode_space(space, thread, src1, mem, addr); + + type_info_key::type_decode(type, size, t); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, + addr_reg.u32, size, src2_data.u32); + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for (k = 0; k < 8; k++) { + if (k % 2 == 0) + nw_v[k].s64 = (v[k / 2].s64 & 0xffff); + else + nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); + } - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + for (k = 0; k < 8; k++) { + if (type == F32_TYPE) { + // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); + mem->write(push_addr, size / 8, &v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, + v[6].s64, v[7].s64); + float temp; + int l; + printf("thread=%d:", thrd); + for (l = 0; l < 8; l++) { + temp = v[l].f32; + printf("%.2f", temp); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; + } else if (wmma_layout == COL) { + // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2 * stride; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + } - if( to_non_generic ) { - switch( space.get_type() ) { - case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; - case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; - case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; - default: abort(); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, + nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); } - } else { - switch( space.get_type() ) { - case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; - case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) - + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call - case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; - default: abort(); - } - } - - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(),to_addr); -} - -void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8 ; break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; - default: assert(0); break; - } - thread->set_operand_value(dst,data, i_type, thread,pI); -} - -void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - printf("DP4A instruction not implemented yet"); - assert(0); - -} - -void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - data.f32 = cuda_math::__powf(2.0, src1_data.f32); - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,data, i_type, thread,pI); -} + } -void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - thread->set_done(); - thread->exitCore(); - thread->registerExit(); -} + delete[] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); + if ((type == F16_TYPE) && + (wmma_layout == COL)) // check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction -void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI,thread); + assert(inst.memory_op == insn_memory_op); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a; - bool t=false; +void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + int t, i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd, stride; + ptx_thread_info *thread; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t src1_data = + thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = + thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride = src2_data.u32; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + + decode_space(space, thread, src1, mem, addr); + type_info_key::type_decode(type, size, t); + + ptx_reg_t data[16]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, + src1_data.u32, size, src2_data.u32); + + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; + + if (wmma_type == LOAD_A) { + for (i = 0; i < 16; i++) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_B) { + for (i = 0; i < 16; i++) { + if (wmma_layout == COL) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == ROW) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_C) { + for (i = 0; i < 8; i++) { + if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * stride * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("mma_ld:wrong_type\n"); + abort(); + } + } else if (type == F32_TYPE) { + // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("wrong type"); + abort(); + } + } + } else { + printf("wrong wmma type\n"); + ; + abort(); + } + // generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); + + if ((wmma_type == LOAD_C) && (type == F16_TYPE) && + (wmma_layout == COL)) // memory address is scattered, check the + // profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert(inst.memory_op == insn_memory_op); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + if (type == F16_TYPE) { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 16; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ", thrd); + float temp; + for (i = 0; i < 16; i++) { + temp = data[i].f16; + printf("%.2f ", temp); + } + printf("\n"); + } else { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%.2f ", data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + } + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); + if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { + thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], + data[3], data[4], data[5], data[6], + data[7]); + } else { + ptx_reg_t nw_data[8]; + int num_reg; - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + if (wmma_type == LOAD_C) + num_reg = 4; + else + num_reg = 8; - switch( space.get_type() ) { - case shared_space: t = isspace_shared( smid, addr ); - case local_space: t = isspace_local( smid, hwtid, addr ); - case global_space: t = isspace_global( addr ); - default: abort(); - } + for (i = 0; i < num_reg; i++) { + nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | + ((data[2 * i + 1].s64 & 0xffff)); + } - ptx_reg_t p; - p.pred = t?1:0; + if (wmma_type == LOAD_C) + thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], + nw_data[2], nw_data[3]); + else + thread->set_wmma_vector_operand_values( + dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], + nw_data[5], nw_data[6], nw_data[7]); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", + data[0].u64, data[1].u64, nw_data[0].u64); + printf( + "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", + data[2].u64, data[3].u64, nw_data[1].u64); + printf( + "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", + data[4].u64, data[5].u64, nw_data[2].u64); + printf( + "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", + data[6].u64, data[7].u64, nw_data[3].u64); + if (wmma_type != LOAD_C) { + printf( + "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", + data[8].u64, data[9].u64, nw_data[4].s64); + printf( + "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" + "llx\n", + data[10].u64, data[11].u64, nw_data[5].u64); + printf( + "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" + "llx\n", + data[12].u64, data[13].u64, nw_data[6].u64); + printf( + "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" + "llx\n", + data[14].u64, data[15].u64, nw_data[3].u64); + } + } + } - thread->set_reg(dst.get_symbol(),p); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) -{ - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - if( space == param_space_unclassified ) { - // need to op to determine whether it refers to a kernel param or local param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if( ti.is_param_kernel() ) - space = param_space_kernel; - else if( ti.is_param_local() ) { - space = param_space_local; - } - //mov r1, param-label - else if (ti.is_reg() ){ - space = param_space_kernel; - } - else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); - abort(); - } - } - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; - addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: mem = thread->get_tex_memory(); break; - case surf_space: mem = thread->get_surf_memory(); break; - case param_space_kernel: mem = thread->get_param_memory(); break; - case shared_space: mem = thread->m_shared_mem; break; - case sstarr_space: mem = thread->m_sstarr_mem; break; - case const_space: mem = thread->get_global_memory(); break; - case generic_space: - if( thread->get_ptx_version().ver() >= 2.0 ) { - // convert generic address to memory space address - space = whichspace(addr); - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; - case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; - case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; - default: abort(); - } - } else { - abort(); - } + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + d.f32 = log(a.f32) / log(2); break; - case param_space_unclassified: - case undefined_space: - default: - abort(); - } -} - -void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - data.u64=0; - type_info_key::type_decode(type,size,t); - if (!vector_spec) { - mem->read(addr,size/8,&data.s64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(data,size,dst); - thread->set_operand_value(dst,data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr,size/8,&data1.s64); - mem->read(addr+size/8,size/8,&data2.s64); - if (vector_spec != V2_TYPE) { //either V3 or V4 - mem->read(addr+2*size/8,size/8,&data3.s64); - if (vector_spec != V3_TYPE) { //v4 - mem->read(addr+3*size/8,size/8,&data4.s64); - thread->set_vector_operand_values(dst,data1,data2,data3,data4); - } else //v3 - thread->set_vector_operand_values(dst,data1,data2,data3,data3); - } else //v2 - thread->set_vector_operand_values(dst,data1,data2,data2,data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; -} - -void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} -void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} - -void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid ; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - for (thrd=0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a= pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t* v= new ptx_reg_t[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - stride = src2_data.u32; - - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - decode_space(space,thread,src1,mem,addr); - - type_info_key::type_decode(type, size, t); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t push_addr; - - ptx_reg_t nw_v[8]; - for(k=0;k<8;k++){ - if(k%2==0) - nw_v[k].s64=(v[k/2].s64&0xffff); - else - nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); - } - - for(k=0;k<8;k++){ - if(type==F32_TYPE){ - //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); - mem->write(push_addr,size/8,&v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); - float temp; - int l; - printf("thread=%d:",thrd); - for(l=0;l<8;l++){ - temp=v[l].f32; - printf("%.2f",temp); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - if(k%2==0) - mem_txn_addr[num_mem_txn++]=push_addr; - } - else if(wmma_layout==COL){ - //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2*stride; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - } - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); - } - } - - delete [] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - - assert( inst.memory_op == insn_memory_op ); - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - int t,i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd,stride; - ptx_thread_info *thread; - - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride=src2_data.u32; - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - - decode_space(space,thread,src1,mem,addr); - type_info_key::type_decode(type, size, t); - - ptx_reg_t data[16]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); - - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - if(wmma_type==LOAD_A){ - for(i=0;i<16;i++){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_B){ - for(i=0;i<16;i++){ - if(wmma_layout==COL){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==ROW){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_C){ - for(i=0;i<8;i++){ - if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*stride*i; - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("mma_ld:wrong_type\n"); - abort(); - } - } - else if(type==F32_TYPE){ - //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("wrong type"); - abort(); - } - } - } - else{ - printf("wrong wmma type\n");; - abort(); - } - //generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert( inst.memory_op == insn_memory_op ); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - if(type==F16_TYPE){ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<16;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - - printf("\nmma_ld:thread%d= ",thrd); - float temp; - for(i=0;i<16;i++){ - temp=data[i].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - else{ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%.2f ",data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - } - } - - if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ - thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); - } - else{ - ptx_reg_t nw_data[8]; - int num_reg; - - if(wmma_type==LOAD_C) - num_reg=4; - else - num_reg=8; - - for(i=0;iset_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); - else - thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); - printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); - printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); - printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); - if(wmma_type!=LOAD_C){ - printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); - printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); - printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); - printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); - } - } - } - - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - d.f32 = log(a.f32)/log(2); - break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert( !pI->is_wide() ); + assert(!pI->is_wide()); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_hi() ) { - d.s64 = (t.s64>>16) + c.s32; - if ( sat_mode ) { - if ( d.s64 > (int)0x7FFFFFFF ) - d.s64 = (int)0x7FFFFFFF; - else if ( d.s64 < (int)0x80000000 ) - d.s64 = (int)0x80000000; - } - } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; - else assert(0); - break; - case U32_TYPE: + if (pI->is_hi()) { + d.s64 = (t.s64 >> 16) + c.s32; + if (sat_mode) { + if (d.s64 > (int)0x7FFFFFFF) + d.s64 = (int)0x7FFFFFFF; + else if (d.s64 < (int)0x80000000) + d.s64 = (int)0x80000000; + } + } else if (pI->is_lo()) + d.s64 = t.s32 + c.s32; + else + assert(0); + break; + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; - else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; - else assert(0); + if (pI->is_hi()) + d.u64 = (t.u64 >> 16) + c.u32; + else if (pI->is_lo()) + d.u64 = t.u32 + c.u32; + else + assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, false); +void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, false); } -void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; - - int carry=0; - int overflow=0; - - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>=2; - } +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + int carry = 0; + int overflow = 0; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>= 2; + } - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; - else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; - else assert(0); - carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; + else if (pI->is_lo()) + d.s16 = t.s16 + c.s16 + carry_bit.pred; + else + assert(0); + carry = + ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; - else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; + else if (pI->is_lo()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; - else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; - else assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if (pI->is_hi()) + d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; + else if (pI->is_lo()) + d.u16 = t.u16 + c.u16 + carry_bit.pred; + else + assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & + 0x100000000) >> + 32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; - else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if (pI->is_hi()) + d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; + else if (pI->is_lo()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else assert(0); - break; - case F16_TYPE:{ - // assert(0); - // break; - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64 + c.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else + assert(0); + break; + case F16_TYPE: { + // assert(0); + // break; + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; } - default: + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); -} - -bool isNaN(float x) -{ - return std::isnan(x); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(double x) -{ - return std::isnan(x); -} +bool isNaN(float x) { return std::isnan(x); } -void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +bool isNaN(double x) { return std::isnan(x); } - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MAX_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MAX_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MAX_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MAX_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MAX_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MAX_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MAX_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MAX_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // handled by timing simulator +void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // handled by timing simulator } -void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MIN_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MIN_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MIN_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MIN_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MIN_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MIN_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MIN_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MIN_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert( src1.is_param_local() == 0 ); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert(src1.is_param_local() == 0); - if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; + if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && + (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - switch( pI->get_type() ) { - case B16_TYPE: nbits_to_move = 16; break; - case B32_TYPE: nbits_to_move = 32; break; - case B64_TYPE: nbits_to_move = 64; break; - default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; - } + switch (pI->get_type()) { + case B16_TYPE: + nbits_to_move = 16; + break; + case B32_TYPE: + nbits_to_move = 32; + break; + case B64_TYPE: + nbits_to_move = 64; + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported type " + "qualifier\n"); + assert(0); + break; + } - if( src1.is_vector() ) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem ); - - unsigned bits_per_src_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_src_elem) { - case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; - case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; - case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; - default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; - } - } - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch( pI->get_type() ) { - case B16_TYPE: tmp_bits.u16 = data.u16; break; - case B32_TYPE: tmp_bits.u32 = data.u32; break; - case B64_TYPE: tmp_bits.u64 = data.u64; break; - default: assert(0); break; - } - } + if (src1.is_vector()) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem); - if( dst.is_vector() ) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_dst_elem) { - case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; - case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; - case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; - default: - printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); - assert(0); - break; - } - } - thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); - } else { - thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); + unsigned bits_per_src_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_src_elem) { + case 8: + tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); + break; + case 16: + tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); + break; + case 32: + tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (src)\n"); + assert(0); + break; + } } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in ptxplus + } else { data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { - - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + switch (pI->get_type()) { + case B16_TYPE: + tmp_bits.u16 = data.u16; + break; + case B32_TYPE: + tmp_bits.u32 = data.u32; + break; + case B64_TYPE: + tmp_bits.u64 = data.u64; + break; + default: + assert(0); + break; + } + } - thread->set_operand_value(dst, data, i_type, thread, pI); + if (dst.is_vector()) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_dst_elem) { + case 8: + v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); + break; + case 16: + v[i].u16 = + (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); + break; + case 32: + v[i].u32 = + (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); + } else { + thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); + } + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in + // ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - } + thread->set_operand_value(dst, data, i_type, thread, pI); + } } -void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + src1_data.mask_and(0, 0x00FFFFFF); + src2_data.mask_and(0, 0x00FFFFFF); - src1_data.mask_and(0,0x00FFFFFF); - src2_data.mask_and(0,0x00FFFFFF); - - switch ( i_type ) { - case S32_TYPE: - if( src1_data.get_bit(23) ) - src1_data.mask_or(0xFFFFFFFF,0xFF000000); - if( src2_data.get_bit(23) ) - src2_data.mask_or(0xFFFFFFFF,0xFF000000); + switch (i_type) { + case S32_TYPE: + if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); + if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf( + "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if ( pI->is_hi() ) { - data.u64 = data.u64 >> 16; - data.mask_and(0,0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0,0xFFFFFFFF); - } + if (pI->is_hi()) { + data.u64 = data.u64 >> 16; + data.mask_and(0, 0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0, 0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if ( pI->is_wide() ) d.s32 = t.s32; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16); - else if ( pI->is_lo() ) d.s16 = t.s16; - else assert(0); + if (pI->is_wide()) + d.s32 = t.s32; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16); + else if (pI->is_lo()) + d.s16 = t.s16; + else + assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if ( pI->is_wide() ) d.s64 = t.s64; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32); - else if ( pI->is_lo() ) d.s32 = t.s32; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32); + else if (pI->is_lo()) + d.s32 = t.s32; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.s64 = t.s64; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.s64 = t.s64; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if ( pI->is_wide() ) d.u32 = t.u32; - else if ( pI->is_lo() ) d.u16 = t.u16; - else if ( pI->is_hi() ) d.u16 = (t.u32>>16); - else assert(0); + if (pI->is_wide()) + d.u32 = t.u32; + else if (pI->is_lo()) + d.u16 = t.u16; + else if (pI->is_hi()) + d.u16 = (t.u32 >> 16); + else + assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if ( pI->is_wide() ) d.u64 = t.u64; - else if ( pI->is_lo() ) d.u32 = t.u32; - else if ( pI->is_hi() ) d.u32 = (t.u64>>32); - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64; + else if (pI->is_lo()) + d.u32 = t.u32; + else if (pI->is_hi()) + d.u32 = (t.u64 >> 32); + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.u64 = t.u64; - else assert(0); - break; - case F16_TYPE:{ - //assert(0); - //break; - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f16 = a.f16 * b.f16; - - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f32 = a.f32 * b.f32; - - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE:{ - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; - } - default: - assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.u64 = t.u64; + else + assert(0); break; - } + case F16_TYPE: { + // assert(0); + // break; + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst, d, i_type, thread, pI); -} + d.f16 = a.f16 * b.f16; -void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + d.f32 = a.f32 * b.f32; - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI); +} - switch ( to_type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); break; - case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; - default: assert(0); break; - } +void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + + switch (to_type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; + break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); + break; + case F16_TYPE: + data.f16 = 0.0f - src1_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = 0.0f - src1_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = 0.0f - src1_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, to_type, thread, pI); + thread->set_operand_value(dst, data, to_type, thread, pI); } -//nandn bitwise negates second operand then bitwise nands with the first operand -void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +// nandn bitwise negates second operand then bitwise nands with the first +// operand +void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); + thread->set_operand_value(dst, data, i_type, thread, pI); +} - thread->set_operand_value(dst,data, i_type, thread, pI); +// norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; -} + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); -//norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - thread->set_operand_value(dst,data, i_type, thread, pI); + switch (i_type) { + case PRED_TYPE: + d.pred = (~(a.pred) & 0x000F); + break; + case B16_TYPE: + d.u16 = ~a.u16; + break; + case B32_TYPE: + d.u32 = ~a.u32; + break; + case B64_TYPE: + d.u64 = ~a.u64; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI); } -void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; - case B16_TYPE: d.u16 = ~a.u16; break; - case B32_TYPE: d.u32 = ~a.u32; break; - case B64_TYPE: d.u64 = ~a.u64; break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); + + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + + switch (i_type) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); +int prmt_mode_present(int mode) { + int returnval = 0; + switch (mode) { + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval = 1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode, int control, int d_sel_index, signed long long value) { + int returnval = 0; + int prmt_f4e_mode[4][4] = { + {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; + int prmt_b4e_mode[4][4] = { + {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; + int prmt_rc8_mode[4][4] = { + {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; + int prmt_ecl_mode[4][4] = { + {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; + int prmt_ecr_mode[4][4] = { + {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; + int prmt_rc16_mode[4][4] = { + {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; + + if (!prmt_mode_present(mode)) { + if (control & 0x8) { + returnval = 0xff; + } else { + returnval = (value >> (8 * control)) & 0xff; + } + } else { + switch (mode) { + case PRMT_F4E_MODE: + returnval = prmt_f4e_mode[control][d_sel_index]; + break; + case PRMT_B4E_MODE: + returnval = prmt_b4e_mode[control][d_sel_index]; + break; + case PRMT_RC8_MODE: + returnval = prmt_rc8_mode[control][d_sel_index]; + break; + case PRMT_ECL_MODE: + returnval = prmt_ecl_mode[control][d_sel_index]; + break; + case PRMT_ECR_MODE: + returnval = prmt_ecr_mode[control][d_sel_index]; + break; + case PRMT_RC16_MODE: + returnval = prmt_rc16_mode[control][d_sel_index]; + break; + // Change the default from printing "ERROR" to just asserting + default: + assert(false); + } + } + return (returnval << 8 * d_sel_index); +} - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; +void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); + int ctl[4]; + + if (!prmt_mode_present(mode)) { + ctl[0] = (src3_data.s32 >> 0) & 0xf; + ctl[1] = (src3_data.s32 >> 4) & 0xf; + ctl[2] = (src3_data.s32 >> 8) & 0xf; + ctl[3] = (src3_data.s32 >> 12) & 0xf; + } else { + ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + data.s32 = 0; + data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First + // byte-0 + data.s32 = + data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 + data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third + // byte-2 + data.s32 = + data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 + + thread->set_operand_value(dst, data, i_type, thread, pI); } -void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); - data.u32 = mask.count(); - } break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,data, i_type, thread, pI); -} -void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -int prmt_mode_present(int mode) -{ - int returnval=0; - switch(mode){ - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval=1; - break; - default: - break; - } - return returnval; -} -int read_byte(int mode, int control, int d_sel_index, signed long long value){ - - int returnval = 0; - int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; - int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; - int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; - int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; - int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; - int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; - - if(!prmt_mode_present(mode)){ - if(control&0x8){ - returnval=0xff; - } - else{ - returnval= (value>>(8*control)) & 0xff; - } - } - else{ - switch(mode){ - case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; - case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; - case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; - case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; - case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; - case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; - // Change the default from printing "ERROR" to just asserting - default: assert(false); - } - } - return (returnval << 8 * d_sel_index); -} - -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - - ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - tmpdata.s64=src1_data.s32|(src2_data.s64<<32); - int ctl[4]; - - if(!prmt_mode_present(mode)){ - ctl[0]=(src3_data.s32>>0)&0xf; - ctl[1]=(src3_data.s32>>4)&0xf; - ctl[2]=(src3_data.s32>>8)&0xf; - ctl[3]=(src3_data.s32>>12)&0xf; - } - else{ - ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; - } - - data.s32=0; - data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 - data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 - data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 - data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 - - thread->set_operand_value(dst,data, i_type, thread, pI); - - -} - -void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} -void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: assert(0); break; - } + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -//Ptxplus version of ret instruction. -void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop_plus(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +// Ptxplus version of ret instruction. +void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop_plus(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf +void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); - break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -#define SAD(d,a,b,c) d = c + ((adst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +#define SAD(d, a, b, c) d = c + ((a < b) ? (b - a) : (a - b)) - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); +void sad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; - case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; - case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; - case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; - case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; - case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; - case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; - case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; - default: + switch (i_type) { + case U16_TYPE: + SAD(d.u16, a.u16, b.u16, c.u16); + break; + case U32_TYPE: + SAD(d.u32, a.u32, b.u32, c.u32); + break; + case U64_TYPE: + SAD(d.u64, a.u64, b.u64, c.u64); + break; + case S16_TYPE: + SAD(d.s16, a.s16, b.s16, c.s16); + break; + case S32_TYPE: + SAD(d.s32, a.s32, b.s32, c.s32); + break; + case S64_TYPE: + SAD(d.s64, a.s64, b.s64, c.s64); + break; + case F32_TYPE: + SAD(d.f32, a.f32, b.f32, c.f32); + break; + case F64_TYPE: + case FF64_TYPE: + SAD(d.f64, a.f64, b.f64, c.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001))?a:b; + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of c.pred must be inverted to get proper + // behavior + d = (!(c.pred & 0x0001)) ? a : b; - thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); } -bool isFloat(int type) -{ - switch ( type ) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) { + switch (type) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) -{ - bool t = false; +bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { + bool t = false; - switch ( type ) { - case B16_TYPE: + switch (type) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s16 == b.s16); break; - case NE_OPTION: t = (a.s16 != b.s16); break; - case LT_OPTION: t = (a.s16 < b.s16); break; - case LE_OPTION: t = (a.s16 <= b.s16); break; - case GT_OPTION: t = (a.s16 > b.s16); break; - case GE_OPTION: t = (a.s16 >= b.s16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s16 == b.s16); + break; + case NE_OPTION: + t = (a.s16 != b.s16); + break; + case LT_OPTION: + t = (a.s16 < b.s16); + break; + case LE_OPTION: + t = (a.s16 <= b.s16); + break; + case GT_OPTION: + t = (a.s16 > b.s16); + break; + case GE_OPTION: + t = (a.s16 >= b.s16); + break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s32 == b.s32); break; - case NE_OPTION: t = (a.s32 != b.s32); break; - case LT_OPTION: t = (a.s32 < b.s32); break; - case LE_OPTION: t = (a.s32 <= b.s32); break; - case GT_OPTION: t = (a.s32 > b.s32); break; - case GE_OPTION: t = (a.s32 >= b.s32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s32 == b.s32); + break; + case NE_OPTION: + t = (a.s32 != b.s32); + break; + case LT_OPTION: + t = (a.s32 < b.s32); + break; + case LE_OPTION: + t = (a.s32 <= b.s32); + break; + case GT_OPTION: + t = (a.s32 > b.s32); + break; + case GE_OPTION: + t = (a.s32 >= b.s32); + break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s64 == b.s64); break; - case NE_OPTION: t = (a.s64 != b.s64); break; - case LT_OPTION: t = (a.s64 < b.s64); break; - case LE_OPTION: t = (a.s64 <= b.s64); break; - case GT_OPTION: t = (a.s64 > b.s64); break; - case GE_OPTION: t = (a.s64 >= b.s64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s64 == b.s64); + break; + case NE_OPTION: + t = (a.s64 != b.s64); + break; + case LT_OPTION: + t = (a.s64 < b.s64); + break; + case LE_OPTION: + t = (a.s64 <= b.s64); + break; + case GT_OPTION: + t = (a.s64 > b.s64); + break; + case GE_OPTION: + t = (a.s64 >= b.s64); + break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - case LT_OPTION: t = (a.u16 < b.u16); break; - case LE_OPTION: t = (a.u16 <= b.u16); break; - case GT_OPTION: t = (a.u16 > b.u16); break; - case GE_OPTION: t = (a.u16 >= b.u16); break; - case LO_OPTION: t = (a.u16 < b.u16); break; - case LS_OPTION: t = (a.u16 <= b.u16); break; - case HI_OPTION: t = (a.u16 > b.u16); break; - case HS_OPTION: t = (a.u16 >= b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + case LT_OPTION: + t = (a.u16 < b.u16); + break; + case LE_OPTION: + t = (a.u16 <= b.u16); + break; + case GT_OPTION: + t = (a.u16 > b.u16); + break; + case GE_OPTION: + t = (a.u16 >= b.u16); + break; + case LO_OPTION: + t = (a.u16 < b.u16); + break; + case LS_OPTION: + t = (a.u16 <= b.u16); + break; + case HI_OPTION: + t = (a.u16 > b.u16); + break; + case HS_OPTION: + t = (a.u16 >= b.u16); + break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - case LT_OPTION: t = (a.u32 < b.u32); break; - case LE_OPTION: t = (a.u32 <= b.u32); break; - case GT_OPTION: t = (a.u32 > b.u32); break; - case GE_OPTION: t = (a.u32 >= b.u32); break; - case LO_OPTION: t = (a.u32 < b.u32); break; - case LS_OPTION: t = (a.u32 <= b.u32); break; - case HI_OPTION: t = (a.u32 > b.u32); break; - case HS_OPTION: t = (a.u32 >= b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + case LT_OPTION: + t = (a.u32 < b.u32); + break; + case LE_OPTION: + t = (a.u32 <= b.u32); + break; + case GT_OPTION: + t = (a.u32 > b.u32); + break; + case GE_OPTION: + t = (a.u32 >= b.u32); + break; + case LO_OPTION: + t = (a.u32 < b.u32); + break; + case LS_OPTION: + t = (a.u32 <= b.u32); + break; + case HI_OPTION: + t = (a.u32 > b.u32); + break; + case HS_OPTION: + t = (a.u32 >= b.u32); + break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - case LT_OPTION: t = (a.u64 < b.u64); break; - case LE_OPTION: t = (a.u64 <= b.u64); break; - case GT_OPTION: t = (a.u64 > b.u64); break; - case GE_OPTION: t = (a.u64 >= b.u64); break; - case LO_OPTION: t = (a.u64 < b.u64); break; - case LS_OPTION: t = (a.u64 <= b.u64); break; - case HI_OPTION: t = (a.u64 > b.u64); break; - case HS_OPTION: t = (a.u64 >= b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + case LT_OPTION: + t = (a.u64 < b.u64); + break; + case LE_OPTION: + t = (a.u64 <= b.u64); + break; + case GT_OPTION: + t = (a.u64 > b.u64); + break; + case GE_OPTION: + t = (a.u64 >= b.u64); + break; + case LO_OPTION: + t = (a.u64 < b.u64); + break; + case LS_OPTION: + t = (a.u64 <= b.u64); + break; + case HI_OPTION: + t = (a.u64 > b.u64); + break; + case HS_OPTION: + t = (a.u64 >= b.u64); + break; + default: + assert(0); } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: + case F16_TYPE: + assert(0); + break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; - case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case NE_OPTION: + t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LT_OPTION: + t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LE_OPTION: + t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GT_OPTION: + t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GE_OPTION: + t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case EQU_OPTION: + t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NEU_OPTION: + t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LTU_OPTION: + t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LEU_OPTION: + t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GTU_OPTION: + t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GEU_OPTION: + t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NUM_OPTION: + t = !isNaN(a.f32) && !isNaN(b.f32); + break; + case NAN_OPTION: + t = isNaN(a.f32) || isNaN(b.f32); + break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; - case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case NE_OPTION: + t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LT_OPTION: + t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LE_OPTION: + t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GT_OPTION: + t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GE_OPTION: + t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case EQU_OPTION: + t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NEU_OPTION: + t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LTU_OPTION: + t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LEU_OPTION: + t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GTU_OPTION: + t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GEU_OPTION: + t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NUM_OPTION: + t = !isNaN(a.f64) && !isNaN(b.f64); + break; + case NAN_OPTION: + t = isNaN(a.f64) || isNaN(b.f64); + break; + default: + assert(0); } break; - default: assert(0); break; - } + default: + assert(0); + break; + } - return t; + return t; } -void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type,a,b,cmpop); + t = CmpOp(type, a, b, cmpop); - ptx_reg_t data; + ptx_reg_t data; - //the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag + // the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = + (t == + 0); // inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); } -void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if(pI->is_abs()) { - switch ( src_type ) { - case S16_TYPE: a.s16 = my_abs(a.s16); break; - case S32_TYPE: a.s32 = my_abs(a.s32); break; - case S64_TYPE: a.s64 = my_abs(a.s64); break; - case U16_TYPE: a.u16 = a.u16; break; - case U32_TYPE: a.u32 = my_abs(a.u32); break; - case U64_TYPE: a.u64 = my_abs(a.u64); break; - case F32_TYPE: a.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; + // Take abs of first operand if needed + if (pI->is_abs()) { + switch (src_type) { + case S16_TYPE: + a.s16 = my_abs(a.s16); + break; + case S32_TYPE: + a.s32 = my_abs(a.s32); + break; + case S64_TYPE: + a.s64 = my_abs(a.s64); + break; + case U16_TYPE: + a.u16 = a.u16; + break; + case U32_TYPE: + a.u32 = my_abs(a.u32); + break; + case U64_TYPE: + a.u64 = my_abs(a.u64); + break; + case F32_TYPE: + a.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + a.f64 = my_abs(a.f64); + break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } - - t = CmpOp(src_type,a,b,cmpop); - - ptx_reg_t data; - if ( isFloat(pI->get_type()) ) { - data.f32 = (t!=0)?1.0f:0.0f; - } else { - data.u32 = (t!=0)?0xFFFFFFFF:0; - } - - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); - -} - -void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - unsigned i_type = pI->get_type(); - int tid; - - if(core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; - - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; - - int src_idx; - unsigned p; - switch(pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; - - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); - - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ - - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } -} - -void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case B16_TYPE: - case U16_TYPE: - if ( b.u16 >= 16 ) - d.u16 = 0; + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } + + t = CmpOp(src_type, a, b, cmpop); + + ptx_reg_t data; + if (isFloat(pI->get_type())) { + data.f32 = (t != 0) ? 1.0f : 0.0f; + } else { + data.u32 = (t != 0) ? 0xFFFFFFFF : 0; + } + + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); +} + +void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + unsigned i_type = pI->get_type(); + int tid; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + unsigned p; + switch (pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf( + "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " + "threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for + predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + +void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case B16_TYPE: + case U16_TYPE: + if (b.u16 >= 16) + d.u16 = 0; else - d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if ( b.u32 >= 32 ) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if (b.u32 >= 32) + d.u32 = 0; else - d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if ( b.u32 >= 64 ) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if (b.u32 >= 64) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case U16_TYPE: - case B16_TYPE: - if ( b.u16 < 16 ) - d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); +void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case U16_TYPE: + case B16_TYPE: + if (b.u16 < 16) + d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if ( b.u32 < 32 ) - d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if (b.u32 < 32) + d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if ( b.u32 < 64 ) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if (b.u32 < 64) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if ( b.u16 < 16 ) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if (b.u16 < 16) + d.s64 = (a.s16 >> b.s16); else { - if ( a.s16 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s16 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if ( b.u32 < 32 ) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if (b.u32 < 32) + d.s64 = (a.s32 >> b.s32); else { - if ( a.s32 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s32 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if ( b.u64 < 64 ) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if (b.u64 < 64) + d.s64 = (a.s64 >> b.u64); else { - if ( a.s64 < 0 ) { - if ( b.s32 < 0 ) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if (a.s64 < 0) { + if (b.s32 < 0) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - ptx_reg_t a, b, c, d; - - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); - - switch ( c_type ) { - case S32_TYPE: t = c.s32 >= 0; break; - case F32_TYPE: t = c.f32 >= 0; break; - default: assert(0); - } - - switch ( i_type ) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: d.u16 = t?a.u16:b.u16; break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: d.u32 = t?a.u32:b.u32; break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: d.u64 = t?a.u64:b.u64; break; - default: assert(0); - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) - d.f32 = nanf(""); + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); + + switch (c_type) { + case S32_TYPE: + t = c.s32 >= 0; + break; + case F32_TYPE: + t = c.f32 >= 0; + break; + default: + assert(0); + } + + switch (i_type) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + d.u16 = t ? a.u16 : b.u16; + break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + d.u32 = t ? a.u32 : b.u32; + break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: + d.u64 = t ? a.u64 : b.u64; + break; + default: + assert(0); + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f64 < 0 ) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f64 < 0) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); break; - default: + d.f64 = sqrt(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_instruction * cpI = const_cast(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - // store data in sstarr memory - mem->write(addr,size/8,&src3_data.s64,thread,pI); - - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; - - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64=0; - mem->read(addr+(tid*4),size/8,&data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } - - // squeeze the zeros out of the array and store data back into original array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space,thread,src1,mem,addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); - mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); - offset++; - } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4*(offset-1); - thread->set_operand_value(dst, data, type, thread, pI); - - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); - offset++; - } - - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; - thread->m_last_memory_space = space; - } -} - -void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); - // TODO: add implementation -} - -void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); //may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - decode_space(space,thread,dst,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr,size/8,&data.s64,thread,pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); - delete [] ptx_regs; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_instruction *cpI = const_cast(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = + src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + // store data in sstarr memory + mem->write(addr, size / 8, &src3_data.s64, thread, pI); + + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; + + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64 = 0; + mem->read(addr + (tid * 4), size / 8, &data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } + + // squeeze the zeros out of the array and store data back into original + // array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space, thread, src1, mem, addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, + pI); + mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, + pI); + offset++; } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4 * (offset - 1); + thread->set_operand_value(dst, data, type, thread, pI); + + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); + offset++; + } + + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; + thread->m_last_memory_space = space; + } +} + +void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // printf("Execution Warning: unimplemented ssy instruction is treated as a + // nop\n"); + // TODO: add implementation } -void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); // may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space, thread, dst, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr, size / 8, &data.s64, thread, pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); + delete[] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} + +void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //performs addition. Sets carry and overflow if needed. - //the constant is added in during subtraction so the carry bit is set properly. - switch ( i_type ) { - case S8_TYPE: + // performs addition. Sets carry and overflow if needed. + // the constant is added in during subtraction so the carry bit is set + // properly. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.s32 & 0x100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.s32 & 0x10000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = ((data.u64)>>32) & 0x0001; - break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; break; - case B8_TYPE: - case U8_TYPE: + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x10000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + + 0x100000000; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = ((data.u64) >> 32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; + break; + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; + break; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + + 0x100000000; + carry = (data.u64 & 0x100000000) >> 32; + break; + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 - src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 - src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 - src2_data.f64; break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; - carry = (data.u64 & 0x100000000)>>32; + default: + assert(0); break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; - default: assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // Do nothing +void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // Do nothing } -void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - +void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision( float x, unsigned bits ) -{ - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1<<23)-1); - int mask = ((1<= mx) nx -= elem_size; - unsigned ny = (y >= my)? my - 1 : y; - return nx + mx*ny; -} - -typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); -float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, - int x, int y, unsigned int width, unsigned int height, size_t elem_size, - float alpha, float beta, texAddr_t b_lim) -{ - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; - - mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); - mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); - mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); - - float sample = (1-alpha)*(1-beta)*Tij + - alpha*(1-beta)*Ti1j + - (1-alpha)*beta*Tij1 + - alpha*beta*Ti1j1; - - return sample; -} - -float textureNormalizeElementSigned(int element, int bits) -{ - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float) element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } -} - -float textureNormalizeElementUnsigned(unsigned int element, int bits) -{ - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } -} - -void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) -{ - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); - datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); - dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); - dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); - datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); - dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); - dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); - } else { - assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); - } -} - -void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned dimension = pI->dimension(); - const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); //the name of the texture - const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched - - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = k.get_texarray(texname); - const struct textureInfo* texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); - - //assume always 2D f32 input - //access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha=0, beta=0; - - type_info_key::type_decode(to_type,size,t); - tex_array_base = cuArray->devPtr32; - - switch (dimension) { - case GEOM_MODIFIER_1D: +float reduce_precision(float x, unsigned bits) { + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1 << 23) - 1); + int mask = ((1 << bits) - 1) << (23 - bits); + int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); + tmp.a = nv; + float result = tmp.b; + return result; +} + +unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = (mx + x) % mx; + unsigned ny = (my + y) % my; + return nx + mx * ny; +} + +unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = x; + while (nx >= mx) nx -= elem_size; + unsigned ny = (y >= my) ? my - 1 : y; + return nx + mx * ny; +} + +typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size); +float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, + int y, unsigned int width, unsigned int height, + size_t elem_size, float alpha, float beta, + texAddr_t b_lim) { + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), + 4, &Ti1j); + mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, + &Tij1); + mem->read( + tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, + &Ti1j1); + + float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + + (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; + + return sample; +} + +float textureNormalizeElementSigned(int element, int bits) { + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float)element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } +} + +float textureNormalizeElementUnsigned(unsigned int element, int bits) { + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } +} + +void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, + ptx_reg_t &datax, ptx_reg_t &datay, + ptx_reg_t &dataz, ptx_reg_t &dataw) { + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); + datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); + dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); + dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); + datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); + dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); + dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); + } else { + assert(0 && + "Undefined texture read mode: cudaReadModeNormalizedFloat expect " + "integer elements"); + } +} + +void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned dimension = pI->dimension(); + const operand_info &dst = + pI->dst(); // the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); // the name of the texture + const operand_info &src2 = + pI->src2(); // the vector registers containing coordinates of the texel + // to be fetched + + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values( + src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, + nelem); // ptx_reg should be 4 entry vector type...coordinates into + // texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference *texref = gpu->get_texref(texname); + const struct cudaArray *cuArray = k.get_texarray(texname); + const struct textureInfo *texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); + + // assume always 2D f32 input + // access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha = 0, beta = 0; + + type_info_key::type_decode(to_type, size, t); + tex_array_base = cuArray->devPtr32; + + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha,9); - beta = 0.0; - - x = (int)floor(xb); - y = 0; - } else { - x = (int) floor(x_f32 * width); - y = 0; - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha, 9); + beta = 0.0; + + x = (int)floor(xb); + y = 0; + } else { + x = (int)floor(x_f32 * width); + y = 0; + } } else { - switch ( c_type ) { - case S32_TYPE: + switch (c_type) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int) x_f32; - break; - default: assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1)? (width - 1) : x; - x = (x < 0)? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - + floor(x_f32); // offset into subtexel (for linear sampling) + x = (int)x_f32; + break; + default: + assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1) ? (width - 1) : x; + x = (x < 0) ? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); - y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); - - if (texref->addressMode[0]) {//clamp - if (x_f32<0) x_f32 = 0; - if (x_f32>=1) x_f32 = 1 - 1/x_f32; - } else {//wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) {//clamp - if (y_f32<0) y_f32 = 0; - if (y_f32>=1) y_f32 = 1 - 1/y_f32; - } else {//wrap - y_f32 = y_f32 - floor(y_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha,9); - beta = reduce_precision(beta,9); - - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int) floor(x_f32 * width); - y = (int) floor(y_f32 * height); - } + x_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); + y_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); + + if (texref->addressMode[0]) { // clamp + if (x_f32 < 0) x_f32 = 0; + if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; + } else { // wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) { // clamp + if (y_f32 < 0) y_f32 = 0; + if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; + } else { // wrap + y_f32 = y_f32 - floor(y_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha, 9); + beta = reduce_precision(beta, 9); + + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int)floor(x_f32 * width); + y = (int)floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); - - x = (int) x_f32; - y = (int) y_f32; - if (texref->addressMode[0]) {//clamp - if (x<0) x = 0; - if (x>= (int)width) x = width-1; - } else {//wrap - x = x % width; - if (x < 0) x*= -1; - } - if (texref->addressMode[1]) {//clamp - if (y<0) y = 0; - if (y>= (int)height) y = height -1; - } else {//wrap - y = y % height; - if (y < 0) y *= -1; - } + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); + + x = (int)x_f32; + y = (int)y_f32; + if (texref->addressMode[0]) { // clamp + if (x < 0) x = 0; + if (x >= (int)width) x = width - 1; + } else { // wrap + x = x % width; + if (x < 0) x *= -1; + } + if (texref->addressMode[1]) { // clamp + if (y < 0) y = 0; + if (y >= (int)height) y = height - 1; + } else { // wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - tex_array_index = tex_array_base + (x + width*y); - break; - default: - assert(0); break; - } - switch ( to_type ) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); - elementOffset += cuArray->desc.x/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + tex_array_index = tex_array_base + (x + width * y); + break; + default: + assert(0); + break; + } + switch (to_type) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); + elementOffset += cuArray->desc.x / 8; if (cuArray->desc.y) { - mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); - elementOffset += cuArray->desc.y/8; - if (cuArray->desc.z) { - mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); - elementOffset += cuArray->desc.z/8; - if (cuArray->desc.w) - mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); - } + mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, + &data2.u32); + elementOffset += cuArray->desc.y / 8; + if (cuArray->desc.z) { + mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, + &data3.u32); + elementOffset += cuArray->desc.z / 8; + if (cuArray->desc.w) + mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, + &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read( tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read(tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.u64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.u64); - } + mem->read(tex_array_index + 8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.u64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); + } } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: { - if( texref->filterMode == cudaFilterModeLinear ) { - texAddr_t b_lim = wrap; - if ( texref->addressMode[0] == cudaAddressModeClamp ) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; - size_t elem_ofst = 0; - - data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - } - } + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + if (texref->filterMode == cudaFilterModeLinear) { + texAddr_t b_lim = wrap; + if (texref->addressMode[0] == cudaAddressModeClamp) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z + cuArray->desc.w) / + 8; + size_t elem_ofst = 0; + + data1.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, + y, width, height, elem_size, alpha, + beta, b_lim); + } + } } else { - mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); - if (cuArray->desc.y) { - mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); - if (cuArray->desc.z) { - mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); - if (cuArray->desc.w) - mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); - } - } + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); + if (cuArray->desc.y) { + mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); + if (cuArray->desc.z) { + mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); + if (cuArray->desc.w) + mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read( tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read(tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.f64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.f64); - } + mem->read(tex_array_index + 8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.f64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); + } } break; - default: assert(0); break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: + assert(0); + break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; + memreqindex = + ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; - blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + + (y % (texInfo->Ty) + << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; + thread->m_last_effective_address = + tex_array_base + memreqindex; // tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; - - // normalize output into floating point numbers according to the texture read mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } - - thread->set_vector_operand_values(dst,data1,data2,data3,data4); -} - -void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list threads_in_warp; - static unsigned last_tid; - - if( first_in_warp ) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset=31; - while( (offset>=0) && !pI->active(offset) ) - offset--; - assert( offset >= 0 ); - last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; - } - - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of src1_data.pred must be inverted to get proper behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); - - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); - - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } - - if( thread->get_hw_tid() == last_tid ) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; + } + thread->m_last_memory_space = tex_space; + + // normalize output into floating point numbers according to the texture read + // mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } - switch( pI->vote_mode() ) { - case ptx_instruction::vote_any: pred_value = or_all; break; - case ptx_instruction::vote_all: pred_value = and_all; break; - case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true - - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); - } + thread->set_vector_operand_values(dst, data1, data2, data3, data4); +} + +void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} + +void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list threads_in_warp; + static unsigned last_tid; + + if (first_in_warp) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset = 31; + while ((offset >= 0) && !pI->active(offset)) offset--; + assert(offset >= 0); + last_tid = + (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + + offset; + } + + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of src1_data.pred must be inverted to get + // proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); + + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); + + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } + + if (thread->get_hw_tid() == last_tid) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); } - first_in_warp = true; - } -} - -void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void inst_not_implemented( const ptx_instruction * pI ) -{ - printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), - pI->source_line(), - pI->get_opcode_cstr() ); - abort(); -} - -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) -{ - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if(opInfo.get_addr_space() == global_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else if(opInfo.get_addr_space() == shared_space) - { - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - - } - else if(opInfo.get_addr_space() == const_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else - { - result = opData; - } - - if(opInfo.get_operand_lohi() == 1) - { - result.u64 = result.u64 & 0xFFFF; - } - else if(opInfo.get_operand_lohi() == 2) - { - result.u64 = (result.u64>>16) & 0xFFFF; - } + } else { + bool pred_value = false; + + switch (pI->vote_mode()) { + case ptx_instruction::vote_any: + pred_value = or_all; + break; + case ptx_instruction::vote_all: + pred_value = and_all; + break; + case ptx_instruction::vote_uni: + pred_value = (or_all ^ and_all); + break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, + // 1 = false and 0 = true + + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); + } + } + first_in_warp = true; + } +} + +void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; - if(opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - return result; +void inst_not_implemented(const ptx_instruction *pI) { + printf( + "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); + abort(); } +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread) { + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if (opInfo.get_addr_space() == global_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else if (opInfo.get_addr_space() == shared_space) { + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + + } else if (opInfo.get_addr_space() == const_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + + mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, + &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else { + result = opData; + } + + if (opInfo.get_operand_lohi() == 1) { + result.u64 = result.u64 & 0xFFFF; + } else if (opInfo.get_operand_lohi() == 2) { + result.u64 = (result.u64 >> 16) & 0xFFFF; + } + + if (opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } + + return result; +} diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 4b2acdf..1323837 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,210 +7,227 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "memory.h" #include -#include "../debug.h" #include "../../libcuda/gpgpu_context.h" +#include "../debug.h" -template memory_space_impl::memory_space_impl( std::string name, unsigned hash_size ) -{ - m_name = name; - MEM_MAP_RESIZE(hash_size); - - m_log2_block_size = -1; - for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { - if( BSIZE & mask ) { - assert( m_log2_block_size == (unsigned)-1 ); - m_log2_block_size = n; - } - } - assert( m_log2_block_size != (unsigned)-1 ); +template +memory_space_impl::memory_space_impl(std::string name, + unsigned hash_size) { + m_name = name; + MEM_MAP_RESIZE(hash_size); + + m_log2_block_size = -1; + for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { + if (BSIZE & mask) { + assert(m_log2_block_size == (unsigned)-1); + m_log2_block_size = n; + } + } + assert(m_log2_block_size != (unsigned)-1); } -template void memory_space_impl::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) -{ - m_data[index].write(offset,length,(const unsigned char*)data); +template +void memory_space_impl::write_only(mem_addr_t offset, mem_addr_t index, + size_t length, const void *data) { + m_data[index].write(offset, length, (const unsigned char *)data); } -template void memory_space_impl::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) -{ - - mem_addr_t index = addr >> m_log2_block_size; - - if ( (addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - m_data[index].write(offset,nbytes,(const unsigned char*)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); - - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if( !m_watchpoints.empty() ) { - std::map::iterator i; - for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { - mem_addr_t wa = i->second; - if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); +template +void memory_space_impl::write(mem_addr_t addr, size_t length, + const void *data, + class ptx_thread_info *thd, + const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; + + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + m_data[index].write(offset, nbytes, (const unsigned char *)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - } + + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, + &((const unsigned char *)data)[src_offset]); + + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } + if (!m_watchpoints.empty()) { + std::map::iterator i; + for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { + mem_addr_t wa = i->second; + if (((addr <= wa) && ((addr + length) > wa)) || + ((addr > wa) && (addr < (wa + 4)))) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( + i->first, thd, pI); + } + } } -template void memory_space_impl::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const -{ - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", - (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if( i == m_data.end() ) { - for( size_t n=0; n < length; n++ ) - ((unsigned char*)data)[n] = (unsigned char) 0; - //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - i->second.read(offset,nbytes,(unsigned char*)data); - } +template +void memory_space_impl::read_single_block(mem_addr_t blk_idx, + mem_addr_t addr, size_t length, + void *data) const { + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf( + "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " + "addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf( + "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " + "index=0x%x, BSIZE=0x%x\n", + (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if (i == m_data.end()) { + for (size_t n = 0; n < length; n++) + ((unsigned char *)data)[n] = (unsigned char)0; + // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized + // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + i->second.read(offset, nbytes, (unsigned char *)data); + } } -template void memory_space_impl::read( mem_addr_t addr, size_t length, void *data ) const -{ - mem_addr_t index = addr >> m_log2_block_size; - if ((addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); - - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; +template +void memory_space_impl::read(mem_addr_t addr, size_t length, + void *data) const { + mem_addr_t index = addr >> m_log2_block_size; + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - assert(nbytes_remain == 0); - } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, + &((unsigned char *)data)[dst_offset]); + + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } } -template void memory_space_impl::print( const char *format, FILE *fout ) const -{ - typename map_t::const_iterator i_page; +template +void memory_space_impl::print(const char *format, FILE *fout) const { + typename map_t::const_iterator i_page; - for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template void memory_space_impl::set_watch( addr_t addr, unsigned watchpoint ) -{ - m_watchpoints[watchpoint]=addr; +template +void memory_space_impl::set_watch(addr_t addr, unsigned watchpoint) { + m_watchpoints[watchpoint] = addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16*1024>; +template class memory_space_impl<16 * 1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) -{ - mem->print(format,fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", + FILE *fout = stdout) { + mem->print(format, fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[] ) -{ - int errors_found=0; - memory_space *mem = new memory_space_impl<32>("test",4); - // write address to [address] - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) - mem->write(addr,4,&addr,NULL,NULL); - - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { - unsigned tmp=0; - mem->read(addr,4,&tmp); - if( tmp != addr ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); - } - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr,1,&val,NULL,NULL); - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned tmp=0; - mem->read(addr,1,&tmp); - unsigned char val = (addr + 128) % 256; - if( tmp != val ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); - } - } - - if( errors_found ) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } +int main(int argc, char *argv[]) { + int errors_found = 0; + memory_space *mem = new memory_space_impl<32>("test", 4); + // write address to [address] + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) + mem->write(addr, 4, &addr, NULL, NULL); + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { + unsigned tmp = 0; + mem->read(addr, 4, &tmp); + if (tmp != addr) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); + } + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr, 1, &val, NULL, NULL); + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned tmp = 0; + mem->read(addr, 1, &tmp); + unsigned char val = (addr + 128) % 256; + if (tmp != val) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, + (unsigned)val); + } + } + + if (errors_found) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index ab588bc..5850aa1 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef memory_h_INCLUDED #define memory_h_INCLUDED @@ -33,101 +34,97 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 - #define MEM_MAP_RESIZE(hash_size) +#define MEM_MAP_RESIZE(hash_size) #else - #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) +#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include -#include #include -#include -#include #include +#include +#include +#include typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4*1024) - -template class mem_storage { -public: - mem_storage( const mem_storage &another ) - { - m_data = (unsigned char*)calloc(1,BSIZE); - memcpy(m_data,another.m_data,BSIZE); - } - mem_storage() - { - m_data = (unsigned char*)calloc(1,BSIZE); - } - ~mem_storage() - { - free(m_data); - } - - void write( unsigned offset, size_t length, const unsigned char *data ) - { - assert( offset + length <= BSIZE ); - memcpy(m_data+offset,data,length); - } - - void read( unsigned offset, size_t length, unsigned char *data ) const - { - assert( offset + length <= BSIZE ); - memcpy(data,m_data+offset,length); - } - - void print( const char *format, FILE *fout ) const - { - unsigned int *i_data = (unsigned int*)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); - } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); +#define MEM_BLOCK_SIZE (4 * 1024) + +template +class mem_storage { + public: + mem_storage(const mem_storage &another) { + m_data = (unsigned char *)calloc(1, BSIZE); + memcpy(m_data, another.m_data, BSIZE); + } + mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } + ~mem_storage() { free(m_data); } + + void write(unsigned offset, size_t length, const unsigned char *data) { + assert(offset + length <= BSIZE); + memcpy(m_data + offset, data, length); + } + + void read(unsigned offset, size_t length, unsigned char *data) const { + assert(offset + length <= BSIZE); + memcpy(data, m_data + offset, length); + } + + void print(const char *format, FILE *fout) const { + unsigned int *i_data = (unsigned int *)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); } - fprintf(fout, "\n"); - fflush(fout); - } - -private: - unsigned m_nbytes; - unsigned char *m_data; + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); + } + fprintf(fout, "\n"); + fflush(fout); + } + + private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space -{ -public: - virtual ~memory_space() {} - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; - virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; - virtual void print( const char *format, FILE *fout ) const = 0; - virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; +class memory_space { + public: + virtual ~memory_space() {} + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI) = 0; + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data) = 0; + virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; + virtual void print(const char *format, FILE *fout) const = 0; + virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; }; -template class memory_space_impl : public memory_space { -public: - memory_space_impl( std::string name, unsigned hash_size ); - - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); - virtual void read( mem_addr_t addr, size_t length, void *data ) const; - virtual void print( const char *format, FILE *fout ) const; - - virtual void set_watch( addr_t addr, unsigned watchpoint ); - -private: - void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map > map_t; - map_t m_data; - std::map m_watchpoints; +template +class memory_space_impl : public memory_space { + public: + memory_space_impl(std::string name, unsigned hash_size); + + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI); + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data); + virtual void read(mem_addr_t addr, size_t length, void *data) const; + virtual void print(const char *format, FILE *fout) const; + + virtual void set_watch(addr_t addr, unsigned watchpoint); + + private: + void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, + void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map > map_t; + map_t m_data; + std::map m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 86d3b99..dc1e8c9 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,71 +7,71 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef opcodes_h_included #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type{ - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 - +enum wmma_type { + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 22517df..9f7e760 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,266 +7,285 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" -#include "ptx_sim.h" #include "ptx-stats.h" -#include "../option_parser.h" #include #include -#include "../tr1_hash_map.h" #include "../../libcuda/gpgpu_context.h" +#include "../option_parser.h" +#include "../tr1_hash_map.h" +#include "ptx_ir.h" +#include "ptx_sim.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) -{ - option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, - &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, - &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { + option_parser_register( + opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register( + opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line -{ -public: - ptx_file_line(const char* s, int l) { - if( s == NULL ) - st = "NULL_NAME"; - else - st = s; - line = l; +class ptx_file_line { + public: + ptx_file_line(const char *s, int l) { + if (s == NULL) + st = "NULL_NAME"; + else + st = s; + line = l; + } + + bool operator<(const ptx_file_line &other) const { + if (st == other.st) { + if (line < other.line) + return true; + else + return false; + } else { + return st < other.st; } + } - bool operator<(const ptx_file_line &other) const { - if( st == other.st ) { - if( line < other.line ) - return true; - else - return false; - } else { - return st < other.st; - } - } + bool operator==(const ptx_file_line &other) const { + return (line == other.line) && (st == other.st); + } - bool operator==(const ptx_file_line &other) const { - return (line==other.line) && (st==other.st); - } - - std::string st; - unsigned line; + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats -{ -public: - ptx_file_line_stats() - : exec_count(0), latency(0), dram_traffic(0), - smem_n_way_bank_conflict_total(0), smem_warp_count(0), - gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), - warp_divergence(0) - { } - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction - unsigned long gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) - unsigned long long warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats { + public: + ptx_file_line_stats() + : exec_count(0), + latency(0), + dram_traffic(0), + smem_n_way_bank_conflict_total(0), + smem_warp_count(0), + gmem_n_access_total(0), + gmem_warp_count(0), + exposed_latency(0), + warp_divergence(0) {} + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long + smem_n_way_bank_conflict_total; // total number of banks accessed by this + // instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in + // total from this instruction + unsigned long + gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles + // (attributed to this instruction) + unsigned long long + warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line -{ - std::size_t operator()(const ptx_file_line & pfline) const { - std::hash hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line { + std::size_t operator()(const ptx_file_line &pfline) const { + std::hash hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() -{ - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; - - ptx_file_line_stats_map_t::iterator it; - FILE * pfile; - - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); - for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); +void ptx_stats::ptx_file_line_stats_write_file() { + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; + + ptx_file_line_stats_map_t::iterator it; + FILE *pfile; + + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf( + pfile, + "kernel line : count latency dram_traffic smem_bk_conflicts smem_warp " + "gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); + for (it = ptx_file_line_stats_tracker.begin(); + it != ptx_file_line_stats_tracker.end(); it++) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) -{ - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent +// in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, + unsigned dram_traffic) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of +// cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( + unsigned pc, unsigned n_way_bkconflict) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory +// requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, + unsigned n_access) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker -{ -public: - typedef std::map insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker { + public: + typedef std::map insn_count_map; - void add_count(const ptx_instruction * pInsn, int count = 1) - { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction *pInsn, int count = 1) { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction * pInsn, int count = 1) - { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction *pInsn, int count = 1) { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); - } + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); } - - void attribute_exposed_latency(int count = 1) - { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; - - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.exposed_latency += count; - } + } + + void attribute_exposed_latency(int count = 1) { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; + + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats &line_stats = + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())]; + line_stats.exposed_latency += count; } + } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) -{ - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) -{ - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory +// instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, + int exposed_latency) { + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; -} +void ptx_stats::ptx_file_line_stats_add_warp_divergence( + unsigned pc, unsigned n_way_divergence) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; +} diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 246b4ce..be77c39 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,33 +7,33 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" - #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); #endif // stat collection interface to gpgpu-sim @@ -41,28 +41,29 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); - class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char * ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char* ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, + unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, + unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index d8943d2..aa1c25a 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,1170 +8,1263 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_parser.h" #include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "opcodes.h" +#include "ptx_parser.h" +typedef void *yyscan_t; +#include #include #include -#include -#include #include +#include #include "assert.h" +#include "opcodes.h" +#include "ptx.tab.h" -#include "cuda-sim.h" #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" #define STR_SIZE 1024 -const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) -{ - if( pc < s_g_pc_to_insn.size() ) - return s_g_pc_to_insn[pc]; - else - return NULL; -} - -unsigned symbol::get_uid() -{ - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { + if (pc < s_g_pc_to_insn.size()) + return s_g_pc_to_insn[pc]; + else + return NULL; } -void symbol::add_initializer( const std::list &init ) -{ - m_initializer = init; -} - -void symbol::print_info(FILE *fp) const -{ - fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); - if( m_address_valid ) - fprintf(fp,"
, "); - if( m_is_label ) - fprintf(fp," is_label "); - if( m_is_shared ) - fprintf(fp," is_shared "); - if( m_is_const ) - fprintf(fp," is_const "); - if( m_is_global ) - fprintf(fp," is_global "); - if( m_is_local ) - fprintf(fp," is_local "); - if( m_is_tex ) - fprintf(fp," is_tex "); - if( m_is_func_addr ) - fprintf(fp," is_func_addr "); - if( m_function ) - fprintf(fp," %p ", m_function ); +unsigned symbol::get_uid() { + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -symbol_table::symbol_table() -{ - assert(0); +void symbol::add_initializer(const std::list &init) { + m_initializer = init; } -symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator=0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; - - //Jin: handle instruction group for cdp - m_inst_group_id = 0; - - m_parent = parent; - if ( m_parent ) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } +void symbol::print_info(FILE *fp) const { + fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), + m_type); + if (m_address_valid) fprintf(fp, "
, "); + if (m_is_label) fprintf(fp, " is_label "); + if (m_is_shared) fprintf(fp, " is_shared "); + if (m_is_const) fprintf(fp, " is_const "); + if (m_is_global) fprintf(fp, " is_global "); + if (m_is_local) fprintf(fp, " is_local "); + if (m_is_tex) fprintf(fp, " is_tex "); + if (m_is_func_addr) fprintf(fp, " is_func_addr "); + if (m_function) fprintf(fp, " %p ", m_function); } -void symbol_table::set_name( const char *name ) -{ - m_scope_name = std::string(name); -} +symbol_table::symbol_table() { assert(0); } -const ptx_version &symbol_table::get_ptx_version() const -{ - if( m_parent == NULL ) return m_ptx_version; - else return m_parent->get_ptx_version(); -} +symbol_table::symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator = 0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; + + // Jin: handle instruction group for cdp + m_inst_group_id = 0; -unsigned symbol_table::get_sm_target() const -{ - if( m_parent == NULL ) - return m_ptx_version.target(); - else return m_parent->get_sm_target(); + m_parent = parent; + if (m_parent) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } } -void symbol_table::set_ptx_version( float ver, unsigned ext ) -{ - m_ptx_version = ptx_version(ver,ext); +void symbol_table::set_name(const char *name) { + m_scope_name = std::string(name); } -void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) -{ - m_ptx_version.set_target(target,ext,ext2); +const ptx_version &symbol_table::get_ptx_version() const { + if (m_parent == NULL) + return m_ptx_version; + else + return m_parent->get_ptx_version(); } -symbol *symbol_table::lookup( const char *identifier ) -{ - std::string key(identifier); - std::map::iterator i = m_symbols.find(key); - if ( i != m_symbols.end() ) { - return i->second; - } - if ( m_parent ) { - return m_parent->lookup(identifier); - } - return NULL; +unsigned symbol_table::get_sm_target() const { + if (m_parent == NULL) + return m_ptx_version.target(); + else + return m_parent->get_sm_target(); } -symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) -{ - char buf[1024]; - std::string key(identifier); - assert( m_symbols.find(key) == m_symbols.end() ); - snprintf(buf,1024,"%s:%u",filename,line); - symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); - m_symbols[ key ] = s; - - if ( type != NULL && type->get_key().is_global() ) { - m_globals.push_back(s); - } - if ( type != NULL && type->get_key().is_const() ) { - m_consts.push_back(s); - } - - return s; +void symbol_table::set_ptx_version(float ver, unsigned ext) { + m_ptx_version = ptx_version(ver, ext); } - -void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) -{ - std::map::iterator i = m_symbols.find( func->get_name() ); - if( i != m_symbols.end() ) - return; - char buf[1024]; - snprintf(buf,1024,"%s:%u",filename,linenumber); - type_info *type = add_type( func ); - symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); - s->set_function(func); - m_symbols[ func->get_name() ] = s; -} - -//Jin: handle instruction group for cdp -symbol_table* symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); - - //previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); - symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - - return sym_table; -} - -symbol_table * symbol_table::end_inst_group() { - symbol_table * sym_table = m_parent; - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; - - return sym_table; -} - -void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl - -bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) -{ - std::string key = std::string(name); - bool prior_decl = false; - if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } - - if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { - assert( prior_decl ); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert( !prior_decl ); - *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info* null_type_info = new type_info( NULL, null_key ); - symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); - null_reg->set_regno(0, 0); - - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert( (*func_info)->get_symtab() == *sym_table ); - register_ptx_function(name,*func_info); - } - return prior_decl; -} - -function_info *symbol_table::lookup_function( std::string name ) -{ - std::string key = std::string(name); - std::map::iterator it = m_function_info_lookup.find(key); - assert ( it != m_function_info_lookup.end() ); - return it->second; -} - -type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) -{ - if( space_spec == param_space_unclassified ) - space_spec = param_space_local; - type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); - type_info *pt; - pt = new type_info(this,t); - return pt; + +void symbol_table::set_sm_target(const char *target, const char *ext, + const char *ext2) { + m_ptx_version.set_target(target, ext, ext2); +} + +symbol *symbol_table::lookup(const char *identifier) { + std::string key(identifier); + std::map::iterator i = m_symbols.find(key); + if (i != m_symbols.end()) { + return i->second; + } + if (m_parent) { + return m_parent->lookup(identifier); + } + return NULL; +} + +symbol *symbol_table::add_variable(const char *identifier, + const type_info *type, unsigned size, + const char *filename, unsigned line) { + char buf[1024]; + std::string key(identifier); + assert(m_symbols.find(key) == m_symbols.end()); + snprintf(buf, 1024, "%s:%u", filename, line); + symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); + m_symbols[key] = s; + + if (type != NULL && type->get_key().is_global()) { + m_globals.push_back(s); + } + if (type != NULL && type->get_key().is_const()) { + m_consts.push_back(s); + } + + return s; +} + +void symbol_table::add_function(function_info *func, const char *filename, + unsigned linenumber) { + std::map::iterator i = + m_symbols.find(func->get_name()); + if (i != m_symbols.end()) return; + char buf[1024]; + snprintf(buf, 1024, "%s:%u", filename, linenumber); + type_info *type = add_type(func); + symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); + s->set_function(func); + m_symbols[func->get_name()] = s; +} + +// Jin: handle instruction group for cdp +symbol_table *symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), + m_inst_group_id); + + // previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == + m_inst_group_symtab.end()); + symbol_table *sym_table = + new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + + return sym_table; +} + +symbol_table *symbol_table::end_inst_group() { + symbol_table *sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; +} + +void register_ptx_function(const char *name, + function_info *impl); // either libcuda or libopencl + +bool symbol_table::add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **sym_table) { + std::string key = std::string(name); + bool prior_decl = false; + if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } + + if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { + assert(prior_decl); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert(!prior_decl); + *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key(reg_space, 0, 0, 0, 0, 0); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info *null_type_info = new type_info(NULL, null_key); + symbol *null_reg = + (*sym_table)->add_variable("_", null_type_info, 0, "", 0); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert((*func_info)->get_symtab() == *sym_table); + register_ptx_function(name, *func_info); + } + return prior_decl; +} + +function_info *symbol_table::lookup_function(std::string name) { + std::string key = std::string(name); + std::map::iterator it = + m_function_info_lookup.find(key); + assert(it != m_function_info_lookup.end()); + return it->second; +} + +type_info *symbol_table::add_type(memory_space_t space_spec, + int scalar_type_spec, int vector_spec, + int alignment_spec, int extern_spec) { + if (space_spec == param_space_unclassified) space_spec = param_space_local; + type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, + extern_spec, 0); + type_info *pt; + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::add_type(function_info *func) { + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::get_array_type(type_info *base_type, + unsigned array_dim) { + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this, t); + // Where else is m_types being used? As of now, I dont find any use of it and + // causing seg fault. So disabling m_types. + // TODO: find where m_types can be used in future and solve the seg fault. + // pt = m_types[t] = new type_info(this,t); + return pt; +} + +void symbol_table::set_label_address(const symbol *label, unsigned addr) { + std::map::iterator i = m_symbols.find(label->name()); + assert(i != m_symbols.end()); + symbol *s = i->second; + s->set_label_address(addr); +} + +void symbol_table::dump() { + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); + std::map::iterator i; + for (i = m_symbols.begin(); i != m_symbols.end(); i++) { + printf("%30s : ", i->first.c_str()); + if (i->second) + i->second->print_info(stdout); + else + printf(" "); + printf("\n"); + } + printf("\n"); } -type_info *symbol_table::add_type( function_info *func ) -{ - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this,t); - return pt; +unsigned operand_info::get_uid() { + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; } -type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) -{ - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this,t); - //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. - //TODO: find where m_types can be used in future and solve the seg fault. - //pt = m_types[t] = new type_info(this,t); - return pt; +std::list::iterator +function_info::find_next_real_instruction( + std::list::iterator i) { + while ((i != m_instructions.end()) && (*i)->is_label()) i++; + return i; } -void symbol_table::set_label_address( const symbol *label, unsigned addr ) -{ - std::map::iterator i=m_symbols.find(label->name()); - assert( i != m_symbols.end() ); - symbol *s = i->second; - s->set_label_address(addr); -} +void function_info::create_basic_blocks() { + std::list leaders; + std::list::iterator i, l; -void symbol_table::dump() -{ - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); - std::map::iterator i; - for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { - printf("%30s : ", i->first.c_str() ); - if( i->second ) - i->second->print_info(stdout); - else - printf(" "); - printf("\n"); - } - printf("\n"); -} - -unsigned operand_info::get_uid() -{ - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; -} - -std::list::iterator function_info::find_next_real_instruction( std::list::iterator i) -{ - while( (i != m_instructions.end()) && (*i)->is_label() ) - i++; - return i; -} - -void function_info::create_basic_blocks() -{ - std::list leaders; - std::list::iterator i, l; - - // first instruction is a leader - i=m_instructions.begin(); - leaders.push_back(*i); - i++; - while( i!=m_instructions.end() ) { - ptx_instruction *pI = *i; - if( pI->is_label() ) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch( pI->get_opcode() ) { - case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: + // first instruction is a leader + i = m_instructions.begin(); + leaders.push_back(*i); + i++; + while (i != m_instructions.end()) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch (pI->get_opcode()) { + case BRA_OP: + case RET_OP: + case EXIT_OP: + case RETP_OP: + case BREAK_OP: + i++; + if (i != m_instructions.end()) leaders.push_back(*i); + i = find_next_real_instruction(i); + break; + case CALL_OP: + case CALLP_OP: + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); + if (i != m_instructions.end()) leaders.push_back(*i); i = find_next_real_instruction(i); - break; - case CALL_OP: case CALLP_OP: - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); - i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); - i = find_next_real_instruction(i); - } else i++; - break; - default: + } else i++; - } - } - } - - if( leaders.empty() ) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); - return; - } - - unsigned bb_id = 0; - l=leaders.begin(); - i=m_instructions.begin(); - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); - ptx_instruction *last_real_inst=*(l++); - - for( ; i!=m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if( l != leaders.end() && *i == *l ) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); - last_real_inst = *find_next_real_instruction(i); - } - // start search for next leader - l++; + break; + default: + i++; } - pI->assign_bb( m_basic_blocks.back() ); - if( !pI->is_label() ) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); -} - -void function_info::print_basic_blocks() -{ - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); - std::list::iterator ptx_itr; - unsigned last_bb=0; - for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { - if( (*ptx_itr)->get_bb() ) { - if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; - } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); - printf("\n"); + } + } + + if (leaders.empty()) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", + m_name.c_str()); + return; + } + + unsigned bb_id = 0; + l = leaders.begin(); + i = m_instructions.begin(); + m_basic_blocks.push_back( + new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); + ptx_instruction *last_real_inst = *(l++); + + for (; i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (l != leaders.end() && *i == *l) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if (find_next_real_instruction(i) != + m_instructions.end()) { // if not bogus trailing label + m_basic_blocks.push_back(new basic_block_t( + bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); + last_real_inst = *find_next_real_instruction(i); } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else printf(" last: NULL\t"); + // start search for next leader + l++; + } + pI->assign_bb(m_basic_blocks.back()); + if (!pI->is_label()) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( + /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); +} + +void function_info::print_basic_blocks() { + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); + std::list::iterator ptx_itr; + unsigned last_bb = 0; + for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); + ptx_itr++) { + if ((*ptx_itr)->get_bb()) { + if ((*ptx_itr)->get_bb()->bb_id != last_bb) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; + } + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); printf("\n"); - } - printf("\n"); -} - -void function_info::print_basic_block_links() -{ - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if ( !(*bb_itr)->predecessor_ids.empty() ) { - printf("Predecessors:"); - std::set::iterator p; - for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { - printf(" %d", *p); - } - printf("\t"); + } + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else + printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else + printf(" last: NULL\t"); + printf("\n"); + } + printf("\n"); +} + +void function_info::print_basic_block_links() { + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if (!(*bb_itr)->predecessor_ids.empty()) { + printf("Predecessors:"); + std::set::iterator p; + for (p = (*bb_itr)->predecessor_ids.begin(); + p != (*bb_itr)->predecessor_ids.end(); p++) { + printf(" %d", *p); } - if ( !(*bb_itr)->successor_ids.empty() ) { - printf("Successors:"); - std::set::iterator s; - for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - printf(" %d", *s); - } + printf("\t"); + } + if (!(*bb_itr)->successor_ids.empty()) { + printf("Successors:"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + printf(" %d", *s); } - printf("\n"); - } + } + printf("\n"); + } } -operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction +operand_info *function_info::find_break_target( + ptx_instruction *p_break_insn) // find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for(const basic_block_t *p_bb = break_bb; - p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) - { - // reverse search through instructions in basic block for breakaddr instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { + // reverse search through instructions in basic block for breakaddr + // instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) + continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); } - } + } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); - insn_addr >= 0; insn_addr--) - { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; + insn_addr--) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together +void function_info::connect_basic_blocks() // iterate across m_basic_blocks of + // function, connecting basic blocks + // together { - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - basic_block_t* exit_bb = m_basic_blocks.back(); - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - } - continue; - } else if (pI->get_opcode() == BRA_OP) { - //find successor and link that basic_block to this one - operand_info &target = pI->dst(); //get operand, e.g. target name - unsigned addr = labels[ target.name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - - if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } -} -bool function_info::connect_break_targets() //connecting break instructions with proper targets -{ - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - bool modified = false; - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set orig_successor_ids = p_bb->successor_ids; - - // erase the previous linkage with old successors - for(std::set::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); - } - p_bb->successor_ids.clear(); - - //find successor and link that basic_block to this one - //successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[ target->name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); - - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); - } - - modified = modified || (orig_successor_ids != p_bb->successor_ids); + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + basic_block_t *exit_bb = m_basic_blocks.back(); + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || + pI->get_opcode() == EXIT_OP) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } } - } - - return modified; -} -void function_info::do_pdom() -{ - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); - - if ( g_debug_execution>=50 ) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if ( g_debug_execution>=2 ) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if ( g_debug_execution>=50 ) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); - for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); - fflush(stdout); - m_assembled = true; -} -void intersect( std::set &A, const std::set &B ) -{ - // return intersection of A and B in A - for( std::set::iterator a=A.begin(); a!=A.end(); ) { - std::set::iterator a_next = a; - a_next++; - if( B.find(*a) == B.end() ) { - A.erase(*a); - a = a_next; - } else - a++; - } -} - -bool is_equal( const std::set &A, const std::set &B ) -{ - if( A.size() != B.size() ) - return false; - for( std::set::iterator b=B.begin(); b!=B.end(); b++ ) - if( A.find(*b) == A.end() ) - return false; - return true; -} + continue; + } else if (pI->get_opcode() == BRA_OP) { + // find successor and link that basic_block to this one + operand_info &target = pI->dst(); // get operand, e.g. target name + unsigned addr = labels[target.name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } -void print_set(const std::set &A) + if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } +} +bool function_info::connect_break_targets() // connecting break instructions + // with proper targets { - std::set::iterator a; - for (a= A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); -} - -void function_info::find_dominators( ) -{ - // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - std::vector::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry - //copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; - } + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + bool modified = false; + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set orig_successor_ids = p_bb->successor_ids; + + // erase the previous linkage with old successors + for (std::set::iterator succ_ids = p_bb->successor_ids.begin(); + succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); } - } - //clean the basic block of dominators of it has no predecessors -- except for entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) - (*bb_itr)->dominator_ids.clear(); - } -} - -void function_info::find_postdominators( ) -{ - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - std::vector::reverse_iterator bb_itr = m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block - for (unsigned i=0; ipostdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; - } + p_bb->successor_ids.clear(); + + // find successor and link that basic_block to this one + // successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[target->name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); + + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); } - } -} -void function_info::find_ipostdominators( ) -{ - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->postdominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = m_basic_blocks.size()-2; n >=0;--n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + modified = modified || (orig_successor_ids != p_bb->successor_ids); + } + } + + return modified; +} +void function_info::do_pdom() { + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if (g_debug_execution >= 50) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if (g_debug_execution >= 2) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if (g_debug_execution >= 50) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", + m_name.c_str()); + for (unsigned ii = 0; ii < m_n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", + m_name.c_str()); + fflush(stdout); + m_assembled = true; +} +void intersect(std::set &A, const std::set &B) { + // return intersection of A and B in A + for (std::set::iterator a = A.begin(); a != A.end();) { + std::set::iterator a_next = a; + a_next++; + if (B.find(*a) == B.end()) { + A.erase(*a); + a = a_next; + } else + a++; + } +} + +bool is_equal(const std::set &A, const std::set &B) { + if (A.size() != B.size()) return false; + for (std::set::iterator b = B.begin(); b != B.end(); b++) + if (A.find(*b) == A.end()) return false; + return true; +} + +void print_set(const std::set &A) { + std::set::iterator a; + for (a = A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); +} + +void function_info::find_dominators() { + // find dominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + std::vector::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert( + (*bb_itr)->bb_id); // the only dominator of the entry block is the entry + // copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = + m_basic_blocks[h]->predecessor_ids.begin(); + s != m_basic_blocks[h]->predecessor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; } - } - unsigned num_ipdoms=0; - for ( int n = m_basic_blocks.size()-1; n >=0;--n) { - assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; + } + } + // clean the basic block of dominators of it has no predecessors -- except for + // entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); + } +} + +void function_info::find_postdominators() { + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + std::vector::reverse_iterator bb_itr = + m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert( + (*bb_itr) + ->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr; bb_itr != m_basic_blocks.rend(); + bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT + // for the exit block + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->postdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = m_basic_blocks[h]->successor_ids.begin(); + s != m_basic_blocks[h]->successor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; } - } - assert( num_ipdoms == m_basic_blocks.size()-1 ); - // the exit node does not have an immediate post dominator, but everyone else should -} - -void function_info::find_idominators( ) -{ - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->dominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + } + } +} + +void function_info::find_ipostdominators() { + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all pdoms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->postdominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - unsigned num_idoms=0; - unsigned num_nopred = 0; - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; + } + } + unsigned num_ipdoms = 0; + for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { + assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatepostdominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; + } + } + assert(num_ipdoms == m_basic_blocks.size() - 1); + // the exit node does not have an immediate post dominator, but everyone else + // should +} + +void function_info::find_idominators() { + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all doms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->dominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - assert( num_idoms == m_basic_blocks.size()-num_nopred ); - // the entry node does not have an immediate dominator, but everyone else should -} - -void function_info::print_dominators() -{ - printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_postdominators() -{ - printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_ipostdominators() -{ - printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } -} - -void function_info::print_idominators() -{ - printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } -} - -unsigned function_info::get_num_reconvergence_pairs() -{ - if (!num_reconvergence_pairs) { - if( m_basic_blocks.size() == 0 ) - return 0; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; - } + } + } + unsigned num_idoms = 0; + unsigned num_nopred = 0; + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatedominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; + } + } + assert(num_idoms == m_basic_blocks.size() - num_nopred); + // the entry node does not have an immediate dominator, but everyone else + // should +} + +void function_info::print_dominators() { + printf("Printing dominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = m_basic_blocks[i]->dominator_ids.begin(); + j != m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_postdominators() { + printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = + m_basic_blocks[i]->postdominator_ids.begin(); + j != m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_ipostdominators() { + printf("Printing immediate postdominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } +} + +void function_info::print_idominators() { + printf("Printing immediate dominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } +} + +unsigned function_info::get_num_reconvergence_pairs() { + if (!num_reconvergence_pairs) { + if (m_basic_blocks.size() == 0) return 0; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; } - } - return num_reconvergence_pairs; + } + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) -{ - unsigned idx=0; //array index - if( m_basic_blocks.size() == 0 ) - return; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { + unsigned idx = 0; // array index + if (m_basic_blocks.size() == 0) return; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); fflush(stdout); + printf("i=%d\n", i); + fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, + m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", + m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { - recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); - recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin) { + recon_points[idx].target_pc = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->get_PC(); + recon_points[idx].target_inst = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); + fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() -{ - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("\t"); - std::set::iterator s; - for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); - } - printf("\n"); - } - printf("}\n"); -} - -unsigned ptx_kernel_shmem_size( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; -} - -unsigned ptx_kernel_nregs( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; -} - -unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const -{ - int type = scalar_type(); - return type_decode(type,size,basic_type); -} - -unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) -{ - switch ( type ) { - case S8_TYPE: size=8; basic_type=1; return 0; - case S16_TYPE: size=16; basic_type=1; return 1; - case S32_TYPE: size=32; basic_type=1; return 2; - case S64_TYPE: size=64; basic_type=1; return 3; - case U8_TYPE: size=8; basic_type=0; return 4; - case U16_TYPE: size=16; basic_type=0; return 5; - case U32_TYPE: size=32; basic_type=0; return 6; - case U64_TYPE: size=64; basic_type=0; return 7; - case F16_TYPE: size=16; basic_type=-1; return 8; - case F32_TYPE: size=32; basic_type=-1; return 9; - case F64_TYPE: size=64; basic_type=-1; return 10; - case FF64_TYPE: size=64; basic_type=-1; return 10; - case PRED_TYPE: size=1; basic_type=2; return 11; - case B8_TYPE: size=8; basic_type=0; return 12; - case B16_TYPE: size=16; basic_type=0; return 13; - case B32_TYPE: size=32; basic_type=0; return 14; - case B64_TYPE: size=64; basic_type=0; return 15; - case BB64_TYPE: size=64; basic_type=0; return 15; - case BB128_TYPE: size=128; basic_type=0; return 16; - case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: - size=32; basic_type=3; return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); - assert(0); +void function_info::print_basic_block_dot() { + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("\t"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); + } + printf("\n"); + } + printf("}\n"); +} + +unsigned ptx_kernel_shmem_size(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; +} + +unsigned ptx_kernel_nregs(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; +} + +unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { + int type = scalar_type(); + return type_decode(type, size, basic_type); +} + +unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { + switch (type) { + case S8_TYPE: + size = 8; + basic_type = 1; + return 0; + case S16_TYPE: + size = 16; + basic_type = 1; + return 1; + case S32_TYPE: + size = 32; + basic_type = 1; + return 2; + case S64_TYPE: + size = 64; + basic_type = 1; + return 3; + case U8_TYPE: + size = 8; + basic_type = 0; + return 4; + case U16_TYPE: + size = 16; + basic_type = 0; + return 5; + case U32_TYPE: + size = 32; + basic_type = 0; + return 6; + case U64_TYPE: + size = 64; + basic_type = 0; + return 7; + case F16_TYPE: + size = 16; + basic_type = -1; + return 8; + case F32_TYPE: + size = 32; + basic_type = -1; + return 9; + case F64_TYPE: + size = 64; + basic_type = -1; + return 10; + case FF64_TYPE: + size = 64; + basic_type = -1; + return 10; + case PRED_TYPE: + size = 1; + basic_type = 2; + return 11; + case B8_TYPE: + size = 8; + basic_type = 0; + return 12; + case B16_TYPE: + size = 16; + basic_type = 0; + return 13; + case B32_TYPE: + size = 32; + basic_type = 0; + return 14; + case B64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB128_TYPE: + size = 128; + basic_type = 0; + return 16; + case TEXREF_TYPE: + case SAMPLERREF_TYPE: + case SURFREF_TYPE: + size = 32; + basic_type = 3; + return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", + decode_token(type)); + assert(0); return 0xDEADBEEF; - } -} - -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) -{ - if( actual_param_op.is_reg() ) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param,actual_param_op,value); - } else if ( actual_param_op.is_param_local() ) { - unsigned size=formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size<1024); - thread->m_local_mem->read(from_addr,size,buffer); - return arg_buffer_t(formal_param,actual_param_op,buffer,size); - } else { - printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); - abort(); - } -} - -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ) -{ - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); - } -} - -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) -{ - if( a.is_reg() ) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(),value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); - } -} - -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) -{ - arg_buffer_list_t::iterator a; - for( a=arg_values.begin(); a != arg_values.end(); a++ ) { - copy_buffer_to_frame(thread, *a); - } -} - - - -static std::list check_operands( int opcode, - const std::list &scalar_type, - const std::list &operands, - gpgpu_context* ctx) -{ - static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if( !g_warn_literal_operands_two_type_inst ) { - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } - } + } +} + +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param) { + if (actual_param_op.is_reg()) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param, actual_param_op, value); + } else if (actual_param_op.is_param_local()) { + unsigned size = formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size < 1024); + thread->m_local_mem->read(from_addr, size, buffer); + return arg_buffer_t(formal_param, actual_param_op, buffer, size); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " + "call/return\n"); + abort(); + } +} + +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values) { + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( + copy_arg_to_buffer(thread, actual_param_op, formal_param)); + } +} + +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { + if (a.is_reg()) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = + operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(), value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); + } +} + +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values) { + arg_buffer_list_t::iterator a; + for (a = arg_values.begin(); a != arg_values.end(); a++) { + copy_buffer_to_frame(thread, *a); + } +} + +static std::list check_operands( + int opcode, const std::list &scalar_type, + const std::list &operands, gpgpu_context *ctx) { + static int g_warn_literal_operands_two_type_inst; + if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || + (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if (!g_warn_literal_operands_two_type_inst) { + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + printf( + "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " + "operand.\n"); + g_warn_literal_operands_two_type_inst = 1; } - } else { - assert( scalar_type.size() < 2 ); - if( scalar_type.size() == 1 ) { - std::list result; - int inst_type = scalar_type.front(); - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); - } - } - return result; - } + } } - return operands; -} - - -ptx_instruction::ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) -{ - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list checked_operands = check_operands(opcode,scalar_type,operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr=0; - std::list::const_iterator i; - unsigned n=1; - for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type=last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++]=last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; - } - } - rr=0; - n=1; - for ( i=options.begin(); i!= options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { + } else { + assert(scalar_type.size() < 2); + if (scalar_type.size() == 1) { + std::list result; + int inst_type = scalar_type.front(); + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; + } + } + return operands; +} + +ptx_instruction::ptx_instruction( + int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, + const std::list &operands, const operand_info &return_var, + const std::list &options, const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx) + : warp_inst_t(config), m_return_var(ctx) { + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list checked_operands = + check_operands(opcode, scalar_type, operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), + checked_operands.end()); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr = 0; + std::list::const_iterator i; + unsigned n = 1; + for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type = last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++] = last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr = 0; + n = 1; + for (i = options.begin(); i != options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1186,16 +1279,16 @@ ptx_instruction::ptx_instruction( int opcode, case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1204,38 +1297,39 @@ ptx_instruction::ptx_instruction( int opcode, case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert( !m_lo ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert(!m_lo); + assert(!m_wide); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert( !m_hi ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert(!m_hi); + assert(!m_wide); + break; case WIDE_OPTION: - m_wide = true; - assert( !m_lo ); - assert( !m_hi ); - break; + m_wide = true; + assert(!m_lo); + assert(!m_hi); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing + // flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1246,223 +1340,225 @@ ptx_instruction::ptx_instruction( int opcode, case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: + case CG_OPTION: + case CS_OPTION: + case LU_OPTION: + case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { - m_space_spec = generic_space; - } - for( std::vector::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { - const operand_info &op = *i; - if( op.get_addr_space() != undefined_space ) - m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst - } - if( opcode == TEX_OP ) - m_space_spec = tex_space; - - m_source_file = file?file:""; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); - - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); - - if (fname =="vprintf"){ - m_is_printf = true; - } - if(fname == "cudaStreamCreateWithFlags") - m_is_cdp = 1; - if(fname == "cudaGetParameterBufferV2") - m_is_cdp = 2; - if(fname == "cudaLaunchDeviceV2") - m_is_cdp = 4; - - } -} - -void ptx_instruction::print_insn() const -{ - print_insn(stdout); - fflush(stdout); -} - -void ptx_instruction::print_insn( FILE *fp ) const -{ - fprintf( fp, "%s", to_string().c_str() ); -} - -std::string ptx_instruction::to_string() const -{ - char buf[ STR_SIZE ]; - unsigned used_bytes = 0; - if( !is_label() ) { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); - } else { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); - } - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, - "(%s:%d) %s", - m_source_file.c_str(), m_source_line, - m_source.c_str() ); - return std::string( buf ); -} -operand_info ptx_instruction::get_pred() const -{ - return operand_info( m_pred, gpgpu_ctx); -} - - -function_info::function_info(int entry_point, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point==1)?true:false; - m_extern = (entry_point==2)?true:false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; //initialize it to false -} - -unsigned function_info::print_insn( unsigned pc, FILE * fp ) const -{ - unsigned inst_size=1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command,1024,"c++filt -p %s",m_name.c_str()); - FILE *p = popen(command,"r"); - buffer[0]=0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp,"%s",buffer); - if ( index >= m_instr_mem_size ) { - fprintf(fp, "", m_start_PC + m_instr_mem_size - 1 ); - } else { - if ( m_instr_mem[index] != NULL ) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "", pc ); - } - pclose(p); - return inst_size; -} - -std::string function_info::get_insn_str( unsigned pc ) const -{ - unsigned index = pc - m_start_PC; - if ( index >= m_instr_mem_size ) { + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && + (space_spec == undefined_space)) { + m_space_spec = generic_space; + } + for (std::vector::const_iterator i = m_operands.begin(); + i != m_operands.end(); ++i) { + const operand_info &op = *i; + if (op.get_addr_space() != undefined_space) + m_space_spec = + op.get_addr_space(); // TODO: can have more than one memory space for + // ptxplus (g8x) inst + } + if (opcode == TEX_OP) m_space_spec = tex_space; + + m_source_file = file ? file : ""; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), + m_source.end()); + + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); + + if (fname == "vprintf") { + m_is_printf = true; + } + if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; + if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; + if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; + } +} + +void ptx_instruction::print_insn() const { + print_insn(stdout); + fflush(stdout); +} + +void ptx_instruction::print_insn(FILE *fp) const { + fprintf(fp, "%s", to_string().c_str()); +} + +std::string ptx_instruction::to_string() const { + char buf[STR_SIZE]; + unsigned used_bytes = 0; + if (!is_label()) { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); + } else { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); + } + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", + m_source_file.c_str(), m_source_line, m_source.c_str()); + return std::string(buf); +} +operand_info ptx_instruction::get_pred() const { + return operand_info(m_pred, gpgpu_ctx); +} + +function_info::function_info(int entry_point, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point == 1) ? true : false; + m_extern = (entry_point == 2) ? true : false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; // initialize it to false +} + +unsigned function_info::print_insn(unsigned pc, FILE *fp) const { + unsigned inst_size = 1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); + FILE *p = popen(command, "r"); + buffer[0] = 0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp, "%s", buffer); + if (index >= m_instr_mem_size) { + fprintf(fp, "", + m_start_PC + m_instr_mem_size - 1); + } else { + if (m_instr_mem[index] != NULL) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "", pc); + } + pclose(p); + return inst_size; +} + +std::string function_info::get_insn_str(unsigned pc) const { + unsigned index = pc - m_start_PC; + if (index >= m_instr_mem_size) { + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", + m_start_PC + m_instr_mem_size - 1); + return std::string(buff); + } else { + if (m_instr_mem[index] != NULL) { + return m_instr_mem[index]->to_string(); + } else { char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", m_start_PC + m_instr_mem_size - 1 ); + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); return std::string(buff); - } else { - if ( m_instr_mem[index] != NULL ) { - return m_instr_mem[index]->to_string(); - } else { - char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", pc ); - return std::string(buff); - } - } -} - -void gpgpu_ptx_assemble( std::string kname, void *kinfo ) -{ - function_info *func_info = (function_info *)kinfo; - if((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); - return; - } - if( func_info->is_extern() ) { - printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); - return; } - func_info->ptx_assemble(); + } +} + +void gpgpu_ptx_assemble(std::string kname, void *kinfo) { + function_info *func_info = (function_info *)kinfo; + if ((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", + kname.c_str()); + return; + } + if (func_info->is_extern()) { + printf( + "GPGPU-Sim PTX: skipping assembly for extern declared function " + "\'%s\'\n", + func_info->get_name().c_str()); + return; + } + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4c5c37..6627847 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,36 +7,37 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ptx_ir_INCLUDED #define ptx_ir_INCLUDED #include "../abstract_hardware_model.h" +#include #include #include -#include #include #include +#include #include -#include //#include "ptx.tab.h" #include "ptx_sim.h" @@ -46,1537 +47,1517 @@ class gpgpu_context; class type_info_key { -public: - type_info_key() - { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) - { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() - { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } - - void set_array_dim( int array_dim ) { m_array_dim = array_dim; } - int get_array_dim() const { assert(m_init); return m_array_dim; } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space;} - bool is_param_kernel() const { return m_space_spec == param_space_kernel;} - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } - bool is_global() const { return m_space_spec == global_space;} - bool is_local() const { return m_space_spec == local_space;} - bool is_shared() const { return m_space_spec == shared_space;} - bool is_const() const { return m_space_spec.get_type() == const_space;} - bool is_tex() const { return m_space_spec == tex_space;} - bool is_func_addr() const { return m_is_function?true:false; } - int scalar_type() const { return m_scalar_type_spec;} - int get_alignment_spec() const { return m_alignment_spec;} - unsigned type_decode( size_t &size, int &t ) const; - static unsigned type_decode( int type, size_t &size, int &t ); - memory_space_t get_memory_space() const { return m_space_spec; } -private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; - - friend struct type_info_key_compare; + public: + type_info_key() { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec, + int array_dim) { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } + + void set_array_dim(int array_dim) { m_array_dim = array_dim; } + int get_array_dim() const { + assert(m_init); + return m_array_dim; + } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } + + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space; } + bool is_param_kernel() const { return m_space_spec == param_space_kernel; } + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { + return m_space_spec == param_space_unclassified; + } + bool is_global() const { return m_space_spec == global_space; } + bool is_local() const { return m_space_spec == local_space; } + bool is_shared() const { return m_space_spec == shared_space; } + bool is_const() const { return m_space_spec.get_type() == const_space; } + bool is_tex() const { return m_space_spec == tex_space; } + bool is_func_addr() const { return m_is_function ? true : false; } + int scalar_type() const { return m_scalar_type_spec; } + int get_alignment_spec() const { return m_alignment_spec; } + unsigned type_decode(size_t &size, int &t) const; + static unsigned type_decode(int type, size_t &size, int &t); + memory_space_t get_memory_space() const { return m_space_spec; } + + private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; + + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()( const type_info_key &a, const type_info_key &b ) const - { - assert( a.m_init && b.m_init ); - if ( a.m_space_spec < b.m_space_spec ) return true; - if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; - if ( a.m_vector_spec < b.m_vector_spec ) return true; - if ( a.m_alignment_spec < b.m_alignment_spec ) return true; - if ( a.m_extern_spec < b.m_extern_spec ) return true; - if ( a.m_array_dim < b.m_array_dim ) return true; - if ( a.m_is_function < b.m_is_function ) return true; - - return false; - } + bool operator()(const type_info_key &a, const type_info_key &b) const { + assert(a.m_init && b.m_init); + if (a.m_space_spec < b.m_space_spec) return true; + if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; + if (a.m_vector_spec < b.m_vector_spec) return true; + if (a.m_alignment_spec < b.m_alignment_spec) return true; + if (a.m_extern_spec < b.m_extern_spec) return true; + if (a.m_array_dim < b.m_array_dim) return true; + if (a.m_is_function < b.m_is_function) return true; + + return false; + } }; class type_info { -public: - type_info( symbol_table *scope, type_info_key t ) - { - m_type_info = t; - } - const type_info_key &get_key() const { return m_type_info;} - -private: - symbol_table *m_scope; - type_info_key m_type_info; + public: + type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } + const type_info_key &get_key() const { return m_type_info; } + + private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, - unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, - v_int_t, v_unsigned_t, undef_t + reg_t, + vector_t, + builtin_t, + address_t, + memory_t, + float_op_t, + double_op_t, + int_t, + unsigned_t, + symbolic_t, + label_t, + v_reg_t, + v_float_op_t, + v_double_op_t, + v_int_t, + v_unsigned_t, + undef_t }; class operand_info; class symbol { -public: - symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) - { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num=(unsigned)-1; - m_arch_reg_num=(unsigned)-1; - m_address=(unsigned)-1; - m_initializer.clear(); - if ( type ) m_is_shared = type->get_key().is_shared(); - if ( type ) m_is_const = type->get_key().is_const(); - if ( type ) m_is_global = type->get_key().is_global(); - if ( type ) m_is_local = type->get_key().is_local(); - if ( type ) m_is_param_local = type->get_key().is_param_local(); - if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); - if ( type ) m_is_tex = type->get_key().is_tex(); - if ( type ) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const - { - return m_size; - } - const std::string &name() const { return m_name;} - const std::string &decl_location() const { return m_decl_location;} - const type_info *type() const { return m_type;} - addr_t get_address() const - { - assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions - assert( m_address_valid ); - return m_address; - } - function_info *get_pc() const - { - return m_function; - } - void set_regno( unsigned regno, unsigned arch_regno ) - { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } - - void set_address( addr_t addr ) - { - m_address_valid = true; - m_address = addr; - } - void set_label_address( addr_t addr) - { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function( function_info *func ) - { - m_function = func; - m_is_func_addr = true; - } - - bool is_label() const { return m_is_label;} - bool is_shared() const { return m_is_shared;} - bool is_sstarr() const { return m_is_sstarr;} - bool is_const() const { return m_is_const;} - bool is_global() const { return m_is_global;} - bool is_local() const { return m_is_local;} - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex;} - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } - - void add_initializer( const std::list &init ); - bool has_initializer() const - { - return m_initializer.size() > 0; - } - std::list get_initializer() const - { - return m_initializer; - } - unsigned reg_num() const - { - assert( m_reg_num_valid ); - return m_reg_num; - } - unsigned arch_reg_num() const - { - assert( m_reg_num_valid ); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; - - unsigned m_address; - function_info *m_function; // used for function symbols - - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; - - std::list m_initializer; + public: + symbol(const char *name, const type_info *type, const char *location, + unsigned size, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num = (unsigned)-1; + m_arch_reg_num = (unsigned)-1; + m_address = (unsigned)-1; + m_initializer.clear(); + if (type) m_is_shared = type->get_key().is_shared(); + if (type) m_is_const = type->get_key().is_const(); + if (type) m_is_global = type->get_key().is_global(); + if (type) m_is_local = type->get_key().is_local(); + if (type) m_is_param_local = type->get_key().is_param_local(); + if (type) m_is_param_kernel = type->get_key().is_param_kernel(); + if (type) m_is_tex = type->get_key().is_tex(); + if (type) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const { return m_size; } + const std::string &name() const { return m_name; } + const std::string &decl_location() const { return m_decl_location; } + const type_info *type() const { return m_type; } + addr_t get_address() const { + assert(m_is_label || + !m_type->get_key().is_reg()); // todo : other assertions + assert(m_address_valid); + return m_address; + } + function_info *get_pc() const { return m_function; } + void set_regno(unsigned regno, unsigned arch_regno) { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } + + void set_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + } + void set_label_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function(function_info *func) { + m_function = func; + m_is_func_addr = true; + } + + bool is_label() const { return m_is_label; } + bool is_shared() const { return m_is_shared; } + bool is_sstarr() const { return m_is_sstarr; } + bool is_const() const { return m_is_const; } + bool is_global() const { return m_is_global; } + bool is_local() const { return m_is_local; } + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex; } + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } + + void add_initializer(const std::list &init); + bool has_initializer() const { return m_initializer.size() > 0; } + std::list get_initializer() const { return m_initializer; } + unsigned reg_num() const { + assert(m_reg_num_valid); + return m_reg_num; + } + unsigned arch_reg_num() const { + assert(m_reg_num_valid); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; + + unsigned m_address; + function_info *m_function; // used for function symbols + + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; + + std::list m_initializer; }; class symbol_table { -public: - symbol_table(); - symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); - void set_name( const char *name ); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version( float ver, unsigned ext ); - void set_sm_target( const char *target, const char *ext, const char *ext2 ); - symbol* lookup( const char *identifier ); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); - void add_function( function_info *func, const char *filename, unsigned linenumber ); - bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); - function_info *lookup_function(std::string name); - type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); - type_info *add_type( function_info *func ); - type_info *get_array_type( type_info *base_type, unsigned array_dim ); - void set_label_address( const symbol *label, unsigned addr ); - unsigned next_reg_num() { return ++m_reg_allocator;} - addr_t get_shared_next() { return m_shared_next;} - addr_t get_sstarr_next() { return m_sstarr_next;} - addr_t get_global_next() { return m_global_next;} - addr_t get_local_next() { return m_local_next;} - addr_t get_tex_next() { return m_tex_next;} - void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} - void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} - void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} - void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} - void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} - - typedef std::list::iterator iterator; - - iterator global_iterator_begin() { return m_globals.begin();} - iterator global_iterator_end() { return m_globals.end();} - - iterator const_iterator_begin() { return m_consts.begin();} - iterator const_iterator_end() { return m_consts.end();} - - void dump(); - - //Jin: handle instruction group for cdp - symbol_table* start_inst_group(); - symbol_table* end_inst_group(); - - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; - - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map m_symbols; //map from name of register to pointers to the registers - std::map m_types; - std::list m_globals; - std::list m_consts; - std::map m_function_info_lookup; - std::map m_function_symtab_lookup; - - //Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map m_inst_group_symtab; + public: + symbol_table(); + symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx); + void set_name(const char *name); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version(float ver, unsigned ext); + void set_sm_target(const char *target, const char *ext, const char *ext2); + symbol *lookup(const char *identifier); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable(const char *identifier, const type_info *type, + unsigned size, const char *filename, unsigned line); + void add_function(function_info *func, const char *filename, + unsigned linenumber); + bool add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **symbol_table); + function_info *lookup_function(std::string name); + type_info *add_type(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec); + type_info *add_type(function_info *func); + type_info *get_array_type(type_info *base_type, unsigned array_dim); + void set_label_address(const symbol *label, unsigned addr); + unsigned next_reg_num() { return ++m_reg_allocator; } + addr_t get_shared_next() { return m_shared_next; } + addr_t get_sstarr_next() { return m_sstarr_next; } + addr_t get_global_next() { return m_global_next; } + addr_t get_local_next() { return m_local_next; } + addr_t get_tex_next() { return m_tex_next; } + void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } + void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } + void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } + void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } + void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } + + typedef std::list::iterator iterator; + + iterator global_iterator_begin() { return m_globals.begin(); } + iterator global_iterator_end() { return m_globals.end(); } + + iterator const_iterator_begin() { return m_consts.begin(); } + iterator const_iterator_end() { return m_consts.end(); } + + void dump(); + + // Jin: handle instruction group for cdp + symbol_table *start_inst_group(); + symbol_table *end_inst_group(); + + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; + + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map + m_symbols; // map from name of register to pointers to the registers + std::map m_types; + std::list m_globals; + std::list m_consts; + std::map m_function_info_lookup; + std::map m_function_symtab_lookup; + + // Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map m_inst_group_symtab; }; class operand_info { -public: - operand_info(gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address=false; - m_addr_offset = 0; - m_value.m_symbolic=NULL; - } - operand_info( const symbol *addr, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if ( addr->is_label() ) { - m_type = label_t; - } else if ( addr->is_shared() ) { - m_type = symbolic_t; - } else if ( addr->is_const() ) { - m_type = symbolic_t; - } else if ( addr->is_global() ) { - m_type = symbolic_t; - } else if ( addr->is_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_kernel() ) { - m_type = symbolic_t; - } else if ( addr->is_tex() ) { - m_type = symbolic_t; - } else if ( addr->is_func_addr() ) { - m_type = symbolic_t; - } else if ( !addr->is_reg() ) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } - - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( unsigned x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=true; - } - operand_info( int x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( float x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( double x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - - void init(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_uid=(unsigned)-1; - m_valid=false; - m_vector=false; - m_type=undef_t; - m_immediate_address=false; - m_addr_space=undefined_space; - m_operand_lohi=0; - m_double_operand_type=0; - m_operand_neg=false; - m_const_mem_offset=(unsigned)-1; - m_value.m_int=0; - m_value.m_unsigned=(unsigned)-1; - m_value.m_float=0; - m_value.m_double=0; - for(unsigned i=0; i<4; i++){ - m_value.m_vint[i]=0; - m_value.m_vunsigned[i]=0; - m_value.m_vfloat[i]=0; - m_value.m_vdouble[i]=0; - } - m_value.m_symbolic=NULL; - m_value.m_vector_symbolic=NULL; - m_addr_offset=0; - m_neg_pred=0; - m_is_return_var=0; - m_is_non_arch_reg=0; - - } - void make_memory_operand() { m_type = memory_t;} - void set_return() { m_is_return_var = true; } - void set_immediate_addr() {m_immediate_address=true;} - const std::string &name() const - { - assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } - - unsigned get_vect_nelem() const - { - assert( is_vector() ); - if( !m_value.m_vector_symbolic[0] ) return 0; - if( !m_value.m_vector_symbolic[1] ) return 1; - if( !m_value.m_vector_symbolic[2] ) return 2; - if( !m_value.m_vector_symbolic[3] ) return 3; - if( !m_value.m_vector_symbolic[4] ) return 4; - if( !m_value.m_vector_symbolic[5] ) return 5; - if( !m_value.m_vector_symbolic[6] ) return 6; - if( !m_value.m_vector_symbolic[7] ) return 7; - return 8; - } - - const symbol* vec_symbol(int idx) const - { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert( result != NULL ); - return result; - } - - const std::string &vec_name1() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } - - const std::string &vec_name2() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } - - const std::string &vec_name3() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } - - const std::string &vec_name4() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } - - bool is_reg() const - { - if ( m_type == reg_t ) { - return true; - } - if ( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } - - bool is_param_kernel() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } - - bool is_vector() const - { - if ( m_vector) return true; + public: + operand_info(gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address = false; + m_addr_offset = 0; + m_value.m_symbolic = NULL; + } + operand_info(const symbol *addr, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if (addr->is_label()) { + m_type = label_t; + } else if (addr->is_shared()) { + m_type = symbolic_t; + } else if (addr->is_const()) { + m_type = symbolic_t; + } else if (addr->is_global()) { + m_type = symbolic_t; + } else if (addr->is_local()) { + m_type = symbolic_t; + } else if (addr->is_param_local()) { + m_type = symbolic_t; + } else if (addr->is_param_kernel()) { + m_type = symbolic_t; + } else if (addr->is_tex()) { + m_type = symbolic_t; + } else if (addr->is_func_addr()) { + m_type = symbolic_t; + } else if (!addr->is_reg()) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } + + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(unsigned x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = true; + } + operand_info(int x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(float x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(double x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, const symbol *s5, const symbol *s6, + const symbol *s7, const symbol *s8, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + + void init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (unsigned)-1; + m_valid = false; + m_vector = false; + m_type = undef_t; + m_immediate_address = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = (unsigned)-1; + m_value.m_int = 0; + m_value.m_unsigned = (unsigned)-1; + m_value.m_float = 0; + m_value.m_double = 0; + for (unsigned i = 0; i < 4; i++) { + m_value.m_vint[i] = 0; + m_value.m_vunsigned[i] = 0; + m_value.m_vfloat[i] = 0; + m_value.m_vdouble[i] = 0; + } + m_value.m_symbolic = NULL; + m_value.m_vector_symbolic = NULL; + m_addr_offset = 0; + m_neg_pred = 0; + m_is_return_var = 0; + m_is_non_arch_reg = 0; + } + void make_memory_operand() { m_type = memory_t; } + void set_return() { m_is_return_var = true; } + void set_immediate_addr() { m_immediate_address = true; } + const std::string &name() const { + assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || + m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } + + unsigned get_vect_nelem() const { + assert(is_vector()); + if (!m_value.m_vector_symbolic[0]) return 0; + if (!m_value.m_vector_symbolic[1]) return 1; + if (!m_value.m_vector_symbolic[2]) return 2; + if (!m_value.m_vector_symbolic[3]) return 3; + if (!m_value.m_vector_symbolic[4]) return 4; + if (!m_value.m_vector_symbolic[5]) return 5; + if (!m_value.m_vector_symbolic[6]) return 6; + if (!m_value.m_vector_symbolic[7]) return 7; + return 8; + } + + const symbol *vec_symbol(int idx) const { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert(result != NULL); + return result; + } + + const std::string &vec_name1() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } + + const std::string &vec_name2() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } + + const std::string &vec_name3() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } + + const std::string &vec_name4() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } + + bool is_reg() const { + if (m_type == reg_t) { + return true; + } + if (m_type != symbolic_t) { return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num();} - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} - int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } - int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } - int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } - int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } - int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } - int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } - bool is_label() const { return m_type == label_t;} - bool is_builtin() const { return m_type == builtin_t;} - - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t;} - - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. - bool is_memory_operand2() const { - return (m_addr_space!=undefined_space); - } - - bool is_immediate_address() const { - return m_immediate_address; - } - - bool is_literal() const { return m_type == int_t || - m_type == float_op_t || - m_type == double_op_t || - m_type == unsigned_t;} - bool is_shared() const { - if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} - bool is_const() const { return m_value.m_symbolic->is_const();} - bool is_global() const { return m_value.m_symbolic->is_global();} - bool is_local() const { return m_value.m_symbolic->is_local();} - bool is_tex() const { return m_value.m_symbolic->is_tex();} - bool is_return_var() const { return m_is_return_var; } - - bool is_function_address() const - { - if( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->is_func_addr(); - } - - ptx_reg_t get_literal_value() const - { - ptx_reg_t result; - switch ( m_type ) { - case int_t: result.s64 = m_value.m_int; break; - case float_op_t: result.f32 = m_value.m_float; break; - case double_op_t: result.f64 = m_value.m_double; break; - case unsigned_t: result.u32 = m_value.m_unsigned; break; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } + + bool is_param_kernel() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + + bool is_vector() const { + if (m_vector) return true; + return false; + } + int reg_num() const { return m_value.m_symbolic->reg_num(); } + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } + int reg3_num() const { + return m_value.m_vector_symbolic[2] + ? m_value.m_vector_symbolic[2]->reg_num() + : 0; + } + int reg4_num() const { + return m_value.m_vector_symbolic[3] + ? m_value.m_vector_symbolic[3]->reg_num() + : 0; + } + int reg5_num() const { + return m_value.m_vector_symbolic[4] + ? m_value.m_vector_symbolic[4]->reg_num() + : 0; + } + int reg6_num() const { + return m_value.m_vector_symbolic[5] + ? m_value.m_vector_symbolic[5]->reg_num() + : 0; + } + int reg7_num() const { + return m_value.m_vector_symbolic[6] + ? m_value.m_vector_symbolic[6]->reg_num() + : 0; + } + int reg8_num() const { + return m_value.m_vector_symbolic[7] + ? m_value.m_vector_symbolic[7]->reg_num() + : 0; + } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { + return (m_value.m_vector_symbolic[n]) + ? m_value.m_vector_symbolic[n]->arch_reg_num() + : -1; + } + bool is_label() const { return m_type == label_t; } + bool is_builtin() const { return m_type == builtin_t; } + + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t; } + + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address + // space during parsing. + bool is_memory_operand2() const { return (m_addr_space != undefined_space); } + + bool is_immediate_address() const { return m_immediate_address; } + + bool is_literal() const { + return m_type == int_t || m_type == float_op_t || m_type == double_op_t || + m_type == unsigned_t; + } + bool is_shared() const { + if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } + bool is_const() const { return m_value.m_symbolic->is_const(); } + bool is_global() const { return m_value.m_symbolic->is_global(); } + bool is_local() const { return m_value.m_symbolic->is_local(); } + bool is_tex() const { return m_value.m_symbolic->is_tex(); } + bool is_return_var() const { return m_is_return_var; } + + bool is_function_address() const { + if (m_type != symbolic_t) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } + + ptx_reg_t get_literal_value() const { + ptx_reg_t result; + switch (m_type) { + case int_t: + result.s64 = m_value.m_int; + break; + case float_op_t: + result.f32 = m_value.m_float; + break; + case double_op_t: + result.f64 = m_value.m_double; + break; + case unsigned_t: + result.u32 = m_value.m_unsigned; + break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int;} - int get_addr_offset() const { return m_addr_offset;} - const symbol *get_symbol() const { return m_value.m_symbolic;} - void set_type( enum operand_type type ) - { - m_type = type; - } - enum operand_type get_type() const { - return m_type; - } - void set_neg_pred() - { - assert( m_valid ); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } - - void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol* m_symbolic; - const symbol** m_vector_symbolic; - } m_value; - - int m_addr_offset; - - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; - - unsigned get_uid(); + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int; } + int get_addr_offset() const { return m_addr_offset; } + const symbol *get_symbol() const { return m_value.m_symbolic; } + void set_type(enum operand_type type) { m_type = type; } + enum operand_type get_type() const { return m_type; } + void set_neg_pred() { + assert(m_valid); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } + + void set_addr_space(enum _memory_space_t set_value) { + m_addr_space = set_value; + } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { + m_double_operand_type = set_value; + } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { + m_const_mem_offset = set_value; + } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol *m_symbolic; + const symbol **m_vector_symbolic; + } m_value; + + int m_addr_offset; + + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; + + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) - { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry=entry; - is_exit=ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } - - ptx_instruction* ptx_begin; - ptx_instruction* ptx_end; - std::set predecessor_ids; //indices of other basic blocks in m_basic_blocks array - std::set successor_ids; - std::set postdominator_ids; - std::set dominator_ids; - std::set Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; - - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } - - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); - } + basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, + bool entry, bool ex) { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry = entry; + is_exit = ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } + + ptx_instruction *ptx_begin; + ptx_instruction *ptx_end; + std::set + predecessor_ids; // indices of other basic blocks in m_basic_blocks array + std::set successor_ids; + std::set postdominator_ids; + std::set dominator_ids; + std::set Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; + + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } + + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != + B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction* source_inst; - class ptx_instruction* target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction *source_inst; + class ptx_instruction *target_inst; }; class ptx_instruction : public warp_inst_t { -public: - ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx); - - void print_insn() const; - virtual void print_insn( FILE *fp ) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid;} - int get_opcode() const { return m_opcode;} - const char *get_opcode_cstr() const - { - if ( m_opcode != -1 ) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str();} - unsigned source_line() const { return m_source_line;} - unsigned get_num_operands() const { return m_operands.size();} - bool has_pred() const { return m_pred != NULL;} - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred;} - int get_pred_mod() const { return m_pred_mod;} - const char *get_source() const { return m_source.c_str();} - - typedef std::vector::const_iterator const_iterator; - - const_iterator op_iter_begin() const - { - return m_operands.begin(); - } - - const_iterator op_iter_end() const - { - return m_operands.end(); - } - - const operand_info &dst() const - { - assert( !m_operands.empty() ); - return m_operands[0]; - } - - const operand_info &func_addr() const - { - assert( !m_operands.empty() ); - if( !m_operands[0].is_return_var() ) { - return m_operands[0]; - } else { - assert( m_operands.size() >= 2 ); - return m_operands[1]; - } - } - - operand_info &dst() - { - assert( !m_operands.empty() ); + public: + ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, + symbol *label, const std::list &operands, + const operand_info &return_var, const std::list &options, + const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx); + + void print_insn() const; + virtual void print_insn(FILE *fp) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid; } + int get_opcode() const { return m_opcode; } + const char *get_opcode_cstr() const { + if (m_opcode != -1) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str(); } + unsigned source_line() const { return m_source_line; } + unsigned get_num_operands() const { return m_operands.size(); } + bool has_pred() const { return m_pred != NULL; } + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred; } + int get_pred_mod() const { return m_pred_mod; } + const char *get_source() const { return m_source.c_str(); } + + typedef std::vector::const_iterator const_iterator; + + const_iterator op_iter_begin() const { return m_operands.begin(); } + + const_iterator op_iter_end() const { return m_operands.end(); } + + const operand_info &dst() const { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &func_addr() const { + assert(!m_operands.empty()); + if (!m_operands[0].is_return_var()) { return m_operands[0]; - } - - const operand_info &src1() const - { - assert( m_operands.size() > 1 ); + } else { + assert(m_operands.size() >= 2); return m_operands[1]; - } - - const operand_info &src2() const - { - assert( m_operands.size() > 2 ); - return m_operands[2]; - } - - const operand_info &src3() const - { - assert( m_operands.size() > 3 ); - return m_operands[3]; - } - const operand_info &src4() const - { - assert( m_operands.size() > 4 ); - return m_operands[4]; - } - const operand_info &src5() const - { - assert( m_operands.size() > 5 ); - return m_operands[5]; - } - const operand_info &src6() const - { - assert( m_operands.size() > 6 ); - return m_operands[6]; - } - const operand_info &src7() const - { - assert( m_operands.size() > 7 ); - return m_operands[7]; - } - const operand_info &src8() const - { - assert( m_operands.size() > 8 ); - return m_operands[8]; - } - - const operand_info &operand_lookup( unsigned n ) const - { - assert( n < m_operands.size() ); - return m_operands[n]; - } - bool has_return() const - { - return m_return_var.is_valid(); - } - - memory_space_t get_space() const { return m_space_spec;} - unsigned get_vector() const { return m_vector_spec;} - unsigned get_atomic() const { return m_atomic_spec;} - - int get_wmma_type() const { - return m_wmma_type; - } - int get_wmma_layout(int index) const { - return m_wmma_layout[index];//0->Matrix D,1->Matrix C - } - int get_type() const - { - assert( !m_scalar_type.empty() ); - return m_scalar_type.front(); - } - - int get_type2() const - { - assert( m_scalar_type.size()==2 ); - return m_scalar_type.back(); - } - - void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t* get_bb() { return m_basic_block;} - void set_m_instr_mem_index(unsigned index) { - m_instr_mem_index = index; - } - void set_PC( addr_t PC ) - { - m_PC = PC; - } - addr_t get_PC() const - { - return m_PC; - } - - unsigned get_m_instr_mem_index() { return m_instr_mem_index;} - unsigned get_cmpop() const { return m_compare_op;} - const symbol *get_label() const { return m_label;} - bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} - bool is_hi() const { return m_hi;} - bool is_lo() const { return m_lo;} - bool is_wide() const { return m_wide;} - bool is_uni() const { return m_uni;} - bool is_exit() const { return m_exit;} - bool is_abs() const { return m_abs;} - bool is_neg() const { return m_neg;} - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode;} - unsigned saturation_mode() const { return m_saturation_mode;} - unsigned dimension() const { return m_geom_spec;} - unsigned barrier_op() const {return m_barrier_op;} - unsigned shfl_op() const {return m_shfl_op;} - unsigned prmt_op() const {return m_prmt_op;} - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } - - int membar_level() const { return m_membar_level; } - - bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands - if( n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands - if( n==0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - - -private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); - - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; - - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector m_operands; - operand_info m_return_var; - - std::list m_options; - std::list m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; - - std::list m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; //index into m_instr_mem array - unsigned m_inst_size; // bytes - - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context* gpgpu_ctx; + } + } + + operand_info &dst() { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &src1() const { + assert(m_operands.size() > 1); + return m_operands[1]; + } + + const operand_info &src2() const { + assert(m_operands.size() > 2); + return m_operands[2]; + } + + const operand_info &src3() const { + assert(m_operands.size() > 3); + return m_operands[3]; + } + const operand_info &src4() const { + assert(m_operands.size() > 4); + return m_operands[4]; + } + const operand_info &src5() const { + assert(m_operands.size() > 5); + return m_operands[5]; + } + const operand_info &src6() const { + assert(m_operands.size() > 6); + return m_operands[6]; + } + const operand_info &src7() const { + assert(m_operands.size() > 7); + return m_operands[7]; + } + const operand_info &src8() const { + assert(m_operands.size() > 8); + return m_operands[8]; + } + + const operand_info &operand_lookup(unsigned n) const { + assert(n < m_operands.size()); + return m_operands[n]; + } + bool has_return() const { return m_return_var.is_valid(); } + + memory_space_t get_space() const { return m_space_spec; } + unsigned get_vector() const { return m_vector_spec; } + unsigned get_atomic() const { return m_atomic_spec; } + + int get_wmma_type() const { return m_wmma_type; } + int get_wmma_layout(int index) const { + return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C + } + int get_type() const { + assert(!m_scalar_type.empty()); + return m_scalar_type.front(); + } + + int get_type2() const { + assert(m_scalar_type.size() == 2); + return m_scalar_type.back(); + } + + void assign_bb( + basic_block_t *basic_block) // assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t *get_bb() { return m_basic_block; } + void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } + void set_PC(addr_t PC) { m_PC = PC; } + addr_t get_PC() const { return m_PC; } + + unsigned get_m_instr_mem_index() { return m_instr_mem_index; } + unsigned get_cmpop() const { return m_compare_op; } + const symbol *get_label() const { return m_label; } + bool is_label() const { + if (m_label) { + assert(m_opcode == -1); + return true; + } + return false; + } + bool is_hi() const { return m_hi; } + bool is_lo() const { return m_lo; } + bool is_wide() const { return m_wide; } + bool is_uni() const { return m_uni; } + bool is_exit() const { return m_exit; } + bool is_abs() const { return m_abs; } + bool is_neg() const { return m_neg; } + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode; } + unsigned saturation_mode() const { return m_saturation_mode; } + unsigned dimension() const { return m_geom_spec; } + unsigned barrier_op() const { return m_barrier_op; } + unsigned shfl_op() const { return m_shfl_op; } + unsigned prmt_op() const { return m_prmt_op; } + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } + + int membar_level() const { return m_membar_level; } + + bool has_memory_read() const { + if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || + m_opcode == MMA_LD_OP) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; op != op_iter_end(); op++, n++) { // process operands + if (n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; (op != op_iter_end() && n < 1); + op++, n++) { // process operands + if (n == 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + + private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); + + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; + + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector m_operands; + operand_info m_return_var; + + std::list m_options; + std::list m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; // if branch instruction, this evaluates to true for uniform + // branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; + + std::list m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; // index into m_instr_mem array + unsigned m_inst_size; // bytes + + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context *gpgpu_ctx; }; class param_info { -public: - param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } - param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) - { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data( param_t v ) { - assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset( unsigned offset ) { m_offset = offset; } - unsigned get_offset() { assert(m_valid); return m_offset; } - std::string get_name() const { assert(m_valid); return m_name; } - int get_type() const { assert(m_valid); return m_type; } - param_t get_value() const { assert(m_value_set); return m_value; } - size_t get_size() const { assert(m_valid); return m_size; } - bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } -private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; + public: + param_info() { + m_valid = false; + m_value_set = false; + m_size = 0; + m_is_ptr = false; + } + param_info(std::string name, int type, size_t size, bool is_ptr, + memory_space_t ptr_space) { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data(param_t v) { + assert((!m_value_set) || + (m_value.size == v.size)); // if this fails concurrent kernel + // launches might execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset(unsigned offset) { m_offset = offset; } + unsigned get_offset() { + assert(m_valid); + return m_offset; + } + std::string get_name() const { + assert(m_valid); + return m_name; + } + int get_type() const { + assert(m_valid); + return m_type; + } + param_t get_value() const { + assert(m_value_set); + return m_value; + } + size_t get_size() const { + assert(m_valid); + return m_size; + } + bool is_ptr_shared() const { + assert(m_valid); + return (m_is_ptr and m_ptr_space == shared_space); + } + + private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { -public: - function_info(int entry_point, gpgpu_context* ctx ); - const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) - { - m_name = name; - } - void set_symtab(symbol_table *symtab ) - { - m_symtab = symtab; - } - std::string get_name() const - { - return m_name; - } - unsigned print_insn( unsigned pc, FILE * fp ) const; - std::string get_insn_str( unsigned pc ) const; - void add_inst( const std::list &instructions ) - { - m_instructions = instructions; - } - std::list::iterator find_next_real_instruction( std::list::iterator i ); - void create_basic_blocks( ); - - void print_basic_blocks(); - - void print_basic_block_links(); - void print_basic_block_dot(); - - operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction - void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together - bool connect_break_targets(); //connecting break instructions with proper targets - - //iterate across m_basic_blocks of function, - //finding dominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators( ); - void print_dominators(); - void find_idominators(); - void print_idominators(); - - //iterate across m_basic_blocks of function, - //finding postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators( ); - void print_postdominators(); - - //iterate across m_basic_blocks of function, - //finding immediate postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators( ); - void print_ipostdominators(); - void do_pdom(); //function to call pdom analysis - - unsigned get_num_reconvergence_pairs(); - - void get_reconvergence_pairs(gpgpu_recon_t* recon_points); - - unsigned get_function_size() { return m_instructions.size();} - - void ptx_assemble(); - - unsigned ptx_get_inst_op( ptx_thread_info *thread ); - void add_param( const char *name, struct param_t value ) - { - m_kernel_params[ name ] = value; - } - void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); - void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); - void add_return_var( const symbol *rv ) - { - m_return_var_sym = rv; - } - void add_arg( const symbol *arg ) - { - assert( arg != NULL ); - m_args.push_back(arg); - } - void remove_args() - { - m_args.clear(); - } - unsigned num_args() const - { - return m_args.size(); - } - unsigned get_args_aligned_size(); - - const symbol* get_arg( unsigned n ) const - { - assert( n < m_args.size() ); - return m_args[n]; - } - bool has_return() const - { - return m_return_var_sym != NULL; - } - const symbol *get_return_var() const - { - return m_return_var_sym; - } - const ptx_instruction *get_instruction( unsigned PC ) const - { - unsigned index = PC - m_start_PC; - if( index < m_instr_mem_size ) - return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const - { - return m_start_PC; - } - - void finalize( memory_space *param_mem ); - void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); - void list_param( FILE *fout ) const; - void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; - - const struct gpgpu_ptx_sim_info* get_kernel_info () const - { - assert (m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } - - const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10*get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() - { - return m_symtab; - } - - unsigned local_mem_framesize() const - { - return m_local_mem_framesize; - } - void set_framesize( unsigned sz ) - { - m_local_mem_framesize = sz; - } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } //return pdom flag - void set_pdom() { pdom_done = true; } //set pdom flag - - void add_config_param( size_t size, unsigned alignment ){ - unsigned offset = 0; - if (m_param_configs.size()>0){ - unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; - //ensure offset matches alignment requirements - offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; - } - m_param_configs.push_back(std::pair(size, offset)); - } - - std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } - - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} - unsigned get_maxnt_id() { return maxnt_id;} - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; //flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map m_kernel_params; - std::map m_ptx_kernel_param_info; - std::vector< std::pair > m_param_configs; - const symbol *m_return_var_sym; - std::vector m_args; - std::list m_instructions; - std::vector m_basic_blocks; - std::list > m_back_edges; - std::map labels; - unsigned num_reconvergence_pairs; - - //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; - - symbol_table *m_symtab; - - //parameter size for device kernels - int m_args_aligned_size; - - addr_t m_n; // offset in m_instr_mem (used in do_pdom) + public: + function_info(int entry_point, gpgpu_context *ctx); + const ptx_version &get_ptx_version() const { + return m_symtab->get_ptx_version(); + } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) { m_name = name; } + void set_symtab(symbol_table *symtab) { m_symtab = symtab; } + std::string get_name() const { return m_name; } + unsigned print_insn(unsigned pc, FILE *fp) const; + std::string get_insn_str(unsigned pc) const; + void add_inst(const std::list &instructions) { + m_instructions = instructions; + } + std::list::iterator find_next_real_instruction( + std::list::iterator i); + void create_basic_blocks(); + + void print_basic_blocks(); + + void print_basic_block_links(); + void print_basic_block_dot(); + + operand_info *find_break_target( + ptx_instruction *p_break_insn); // find the target of a break instruction + void connect_basic_blocks(); // iterate across m_basic_blocks of function, + // connecting basic blocks together + bool + connect_break_targets(); // connecting break instructions with proper targets + + // iterate across m_basic_blocks of function, + // finding dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators(); + void print_dominators(); + void find_idominators(); + void print_idominators(); + + // iterate across m_basic_blocks of function, + // finding postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators(); + void print_postdominators(); + + // iterate across m_basic_blocks of function, + // finding immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators(); + void print_ipostdominators(); + void do_pdom(); // function to call pdom analysis + + unsigned get_num_reconvergence_pairs(); + + void get_reconvergence_pairs(gpgpu_recon_t *recon_points); + + unsigned get_function_size() { return m_instructions.size(); } + + void ptx_assemble(); + + unsigned ptx_get_inst_op(ptx_thread_info *thread); + void add_param(const char *name, struct param_t value) { + m_kernel_params[name] = value; + } + void add_param_name_type_size(unsigned index, std::string name, int type, + size_t size, bool ptr, memory_space_t space); + void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); + void add_return_var(const symbol *rv) { m_return_var_sym = rv; } + void add_arg(const symbol *arg) { + assert(arg != NULL); + m_args.push_back(arg); + } + void remove_args() { m_args.clear(); } + unsigned num_args() const { return m_args.size(); } + unsigned get_args_aligned_size(); + + const symbol *get_arg(unsigned n) const { + assert(n < m_args.size()); + return m_args[n]; + } + bool has_return() const { return m_return_var_sym != NULL; } + const symbol *get_return_var() const { return m_return_var_sym; } + const ptx_instruction *get_instruction(unsigned PC) const { + unsigned index = PC - m_start_PC; + if (index < m_instr_mem_size) return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const { return m_start_PC; } + + void finalize(memory_space *param_mem); + void param_to_shared(memory_space *shared_mem, symbol_table *symtab); + void list_param(FILE *fout) const; + void ptx_jit_config(std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, + dim3 blockDim); + + const struct gpgpu_ptx_sim_info *get_kernel_info() const { + assert(m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } + + const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() { return m_symtab; } + + unsigned local_mem_framesize() const { return m_local_mem_framesize; } + void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } // return pdom flag + void set_pdom() { pdom_done = true; } // set pdom flag + + void add_config_param(size_t size, unsigned alignment) { + unsigned offset = 0; + if (m_param_configs.size() > 0) { + unsigned offset_nom = + m_param_configs.back().first + m_param_configs.back().second; + // ensure offset matches alignment requirements + offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment + : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { + return m_param_configs[param_num]; + } + + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } + unsigned get_maxnt_id() { return maxnt_id; } + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; // flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map m_kernel_params; + std::map m_ptx_kernel_param_info; + std::vector > m_param_configs; + const symbol *m_return_var_sym; + std::vector m_args; + std::list m_instructions; + std::vector m_basic_blocks; + std::list > m_back_edges; + std::map labels; + unsigned num_reconvergence_pairs; + + // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along + // with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; + + symbol_table *m_symtab; + + // parameter size for device kernels + int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { -public: - arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) - { - m_is_reg=false; - m_is_param=false; - m_param_value=NULL; - m_reg_value=ptx_reg_t(); - } - arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) - { - make_copy(another); - } - void make_copy( const arg_buffer_t &another ) - { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if( m_is_param ) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value,another.m_param_value,m_param_bytes); - } - } - void operator=( const arg_buffer_t &another ) - { - make_copy(another); - } - ~arg_buffer_t() - { - if( m_is_param ) - free(m_param_value); - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) - { - m_dst = dst_sym; - m_reg_value=ptx_reg_t(); - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_reg() ); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t),1); - //new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); + public: + arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { + m_is_reg = false; + m_is_param = false; + m_param_value = NULL; + m_reg_value = ptx_reg_t(); + } + arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) + : m_src_op(ctx) { + make_copy(another); + } + void make_copy(const arg_buffer_t &another) { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if (m_is_param) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value, another.m_param_value, m_param_bytes); + } + } + void operator=(const arg_buffer_t &another) { make_copy(another); } + ~arg_buffer_t() { + if (m_is_param) free(m_param_value); + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + ptx_reg_t source_value) + : m_src_op(src_op) { + m_dst = dst_sym; + m_reg_value = ptx_reg_t(); + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_reg()); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t), 1); + // new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); + } + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + void *source_param_value_array, unsigned array_size) + : m_src_op(src_op) { + m_dst = dst_sym; + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_param_local()); + assert(dst_sym->get_size_in_bytes() == array_size); + switch (array_size) { + case 1: + m_reg_value.u8 = *(unsigned char *)source_param_value_array; + break; + case 2: + m_reg_value.u16 = *(unsigned short *)source_param_value_array; + break; + case 4: + m_reg_value.u32 = *(unsigned int *)source_param_value_array; + break; + case 8: + m_reg_value.u64 = *(unsigned long long *)source_param_value_array; + break; + default: + printf( + "GPGPU-Sim PTX: ERROR ** source param size does not match known " + "register sizes\n"); + break; } - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) - { - m_dst = dst_sym; - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_param_local() ); - assert( dst_sym->get_size_in_bytes() == array_size ); - switch( array_size ) { - case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; - case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; - case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; - case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; - default: - printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); - break; - } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size,1); - m_param_bytes = array_size; - memcpy(m_param_value,source_param_value_array,array_size); - } - } - - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const - { - assert(m_is_reg); - return m_reg_value; - } - - const void *get_param_buffer() const - { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const - { - assert(m_is_param); - return m_param_bytes; - } - - const symbol *get_dst() const { return m_dst; } - -private: - // destination of copy - const symbol *m_dst; - - // source operand - operand_info m_src_op; - - // source information - bool m_is_reg; - bool m_is_param; - - // source is register - ptx_reg_t m_reg_value; - - // source is param - void *m_param_value; - unsigned m_param_bytes; + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size, 1); + m_param_bytes = array_size; + memcpy(m_param_value, source_param_value_array, array_size); + } + } + + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const { + assert(m_is_reg); + return m_reg_value; + } + + const void *get_param_buffer() const { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const { + assert(m_is_param); + return m_param_bytes; + } + + const symbol *get_dst() const { return m_dst; } + + private: + // destination of copy + const symbol *m_dst; + + // source operand + operand_info m_src_op; + + // source information + bool m_is_reg; + bool m_is_param; + + // source is register + ptx_reg_t m_reg_value; + + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list< arg_buffer_t > arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ); -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); - +typedef std::list arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param); +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values); +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); struct textureInfo { - unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits,Ty_numbits; //log2(T) - unsigned int texel_size_numbits; //log2(texel_size) + unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx, + Ty; // tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits, Ty_numbits; // log2(T) + unsigned int texel_size_numbits; // log2(texel_size) }; -extern std::map g_sym_name_to_symbol_table; - +extern std::map g_sym_name_to_symbol_table; -void gpgpu_ptx_assemble( std::string kname, void *kinfo ); +void gpgpu_ptx_assemble(std::string kname, void *kinfo); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size( void *kernel_impl ); -unsigned ptx_kernel_nregs( void *kernel_impl ); +unsigned ptx_kernel_shmem_size(void *kernel_impl); +unsigned ptx_kernel_nregs(void *kernel_impl); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 33bcf45..372bda4 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,522 +7,576 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include "ptx_ir.h" -#include "cuda-sim.h" -#include "ptx_parser.h" -#include #include +#include #include #include #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" +#include "ptx_ir.h" +#include "ptx_parser.h" /// extern prototypes -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char*, yyscan_t scanner); +extern int ptx__scan_string(const char *, yyscan_t scanner); -extern std::map get_duplicate(); +extern std::map get_duplicate(); -typedef void * yyscan_t; -extern int ptxinfo_lex_init(yyscan_t* scanner); -extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); +typedef void *yyscan_t; +extern int ptxinfo_lex_init(yyscan_t *scanner); +extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} - -void gpgpu_context::ptx_reg_options(option_parser_t opp) -{ - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, - "saves ptx files embedded in binary as .ptx", - "0"); - option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", - "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +bool ptxinfo_data::keep_intermediate_files() { + return g_keep_intermediate_files; } -void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) -{ - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); - char *s = strdup(p); - char *t = s; - unsigned n=1; - while ( *t != '\0' ) { - char *u = t; - while ( (*u != '\n') && (*u != '\0') ) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); - char pc[64]; - if( pI && pI->get_PC() ) - snprintf(pc,64,"%4u", pI->get_PC() ); - else - snprintf(pc,64," "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); - if ( last ) break; - t = u+1; - n++; - } - free(s); - fflush(stdout); +void gpgpu_context::ptx_reg_options(option_parser_t opp) { + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, + &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", "0"); + option_parser_register(opp, "-keep", OPT_BOOL, + &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when " + "interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, + &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register " + "usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) -{ - - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - - char fname_ptxplus[1024]; - snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); - int fd4=mkstemp(fname_ptxplus); - close(fd4); - - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), - sassfilename.c_str(), - elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); - result = system(commandline); - if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} - - - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while(getline(fileStream,line)) { - text += (line + "\n"); - } - fileStream.close(); - - char* ptxplus_str = new char [strlen(text.c_str())+1]; - strcpy(ptxplus_str, text.c_str()); - - if (!m_ptx_save_converted_ptxplus){ - char rm_commandline[1024]; - - snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); - - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); - int rm_result = system(rm_commandline); - if( rm_result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); - exit(1); - } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - - return ptxplus_str; +void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, + const char *filename) { + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); + char *s = strdup(p); + char *t = s; + unsigned n = 1; + while (*t != '\0') { + char *u = t; + while ((*u != '\n') && (*u != '\0')) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); + char pc[64]; + if (pI && pI->get_PC()) + snprintf(pc, 64, "%4u", pI->get_PC()); + else + snprintf(pc, 64, " "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); + if (last) break; + t = u + 1; + n++; + } + free(s); + fflush(stdout); } - -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) -{ - char buf[1024]; - snprintf(buf,1024,"_%u.ptx", source_num ); - if( g_save_embedded_ptx ) { - FILE *fp = fopen(buf,"w"); - fprintf(fp,"%s",p); - fclose(fp); +char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptxfilename, const std::string elffilename, + const std::string sassfilename) { + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); + + char fname_ptxplus[1024]; + snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); + int fd4 = mkstemp(fname_ptxplus); + close(fd4); + + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, + "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" + "cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", + commandline); + result = system(commandline); + if (result) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", + commandline); + exit(1); + } + + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while (getline(fileStream, line)) { + text += (line + "\n"); + } + fileStream.close(); + + char *ptxplus_str = new char[strlen(text.c_str()) + 1]; + strcpy(ptxplus_str, text.c_str()); + + if (!m_ptx_save_converted_ptxplus) { + char rm_commandline[1024]; + + snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", + rm_commandline); + int rm_result = system(rm_commandline); + if (rm_result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", + rm_result); + exit(1); } - symbol_table *symtab=init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse (ptx_parser->scanner, ptx_parser); - if ( errors ) { - char fname[1024]; - snprintf(fname,1024,"_ptx_errors_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p ); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - if ( g_debug_execution >= 100 ) - print_ptx_file(p,source_num,buf); + return ptxplus_str; +} - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( + const char *p, unsigned source_num) { + char buf[1024]; + snprintf(buf, 1024, "_%u.ptx", source_num); + if (g_save_embedded_ptx) { + FILE *fp = fopen(buf, "w"); + fprintf(fp, "%s", p); + fclose(fp); + } + symbol_table *symtab = init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse(ptx_parser->scanner, ptx_parser); + if (errors) { + char fname[1024]; + snprintf(fname, 1024, "_ptx_errors_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + printf( + "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " + ".ptx to \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); + + if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); + + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) -{ - symbol_table *symtab=init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( + const char *filename) { + symbol_table *symtab = init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; - - // change the name of the ptx file to _temp_ptx - snprintf(commandline,1024,"mv %s %s",fname2,tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); - exit(1); - } - - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile,"r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); - - FILE *ptxdest = fopen(fname2,"w"); - std::map duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; - - // recreate the ptx file without duplications - for ( std::map::iterator iter = duplicate.begin(); - iter != duplicate.end(); - iter++){ - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } - - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } - - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; - - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - - //find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') bracket++; - else if (*(lineptr + offset) == '}') bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - - //find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); - } - - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); - - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline,1024,"rm -f %s",tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } -} + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; + + // change the name of the ptx file to _temp_ptx + snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", + fname2, tempfile); + exit(1); + } + + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile, "r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); + + FILE *ptxdest = fopen(fname2, "w"); + std::map duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; + + // recreate the ptx file without duplications + for (std::map::iterator iter = duplicate.begin(); + iter != duplicate.end(); iter++) { + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } + + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; + + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + + // find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') + bracket++; + else if (*(lineptr + offset) == '}') + bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + // find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", + iter->second); + } + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline, 1024, "rm -f %s", tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } +} -//we need the application name here too. -char* get_app_binary_name(){ - char exe_path[1025]; - char *self_exe_path; +// we need the application name here too. +char *get_app_binary_name() { + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - //AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + // AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; - - char *token = strtok(exe_path, "/"); - while(token !=NULL){ - self_exe_path = token; - token = strtok(NULL,"/"); - } + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while (token != NULL) { + self_exe_path = token; + token = strtok(NULL, "/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) -{ - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0]=0; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); +void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, + unsigned sm_version) { + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0] = 0; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); + else + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); + snprintf( + buff, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) -{ - //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name=get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index=1; index <= no_of_ptx; index++){ - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); - if (system(commandline) !=0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; +void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, + unsigned source_num, + unsigned sm_version, + int no_of_ptx) { + // do ptxas for individual files instead of one big embedded ptx. This + // prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name = get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], + final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index = 1; index <= no_of_ptx; index++) { + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); + if (system(commandline) != 0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; #if CUDART_VERSION >= 3000 - if ( g_occupancy_sm_number == 0 ) { - fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences." ); - exit(1); + if (g_occupancy_sm_number == 0) { + fprintf( + stderr, + "gpgpusim.config must specify the sm version for the GPU that you " + "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used " + "to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences."); + exit(1); } - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + g_occupancy_sm_number); #endif - snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + snprintf(commandline, 1024, + "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if( result != 0 ) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo,"r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - fix_duplicate_errors(fname2); - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - } + if (result != 0) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo, "r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + fix_duplicate_errors(fname2); + snprintf(commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", + commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } - - //TODO: duplicate code! move it into a function so that it can be reused! - if(no_of_ptx==0) { - //For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p_for_info); - fclose(ptxfile); - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - //char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; - - #if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - #endif - - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + } + + // TODO: duplicate code! move it into a function so that it can be reused! + if (no_of_ptx == 0) { + // For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); } + // char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; - //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. - if(no_of_ptx>0){ - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); - snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3)!=0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n",commandline3); - exit(1); - } - } - - if(no_of_ptx>0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; +#if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - snprintf(commandline,1024,"rm -f *info"); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + sm_version); +#endif + + snprintf( + commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); } - if( ! g_save_embedded_ptx ) { - if(no_of_ptx>0) - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); - else - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); - } + } + + // Now that we got resource usage per kernel in a ptx file, we dump all into + // one file and pass it to rest of the code as usual. + if (no_of_ptx > 0) { + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); + snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3) != 0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n", commandline3); + exit(1); + } + } + + if (no_of_ptx > 0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; + else + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + snprintf(commandline, 1024, "rm -f *info"); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if (!g_save_embedded_ptx) { + if (no_of_ptx > 0) + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + final_tempfile_ptxinfo); + else + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); } + } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index d8f1cbc..ff5d7ed 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef PTX_LOADER_H_INCLUDED #define PTX_LOADER_H_INCLUDED @@ -31,22 +32,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void * yyscan_t; -class ptxinfo_data{ - public: - ptxinfo_data(gpgpu_context* ctx) { - gpgpu_ctx = ctx; - } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char *g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); +typedef void* yyscan_t; +class ptxinfo_data { + public: + ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char* g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptx_str, const std::string sass_str, + const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a4f4a0c..3ae8de3 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,1017 +7,991 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "ptx_ir.h" #include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" +typedef void *yyscan_t; #include +#include "ptx.tab.h" -extern int ptx_get_lineno (yyscan_t yyscanner ); -extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern FILE *ptx_get_in (yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_get_lineno(yyscan_t yyscanner); +extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern FILE *ptx_get_in(yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) -{ - g_shader_core_config=warp_size; -} - - -#define PTX_PARSE_DPRINTF(...) \ - if( g_debug_ir_generation ) { \ - printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } - -static std::map g_ptx_token_decode; - -const char *decode_token( int type ) -{ - return g_ptx_token_decode[type].c_str(); -} - -void ptx_recognizer::read_parser_environment_variables() -{ - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - int debug_execution=0; - sscanf(dbg_level,"%d", &debug_execution); - if ( debug_execution >= 30 ) - g_debug_ir_generation=true; - } -} - -void ptx_recognizer::init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void ptx_recognizer::init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); -} - -symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) -{ - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; - } - /*else { - g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - }*/ - -#define DEF(X,Y) g_ptx_token_decode[X] = Y; +void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { + g_shader_core_config = warp_size; +} + +#define PTX_PARSE_DPRINTF(...) \ + if (g_debug_ir_generation) { \ + printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } + +static std::map g_ptx_token_decode; + +const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } + +void ptx_recognizer::read_parser_environment_variables() { + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + int debug_execution = 0; + sscanf(dbg_level, "%d", &debug_execution); + if (debug_execution >= 30) g_debug_ir_generation = true; + } +} + +void ptx_recognizer::init_directive_state() { + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_opcode = -1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void ptx_recognizer::init_instruction_state() { + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); +} + +symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = + new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = + g_global_allfiles_symbol_table; + } + /*else { + g_global_symbol_table = g_current_symbol_table = new + symbol_table("global",0,g_global_allfiles_symbol_table); + }*/ + +#define DEF(X, Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; - - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); - - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; -} - - -void ptx_recognizer::start_function( int entry_point ) -{ - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index=0; -} - -void ptx_recognizer::add_function_name( const char *name ) -{ - PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); - bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); - if( g_add_identifier_cached__identifier ) { - add_identifier( g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident ); - free( g_add_identifier_cached__identifier ); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var( g_last_symbol ); - init_directive_state(); - } - if( prior_decl ) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); -} - -//Jin: handle instruction group for cdp + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; + + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); + + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; +} + +void ptx_recognizer::start_function(int entry_point) { + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index = 0; +} + +void ptx_recognizer::add_function_name(const char *name) { + PTX_PARSE_DPRINTF( + "add_function_name %s %s", name, + ((g_entry_point == 1) ? "(entrypoint)" + : ((g_entry_point == 2) ? "(extern)" : ""))); + bool prior_decl = g_global_symbol_table->add_function_decl( + name, g_entry_point, &g_func_info, &g_current_symbol_table); + if (g_add_identifier_cached__identifier) { + add_identifier(g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident); + free(g_add_identifier_cached__identifier); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var(g_last_symbol); + init_directive_state(); + } + if (prior_decl) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, + ptx_get_lineno(scanner)); +} + +// Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); -} - -void ptx_recognizer::add_directive() -{ - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); -} - -#define mymax(a,b) ((a)>(b)?(a):(b)) - -void ptx_recognizer::end_function() -{ - PTX_PARSE_DPRINTF("end_function"); - - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax( g_max_regs_per_thread, (g_current_symbol_table->next_reg_num()-1)); - g_func_info->add_inst( g_instructions ); - g_instructions.clear(); - gpgpu_ptx_assemble( g_func_info->get_name(), g_func_info ); - g_current_symbol_table = g_global_symbol_table; - - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), g_func_info->get_start_PC()); -} - -#define parse_error(msg, ...) parse_error_impl(__FILE__,__LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond,msg, ...) parse_assert_impl((cond),__FILE__,__LINE__, msg, ##__VA_ARGS__) - -void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); -} - -void ptx_recognizer::parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - if ( test_value == 0 ) - parse_error_impl(file,line, msg); -} - - - -void ptx_recognizer::set_return() -{ - parse_assert( (g_opcode == CALL_OP || g_opcode == CALLP_OP), "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); -} - - -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( const char *filename, unsigned linenumber ) -{ - std::map >::iterator f=g_inst_lookup.find(filename); - if( f == g_inst_lookup.end() ) - return NULL; - std::map::iterator l=f->second.find(linenumber); - if( l == f->second.end() ) - return NULL; - return l->second; -} - -void ptx_recognizer::add_instruction() -{ - PTX_PARSE_DPRINTF("add_instruction: %s", ((g_opcode>0)?g_opcode_string[g_opcode]:"