diff options
Diffstat (limited to 'src')
81 files changed, 39048 insertions, 36421 deletions
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 07232ee..4bfe3c9 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -7,1203 +7,1210 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "abstract_hardware_model.h" +#include <sys/stat.h> +#include <algorithm> +#include <iostream> +#include <sstream> +#include "../libcuda/gpgpu_context.h" +#include "cuda-sim/cuda-sim.h" #include "cuda-sim/memory.h" -#include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx-stats.h" -#include "cuda-sim/cuda-sim.h" +#include "cuda-sim/ptx_ir.h" #include "gpgpu-sim/gpu-sim.h" -#include "option_parser.h" #include "gpgpusim_entrypoint.h" -#include <algorithm> -#include <sys/stat.h> -#include <sstream> -#include <iostream> -#include "../libcuda/gpgpu_context.h" +#include "option_parser.h" -void mem_access_t::init(gpgpu_context* ctx) -{ - gpgpu_ctx = ctx; - m_uid=++(gpgpu_ctx->sm_next_access_uid); - m_addr=0; - m_req_size=0; +void mem_access_t::init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = ++(gpgpu_ctx->sm_next_access_uid); + m_addr = 0; + m_req_size = 0; } -void warp_inst_t::issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ) -{ - m_warp_active_mask = mask; - m_warp_issued_mask = mask; - m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); - m_warp_id = warp_id; - m_dynamic_warp_id = dynamic_warp_id; - issue_cycle = cycle; - cycles = initiation_interval; - m_cache_hit=false; - m_empty=false; - m_scheduler_id=sch_id; +void warp_inst_t::issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, + int sch_id) { + m_warp_active_mask = mask; + m_warp_issued_mask = mask; + m_uid = ++(m_config->gpgpu_ctx->warp_inst_sm_next_uid); + m_warp_id = warp_id; + m_dynamic_warp_id = dynamic_warp_id; + issue_cycle = cycle; + cycles = initiation_interval; + m_cache_hit = false; + m_empty = false; + m_scheduler_id = sch_id; } -checkpoint::checkpoint() -{ - - struct stat st = {0}; - - if (stat("checkpoint_files", &st) == -1) { - mkdir("checkpoint_files", 0777); - } +checkpoint::checkpoint() { + struct stat st = {0}; + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); + } } -void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) -{ +void checkpoint::load_global_mem(class memory_space *temp_mem, char *f1name) { + FILE *fp2 = fopen(f1name, "r"); + assert(fp2 != NULL); + char line[128]; /* or other suitable maximum line size */ + unsigned int offset; + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + unsigned int index; + char *pch; + pch = strtok(line, " "); + if (pch[0] == 'g' || pch[0] == 's' || pch[0] == 'l') { + pch = strtok(NULL, " "); - FILE * fp2 = fopen(f1name, "r"); - assert(fp2!=NULL); - char line [ 128 ]; /* or other suitable maximum line size */ - unsigned int offset ; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - unsigned int index; - char * pch; - pch = strtok (line," "); - if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') - { + std::stringstream ss; + ss << std::hex << pch; + ss >> index; - pch = strtok (NULL, " "); - - std::stringstream ss; - ss << std::hex << pch; - ss >> index; - - offset=0; - } - else { - unsigned int data; - std::stringstream ss; - ss << std::hex << pch; - ss >> data; - temp_mem->write_only(offset,index, 4,&data); - offset= offset+4; - } - //fputs ( line, stdout ); /* write the line */ - } - fclose ( fp2 ); + offset = 0; + } else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset, index, 4, &data); + offset = offset + 4; + } + // fputs ( line, stdout ); /* write the line */ + } + fclose(fp2); } -void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) -{ - - FILE * fp3 = fopen(fname, "w"); - assert(fp3!=NULL); - mem->print(format,fp3); - fclose(fp3); +void checkpoint::store_global_mem(class memory_space *mem, char *fname, + char *format) { + FILE *fp3 = fopen(fname, "w"); + assert(fp3 != NULL); + mem->print(format, fp3); + fclose(fp3); } -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) -{ - assert( dst->empty() ); - warp_inst_t* temp = dst; - dst = src; - src = temp; - src->clear(); +void move_warp(warp_inst_t *&dst, warp_inst_t *&src) { + assert(dst->empty()); + warp_inst_t *temp = dst; + dst = src; + src = temp; + src->clear(); } - -void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) -{ - option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, - &m_ptx_use_cuobjdump, - "Use cuobjdump to extract ptx and sass from binaries", +void gpgpu_functional_sim_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-gpgpu_ptx_use_cuobjdump", OPT_BOOL, + &m_ptx_use_cuobjdump, + "Use cuobjdump to extract ptx and sass from binaries", #if (CUDART_VERSION >= 4000) - "1" + "1" #else - "0" + "0" #endif - ); - option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, - &m_experimental_lib_support, - "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", - "0"); - option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, - " checkpointing flag (0 = no checkpoint)", - "0"); - option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, - " checkpointing during execution of which kernel (1- 1st kernel)", - "1"); - option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, - " checkpointing after # of CTA (< less than total CTA)", - "0"); - option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, - " resume flag (0 = no resume)", - "0"); - option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, - " Resume from which kernel (1= 1st kernel)", - "0"); - option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, - " resume from which CTA ", - "0"); - option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, - " resume from which CTA ", - "0"); + ); + option_parser_register(opp, "-gpgpu_experimental_lib_support", OPT_BOOL, + &m_experimental_lib_support, + "Try to extract code from cuda libraries [Broken " + "because of unknown cudaGetExportTable]", + "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, + &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", "0"); + option_parser_register( + opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", "1"); + option_parser_register( + opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, + &checkpoint_insn_Y, " resume from which CTA ", "0"); - option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, - &m_ptx_convert_to_ptxplus, - "Convert SASS (native ISA) to ptxplus and run ptxplus", - "0"); - option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, - &m_ptx_force_max_capability, - "Force maximum compute capability", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, - &g_ptx_inst_debug_to_file, - "Dump executed instructions' debug information to file", - "0"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, - "Executed instructions' debug output file", - "inst_debug.txt"); - option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, &g_ptx_inst_debug_thread_uid, - "Thread UID for executed instructions' debug output", - "1"); + option_parser_register( + opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, + "Convert SASS (native ISA) to ptxplus and run ptxplus", "0"); + option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32, + &m_ptx_force_max_capability, + "Force maximum compute capability", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, &g_ptx_inst_debug_to_file, + "Dump executed instructions' debug information to file", "0"); + option_parser_register( + opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, + "Executed instructions' debug output file", "inst_debug.txt"); + option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, + &g_ptx_inst_debug_thread_uid, + "Thread UID for executed instructions' debug output", + "1"); } -void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize(unsigned linesize) -{ - m_texcache_linesize = linesize; +void gpgpu_functional_sim_config::ptx_set_tex_cache_linesize( + unsigned linesize) { + m_texcache_linesize = linesize; } -gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ) - : m_function_model_config(config) -{ - gpgpu_ctx = ctx; - m_global_mem = new memory_space_impl<8192>("global",64*1024); - - m_tex_mem = new memory_space_impl<8192>("tex",64*1024); - m_surf_mem = new memory_space_impl<8192>("surf",64*1024); +gpgpu_t::gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx) + : m_function_model_config(config) { + gpgpu_ctx = ctx; + m_global_mem = new memory_space_impl<8192>("global", 64 * 1024); + + m_tex_mem = new memory_space_impl<8192>("tex", 64 * 1024); + m_surf_mem = new memory_space_impl<8192>("surf", 64 * 1024); - m_dev_malloc=GLOBAL_HEAP_START; - checkpoint_option = m_function_model_config.get_checkpoint_option(); - checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); - checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); - resume_option = m_function_model_config.get_resume_option(); - resume_kernel = m_function_model_config.get_resume_kernel(); - resume_CTA = m_function_model_config.get_resume_CTA(); - checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); - checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + m_dev_malloc = GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); - // initialize texture mappings to empty - m_NameToTextureInfo.clear(); - m_NameToCudaArray.clear(); - m_TextureRefToName.clear(); - m_NameToAttribute.clear(); + // initialize texture mappings to empty + m_NameToTextureInfo.clear(); + m_NameToCudaArray.clear(); + m_TextureRefToName.clear(); + m_NameToAttribute.clear(); - if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) - ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); + if (m_function_model_config.get_ptx_inst_debug_to_file() != 0) + ptx_inst_debug_file = + fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); - gpu_sim_cycle=0; - gpu_tot_sim_cycle=0; + gpu_sim_cycle = 0; + gpu_tot_sim_cycle = 0; } -address_type line_size_based_tag_func(new_addr_type address, new_addr_type line_size) -{ - //gives the tag for an address based on a given line size - return address & ~(line_size-1); +address_type line_size_based_tag_func(new_addr_type address, + new_addr_type line_size) { + // gives the tag for an address based on a given line size + return address & ~(line_size - 1); } -const char * mem_access_type_str(enum mem_access_type access_type) -{ - #define MA_TUP_BEGIN(X) static const char* access_type_str[] = { - #define MA_TUP(X) #X - #define MA_TUP_END(X) }; - MEM_ACCESS_TYPE_TUP_DEF - #undef MA_TUP_BEGIN - #undef MA_TUP - #undef MA_TUP_END +const char *mem_access_type_str(enum mem_access_type access_type) { +#define MA_TUP_BEGIN(X) static const char *access_type_str[] = { +#define MA_TUP(X) #X +#define MA_TUP_END(X) \ + } \ + ; + MEM_ACCESS_TYPE_TUP_DEF +#undef MA_TUP_BEGIN +#undef MA_TUP +#undef MA_TUP_END - assert(access_type < NUM_MEM_ACCESS_TYPE); + assert(access_type < NUM_MEM_ACCESS_TYPE); - return access_type_str[access_type]; + return access_type_str[access_type]; } - -void warp_inst_t::clear_active( const active_mask_t &inactive ) { - active_mask_t test = m_warp_active_mask; - test &= inactive; - assert( test == inactive ); // verify threads being disabled were active - m_warp_active_mask &= ~inactive; +void warp_inst_t::clear_active(const active_mask_t &inactive) { + active_mask_t test = m_warp_active_mask; + test &= inactive; + assert(test == inactive); // verify threads being disabled were active + m_warp_active_mask &= ~inactive; } -void warp_inst_t::set_not_active( unsigned lane_id ) { - m_warp_active_mask.reset(lane_id); +void warp_inst_t::set_not_active(unsigned lane_id) { + m_warp_active_mask.reset(lane_id); } -void warp_inst_t::set_active( const active_mask_t &active ) { - m_warp_active_mask = active; - if( m_isatomic ) { - for( unsigned i=0; i < m_config->warp_size; i++ ) { - if( !m_warp_active_mask.test(i) ) { - m_per_scalar_thread[i].callback.function = NULL; - m_per_scalar_thread[i].callback.instruction = NULL; - m_per_scalar_thread[i].callback.thread = NULL; - } +void warp_inst_t::set_active(const active_mask_t &active) { + m_warp_active_mask = active; + if (m_isatomic) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (!m_warp_active_mask.test(i)) { + m_per_scalar_thread[i].callback.function = NULL; + m_per_scalar_thread[i].callback.instruction = NULL; + m_per_scalar_thread[i].callback.thread = NULL; } - } + } + } } void warp_inst_t::do_atomic(bool forceDo) { - do_atomic( m_warp_active_mask,forceDo ); + do_atomic(m_warp_active_mask, forceDo); } - -void warp_inst_t::do_atomic( const active_mask_t& access_mask,bool forceDo ) { - assert( m_isatomic && (!m_empty||forceDo) ); - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ) - cb.function(cb.instruction, cb.thread); - } +void warp_inst_t::do_atomic(const active_mask_t &access_mask, bool forceDo) { + assert(m_isatomic && (!m_empty || forceDo)); + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) cb.function(cb.instruction, cb.thread); } + } } -void warp_inst_t::broadcast_barrier_reduction(const active_mask_t& access_mask) -{ - for( unsigned i=0; i < m_config->warp_size; i++ ) - { - if( access_mask.test(i) ) - { - dram_callback_t &cb = m_per_scalar_thread[i].callback; - if( cb.thread ){ - cb.function(cb.instruction, cb.thread); - } - } +void warp_inst_t::broadcast_barrier_reduction( + const active_mask_t &access_mask) { + for (unsigned i = 0; i < m_config->warp_size; i++) { + if (access_mask.test(i)) { + dram_callback_t &cb = m_per_scalar_thread[i].callback; + if (cb.thread) { + cb.function(cb.instruction, cb.thread); + } } + } } -void warp_inst_t::generate_mem_accesses() -{ - if( empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created ) - return; - if (!((op == LOAD_OP) || (op==TENSOR_CORE_LOAD_OP) || (op == STORE_OP)||(op==TENSOR_CORE_STORE_OP))) - return; - if( m_warp_active_mask.count() == 0 ) - return; // predicated off +void warp_inst_t::generate_mem_accesses() { + if (empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created) return; + if (!((op == LOAD_OP) || (op == TENSOR_CORE_LOAD_OP) || (op == STORE_OP) || + (op == TENSOR_CORE_STORE_OP))) + return; + if (m_warp_active_mask.count() == 0) return; // predicated off - const size_t starting_queue_size = m_accessq.size(); + const size_t starting_queue_size = m_accessq.size(); - assert( is_load() || is_store() ); - assert( m_per_scalar_thread_valid ); // need address information per thread + assert(is_load() || is_store()); + assert(m_per_scalar_thread_valid); // need address information per thread - bool is_write = is_store(); + bool is_write = is_store(); - mem_access_type access_type; - switch (space.get_type()) { + mem_access_type access_type; + switch (space.get_type()) { case const_space: - case param_space_kernel: - access_type = CONST_ACC_R; - break; - case tex_space: - access_type = TEXTURE_ACC_R; - break; - case global_space: - access_type = is_write? GLOBAL_ACC_W: GLOBAL_ACC_R; - break; + case param_space_kernel: + access_type = CONST_ACC_R; + break; + case tex_space: + access_type = TEXTURE_ACC_R; + break; + case global_space: + access_type = is_write ? GLOBAL_ACC_W : GLOBAL_ACC_R; + break; case local_space: - case param_space_local: - access_type = is_write? LOCAL_ACC_W: LOCAL_ACC_R; - break; - case shared_space: break; - case sstarr_space: break; - default: assert(0); break; - } + case param_space_local: + access_type = is_write ? LOCAL_ACC_W : LOCAL_ACC_R; + break; + case shared_space: + break; + case sstarr_space: + break; + default: + assert(0); + break; + } - // Calculate memory accesses generated by this warp - new_addr_type cache_block_size = 0; // in bytes + // Calculate memory accesses generated by this warp + new_addr_type cache_block_size = 0; // in bytes - switch( space.get_type() ) { + switch (space.get_type()) { case shared_space: case sstarr_space: { - unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; - unsigned total_accesses=0; - for( unsigned subwarp=0; subwarp < m_config->mem_warp_parts; subwarp++ ) { + unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; + unsigned total_accesses = 0; + for (unsigned subwarp = 0; subwarp < m_config->mem_warp_parts; + subwarp++) { + // data structures used per part warp + std::map<unsigned, std::map<new_addr_type, unsigned> > + bank_accs; // bank -> word address -> access count - // data structures used per part warp - std::map<unsigned,std::map<new_addr_type,unsigned> > bank_accs; // bank -> word address -> access count + // step 1: compute accesses to words in banks + for (unsigned thread = subwarp * subwarp_size; + thread < (subwarp + 1) * subwarp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + // FIXME: deferred allocation of shared memory should not accumulate + // across kernel launches assert( addr < m_config->gpgpu_shmem_size ); + unsigned bank = m_config->shmem_bank_func(addr); + new_addr_type word = + line_size_based_tag_func(addr, m_config->WORD_SIZE); + bank_accs[bank][word]++; + } - // step 1: compute accesses to words in banks - for( unsigned thread=subwarp*subwarp_size; thread < (subwarp+1)*subwarp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - //FIXME: deferred allocation of shared memory should not accumulate across kernel launches - //assert( addr < m_config->gpgpu_shmem_size ); - unsigned bank = m_config->shmem_bank_func(addr); - new_addr_type word = line_size_based_tag_func(addr,m_config->WORD_SIZE); - bank_accs[bank][word]++; + if (m_config->shmem_limited_broadcast) { + // step 2: look for and select a broadcast bank/word if one occurs + bool broadcast_detected = false; + new_addr_type broadcast_word = (new_addr_type)-1; + unsigned broadcast_bank = (unsigned)-1; + std::map<unsigned, std::map<new_addr_type, unsigned> >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank = b->first; + std::map<new_addr_type, unsigned> &access_set = b->second; + std::map<new_addr_type, unsigned>::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->second > 1) { + // found a broadcast + broadcast_detected = true; + broadcast_bank = bank; + broadcast_word = w->first; + break; + } } + if (broadcast_detected) break; + } - if (m_config->shmem_limited_broadcast) { - // step 2: look for and select a broadcast bank/word if one occurs - bool broadcast_detected = false; - new_addr_type broadcast_word=(new_addr_type)-1; - unsigned broadcast_bank=(unsigned)-1; - std::map<unsigned,std::map<new_addr_type,unsigned> >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank = b->first; - std::map<new_addr_type,unsigned> &access_set = b->second; - std::map<new_addr_type,unsigned>::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->second > 1 ) { - // found a broadcast - broadcast_detected=true; - broadcast_bank=bank; - broadcast_word=w->first; - break; - } - } - if( broadcast_detected ) - break; - } - - // step 3: figure out max bank accesses performed, taking account of broadcast case - unsigned max_bank_accesses=0; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - unsigned bank_accesses=0; - std::map<new_addr_type,unsigned> &access_set = b->second; - std::map<new_addr_type,unsigned>::iterator w; - for( w=access_set.begin(); w != access_set.end(); ++w ) - bank_accesses += w->second; - if( broadcast_detected && broadcast_bank == b->first ) { - for( w=access_set.begin(); w != access_set.end(); ++w ) { - if( w->first == broadcast_word ) { - unsigned n = w->second; - assert(n > 1); // or this wasn't a broadcast - assert(bank_accesses >= (n-1)); - bank_accesses -= (n-1); - break; - } - } - } - if( bank_accesses > max_bank_accesses ) - max_bank_accesses = bank_accesses; + // step 3: figure out max bank accesses performed, taking account of + // broadcast case + unsigned max_bank_accesses = 0; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + unsigned bank_accesses = 0; + std::map<new_addr_type, unsigned> &access_set = b->second; + std::map<new_addr_type, unsigned>::iterator w; + for (w = access_set.begin(); w != access_set.end(); ++w) + bank_accesses += w->second; + if (broadcast_detected && broadcast_bank == b->first) { + for (w = access_set.begin(); w != access_set.end(); ++w) { + if (w->first == broadcast_word) { + unsigned n = w->second; + assert(n > 1); // or this wasn't a broadcast + assert(bank_accesses >= (n - 1)); + bank_accesses -= (n - 1); + break; } + } + } + if (bank_accesses > max_bank_accesses) + max_bank_accesses = bank_accesses; + } - // step 4: accumulate - total_accesses+= max_bank_accesses; - } else { - // step 2: look for the bank with the maximum number of access to different words - unsigned max_bank_accesses=0; - std::map<unsigned,std::map<new_addr_type,unsigned> >::iterator b; - for( b=bank_accs.begin(); b != bank_accs.end(); b++ ) { - max_bank_accesses = std::max(max_bank_accesses, (unsigned)b->second.size()); - } + // step 4: accumulate + total_accesses += max_bank_accesses; + } else { + // step 2: look for the bank with the maximum number of access to + // different words + unsigned max_bank_accesses = 0; + std::map<unsigned, std::map<new_addr_type, unsigned> >::iterator b; + for (b = bank_accs.begin(); b != bank_accs.end(); b++) { + max_bank_accesses = + std::max(max_bank_accesses, (unsigned)b->second.size()); + } - // step 3: accumulate - total_accesses+= max_bank_accesses; - } + // step 3: accumulate + total_accesses += max_bank_accesses; } - assert( total_accesses > 0 && total_accesses <= m_config->warp_size ); - cycles = total_accesses; // shared memory conflicts modeled as larger initiation interval - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( pc, total_accesses ); - break; + } + assert(total_accesses > 0 && total_accesses <= m_config->warp_size); + cycles = total_accesses; // shared memory conflicts modeled as larger + // initiation interval + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_smem_bank_conflict( + pc, total_accesses); + break; } - case tex_space: - cache_block_size = m_config->gpgpu_cache_texl1_linesize; - break; - case const_space: case param_space_kernel: - cache_block_size = m_config->gpgpu_cache_constl1_linesize; - break; + case tex_space: + cache_block_size = m_config->gpgpu_cache_texl1_linesize; + break; + case const_space: + case param_space_kernel: + cache_block_size = m_config->gpgpu_cache_constl1_linesize; + break; - case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch >= 13) { - if(isatomic()) - memory_coalescing_arch_atomic(is_write, access_type); - else - memory_coalescing_arch(is_write, access_type); - } else abort(); + case global_space: + case local_space: + case param_space_local: + if (m_config->gpgpu_coalesce_arch >= 13) { + if (isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else + abort(); - break; + break; default: - abort(); - } + abort(); + } - if( cache_block_size ) { - assert( m_accessq.empty() ); - mem_access_byte_mask_t byte_mask; - std::map<new_addr_type,active_mask_t> accesses; // block address -> set of thread offsets in warp - std::map<new_addr_type,active_mask_t>::iterator a; - for( unsigned thread=0; thread < m_config->warp_size; thread++ ) { - if( !active(thread) ) - continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr,cache_block_size); - accesses[block_address].set(thread); - unsigned idx = addr-block_address; - for( unsigned i=0; i < data_size; i++ ) - byte_mask.set(idx+i); - } - for( a=accesses.begin(); a != accesses.end(); ++a ) - m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); + if (cache_block_size) { + assert(m_accessq.empty()); + mem_access_byte_mask_t byte_mask; + std::map<new_addr_type, active_mask_t> + accesses; // block address -> set of thread offsets in warp + std::map<new_addr_type, active_mask_t>::iterator a; + for (unsigned thread = 0; thread < m_config->warp_size; thread++) { + if (!active(thread)) continue; + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, cache_block_size); + accesses[block_address].set(thread); + unsigned idx = addr - block_address; + for (unsigned i = 0; i < data_size; i++) byte_mask.set(idx + i); } + for (a = accesses.begin(); a != accesses.end(); ++a) + m_accessq.push_back(mem_access_t( + access_type, a->first, cache_block_size, is_write, a->second, + byte_mask, mem_access_sector_mask_t(), m_config->gpgpu_ctx)); + } - if ( space.get_type() == global_space ) { - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( pc, m_accessq.size() - starting_queue_size ); - } - m_mem_accesses_created=true; + if (space.get_type() == global_space) { + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_uncoalesced_gmem( + pc, m_accessq.size() - starting_queue_size); + } + m_mem_accesses_created = true; } -void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_type ) -{ - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; +void warp_inst_t::memory_coalescing_arch(bool is_write, + mem_access_type access_type) { + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } - - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map<new_addr_type,transaction_info> subwarp_transactions; + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) { - if( !active(thread) ) - continue; + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map<new_addr_type, transaction_info> subwarp_transactions; - unsigned data_size_coales = data_size; - unsigned num_accesses = 1; + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; - if( space.get_type() == local_space || space.get_type() == param_space_local ) { - // Local memory accesses >4B were split into 4B chunks - if(data_size >= 4) { - data_size_coales = 4; - num_accesses = data_size/4; - } - // Otherwise keep the same data_size for sub-4B access to local memory - } + unsigned data_size_coales = data_size; + unsigned num_accesses = 1; + if (space.get_type() == local_space || + space.get_type() == param_space_local) { + // Local memory accesses >4B were split into 4B chunks + if (data_size >= 4) { + data_size_coales = 4; + num_accesses = data_size / 4; + } + // Otherwise keep the same data_size for sub-4B access to local memory + } - assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); + assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); -// for(unsigned access=0; access<num_accesses; access++) { - for(unsigned access=0; (access<MAX_ACCESSES_PER_INSN_PER_THREAD)&&(m_per_scalar_thread[thread].memreqaddr[access]!=0); access++) { - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access]; - unsigned block_address = line_size_based_tag_func(addr,segment_size); - unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? - transaction_info &info = subwarp_transactions[block_address]; + // for(unsigned access=0; access<num_accesses; access++) { + for (unsigned access = 0; + (access < MAX_ACCESSES_PER_INSN_PER_THREAD) && + (m_per_scalar_thread[thread].memreqaddr[access] != 0); + access++) { + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access]; + unsigned block_address = line_size_based_tag_func(addr, segment_size); + unsigned chunk = + (addr & 127) / 32; // which 32-byte chunk within in a 128-byte + // chunk does this thread access? + transaction_info &info = subwarp_transactions[block_address]; - // can only write to one segment - assert(block_address == line_size_based_tag_func(addr+data_size_coales-1,segment_size)); + // can only write to one segment + assert(block_address == line_size_based_tag_func( + addr + data_size_coales - 1, segment_size)); - info.chunks.set(chunk); - info.active.set(thread); - unsigned idx = (addr&127); - for( unsigned i=0; i < data_size_coales; i++ ) - info.bytes.set(idx+i); - } - } - - // step 2: reduce each transaction size, if possible - std::map< new_addr_type, transaction_info >::iterator t; - for( t=subwarp_transactions.begin(); t !=subwarp_transactions.end(); t++ ) { - new_addr_type addr = t->first; - const transaction_info &info = t->second; + info.chunks.set(chunk); + info.active.set(thread); + unsigned idx = (addr & 127); + for (unsigned i = 0; i < data_size_coales; i++) info.bytes.set(idx + i); + } + } - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); + // step 2: reduce each transaction size, if possible + std::map<new_addr_type, transaction_info>::iterator t; + for (t = subwarp_transactions.begin(); t != subwarp_transactions.end(); + t++) { + new_addr_type addr = t->first; + const transaction_info &info = t->second; - } + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, + segment_size); } + } } -void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) -{ +void warp_inst_t::memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type) { + assert(space.get_type() == + global_space); // Atomics allowed only for global memory - assert(space.get_type() == global_space); // Atomics allowed only for global memory + // see the CUDA manual where it discusses coalescing rules before reading this + unsigned segment_size = 0; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; - // see the CUDA manual where it discusses coalescing rules before reading this - unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; - bool sector_segment_size = false; - - if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) - { - //Fermi and Kepler, L1 is normal and L2 is sector - if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) - sector_segment_size = true; - else - sector_segment_size = false; - } - else if(m_config->gpgpu_coalesce_arch >= 40) - { - //Maxwell, Pascal and Volta, L1 and L2 are sectors - //all requests should be 32 bytes - sector_segment_size = true; - } + if (m_config->gpgpu_coalesce_arch >= 20 && + m_config->gpgpu_coalesce_arch < 39) { + // Fermi and Kepler, L1 is normal and L2 is sector + if (m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } else if (m_config->gpgpu_coalesce_arch >= 40) { + // Maxwell, Pascal and Volta, L1 and L2 are sectors + // all requests should be 32 bytes + sector_segment_size = true; + } - switch( data_size ) { - case 1: segment_size = 32; break; - case 2: segment_size = sector_segment_size? 32 : 64; break; - case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; - } - unsigned subwarp_size = m_config->warp_size / warp_parts; + switch (data_size) { + case 1: + segment_size = 32; + break; + case 2: + segment_size = sector_segment_size ? 32 : 64; + break; + case 4: + case 8: + case 16: + segment_size = sector_segment_size ? 32 : 128; + break; + } + unsigned subwarp_size = m_config->warp_size / warp_parts; - for( unsigned subwarp=0; subwarp < warp_parts; subwarp++ ) { - std::map<new_addr_type,std::list<transaction_info> > subwarp_transactions; // each block addr maps to a list of transactions + for (unsigned subwarp = 0; subwarp < warp_parts; subwarp++) { + std::map<new_addr_type, std::list<transaction_info> > + subwarp_transactions; // each block addr maps to a list of transactions - // step 1: find all transactions generated by this subwarp - for( unsigned thread=subwarp*subwarp_size; thread<subwarp_size*(subwarp+1); thread++ ) { - if( !active(thread) ) - continue; + // step 1: find all transactions generated by this subwarp + for (unsigned thread = subwarp * subwarp_size; + thread < subwarp_size * (subwarp + 1); thread++) { + if (!active(thread)) continue; - new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; - unsigned block_address = line_size_based_tag_func(addr,segment_size); - unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? + new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0]; + unsigned block_address = line_size_based_tag_func(addr, segment_size); + unsigned chunk = + (addr & 127) / 32; // which 32-byte chunk within in a 128-byte chunk + // does this thread access? - // can only write to one segment - assert(block_address == line_size_based_tag_func(addr+data_size-1,segment_size)); + // can only write to one segment + assert(block_address == + line_size_based_tag_func(addr + data_size - 1, segment_size)); - // Find a transaction that does not conflict with this thread's accesses - bool new_transaction = true; - std::list<transaction_info>::iterator it; - transaction_info* info; - for(it=subwarp_transactions[block_address].begin(); it!=subwarp_transactions[block_address].end(); it++) { - unsigned idx = (addr&127); - if(not it->test_bytes(idx,idx+data_size-1)) { - new_transaction = false; - info = &(*it); - break; - } - } - if(new_transaction) { - // Need a new transaction - subwarp_transactions[block_address].push_back(transaction_info()); - info = &subwarp_transactions[block_address].back(); - } - assert(info); + // Find a transaction that does not conflict with this thread's accesses + bool new_transaction = true; + std::list<transaction_info>::iterator it; + transaction_info *info; + for (it = subwarp_transactions[block_address].begin(); + it != subwarp_transactions[block_address].end(); it++) { + unsigned idx = (addr & 127); + if (not it->test_bytes(idx, idx + data_size - 1)) { + new_transaction = false; + info = &(*it); + break; + } + } + if (new_transaction) { + // Need a new transaction + subwarp_transactions[block_address].push_back(transaction_info()); + info = &subwarp_transactions[block_address].back(); + } + assert(info); - info->chunks.set(chunk); - info->active.set(thread); - unsigned idx = (addr&127); - for( unsigned i=0; i < data_size; i++ ) { - assert(!info->bytes.test(idx+i)); - info->bytes.set(idx+i); - } - } + info->chunks.set(chunk); + info->active.set(thread); + unsigned idx = (addr & 127); + for (unsigned i = 0; i < data_size; i++) { + assert(!info->bytes.test(idx + i)); + info->bytes.set(idx + i); + } + } - // step 2: reduce each transaction size, if possible - std::map< new_addr_type, std::list<transaction_info> >::iterator t_list; - for( t_list=subwarp_transactions.begin(); t_list !=subwarp_transactions.end(); t_list++ ) { - // For each block addr - new_addr_type addr = t_list->first; - const std::list<transaction_info>& transaction_list = t_list->second; + // step 2: reduce each transaction size, if possible + std::map<new_addr_type, std::list<transaction_info> >::iterator t_list; + for (t_list = subwarp_transactions.begin(); + t_list != subwarp_transactions.end(); t_list++) { + // For each block addr + new_addr_type addr = t_list->first; + const std::list<transaction_info> &transaction_list = t_list->second; - std::list<transaction_info>::const_iterator t; - for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { - // For each transaction - const transaction_info &info = *t; - memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); - } - } - } + std::list<transaction_info>::const_iterator t; + for (t = transaction_list.begin(); t != transaction_list.end(); t++) { + // For each transaction + const transaction_info &info = *t; + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, + addr, segment_size); + } + } + } } -void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) -{ - assert( (addr & (segment_size-1)) == 0 ); +void warp_inst_t::memory_coalescing_arch_reduce_and_send( + bool is_write, mem_access_type access_type, const transaction_info &info, + new_addr_type addr, unsigned segment_size) { + assert((addr & (segment_size - 1)) == 0); - const std::bitset<4> &q = info.chunks; - assert( q.count() >= 1 ); - std::bitset<2> h; // halves (used to check if 64 byte segment can be compressed into a single 32 byte segment) + const std::bitset<4> &q = info.chunks; + assert(q.count() >= 1); + std::bitset<2> h; // halves (used to check if 64 byte segment can be + // compressed into a single 32 byte segment) - unsigned size=segment_size; - if( segment_size == 128 ) { - bool lower_half_used = q[0] || q[1]; - bool upper_half_used = q[2] || q[3]; - if( lower_half_used && !upper_half_used ) { - // only lower 64 bytes used - size = 64; - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else if ( (!lower_half_used) && upper_half_used ) { - // only upper 64 bytes used - addr = addr+64; - size = 64; - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } else { - assert(lower_half_used && upper_half_used); - } - } else if( segment_size == 64 ) { - // need to set halves - if( (addr % 128) == 0 ) { - if(q[0]) h.set(0); - if(q[1]) h.set(1); - } else { - assert( (addr % 128) == 64 ); - if(q[2]) h.set(0); - if(q[3]) h.set(1); - } - } - if( size == 64 ) { - bool lower_half_used = h[0]; - bool upper_half_used = h[1]; - if( lower_half_used && !upper_half_used ) { - size = 32; - } else if ( (!lower_half_used) && upper_half_used ) { - addr = addr+32; - size = 32; - } else { - assert(lower_half_used && upper_half_used); - } - } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks,m_config->gpgpu_ctx) ); + unsigned size = segment_size; + if (segment_size == 128) { + bool lower_half_used = q[0] || q[1]; + bool upper_half_used = q[2] || q[3]; + if (lower_half_used && !upper_half_used) { + // only lower 64 bytes used + size = 64; + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else if ((!lower_half_used) && upper_half_used) { + // only upper 64 bytes used + addr = addr + 64; + size = 64; + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } else { + assert(lower_half_used && upper_half_used); + } + } else if (segment_size == 64) { + // need to set halves + if ((addr % 128) == 0) { + if (q[0]) h.set(0); + if (q[1]) h.set(1); + } else { + assert((addr % 128) == 64); + if (q[2]) h.set(0); + if (q[3]) h.set(1); + } + } + if (size == 64) { + bool lower_half_used = h[0]; + bool upper_half_used = h[1]; + if (lower_half_used && !upper_half_used) { + size = 32; + } else if ((!lower_half_used) && upper_half_used) { + addr = addr + 32; + size = 32; + } else { + assert(lower_half_used && upper_half_used); + } + } + m_accessq.push_back(mem_access_t(access_type, addr, size, is_write, + info.active, info.bytes, info.chunks, + m_config->gpgpu_ctx)); } -void warp_inst_t::completed( unsigned long long cycle ) const -{ - unsigned long long latency = cycle - issue_cycle; - assert(latency <= cycle); // underflow detection - m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency(pc, latency * active_count()); +void warp_inst_t::completed(unsigned long long cycle) const { + unsigned long long latency = cycle - issue_cycle; + assert(latency <= cycle); // underflow detection + m_config->gpgpu_ctx->stats->ptx_file_line_stats_add_latency( + pc, latency * active_count()); } +kernel_info_t::kernel_info_t(dim3 gridDim, dim3 blockDim, + class function_info *entry) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; - - volta_cache_config_set=false; + volta_cache_config_set = false; } -/*A snapshot of the texture mappings needs to be stored in the kernel's info as +/*A snapshot of the texture mappings needs to be stored in the kernel's info as kernels should use the texture bindings seen at the time of launch and textures can be bound/unbound asynchronously with respect to streams. */ -kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map<std::string, const struct cudaArray*> nameToCudaArray, std::map<std::string, const struct textureInfo*> nameToTextureInfo) -{ - m_kernel_entry=entry; - m_grid_dim=gridDim; - m_block_dim=blockDim; - m_next_cta.x=0; - m_next_cta.y=0; - m_next_cta.z=0; - m_next_tid=m_next_cta; - m_num_cores_running=0; - m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; - m_param_mem = new memory_space_impl<8192>("param",64*1024); +kernel_info_t::kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map<std::string, const struct cudaArray *> nameToCudaArray, + std::map<std::string, const struct textureInfo *> nameToTextureInfo) { + m_kernel_entry = entry; + m_grid_dim = gridDim; + m_block_dim = blockDim; + m_next_cta.x = 0; + m_next_cta.y = 0; + m_next_cta.z = 0; + m_next_tid = m_next_cta; + m_num_cores_running = 0; + m_uid = (entry->gpgpu_ctx->kernel_info_m_next_uid)++; + m_param_mem = new memory_space_impl<8192>("param", 64 * 1024); - //Jin: parent and child kernel management for CDP - m_parent_kernel = NULL; - - //Jin: launch latency management - m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; + // Jin: parent and child kernel management for CDP + m_parent_kernel = NULL; - volta_cache_config_set=false; - m_NameToCudaArray = nameToCudaArray; - m_NameToTextureInfo = nameToTextureInfo; -} + // Jin: launch latency management + m_launch_latency = entry->gpgpu_ctx->device_runtime->g_kernel_launch_latency; -kernel_info_t::~kernel_info_t() -{ - assert( m_active_threads.empty() ); - destroy_cta_streams(); - delete m_param_mem; + volta_cache_config_set = false; + m_NameToCudaArray = nameToCudaArray; + m_NameToTextureInfo = nameToTextureInfo; } -std::string kernel_info_t::name() const -{ - return m_kernel_entry->get_name(); +kernel_info_t::~kernel_info_t() { + assert(m_active_threads.empty()); + destroy_cta_streams(); + delete m_param_mem; } -//Jin: parent and child kernel management for CDP -void kernel_info_t::set_parent(kernel_info_t * parent, - dim3 parent_ctaid, dim3 parent_tid) { - m_parent_kernel = parent; - m_parent_ctaid = parent_ctaid; - m_parent_tid = parent_tid; - parent->set_child(this); +std::string kernel_info_t::name() const { return m_kernel_entry->get_name(); } + +// Jin: parent and child kernel management for CDP +void kernel_info_t::set_parent(kernel_info_t *parent, dim3 parent_ctaid, + dim3 parent_tid) { + m_parent_kernel = parent; + m_parent_ctaid = parent_ctaid; + m_parent_tid = parent_tid; + parent->set_child(this); } -void kernel_info_t::set_child(kernel_info_t * child) { - m_child_kernels.push_back(child); +void kernel_info_t::set_child(kernel_info_t *child) { + m_child_kernels.push_back(child); } -void kernel_info_t::remove_child(kernel_info_t * child) { - assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) - != m_child_kernels.end()); - m_child_kernels.remove(child); +void kernel_info_t::remove_child(kernel_info_t *child) { + assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child) != + m_child_kernels.end()); + m_child_kernels.remove(child); } bool kernel_info_t::is_finished() { - if(done() && children_all_finished()) - return true; + if (done() && children_all_finished()) + return true; else - return false; + return false; } bool kernel_info_t::children_all_finished() { - if(!m_child_kernels.empty()) - return false; - - return true; + if (!m_child_kernels.empty()) return false; + + return true; } void kernel_info_t::notify_parent_finished() { - if(m_parent_kernel) { - m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= ((m_kernel_entry->get_args_aligned_size() + 255)/256*256); - m_parent_kernel->remove_child(this); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(m_parent_kernel->get_uid()); - } + if (m_parent_kernel) { + m_kernel_entry->gpgpu_ctx->device_runtime->g_total_param_size -= + ((m_kernel_entry->get_args_aligned_size() + 255) / 256 * 256); + m_parent_kernel->remove_child(this); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager + ->register_finished_kernel(m_parent_kernel->get_uid()); + } } -CUstream_st * kernel_info_t::create_stream_cta(dim3 ctaid) { - assert(get_default_stream_cta(ctaid)); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); - assert(m_cta_streams[ctaid].size() >= 1); //must have default stream - m_cta_streams[ctaid].push_back(stream); +CUstream_st *kernel_info_t::create_stream_cta(dim3 ctaid) { + assert(get_default_stream_cta(ctaid)); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); + assert(m_cta_streams.find(ctaid) != m_cta_streams.end()); + assert(m_cta_streams[ctaid].size() >= 1); // must have default stream + m_cta_streams[ctaid].push_back(stream); - return stream; + return stream; } -CUstream_st * kernel_info_t::get_default_stream_cta(dim3 ctaid) { - if(m_cta_streams.find(ctaid) != m_cta_streams.end()) { - assert(m_cta_streams[ctaid].size() >= 1); //already created, must have default stream - return *(m_cta_streams[ctaid].begin()); - } - else { - m_cta_streams[ctaid] = std::list<CUstream_st *>(); - CUstream_st * stream = new CUstream_st(); - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream(stream); - m_cta_streams[ctaid].push_back(stream); - return stream; - } +CUstream_st *kernel_info_t::get_default_stream_cta(dim3 ctaid) { + if (m_cta_streams.find(ctaid) != m_cta_streams.end()) { + assert(m_cta_streams[ctaid].size() >= + 1); // already created, must have default stream + return *(m_cta_streams[ctaid].begin()); + } else { + m_cta_streams[ctaid] = std::list<CUstream_st *>(); + CUstream_st *stream = new CUstream_st(); + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->add_stream( + stream); + m_cta_streams[ctaid].push_back(stream); + return stream; + } } -bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st* stream) { - if(m_cta_streams.find(ctaid) == m_cta_streams.end()) - return false; +bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st *stream) { + if (m_cta_streams.find(ctaid) == m_cta_streams.end()) return false; - std::list<CUstream_st *> &stream_list = m_cta_streams[ctaid]; - if(std::find(stream_list.begin(), stream_list.end(), stream) - == stream_list.end()) - return false; - else - return true; + std::list<CUstream_st *> &stream_list = m_cta_streams[ctaid]; + if (std::find(stream_list.begin(), stream_list.end(), stream) == + stream_list.end()) + return false; + else + return true; } void kernel_info_t::print_parent_info() { - if(m_parent_kernel) { - printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", - m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), - m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, - m_parent_tid.x, m_parent_tid.y, m_parent_tid.z); - } + if (m_parent_kernel) { + printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", + m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), + m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z, m_parent_tid.x, + m_parent_tid.y, m_parent_tid.z); + } } void kernel_info_t::destroy_cta_streams() { - printf("Destroy streams for kernel %d: ", get_uid()); size_t stream_size = 0; - for(auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { - stream_size += s->second.size(); - for(auto ss = s->second.begin(); ss != s->second.end(); ss++) - m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream(*ss); - s->second.clear(); - } - printf("size %lu\n", stream_size); - m_cta_streams.clear(); + printf("Destroy streams for kernel %d: ", get_uid()); + size_t stream_size = 0; + for (auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) { + stream_size += s->second.size(); + for (auto ss = s->second.begin(); ss != s->second.end(); ss++) + m_kernel_entry->gpgpu_ctx->the_gpgpusim->g_stream_manager->destroy_stream( + *ss); + s->second.clear(); + } + printf("size %lu\n", stream_size); + m_cta_streams.clear(); } -simt_stack::simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu) -{ - m_warp_id=wid; - m_warp_size = warpSize; - m_gpu=gpu; - reset(); +simt_stack::simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu) { + m_warp_id = wid; + m_warp_size = warpSize; + m_gpu = gpu; + reset(); } -void simt_stack::reset() -{ - m_stack.clear(); -} +void simt_stack::reset() { m_stack.clear(); } -void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) -{ - reset(); - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = start_pc; - new_stack_entry.m_calldepth = 1; - new_stack_entry.m_active_mask = active_mask; - new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; - m_stack.push_back(new_stack_entry); +void simt_stack::launch(address_type start_pc, const simt_mask_t &active_mask) { + reset(); + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = start_pc; + new_stack_entry.m_calldepth = 1; + new_stack_entry.m_active_mask = active_mask; + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + m_stack.push_back(new_stack_entry); } -void simt_stack::resume( char * fname ) -{ - reset(); - - +void simt_stack::resume(char *fname) { + reset(); - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); + FILE *fp2 = fopen(fname, "r"); + assert(fp2 != NULL); - char line [ 200 ]; /* or other suitable maximum line size */ + char line[200]; /* or other suitable maximum line size */ - while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ - { - simt_stack_entry new_stack_entry; - char * pch; - pch = strtok (line," "); - for (unsigned j=0; j<m_warp_size; j++) - { - if (pch[0]=='1') - new_stack_entry.m_active_mask.set(j); - else - new_stack_entry.m_active_mask.reset(j); - pch = strtok (NULL," "); - - } - - new_stack_entry.m_pc=atoi(pch); - pch = strtok (NULL," "); - new_stack_entry.m_calldepth=atoi(pch); - pch = strtok (NULL," "); - new_stack_entry.m_recvg_pc=atoi(pch); - pch = strtok (NULL," "); - new_stack_entry.m_branch_div_cycle=atoi(pch); - pch = strtok (NULL," "); - if(pch[0]=='0') - new_stack_entry.m_type= STACK_ENTRY_TYPE_NORMAL; - else - new_stack_entry.m_type= STACK_ENTRY_TYPE_CALL; - m_stack.push_back(new_stack_entry); - } - fclose ( fp2 ); + while (fgets(line, sizeof line, fp2) != NULL) /* read a line */ + { + simt_stack_entry new_stack_entry; + char *pch; + pch = strtok(line, " "); + for (unsigned j = 0; j < m_warp_size; j++) { + if (pch[0] == '1') + new_stack_entry.m_active_mask.set(j); + else + new_stack_entry.m_active_mask.reset(j); + pch = strtok(NULL, " "); + } - + new_stack_entry.m_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_calldepth = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_recvg_pc = atoi(pch); + pch = strtok(NULL, " "); + new_stack_entry.m_branch_div_cycle = atoi(pch); + pch = strtok(NULL, " "); + if (pch[0] == '0') + new_stack_entry.m_type = STACK_ENTRY_TYPE_NORMAL; + else + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + } + fclose(fp2); } -const simt_mask_t &simt_stack::get_active_mask() const -{ - assert(m_stack.size() > 0); - return m_stack.back().m_active_mask; +const simt_mask_t &simt_stack::get_active_mask() const { + assert(m_stack.size() > 0); + return m_stack.back().m_active_mask; } -void simt_stack::get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const -{ - assert(m_stack.size() > 0); - *pc = m_stack.back().m_pc; - *rpc = m_stack.back().m_recvg_pc; +void simt_stack::get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const { + assert(m_stack.size() > 0); + *pc = m_stack.back().m_pc; + *rpc = m_stack.back().m_recvg_pc; } -unsigned simt_stack::get_rp() const -{ - assert(m_stack.size() > 0); - return m_stack.back().m_recvg_pc; +unsigned simt_stack::get_rp() const { + assert(m_stack.size() > 0); + return m_stack.back().m_recvg_pc; } -void simt_stack::print (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - if ( k==0 ) { - fprintf(fout, "w%02d %1u ", m_warp_id, k ); - } else { - fprintf(fout, " %1u ", k ); - } - for (unsigned j=0; j<m_warp_size; j++) - fprintf(fout, "%c", (stack_entry.m_active_mask.test(j)?'1':'0') ); - fprintf(fout, " pc: 0x%03x", stack_entry.m_pc ); - if ( stack_entry.m_recvg_pc == (unsigned)-1 ) { - fprintf(fout," rp: ---- tp: %s cd: %2u ", (stack_entry.m_type==STACK_ENTRY_TYPE_CALL?"C":"N"), stack_entry.m_calldepth ); - } else { - fprintf(fout," rp: %4u tp: %s cd: %2u ", stack_entry.m_recvg_pc, (stack_entry.m_type==STACK_ENTRY_TYPE_CALL?"C":"N"), stack_entry.m_calldepth ); - } - if ( stack_entry.m_branch_div_cycle != 0 ) { - fprintf(fout," bd@%6u ", (unsigned) stack_entry.m_branch_div_cycle ); - } else { - fprintf(fout," " ); - } - m_gpu->gpgpu_ctx->func_sim->ptx_print_insn( stack_entry.m_pc, fout ); - fprintf(fout,"\n"); +void simt_stack::print(FILE *fout) const { + for (unsigned k = 0; k < m_stack.size(); k++) { + simt_stack_entry stack_entry = m_stack[k]; + if (k == 0) { + fprintf(fout, "w%02d %1u ", m_warp_id, k); + } else { + fprintf(fout, " %1u ", k); } - -} - -void simt_stack::print_checkpoint (FILE *fout) const -{ - for ( unsigned k=0; k < m_stack.size(); k++ ) { - simt_stack_entry stack_entry = m_stack[k]; - - for (unsigned j=0; j<m_warp_size; j++) - fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j)?'1':'0') ); - fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc,stack_entry.m_calldepth,stack_entry.m_recvg_pc,stack_entry.m_branch_div_cycle,stack_entry.m_type ); - fprintf(fout, "%d %d\n",m_warp_id, m_warp_size ); - + for (unsigned j = 0; j < m_warp_size; j++) + fprintf(fout, "%c", (stack_entry.m_active_mask.test(j) ? '1' : '0')); + fprintf(fout, " pc: 0x%03x", stack_entry.m_pc); + if (stack_entry.m_recvg_pc == (unsigned)-1) { + fprintf(fout, " rp: ---- tp: %s cd: %2u ", + (stack_entry.m_type == STACK_ENTRY_TYPE_CALL ? "C" : "N"), + stack_entry.m_calldepth); + } else { + fprintf(fout, " rp: %4u tp: %s cd: %2u ", stack_entry.m_recvg_pc, + (stack_entry.m_type == STACK_ENTRY_TYPE_CALL ? "C" : "N"), + stack_entry.m_calldepth); + } + if (stack_entry.m_branch_div_cycle != 0) { + fprintf(fout, " bd@%6u ", (unsigned)stack_entry.m_branch_div_cycle); + } else { + fprintf(fout, " "); } + m_gpu->gpgpu_ctx->func_sim->ptx_print_insn(stack_entry.m_pc, fout); + fprintf(fout, "\n"); + } } -void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ) -{ - assert(m_stack.size() > 0); +void simt_stack::print_checkpoint(FILE *fout) const { + for (unsigned k = 0; k < m_stack.size(); k++) { + simt_stack_entry stack_entry = m_stack[k]; - assert( next_pc.size() == m_warp_size ); + for (unsigned j = 0; j < m_warp_size; j++) + fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j) ? '1' : '0')); + fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc, + stack_entry.m_calldepth, stack_entry.m_recvg_pc, + stack_entry.m_branch_div_cycle, stack_entry.m_type); + fprintf(fout, "%d %d\n", m_warp_id, m_warp_size); + } +} - simt_mask_t top_active_mask = m_stack.back().m_active_mask; - address_type top_recvg_pc = m_stack.back().m_recvg_pc; - address_type top_pc = m_stack.back().m_pc; // the pc of the instruction just executed - stack_entry_type top_type = m_stack.back().m_type; - assert(top_pc==next_inst_pc); - assert(top_active_mask.any()); +void simt_stack::update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc) { + assert(m_stack.size() > 0); - const address_type null_pc = -1; - bool warp_diverged = false; - address_type new_recvg_pc = null_pc; - unsigned num_divergent_paths=0; + assert(next_pc.size() == m_warp_size); - std::map<address_type,simt_mask_t> divergent_paths; - while (top_active_mask.any()) { + simt_mask_t top_active_mask = m_stack.back().m_active_mask; + address_type top_recvg_pc = m_stack.back().m_recvg_pc; + address_type top_pc = + m_stack.back().m_pc; // the pc of the instruction just executed + stack_entry_type top_type = m_stack.back().m_type; + assert(top_pc == next_inst_pc); + assert(top_active_mask.any()); - // extract a group of threads with the same next PC among the active threads in the warp - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - for (int i = m_warp_size - 1; i >= 0; i--) { - if ( top_active_mask.test(i) ) { // is this thread active? - if (thread_done.test(i)) { - top_active_mask.reset(i); // remove completed thread from active mask - } else if (tmp_next_pc == null_pc) { - tmp_next_pc = next_pc[i]; - tmp_active_mask.set(i); - top_active_mask.reset(i); - } else if (tmp_next_pc == next_pc[i]) { - tmp_active_mask.set(i); - top_active_mask.reset(i); - } - } - } + const address_type null_pc = -1; + bool warp_diverged = false; + address_type new_recvg_pc = null_pc; + unsigned num_divergent_paths = 0; - if(tmp_next_pc == null_pc) { - assert(!top_active_mask.any()); // all threads done - continue; + std::map<address_type, simt_mask_t> divergent_paths; + while (top_active_mask.any()) { + // extract a group of threads with the same next PC among the active threads + // in the warp + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + for (int i = m_warp_size - 1; i >= 0; i--) { + if (top_active_mask.test(i)) { // is this thread active? + if (thread_done.test(i)) { + top_active_mask.reset(i); // remove completed thread from active mask + } else if (tmp_next_pc == null_pc) { + tmp_next_pc = next_pc[i]; + tmp_active_mask.set(i); + top_active_mask.reset(i); + } else if (tmp_next_pc == next_pc[i]) { + tmp_active_mask.set(i); + top_active_mask.reset(i); } - - divergent_paths[tmp_next_pc]=tmp_active_mask; - num_divergent_paths++; + } } + if (tmp_next_pc == null_pc) { + assert(!top_active_mask.any()); // all threads done + continue; + } - address_type not_taken_pc = next_inst_pc+next_inst_size; - assert(num_divergent_paths<=2); - for(unsigned i=0; i<num_divergent_paths; i++){ - address_type tmp_next_pc = null_pc; - simt_mask_t tmp_active_mask; - tmp_active_mask.reset(); - if(divergent_paths.find(not_taken_pc)!=divergent_paths.end()){ - assert(i==0); - tmp_next_pc=not_taken_pc; - tmp_active_mask=divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); - }else{ - std::map<address_type,simt_mask_t>:: iterator it=divergent_paths.begin(); - tmp_next_pc=it->first; - tmp_active_mask=divergent_paths[tmp_next_pc]; - divergent_paths.erase(tmp_next_pc); - } - - // HANDLE THE SPECIAL CASES FIRST - if (next_inst_op== CALL_OPS){ - // Since call is not a divergent instruction, all threads should have executed a call instruction - assert(num_divergent_paths == 1); - - simt_stack_entry new_stack_entry; - new_stack_entry.m_pc = tmp_next_pc; - new_stack_entry.m_active_mask = tmp_active_mask; - new_stack_entry.m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; - new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; - m_stack.push_back(new_stack_entry); - return; - }else if(next_inst_op == RET_OPS && top_type==STACK_ENTRY_TYPE_CALL){ - // pop the CALL Entry - assert(num_divergent_paths == 1); - m_stack.pop_back(); + divergent_paths[tmp_next_pc] = tmp_active_mask; + num_divergent_paths++; + } - assert(m_stack.size() > 0); - m_stack.back().m_pc=tmp_next_pc;// set the PC of the stack top entry to return PC from the call stack; - // Check if the New top of the stack is reconverging - if (tmp_next_pc == m_stack.back().m_recvg_pc && m_stack.back().m_type!=STACK_ENTRY_TYPE_CALL){ - assert(m_stack.back().m_type==STACK_ENTRY_TYPE_NORMAL); - m_stack.pop_back(); - } - return; - } + address_type not_taken_pc = next_inst_pc + next_inst_size; + assert(num_divergent_paths <= 2); + for (unsigned i = 0; i < num_divergent_paths; i++) { + address_type tmp_next_pc = null_pc; + simt_mask_t tmp_active_mask; + tmp_active_mask.reset(); + if (divergent_paths.find(not_taken_pc) != divergent_paths.end()) { + assert(i == 0); + tmp_next_pc = not_taken_pc; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } else { + std::map<address_type, simt_mask_t>::iterator it = + divergent_paths.begin(); + tmp_next_pc = it->first; + tmp_active_mask = divergent_paths[tmp_next_pc]; + divergent_paths.erase(tmp_next_pc); + } - // discard the new entry if its PC matches with reconvergence PC - // that automatically reconverges the entry - // If the top stack entry is CALL, dont reconverge. - if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) continue; + // HANDLE THE SPECIAL CASES FIRST + if (next_inst_op == CALL_OPS) { + // Since call is not a divergent instruction, all threads should have + // executed a call instruction + assert(num_divergent_paths == 1); - // this new entry is not converging - // if this entry does not include thread from the warp, divergence occurs - if ((num_divergent_paths>1) && !warp_diverged ) { - warp_diverged = true; - // modify the existing top entry into a reconvergence entry in the pdom stack - new_recvg_pc = recvg_pc; - if (new_recvg_pc != top_recvg_pc) { - m_stack.back().m_pc = new_recvg_pc; - m_stack.back().m_branch_div_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle; + simt_stack_entry new_stack_entry; + new_stack_entry.m_pc = tmp_next_pc; + new_stack_entry.m_active_mask = tmp_active_mask; + new_stack_entry.m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; + new_stack_entry.m_type = STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + return; + } else if (next_inst_op == RET_OPS && top_type == STACK_ENTRY_TYPE_CALL) { + // pop the CALL Entry + assert(num_divergent_paths == 1); + m_stack.pop_back(); - m_stack.push_back(simt_stack_entry()); - } - } + assert(m_stack.size() > 0); + m_stack.back().m_pc = tmp_next_pc; // set the PC of the stack top entry + // to return PC from the call stack; + // Check if the New top of the stack is reconverging + if (tmp_next_pc == m_stack.back().m_recvg_pc && + m_stack.back().m_type != STACK_ENTRY_TYPE_CALL) { + assert(m_stack.back().m_type == STACK_ENTRY_TYPE_NORMAL); + m_stack.pop_back(); + } + return; + } - // discard the new entry if its PC matches with reconvergence PC - if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // discard the new entry if its PC matches with reconvergence PC + // that automatically reconverges the entry + // If the top stack entry is CALL, dont reconverge. + if (tmp_next_pc == top_recvg_pc && (top_type != STACK_ENTRY_TYPE_CALL)) + continue; - // update the current top of pdom stack - m_stack.back().m_pc = tmp_next_pc; - m_stack.back().m_active_mask = tmp_active_mask; - if (warp_diverged) { - m_stack.back().m_calldepth = 0; - m_stack.back().m_recvg_pc = new_recvg_pc; - } else { - m_stack.back().m_recvg_pc = top_recvg_pc; - } + // this new entry is not converging + // if this entry does not include thread from the warp, divergence occurs + if ((num_divergent_paths > 1) && !warp_diverged) { + warp_diverged = true; + // modify the existing top entry into a reconvergence entry in the pdom + // stack + new_recvg_pc = recvg_pc; + if (new_recvg_pc != top_recvg_pc) { + m_stack.back().m_pc = new_recvg_pc; + m_stack.back().m_branch_div_cycle = + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; m_stack.push_back(simt_stack_entry()); + } } - assert(m_stack.size() > 0); - m_stack.pop_back(); + // discard the new entry if its PC matches with reconvergence PC + if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + // update the current top of pdom stack + m_stack.back().m_pc = tmp_next_pc; + m_stack.back().m_active_mask = tmp_active_mask; if (warp_diverged) { - m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + m_stack.back().m_calldepth = 0; + m_stack.back().m_recvg_pc = new_recvg_pc; + } else { + m_stack.back().m_recvg_pc = top_recvg_pc; } + + m_stack.push_back(simt_stack_entry()); + } + assert(m_stack.size() > 0); + m_stack.pop_back(); + + if (warp_diverged) { + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_warp_divergence(top_pc, 1); + } } -void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) -{ - for ( unsigned t=0; t < m_warp_size; t++ ) { - if( inst.active(t) ) { - if(warpId==(unsigned (-1))) - warpId = inst.warp_id(); - unsigned tid=m_warp_size*warpId+t; - m_thread[tid]->ptx_exec_inst(inst,t); - - //virtual function - checkExecutionStatusAndUpdate(inst,t,tid); - } - } +void core_t::execute_warp_inst_t(warp_inst_t &inst, unsigned warpId) { + for (unsigned t = 0; t < m_warp_size; t++) { + if (inst.active(t)) { + if (warpId == (unsigned(-1))) warpId = inst.warp_id(); + unsigned tid = m_warp_size * warpId + t; + m_thread[tid]->ptx_exec_inst(inst, t); + + // virtual function + checkExecutionStatusAndUpdate(inst, t, tid); + } + } } - -bool core_t::ptx_thread_done( unsigned hw_thread_id ) const -{ - return ((m_thread[ hw_thread_id ]==NULL) || m_thread[ hw_thread_id ]->is_done()); + +bool core_t::ptx_thread_done(unsigned hw_thread_id) const { + return ((m_thread[hw_thread_id] == NULL) || + m_thread[hw_thread_id]->is_done()); } - -void core_t::updateSIMTStack(unsigned warpId, warp_inst_t * inst) -{ - simt_mask_t thread_done; - addr_vector_t next_pc; - unsigned wtid = warpId * m_warp_size; - for (unsigned i = 0; i < m_warp_size; i++) { - if( ptx_thread_done(wtid+i) ) { - thread_done.set(i); - next_pc.push_back( (address_type)-1 ); - } else { - if( inst->reconvergence_pc == RECONVERGE_RETURN_PC ) - inst->reconvergence_pc = get_return_pc(m_thread[wtid+i]); - next_pc.push_back( m_thread[wtid+i]->get_pc() ); - } + +void core_t::updateSIMTStack(unsigned warpId, warp_inst_t *inst) { + simt_mask_t thread_done; + addr_vector_t next_pc; + unsigned wtid = warpId * m_warp_size; + for (unsigned i = 0; i < m_warp_size; i++) { + if (ptx_thread_done(wtid + i)) { + thread_done.set(i); + next_pc.push_back((address_type)-1); + } else { + if (inst->reconvergence_pc == RECONVERGE_RETURN_PC) + inst->reconvergence_pc = get_return_pc(m_thread[wtid + i]); + next_pc.push_back(m_thread[wtid + i]->get_pc()); } - m_simt_stack[warpId]->update(thread_done,next_pc,inst->reconvergence_pc, inst->op,inst->isize,inst->pc); + } + m_simt_stack[warpId]->update(thread_done, next_pc, inst->reconvergence_pc, + inst->op, inst->isize, inst->pc); } //! Get the warp to be executed using the data taken form the SIMT stack -warp_inst_t core_t::getExecuteWarp(unsigned warpId) -{ - unsigned pc,rpc; - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - warp_inst_t wi= *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); - wi.set_active(m_simt_stack[warpId]->get_active_mask()); - return wi; +warp_inst_t core_t::getExecuteWarp(unsigned warpId) { + unsigned pc, rpc; + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + warp_inst_t wi = *(m_gpu->gpgpu_ctx->ptx_fetch_inst(pc)); + wi.set_active(m_simt_stack[warpId]->get_active_mask()); + return wi; } -void core_t::deleteSIMTStack() -{ - if ( m_simt_stack ) { - for (unsigned i = 0; i < m_warp_count; ++i) - delete m_simt_stack[i]; - delete[] m_simt_stack; - m_simt_stack = NULL; - } +void core_t::deleteSIMTStack() { + if (m_simt_stack) { + for (unsigned i = 0; i < m_warp_count; ++i) delete m_simt_stack[i]; + delete[] m_simt_stack; + m_simt_stack = NULL; + } } -void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) -{ - m_simt_stack = new simt_stack*[warp_count]; - for (unsigned i = 0; i < warp_count; ++i) - m_simt_stack[i] = new simt_stack(i,warp_size,m_gpu); - m_warp_size = warp_size; - m_warp_count = warp_count; +void core_t::initilizeSIMTStack(unsigned warp_count, unsigned warp_size) { + m_simt_stack = new simt_stack *[warp_count]; + for (unsigned i = 0; i < warp_count; ++i) + m_simt_stack[i] = new simt_stack(i, warp_size, m_gpu); + m_warp_size = warp_size; + m_warp_count = warp_count; } -void core_t::get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const -{ - m_simt_stack[warpId]->get_pdom_stack_top_info(pc,rpc); +void core_t::get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const { + m_simt_stack[warpId]->get_pdom_stack_top_info(pc, rpc); } diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 29e4a30..dda4ead 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED #define ABSTRACT_HARDWARE_MODEL_INCLUDED @@ -33,45 +34,41 @@ class gpgpu_sim; class kernel_info_t; class gpgpu_context; - -//Set a hard limit of 32 CTAs per shader [cuda only has 8] +// Set a hard limit of 32 CTAs per shader [cuda only has 8] #define MAX_CTA_PER_SHADER 32 #define MAX_BARRIERS_PER_CTA 16 -//After expanding the vector input and output operands +// After expanding the vector input and output operands #define MAX_INPUT_VALUES 24 #define MAX_OUTPUT_VALUES 8 enum _memory_space_t { - undefined_space=0, - reg_space, - local_space, - shared_space, - sstarr_space, - param_space_unclassified, - param_space_kernel, /* global to all threads in a kernel : read-only */ - param_space_local, /* local to a thread : read-writable */ - const_space, - tex_space, - surf_space, - global_space, - generic_space, - instruction_space + undefined_space = 0, + reg_space, + local_space, + shared_space, + sstarr_space, + param_space_unclassified, + param_space_kernel, /* global to all threads in a kernel : read-only */ + param_space_local, /* local to a thread : read-writable */ + const_space, + tex_space, + surf_space, + global_space, + generic_space, + instruction_space }; - -enum FuncCache -{ +enum FuncCache { FuncCachePreferNone = 0, FuncCachePreferShared = 1, FuncCachePreferL1 = 2 }; - #ifdef __cplusplus -#include <string.h> #include <stdio.h> +#include <string.h> #include <set> typedef unsigned long long new_addr_type; @@ -79,371 +76,357 @@ typedef unsigned long long cudaTextureObject_t; typedef unsigned address_type; typedef unsigned addr_t; -// the following are operations the timing model can see +// the following are operations the timing model can see enum uarch_op_t { - NO_OP=-1, - ALU_OP=1, - SFU_OP, - TENSOR_CORE_OP, - DP_OP, - SP_OP, - INTP_OP, - ALU_SFU_OP, - LOAD_OP, - TENSOR_CORE_LOAD_OP, - TENSOR_CORE_STORE_OP, - STORE_OP, - BRANCH_OP, - BARRIER_OP, - MEMORY_BARRIER_OP, - CALL_OPS, - RET_OPS + NO_OP = -1, + ALU_OP = 1, + SFU_OP, + TENSOR_CORE_OP, + DP_OP, + SP_OP, + INTP_OP, + ALU_SFU_OP, + LOAD_OP, + TENSOR_CORE_LOAD_OP, + TENSOR_CORE_STORE_OP, + STORE_OP, + BRANCH_OP, + BARRIER_OP, + MEMORY_BARRIER_OP, + CALL_OPS, + RET_OPS }; typedef enum uarch_op_t op_type; - -enum uarch_bar_t { - NOT_BAR=-1, - SYNC=1, - ARRIVE, - RED -}; +enum uarch_bar_t { NOT_BAR = -1, SYNC = 1, ARRIVE, RED }; typedef enum uarch_bar_t barrier_type; -enum uarch_red_t { - NOT_RED=-1, - POPC_RED=1, - AND_RED, - OR_RED -}; +enum uarch_red_t { NOT_RED = -1, POPC_RED = 1, AND_RED, OR_RED }; typedef enum uarch_red_t reduction_type; - -enum uarch_operand_type_t { - UN_OP=-1, - INT_OP, - FP_OP -}; +enum uarch_operand_type_t { UN_OP = -1, INT_OP, FP_OP }; typedef enum uarch_operand_type_t types_of_operands; enum special_operations_t { - OTHER_OP, - INT__OP, - INT_MUL24_OP, - INT_MUL32_OP, - INT_MUL_OP, - INT_DIV_OP, - FP_MUL_OP, - FP_DIV_OP, - FP__OP, - FP_SQRT_OP, - FP_LG_OP, - FP_SIN_OP, - FP_EXP_OP + OTHER_OP, + INT__OP, + INT_MUL24_OP, + INT_MUL32_OP, + INT_MUL_OP, + INT_DIV_OP, + FP_MUL_OP, + FP_DIV_OP, + FP__OP, + FP_SQRT_OP, + FP_LG_OP, + FP_SIN_OP, + FP_EXP_OP }; -typedef enum special_operations_t special_ops; // Required to identify for the power model +typedef enum special_operations_t + special_ops; // Required to identify for the power model enum operation_pipeline_t { - UNKOWN_OP, - SP__OP, - DP__OP, - INTP__OP, - SFU__OP, - TENSOR_CORE__OP, - MEM__OP + UNKOWN_OP, + SP__OP, + DP__OP, + INTP__OP, + SFU__OP, + TENSOR_CORE__OP, + MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; -enum mem_operation_t { - NOT_TEX, - TEX -}; +enum mem_operation_t { NOT_TEX, TEX }; typedef enum mem_operation_t mem_operation; -enum _memory_op_t { - no_memory_op = 0, - memory_load, - memory_store -}; +enum _memory_op_t { no_memory_op = 0, memory_load, memory_store }; -#include <bitset> -#include <list> -#include <vector> #include <assert.h> #include <stdlib.h> -#include <map> -#include <deque> #include <algorithm> +#include <bitset> +#include <deque> +#include <list> +#include <map> +#include <vector> #if !defined(__VECTOR_TYPES_H__) #include "vector_types.h" #endif struct dim3comp { - bool operator() (const dim3 & a, const dim3 & b) const - { - if(a.z < b.z) - return true; - else if(a.y < b.y) - return true; - else if (a.x < b.x) - return true; - else - return false; - } + bool operator()(const dim3 &a, const dim3 &b) const { + if (a.z < b.z) + return true; + else if (a.y < b.y) + return true; + else if (a.x < b.x) + return true; + else + return false; + } }; -void increment_x_then_y_then_z( dim3 &i, const dim3 &bound); +void increment_x_then_y_then_z(dim3 &i, const dim3 &bound); -//Jin: child kernel information for CDP +// Jin: child kernel information for CDP #include "stream_manager.h" class stream_manager; struct CUstream_st; -//extern stream_manager * g_stream_manager; -//support for pinned memories added -extern std::map<void *,void **> pinned_memory; +// extern stream_manager * g_stream_manager; +// support for pinned memories added +extern std::map<void *, void **> pinned_memory; extern std::map<void *, size_t> pinned_memory_size; class kernel_info_t { -public: -// kernel_info_t() -// { -// m_valid=false; -// m_kernel_entry=NULL; -// m_uid=0; -// m_num_cores_running=0; -// m_param_mem=NULL; -// } - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry); - kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *entry, std::map<std::string, const struct cudaArray*> nameToCudaArray, std::map<std::string, const struct textureInfo*> nameToTextureInfo); - ~kernel_info_t(); + public: + // kernel_info_t() + // { + // m_valid=false; + // m_kernel_entry=NULL; + // m_uid=0; + // m_num_cores_running=0; + // m_param_mem=NULL; + // } + kernel_info_t(dim3 gridDim, dim3 blockDim, class function_info *entry); + kernel_info_t( + dim3 gridDim, dim3 blockDim, class function_info *entry, + std::map<std::string, const struct cudaArray *> nameToCudaArray, + std::map<std::string, const struct textureInfo *> nameToTextureInfo); + ~kernel_info_t(); - void inc_running() { m_num_cores_running++; } - void dec_running() - { - assert( m_num_cores_running > 0 ); - m_num_cores_running--; - } - bool running() const { return m_num_cores_running>0; } - bool done() const - { - return no_more_ctas_to_run() && !running(); - } - class function_info *entry() { return m_kernel_entry; } - const class function_info *entry() const { return m_kernel_entry; } + void inc_running() { m_num_cores_running++; } + void dec_running() { + assert(m_num_cores_running > 0); + m_num_cores_running--; + } + bool running() const { return m_num_cores_running > 0; } + bool done() const { return no_more_ctas_to_run() && !running(); } + class function_info *entry() { + return m_kernel_entry; + } + const class function_info *entry() const { return m_kernel_entry; } - size_t num_blocks() const - { - return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; - } + size_t num_blocks() const { + return m_grid_dim.x * m_grid_dim.y * m_grid_dim.z; + } - size_t threads_per_cta() const - { - return m_block_dim.x * m_block_dim.y * m_block_dim.z; - } + size_t threads_per_cta() const { + return m_block_dim.x * m_block_dim.y * m_block_dim.z; + } - dim3 get_grid_dim() const { return m_grid_dim; } - dim3 get_cta_dim() const { return m_block_dim; } + dim3 get_grid_dim() const { return m_grid_dim; } + dim3 get_cta_dim() const { return m_block_dim; } - void increment_cta_id() - { - increment_x_then_y_then_z(m_next_cta,m_grid_dim); - m_next_tid.x=0; - m_next_tid.y=0; - m_next_tid.z=0; - } - dim3 get_next_cta_id() const { return m_next_cta; } - unsigned get_next_cta_id_single() const - { - return m_next_cta.x + m_grid_dim.x*m_next_cta.y + m_grid_dim.x*m_grid_dim.y*m_next_cta.z; - } - bool no_more_ctas_to_run() const - { - return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || m_next_cta.z >= m_grid_dim.z ); - } + void increment_cta_id() { + increment_x_then_y_then_z(m_next_cta, m_grid_dim); + m_next_tid.x = 0; + m_next_tid.y = 0; + m_next_tid.z = 0; + } + dim3 get_next_cta_id() const { return m_next_cta; } + unsigned get_next_cta_id_single() const { + return m_next_cta.x + m_grid_dim.x * m_next_cta.y + + m_grid_dim.x * m_grid_dim.y * m_next_cta.z; + } + bool no_more_ctas_to_run() const { + return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || + m_next_cta.z >= m_grid_dim.z); + } + + void increment_thread_id() { + increment_x_then_y_then_z(m_next_tid, m_block_dim); + } + dim3 get_next_thread_id_3d() const { return m_next_tid; } + unsigned get_next_thread_id() const { + return m_next_tid.x + m_block_dim.x * m_next_tid.y + + m_block_dim.x * m_block_dim.y * m_next_tid.z; + } + bool more_threads_in_cta() const { + return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && + m_next_tid.x < m_block_dim.x; + } + unsigned get_uid() const { return m_uid; } + std::string name() const; + + std::list<class ptx_thread_info *> &active_threads() { + return m_active_threads; + } + class memory_space *get_param_memory() { + return m_param_mem; + } - void increment_thread_id() { increment_x_then_y_then_z(m_next_tid,m_block_dim); } - dim3 get_next_thread_id_3d() const { return m_next_tid; } - unsigned get_next_thread_id() const - { - return m_next_tid.x + m_block_dim.x*m_next_tid.y + m_block_dim.x*m_block_dim.y*m_next_tid.z; - } - bool more_threads_in_cta() const - { - return m_next_tid.z < m_block_dim.z && m_next_tid.y < m_block_dim.y && m_next_tid.x < m_block_dim.x; - } - unsigned get_uid() const { return m_uid; } - std::string name() const; + // The following functions access texture bindings present at the kernel's + // launch - std::list<class ptx_thread_info *> &active_threads() { return m_active_threads; } - class memory_space *get_param_memory() { return m_param_mem; } + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map<std::string, const struct cudaArray *>::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } - - //The following functions access texture bindings present at the kernel's launch - - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map<std::string, const struct textureInfo *>::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map<std::string, const struct textureInfo*>::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } + private: + kernel_info_t(const kernel_info_t &); // disable copy constructor + void operator=(const kernel_info_t &); // disable copy operator -private: - kernel_info_t( const kernel_info_t & ); // disable copy constructor - void operator=( const kernel_info_t & ); // disable copy operator + class function_info *m_kernel_entry; - class function_info *m_kernel_entry; + unsigned m_uid; - unsigned m_uid; - - //These maps contain the snapshot of the texture mappings at kernel launch - std::map<std::string, const struct cudaArray*> m_NameToCudaArray; - std::map<std::string, const struct textureInfo*> m_NameToTextureInfo; + // These maps contain the snapshot of the texture mappings at kernel launch + std::map<std::string, const struct cudaArray *> m_NameToCudaArray; + std::map<std::string, const struct textureInfo *> m_NameToTextureInfo; - dim3 m_grid_dim; - dim3 m_block_dim; - dim3 m_next_cta; - dim3 m_next_tid; + dim3 m_grid_dim; + dim3 m_block_dim; + dim3 m_next_cta; + dim3 m_next_tid; - unsigned m_num_cores_running; + unsigned m_num_cores_running; - std::list<class ptx_thread_info *> m_active_threads; - class memory_space *m_param_mem; + std::list<class ptx_thread_info *> m_active_threads; + class memory_space *m_param_mem; -public: - //Jin: parent and child kernel management for CDP - void set_parent(kernel_info_t * parent, dim3 parent_ctaid, dim3 parent_tid); - void set_child(kernel_info_t * child); - void remove_child(kernel_info_t * child); - bool is_finished(); - bool children_all_finished(); - void notify_parent_finished(); - CUstream_st * create_stream_cta(dim3 ctaid); - CUstream_st * get_default_stream_cta(dim3 ctaid); - bool cta_has_stream(dim3 ctaid, CUstream_st* stream); - void destroy_cta_streams(); - void print_parent_info(); - kernel_info_t * get_parent() { return m_parent_kernel; } + public: + // Jin: parent and child kernel management for CDP + void set_parent(kernel_info_t *parent, dim3 parent_ctaid, dim3 parent_tid); + void set_child(kernel_info_t *child); + void remove_child(kernel_info_t *child); + bool is_finished(); + bool children_all_finished(); + void notify_parent_finished(); + CUstream_st *create_stream_cta(dim3 ctaid); + CUstream_st *get_default_stream_cta(dim3 ctaid); + bool cta_has_stream(dim3 ctaid, CUstream_st *stream); + void destroy_cta_streams(); + void print_parent_info(); + kernel_info_t *get_parent() { return m_parent_kernel; } -private: - kernel_info_t * m_parent_kernel; - dim3 m_parent_ctaid; - dim3 m_parent_tid; - std::list<kernel_info_t *> m_child_kernels; //child kernel launched - std::map< dim3, std::list<CUstream_st *>, dim3comp > m_cta_streams; //streams created in each CTA + private: + kernel_info_t *m_parent_kernel; + dim3 m_parent_ctaid; + dim3 m_parent_tid; + std::list<kernel_info_t *> m_child_kernels; // child kernel launched + std::map<dim3, std::list<CUstream_st *>, dim3comp> + m_cta_streams; // streams created in each CTA -//Jin: kernel timing -public: - unsigned long long launch_cycle; - unsigned long long start_cycle; - unsigned long long end_cycle; - unsigned m_launch_latency; + // Jin: kernel timing + public: + unsigned long long launch_cycle; + unsigned long long start_cycle; + unsigned long long end_cycle; + unsigned m_launch_latency; - mutable bool volta_cache_config_set; + mutable bool volta_cache_config_set; }; class core_config { - public: - core_config(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_valid = false; - num_shmem_bank=16; - shmem_limited_broadcast = false; - gpgpu_shmem_sizeDefault=(unsigned)-1; - gpgpu_shmem_sizePrefL1=(unsigned)-1; - gpgpu_shmem_sizePrefShared=(unsigned)-1; - } - virtual void init() = 0; + public: + core_config(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_valid = false; + num_shmem_bank = 16; + shmem_limited_broadcast = false; + gpgpu_shmem_sizeDefault = (unsigned)-1; + gpgpu_shmem_sizePrefL1 = (unsigned)-1; + gpgpu_shmem_sizePrefShared = (unsigned)-1; + } + virtual void init() = 0; - bool m_valid; - unsigned warp_size; - // backward pointer - class gpgpu_context* gpgpu_ctx; + bool m_valid; + unsigned warp_size; + // backward pointer + class gpgpu_context *gpgpu_ctx; - // off-chip memory request architecture parameters - int gpgpu_coalesce_arch; + // off-chip memory request architecture parameters + int gpgpu_coalesce_arch; - // shared memory bank conflict checking parameters - bool shmem_limited_broadcast; - static const address_type WORD_SIZE=4; - unsigned num_shmem_bank; - unsigned shmem_bank_func(address_type addr) const - { - return ((addr/WORD_SIZE) % num_shmem_bank); - } - unsigned mem_warp_parts; - mutable unsigned gpgpu_shmem_size; - unsigned gpgpu_shmem_sizeDefault; - unsigned gpgpu_shmem_sizePrefL1; - unsigned gpgpu_shmem_sizePrefShared; - unsigned mem_unit_ports; + // shared memory bank conflict checking parameters + bool shmem_limited_broadcast; + static const address_type WORD_SIZE = 4; + unsigned num_shmem_bank; + unsigned shmem_bank_func(address_type addr) const { + return ((addr / WORD_SIZE) % num_shmem_bank); + } + unsigned mem_warp_parts; + mutable unsigned gpgpu_shmem_size; + unsigned gpgpu_shmem_sizeDefault; + unsigned gpgpu_shmem_sizePrefL1; + unsigned gpgpu_shmem_sizePrefShared; + unsigned mem_unit_ports; - // texture and constant cache line sizes (used to determine number of memory accesses) - unsigned gpgpu_cache_texl1_linesize; - unsigned gpgpu_cache_constl1_linesize; + // texture and constant cache line sizes (used to determine number of memory + // accesses) + unsigned gpgpu_cache_texl1_linesize; + unsigned gpgpu_cache_constl1_linesize; - unsigned gpgpu_max_insn_issue_per_warp; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache + unsigned gpgpu_max_insn_issue_per_warp; + bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - bool adaptive_volta_cache_config; + bool adaptive_volta_cache_config; }; -// bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper +// bounded stack that implements simt reconvergence using pdom mechanism from +// MICRO'07 paper const unsigned MAX_WARP_SIZE = 32; typedef std::bitset<MAX_WARP_SIZE> active_mask_t; -#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE +#define MAX_WARP_SIZE_SIMT_STACK MAX_WARP_SIZE typedef std::bitset<MAX_WARP_SIZE_SIMT_STACK> simt_mask_t; typedef std::vector<address_type> addr_vector_t; class simt_stack { -public: - simt_stack( unsigned wid, unsigned warpSize, class gpgpu_sim * gpu); - - void reset(); - void launch( address_type start_pc, const simt_mask_t &active_mask ); - void update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ); + public: + simt_stack(unsigned wid, unsigned warpSize, class gpgpu_sim *gpu); - const simt_mask_t &get_active_mask() const; - void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; - unsigned get_rp() const; - void print(FILE *fp) const; - void resume(char * fname) ; - void print_checkpoint (FILE *fout) const; + void reset(); + void launch(address_type start_pc, const simt_mask_t &active_mask); + void update(simt_mask_t &thread_done, addr_vector_t &next_pc, + address_type recvg_pc, op_type next_inst_op, + unsigned next_inst_size, address_type next_inst_pc); -protected: - unsigned m_warp_id; - unsigned m_warp_size; + const simt_mask_t &get_active_mask() const; + void get_pdom_stack_top_info(unsigned *pc, unsigned *rpc) const; + unsigned get_rp() const; + void print(FILE *fp) const; + void resume(char *fname); + void print_checkpoint(FILE *fout) const; + protected: + unsigned m_warp_id; + unsigned m_warp_size; - enum stack_entry_type { - STACK_ENTRY_TYPE_NORMAL = 0, - STACK_ENTRY_TYPE_CALL - }; + enum stack_entry_type { STACK_ENTRY_TYPE_NORMAL = 0, STACK_ENTRY_TYPE_CALL }; - struct simt_stack_entry { - address_type m_pc; - unsigned int m_calldepth; - simt_mask_t m_active_mask; - address_type m_recvg_pc; - unsigned long long m_branch_div_cycle; - stack_entry_type m_type; - simt_stack_entry() : - m_pc(-1), m_calldepth(0), m_active_mask(), m_recvg_pc(-1), m_branch_div_cycle(0), m_type(STACK_ENTRY_TYPE_NORMAL) { }; - }; + struct simt_stack_entry { + address_type m_pc; + unsigned int m_calldepth; + simt_mask_t m_active_mask; + address_type m_recvg_pc; + unsigned long long m_branch_div_cycle; + stack_entry_type m_type; + simt_stack_entry() + : m_pc(-1), + m_calldepth(0), + m_active_mask(), + m_recvg_pc(-1), + m_branch_div_cycle(0), + m_type(STACK_ENTRY_TYPE_NORMAL){}; + }; - std::deque<simt_stack_entry> m_stack; + std::deque<simt_stack_entry> m_stack; - class gpgpu_sim * m_gpu; + class gpgpu_sim *m_gpu; }; // Let's just upgrade to C++11 so we can use constexpr here... -// start allocating from this address (lower values used for allocating globals in .ptx file) +// start allocating from this address (lower values used for allocating globals +// in .ptx file) const unsigned long long GLOBAL_HEAP_START = 0xC0000000; // Volta max shmem size is 96kB const unsigned long long SHARED_MEM_SIZE_MAX = 96 * (1 << 10); @@ -455,873 +438,928 @@ const unsigned MAX_STREAMING_MULTIPROCESSORS = 80; const unsigned MAX_THREAD_PER_SM = 1 << 11; // MAX 64 warps / SM const unsigned MAX_WARP_PER_SM = 1 << 6; -const unsigned long long TOTAL_LOCAL_MEM_PER_SM = MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long TOTAL_SHARED_MEM = MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; -const unsigned long long TOTAL_LOCAL_MEM = MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; -const unsigned long long SHARED_GENERIC_START = GLOBAL_HEAP_START - TOTAL_SHARED_MEM; -const unsigned long long LOCAL_GENERIC_START = SHARED_GENERIC_START - TOTAL_LOCAL_MEM; -const unsigned long long STATIC_ALLOC_LIMIT = GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); +const unsigned long long TOTAL_LOCAL_MEM_PER_SM = + MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long TOTAL_SHARED_MEM = + MAX_STREAMING_MULTIPROCESSORS * SHARED_MEM_SIZE_MAX; +const unsigned long long TOTAL_LOCAL_MEM = + MAX_STREAMING_MULTIPROCESSORS * MAX_THREAD_PER_SM * LOCAL_MEM_SIZE_MAX; +const unsigned long long SHARED_GENERIC_START = + GLOBAL_HEAP_START - TOTAL_SHARED_MEM; +const unsigned long long LOCAL_GENERIC_START = + SHARED_GENERIC_START - TOTAL_LOCAL_MEM; +const unsigned long long STATIC_ALLOC_LIMIT = + GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM + TOTAL_SHARED_MEM); #if !defined(__CUDA_RUNTIME_API_H__) #include "builtin_types.h" struct cudaArray { - void *devPtr; - int devPtr32; - struct cudaChannelFormatDesc desc; - int width; - int height; - int size; //in bytes - unsigned dimensions; + void *devPtr; + int devPtr32; + struct cudaChannelFormatDesc desc; + int width; + int height; + int size; // in bytes + unsigned dimensions; }; #endif -// Struct that record other attributes in the textureReference declaration +// Struct that record other attributes in the textureReference declaration // - These attributes are passed thru __cudaRegisterTexture() struct textureReferenceAttr { - const struct textureReference *m_texref; - int m_dim; - enum cudaTextureReadMode m_readmode; - int m_ext; - textureReferenceAttr(const struct textureReference *texref, - int dim, - enum cudaTextureReadMode readmode, - int ext) - : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) - { } + const struct textureReference *m_texref; + int m_dim; + enum cudaTextureReadMode m_readmode; + int m_ext; + textureReferenceAttr(const struct textureReference *texref, int dim, + enum cudaTextureReadMode readmode, int ext) + : m_texref(texref), m_dim(dim), m_readmode(readmode), m_ext(ext) {} }; -class gpgpu_functional_sim_config -{ -public: - void reg_options(class OptionParser * opp); +class gpgpu_functional_sim_config { + public: + void reg_options(class OptionParser *opp); - void ptx_set_tex_cache_linesize(unsigned linesize); + void ptx_set_tex_cache_linesize(unsigned linesize); - unsigned get_forced_max_capability() const { return m_ptx_force_max_capability; } - bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } - bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } - bool experimental_lib_support() const { return m_experimental_lib_support; } + unsigned get_forced_max_capability() const { + return m_ptx_force_max_capability; + } + bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; } + bool use_cuobjdump() const { return m_ptx_use_cuobjdump; } + bool experimental_lib_support() const { return m_experimental_lib_support; } - int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } - const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } - int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } - unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const {return checkpoint_option; } - int get_checkpoint_kernel() const {return checkpoint_kernel; } - int get_checkpoint_CTA() const {return checkpoint_CTA; } - int get_resume_option() const {return resume_option; } - int get_resume_kernel() const {return resume_kernel; } - int get_resume_CTA() const {return resume_CTA; } - int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } -private: - // PTX options - int m_ptx_convert_to_ptxplus; - int m_ptx_use_cuobjdump; - int m_experimental_lib_support; - unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; - int g_ptx_inst_debug_to_file; - char* g_ptx_inst_debug_file; - int g_ptx_inst_debug_thread_uid; + int get_ptx_inst_debug_to_file() const { return g_ptx_inst_debug_to_file; } + const char *get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } + int get_ptx_inst_debug_thread_uid() const { + return g_ptx_inst_debug_thread_uid; + } + unsigned get_texcache_linesize() const { return m_texcache_linesize; } + int get_checkpoint_option() const { return checkpoint_option; } + int get_checkpoint_kernel() const { return checkpoint_kernel; } + int get_checkpoint_CTA() const { return checkpoint_CTA; } + int get_resume_option() const { return resume_option; } + int get_resume_kernel() const { return resume_kernel; } + int get_resume_CTA() const { return resume_CTA; } + int get_checkpoint_CTA_t() const { return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const { return checkpoint_insn_Y; } - unsigned m_texcache_linesize; -}; + private: + // PTX options + int m_ptx_convert_to_ptxplus; + int m_ptx_use_cuobjdump; + int m_experimental_lib_support; + unsigned m_ptx_force_max_capability; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; + char *g_ptx_inst_debug_file; + int g_ptx_inst_debug_thread_uid; + unsigned m_texcache_linesize; +}; class gpgpu_t { -public: - gpgpu_t( const gpgpu_functional_sim_config &config, gpgpu_context* ctx ); - // backward pointer - class gpgpu_context* gpgpu_ctx; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - unsigned resume_option; - unsigned resume_kernel; - unsigned resume_CTA; - unsigned checkpoint_CTA_t; - int checkpoint_insn_Y; + public: + gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx); + // backward pointer + class gpgpu_context *gpgpu_ctx; + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + unsigned resume_option; + unsigned resume_kernel; + unsigned resume_CTA; + unsigned checkpoint_CTA_t; + int checkpoint_insn_Y; - //Move some cycle core stats here instead of being global - unsigned long long gpu_sim_cycle; - unsigned long long gpu_tot_sim_cycle; + // Move some cycle core stats here instead of being global + unsigned long long gpu_sim_cycle; + unsigned long long gpu_tot_sim_cycle; + void *gpu_malloc(size_t size); + void *gpu_mallocarray(size_t count); + void gpu_memset(size_t dst_start_addr, int c, size_t count); + void memcpy_to_gpu(size_t dst_start_addr, const void *src, size_t count); + void memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count); + void memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count); - void* gpu_malloc( size_t size ); - void* gpu_mallocarray( size_t count ); - void gpu_memset( size_t dst_start_addr, int c, size_t count ); - void memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ); - void memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ); - void memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ); - - class memory_space *get_global_memory() { return m_global_mem; } - class memory_space *get_tex_memory() { return m_tex_mem; } - class memory_space *get_surf_memory() { return m_surf_mem; } + class memory_space *get_global_memory() { + return m_global_mem; + } + class memory_space *get_tex_memory() { + return m_tex_mem; + } + class memory_space *get_surf_memory() { + return m_surf_mem; + } - void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array); - void gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext); - void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); - const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); + void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference *texref, + const struct cudaArray *array); + void gpgpu_ptx_sim_bindNameToTexture(const char *name, + const struct textureReference *texref, + int dim, int readmode, int ext); + void gpgpu_ptx_sim_unbindTexture(const struct textureReference *texref); + const char *gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref); - const struct textureReference* get_texref( const std::string &texname ) const - { - std::map<std::string, std::set<const struct textureReference*> >::const_iterator t=m_NameToTextureRef.find(texname); - assert( t != m_NameToTextureRef.end() ); - return *(t->second.begin()); - } + const struct textureReference *get_texref(const std::string &texname) const { + std::map<std::string, + std::set<const struct textureReference *> >::const_iterator t = + m_NameToTextureRef.find(texname); + assert(t != m_NameToTextureRef.end()); + return *(t->second.begin()); + } - const struct cudaArray* get_texarray( const std::string &texname ) const - { - std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); - assert(t != m_NameToCudaArray.end()); - return t->second; - } + const struct cudaArray *get_texarray(const std::string &texname) const { + std::map<std::string, const struct cudaArray *>::const_iterator t = + m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); + return t->second; + } - const struct textureInfo* get_texinfo( const std::string &texname ) const - { - std::map<std::string, const struct textureInfo*>::const_iterator t=m_NameToTextureInfo.find(texname); - assert(t != m_NameToTextureInfo.end()); - return t->second; - } + const struct textureInfo *get_texinfo(const std::string &texname) const { + std::map<std::string, const struct textureInfo *>::const_iterator t = + m_NameToTextureInfo.find(texname); + assert(t != m_NameToTextureInfo.end()); + return t->second; + } - const struct textureReferenceAttr* get_texattr( const std::string &texname ) const - { - std::map<std::string, const struct textureReferenceAttr*>::const_iterator t=m_NameToAttribute.find(texname); - assert(t != m_NameToAttribute.end()); - return t->second; - } + const struct textureReferenceAttr *get_texattr( + const std::string &texname) const { + std::map<std::string, const struct textureReferenceAttr *>::const_iterator + t = m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); + return t->second; + } - const gpgpu_functional_sim_config &get_config() const { return m_function_model_config; } - FILE* get_ptx_inst_debug_file() { return ptx_inst_debug_file; } - - // These maps return the current texture mappings for the GPU at any given time. - std::map<std::string, const struct cudaArray*> getNameArrayMapping() {return m_NameToCudaArray;} - std::map<std::string, const struct textureInfo*> getNameInfoMapping() {return m_NameToTextureInfo;} + const gpgpu_functional_sim_config &get_config() const { + return m_function_model_config; + } + FILE *get_ptx_inst_debug_file() { return ptx_inst_debug_file; } -protected: - const gpgpu_functional_sim_config &m_function_model_config; - FILE* ptx_inst_debug_file; + // These maps return the current texture mappings for the GPU at any given + // time. + std::map<std::string, const struct cudaArray *> getNameArrayMapping() { + return m_NameToCudaArray; + } + std::map<std::string, const struct textureInfo *> getNameInfoMapping() { + return m_NameToTextureInfo; + } - class memory_space *m_global_mem; - class memory_space *m_tex_mem; - class memory_space *m_surf_mem; + protected: + const gpgpu_functional_sim_config &m_function_model_config; + FILE *ptx_inst_debug_file; - unsigned long long m_dev_malloc; - // These maps contain the current texture mappings for the GPU at any given time. - std::map<std::string, std::set<const struct textureReference*> > m_NameToTextureRef; - std::map<const struct textureReference*, std::string> m_TextureRefToName; - std::map<std::string, const struct cudaArray*> m_NameToCudaArray; - std::map<std::string, const struct textureInfo*> m_NameToTextureInfo; - std::map<std::string, const struct textureReferenceAttr*> m_NameToAttribute; -}; + class memory_space *m_global_mem; + class memory_space *m_tex_mem; + class memory_space *m_surf_mem; -struct gpgpu_ptx_sim_info -{ - // Holds properties of the kernel (Kernel's resource use). - // These will be set to zero if a ptxinfo file is not present. - int lmem; - int smem; - int cmem; - int gmem; - int regs; - unsigned maxthreads; - unsigned ptx_version; - unsigned sm_target; + unsigned long long m_dev_malloc; + // These maps contain the current texture mappings for the GPU at any given + // time. + std::map<std::string, std::set<const struct textureReference *> > + m_NameToTextureRef; + std::map<const struct textureReference *, std::string> m_TextureRefToName; + std::map<std::string, const struct cudaArray *> m_NameToCudaArray; + std::map<std::string, const struct textureInfo *> m_NameToTextureInfo; + std::map<std::string, const struct textureReferenceAttr *> m_NameToAttribute; }; +struct gpgpu_ptx_sim_info { + // Holds properties of the kernel (Kernel's resource use). + // These will be set to zero if a ptxinfo file is not present. + int lmem; + int smem; + int cmem; + int gmem; + int regs; + unsigned maxthreads; + unsigned ptx_version; + unsigned sm_target; +}; struct gpgpu_ptx_sim_arg { - gpgpu_ptx_sim_arg() { m_start=NULL; } - gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) - { - m_start=arg; - m_nbytes=size; - m_offset=offset; - } - const void *m_start; - size_t m_nbytes; - size_t m_offset; + gpgpu_ptx_sim_arg() { m_start = NULL; } + gpgpu_ptx_sim_arg(const void *arg, size_t size, size_t offset) { + m_start = arg; + m_nbytes = size; + m_offset = offset; + } + const void *m_start; + size_t m_nbytes; + size_t m_offset; }; typedef std::list<gpgpu_ptx_sim_arg> gpgpu_ptx_sim_arg_list_t; class memory_space_t { -public: - memory_space_t() { m_type = undefined_space; m_bank=0; } - memory_space_t( const enum _memory_space_t &from ) { m_type = from; m_bank = 0; } - bool operator==( const memory_space_t &x ) const { return (m_bank == x.m_bank) && (m_type == x.m_type); } - bool operator!=( const memory_space_t &x ) const { return !(*this == x); } - bool operator<( const memory_space_t &x ) const - { - if(m_type < x.m_type) - return true; - else if(m_type > x.m_type) - return false; - else if( m_bank < x.m_bank ) - return true; + public: + memory_space_t() { + m_type = undefined_space; + m_bank = 0; + } + memory_space_t(const enum _memory_space_t &from) { + m_type = from; + m_bank = 0; + } + bool operator==(const memory_space_t &x) const { + return (m_bank == x.m_bank) && (m_type == x.m_type); + } + bool operator!=(const memory_space_t &x) const { return !(*this == x); } + bool operator<(const memory_space_t &x) const { + if (m_type < x.m_type) + return true; + else if (m_type > x.m_type) return false; - } - enum _memory_space_t get_type() const { return m_type; } - void set_type( enum _memory_space_t t ) { m_type = t; } - unsigned get_bank() const { return m_bank; } - void set_bank( unsigned b ) { m_bank = b; } - bool is_const() const { return (m_type == const_space) || (m_type == param_space_kernel); } - bool is_local() const { return (m_type == local_space) || (m_type == param_space_local); } - bool is_global() const { return (m_type == global_space); } + else if (m_bank < x.m_bank) + return true; + return false; + } + enum _memory_space_t get_type() const { return m_type; } + void set_type(enum _memory_space_t t) { m_type = t; } + unsigned get_bank() const { return m_bank; } + void set_bank(unsigned b) { m_bank = b; } + bool is_const() const { + return (m_type == const_space) || (m_type == param_space_kernel); + } + bool is_local() const { + return (m_type == local_space) || (m_type == param_space_local); + } + bool is_global() const { return (m_type == global_space); } -private: - enum _memory_space_t m_type; - unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 manual, sec. 5.1.3) + private: + enum _memory_space_t m_type; + unsigned m_bank; // n in ".const[n]"; note .const == .const[0] (see PTX 2.1 + // manual, sec. 5.1.3) }; const unsigned MAX_MEMORY_ACCESS_SIZE = 128; typedef std::bitset<MAX_MEMORY_ACCESS_SIZE> mem_access_byte_mask_t; -const unsigned SECTOR_CHUNCK_SIZE = 4; //four sectors -const unsigned SECTOR_SIZE = 32 ; //sector is 32 bytes width +const unsigned SECTOR_CHUNCK_SIZE = 4; // four sectors +const unsigned SECTOR_SIZE = 32; // sector is 32 bytes width typedef std::bitset<SECTOR_CHUNCK_SIZE> mem_access_sector_mask_t; #define NO_PARTIAL_WRITE (mem_access_byte_mask_t()) -#define MEM_ACCESS_TYPE_TUP_DEF \ -MA_TUP_BEGIN( mem_access_type ) \ - MA_TUP( GLOBAL_ACC_R ), \ - MA_TUP( LOCAL_ACC_R ), \ - MA_TUP( CONST_ACC_R ), \ - MA_TUP( TEXTURE_ACC_R ), \ - MA_TUP( GLOBAL_ACC_W ), \ - MA_TUP( LOCAL_ACC_W ), \ - MA_TUP( L1_WRBK_ACC ), \ - MA_TUP( L2_WRBK_ACC ), \ - MA_TUP( INST_ACC_R ), \ - MA_TUP( L1_WR_ALLOC_R ), \ - MA_TUP( L2_WR_ALLOC_R ), \ - MA_TUP( NUM_MEM_ACCESS_TYPE ) \ -MA_TUP_END( mem_access_type ) +#define MEM_ACCESS_TYPE_TUP_DEF \ + MA_TUP_BEGIN(mem_access_type) \ + MA_TUP(GLOBAL_ACC_R), MA_TUP(LOCAL_ACC_R), MA_TUP(CONST_ACC_R), \ + MA_TUP(TEXTURE_ACC_R), MA_TUP(GLOBAL_ACC_W), MA_TUP(LOCAL_ACC_W), \ + MA_TUP(L1_WRBK_ACC), MA_TUP(L2_WRBK_ACC), MA_TUP(INST_ACC_R), \ + MA_TUP(L1_WR_ALLOC_R), MA_TUP(L2_WR_ALLOC_R), \ + MA_TUP(NUM_MEM_ACCESS_TYPE) MA_TUP_END(mem_access_type) #define MA_TUP_BEGIN(X) enum X { #define MA_TUP(X) X -#define MA_TUP_END(X) }; +#define MA_TUP_END(X) \ + } \ + ; MEM_ACCESS_TYPE_TUP_DEF #undef MA_TUP_BEGIN #undef MA_TUP #undef MA_TUP_END -const char * mem_access_type_str(enum mem_access_type access_type); +const char *mem_access_type_str(enum mem_access_type access_type); enum cache_operator_type { - CACHE_UNDEFINED, + CACHE_UNDEFINED, + + // loads + CACHE_ALL, // .ca + CACHE_LAST_USE, // .lu + CACHE_VOLATILE, // .cv + CACHE_L1, // .nc - // loads - CACHE_ALL, // .ca - CACHE_LAST_USE, // .lu - CACHE_VOLATILE, // .cv - CACHE_L1, // .nc - - // loads and stores - CACHE_STREAMING, // .cs - CACHE_GLOBAL, // .cg + // loads and stores + CACHE_STREAMING, // .cs + CACHE_GLOBAL, // .cg - // stores - CACHE_WRITE_BACK, // .wb - CACHE_WRITE_THROUGH // .wt + // stores + CACHE_WRITE_BACK, // .wb + CACHE_WRITE_THROUGH // .wt }; class mem_access_t { -public: - mem_access_t(gpgpu_context* ctx) { init(ctx); } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - gpgpu_context* ctx) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } - mem_access_t( mem_access_type type, - new_addr_type address, - unsigned size, - bool wr, - const active_mask_t &active_mask, - const mem_access_byte_mask_t &byte_mask, - const mem_access_sector_mask_t §or_mask, - gpgpu_context* ctx) - : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) - { - init(ctx); - m_type = type; - m_addr = address; - m_req_size = size; - m_write = wr; - } + public: + mem_access_t(gpgpu_context *ctx) { init(ctx); } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, gpgpu_context *ctx) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } + mem_access_t(mem_access_type type, new_addr_type address, unsigned size, + bool wr, const active_mask_t &active_mask, + const mem_access_byte_mask_t &byte_mask, + const mem_access_sector_mask_t §or_mask, gpgpu_context *ctx) + : m_warp_mask(active_mask), + m_byte_mask(byte_mask), + m_sector_mask(sector_mask) { + init(ctx); + m_type = type; + m_addr = address; + m_req_size = size; + m_write = wr; + } - new_addr_type get_addr() const { return m_addr; } - void set_addr(new_addr_type addr) {m_addr=addr;} - unsigned get_size() const { return m_req_size; } - const active_mask_t &get_warp_mask() const { return m_warp_mask; } - bool is_write() const { return m_write; } - enum mem_access_type get_type() const { return m_type; } - mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } - mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } + new_addr_type get_addr() const { return m_addr; } + void set_addr(new_addr_type addr) { m_addr = addr; } + unsigned get_size() const { return m_req_size; } + const active_mask_t &get_warp_mask() const { return m_warp_mask; } + bool is_write() const { return m_write; } + enum mem_access_type get_type() const { return m_type; } + mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } + mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } - void print(FILE *fp) const - { - fprintf(fp,"addr=0x%llx, %s, size=%u, ", m_addr, m_write?"store":"load ", m_req_size ); - switch(m_type) { - case GLOBAL_ACC_R: fprintf(fp,"GLOBAL_R"); break; - case LOCAL_ACC_R: fprintf(fp,"LOCAL_R "); break; - case CONST_ACC_R: fprintf(fp,"CONST "); break; - case TEXTURE_ACC_R: fprintf(fp,"TEXTURE "); break; - case GLOBAL_ACC_W: fprintf(fp,"GLOBAL_W"); break; - case LOCAL_ACC_W: fprintf(fp,"LOCAL_W "); break; - case L2_WRBK_ACC: fprintf(fp,"L2_WRBK "); break; - case INST_ACC_R: fprintf(fp,"INST "); break; - case L1_WRBK_ACC: fprintf(fp,"L1_WRBK "); break; - default: fprintf(fp,"unknown "); break; - } - } + void print(FILE *fp) const { + fprintf(fp, "addr=0x%llx, %s, size=%u, ", m_addr, + m_write ? "store" : "load ", m_req_size); + switch (m_type) { + case GLOBAL_ACC_R: + fprintf(fp, "GLOBAL_R"); + break; + case LOCAL_ACC_R: + fprintf(fp, "LOCAL_R "); + break; + case CONST_ACC_R: + fprintf(fp, "CONST "); + break; + case TEXTURE_ACC_R: + fprintf(fp, "TEXTURE "); + break; + case GLOBAL_ACC_W: + fprintf(fp, "GLOBAL_W"); + break; + case LOCAL_ACC_W: + fprintf(fp, "LOCAL_W "); + break; + case L2_WRBK_ACC: + fprintf(fp, "L2_WRBK "); + break; + case INST_ACC_R: + fprintf(fp, "INST "); + break; + case L1_WRBK_ACC: + fprintf(fp, "L1_WRBK "); + break; + default: + fprintf(fp, "unknown "); + break; + } + } - gpgpu_context* gpgpu_ctx; -private: - void init(gpgpu_context* ctx); + gpgpu_context *gpgpu_ctx; - unsigned m_uid; - new_addr_type m_addr; // request address - bool m_write; - unsigned m_req_size; // bytes - mem_access_type m_type; - active_mask_t m_warp_mask; - mem_access_byte_mask_t m_byte_mask; - mem_access_sector_mask_t m_sector_mask; + private: + void init(gpgpu_context *ctx); + + unsigned m_uid; + new_addr_type m_addr; // request address + bool m_write; + unsigned m_req_size; // bytes + mem_access_type m_type; + active_mask_t m_warp_mask; + mem_access_byte_mask_t m_byte_mask; + mem_access_sector_mask_t m_sector_mask; }; class mem_fetch; class mem_fetch_interface { -public: - virtual bool full( unsigned size, bool write ) const = 0; - virtual void push( mem_fetch *mf ) = 0; + public: + virtual bool full(unsigned size, bool write) const = 0; + virtual void push(mem_fetch *mf) = 0; }; class mem_fetch_allocator { -public: - virtual mem_fetch *alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const = 0; - virtual mem_fetch *alloc( const class warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle ) const = 0; + public: + virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type, + unsigned size, bool wr, + unsigned long long cycle) const = 0; + virtual mem_fetch *alloc(const class warp_inst_t &inst, + const mem_access_t &access, + unsigned long long cycle) const = 0; }; -// the maximum number of destination, source, or address uarch operands in a instruction -#define MAX_REG_OPERANDS 32 +// the maximum number of destination, source, or address uarch operands in a +// instruction +#define MAX_REG_OPERANDS 32 struct dram_callback_t { - dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } - void (*function)(const class inst_t*, class ptx_thread_info*); + dram_callback_t() { + function = NULL; + instruction = NULL; + thread = NULL; + } + void (*function)(const class inst_t *, class ptx_thread_info *); - const class inst_t* instruction; - class ptx_thread_info *thread; + const class inst_t *instruction; + class ptx_thread_info *thread; }; class inst_t { -public: - inst_t() - { - m_decoded=false; - pc=(address_type)-1; - reconvergence_pc=(address_type)-1; - op=NO_OP; - bar_type=NOT_BAR; - red_type=NOT_RED; - bar_id=(unsigned)-1; - bar_count=(unsigned)-1; - oprnd_type=UN_OP; - sp_op=OTHER_OP; - op_pipe=UNKOWN_OP; - mem_op=NOT_TEX; - num_operands=0; - num_regs=0; - memset(out, 0, sizeof(unsigned)); - memset(in, 0, sizeof(unsigned)); - is_vectorin=0; - is_vectorout=0; - space = memory_space_t(); - cache_op = CACHE_UNDEFINED; - latency = 1; - initiation_interval = 1; - for( unsigned i=0; i < MAX_REG_OPERANDS; i++ ) { - arch_reg.src[i] = -1; - arch_reg.dst[i] = -1; - } - isize=0; + public: + inst_t() { + m_decoded = false; + pc = (address_type)-1; + reconvergence_pc = (address_type)-1; + op = NO_OP; + bar_type = NOT_BAR; + red_type = NOT_RED; + bar_id = (unsigned)-1; + bar_count = (unsigned)-1; + oprnd_type = UN_OP; + sp_op = OTHER_OP; + op_pipe = UNKOWN_OP; + mem_op = NOT_TEX; + num_operands = 0; + num_regs = 0; + memset(out, 0, sizeof(unsigned)); + memset(in, 0, sizeof(unsigned)); + is_vectorin = 0; + is_vectorout = 0; + space = memory_space_t(); + cache_op = CACHE_UNDEFINED; + latency = 1; + initiation_interval = 1; + for (unsigned i = 0; i < MAX_REG_OPERANDS; i++) { + arch_reg.src[i] = -1; + arch_reg.dst[i] = -1; } - bool valid() const { return m_decoded; } - virtual void print_insn( FILE *fp ) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); - } - bool is_load() const { return (op == LOAD_OP ||op==TENSOR_CORE_LOAD_OP || memory_op == memory_load); } - bool is_store() const { return (op == STORE_OP ||op==TENSOR_CORE_STORE_OP || memory_op == memory_store); } - unsigned get_num_operands() const {return num_operands;} - unsigned get_num_regs() const {return num_regs;} - void set_num_regs(unsigned num) {num_regs=num;} - void set_num_operands(unsigned num) {num_operands=num;} - void set_bar_id(unsigned id) {bar_id=id;} - void set_bar_count(unsigned count) {bar_count=count;} + isize = 0; + } + bool valid() const { return m_decoded; } + virtual void print_insn(FILE *fp) const { + fprintf(fp, " [inst @ pc=0x%04x] ", pc); + } + bool is_load() const { + return (op == LOAD_OP || op == TENSOR_CORE_LOAD_OP || + memory_op == memory_load); + } + bool is_store() const { + return (op == STORE_OP || op == TENSOR_CORE_STORE_OP || + memory_op == memory_store); + } + unsigned get_num_operands() const { return num_operands; } + unsigned get_num_regs() const { return num_regs; } + void set_num_regs(unsigned num) { num_regs = num; } + void set_num_operands(unsigned num) { num_operands = num; } + void set_bar_id(unsigned id) { bar_id = id; } + void set_bar_count(unsigned count) { bar_count = count; } - address_type pc; // program counter address of instruction - unsigned isize; // size of instruction in bytes - op_type op; // opcode (uarch visible) + address_type pc; // program counter address of instruction + unsigned isize; // size of instruction in bytes + op_type op; // opcode (uarch visible) - barrier_type bar_type; - reduction_type red_type; - unsigned bar_id; - unsigned bar_count; + barrier_type bar_type; + reduction_type red_type; + unsigned bar_id; + unsigned bar_count; - types_of_operands oprnd_type; // code (uarch visible) identify if the operation is an interger or a floating point - special_ops sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... - operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of the operation (SP, SFU or MEM) - mem_operation mem_op; // code (uarch visible) identify memory type - _memory_op_t memory_op; // memory_op used by ptxplus - unsigned num_operands; - unsigned num_regs; // count vector operand as one register operand + types_of_operands oprnd_type; // code (uarch visible) identify if the + // operation is an interger or a floating point + special_ops + sp_op; // code (uarch visible) identify if int_alu, fp_alu, int_mul .... + operation_pipeline op_pipe; // code (uarch visible) identify the pipeline of + // the operation (SP, SFU or MEM) + mem_operation mem_op; // code (uarch visible) identify memory type + _memory_op_t memory_op; // memory_op used by ptxplus + unsigned num_operands; + unsigned num_regs; // count vector operand as one register operand - address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address - - unsigned out[8]; - unsigned outcount; - unsigned in[24]; - unsigned incount; - unsigned char is_vectorin; - unsigned char is_vectorout; - int pred; // predicate register number - int ar1, ar2; - // register number for bank conflict evaluation - struct { - int dst[MAX_REG_OPERANDS]; - int src[MAX_REG_OPERANDS]; - } arch_reg; - //int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation - unsigned latency; // operation latency - unsigned initiation_interval; + address_type reconvergence_pc; // -1 => not a branch, -2 => use function + // return address - unsigned data_size; // what is the size of the word being operated on? - memory_space_t space; - cache_operator_type cache_op; + unsigned out[8]; + unsigned outcount; + unsigned in[24]; + unsigned incount; + unsigned char is_vectorin; + unsigned char is_vectorout; + int pred; // predicate register number + int ar1, ar2; + // register number for bank conflict evaluation + struct { + int dst[MAX_REG_OPERANDS]; + int src[MAX_REG_OPERANDS]; + } arch_reg; + // int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict + // evaluation + unsigned latency; // operation latency + unsigned initiation_interval; -protected: - bool m_decoded; - virtual void pre_decode() {} -}; + unsigned data_size; // what is the size of the word being operated on? + memory_space_t space; + cache_operator_type cache_op; -enum divergence_support_t { - POST_DOMINATOR = 1, - NUM_SIMD_MODEL + protected: + bool m_decoded; + virtual void pre_decode() {} }; +enum divergence_support_t { POST_DOMINATOR = 1, NUM_SIMD_MODEL }; + const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; -class warp_inst_t: public inst_t { -public: - // constructors - warp_inst_t() - { - m_uid=0; - m_empty=true; - m_config=NULL; - } - warp_inst_t( const core_config *config ) - { - m_uid=0; - assert(config->warp_size<=MAX_WARP_SIZE); - m_config=config; - m_empty=true; - m_isatomic=false; - m_per_scalar_thread_valid=false; - m_mem_accesses_created=false; - m_cache_hit=false; - m_is_printf=false; - m_is_cdp = 0; - } - virtual ~warp_inst_t(){ - } +class warp_inst_t : public inst_t { + public: + // constructors + warp_inst_t() { + m_uid = 0; + m_empty = true; + m_config = NULL; + } + warp_inst_t(const core_config *config) { + m_uid = 0; + assert(config->warp_size <= MAX_WARP_SIZE); + m_config = config; + m_empty = true; + m_isatomic = false; + m_per_scalar_thread_valid = false; + m_mem_accesses_created = false; + m_cache_hit = false; + m_is_printf = false; + m_is_cdp = 0; + } + virtual ~warp_inst_t() {} - // modifiers - void broadcast_barrier_reduction( const active_mask_t& access_mask); - void do_atomic(bool forceDo=false); - void do_atomic( const active_mask_t& access_mask, bool forceDo=false ); - void clear() - { - m_empty=true; - } + // modifiers + void broadcast_barrier_reduction(const active_mask_t &access_mask); + void do_atomic(bool forceDo = false); + void do_atomic(const active_mask_t &access_mask, bool forceDo = false); + void clear() { m_empty = true; } - void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ); + void issue(const active_mask_t &mask, unsigned warp_id, + unsigned long long cycle, int dynamic_warp_id, int sch_id); - const active_mask_t & get_active_mask() const - { - return m_warp_active_mask; - } - void completed( unsigned long long cycle ) const; // stat collection: called when the instruction is completed + const active_mask_t &get_active_mask() const { return m_warp_active_mask; } + void completed(unsigned long long cycle) + const; // stat collection: called when the instruction is completed - void set_addr( unsigned n, new_addr_type addr ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - m_per_scalar_thread[n].memreqaddr[0] = addr; + void set_addr(unsigned n, new_addr_type addr) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - void set_addr( unsigned n, new_addr_type* addr, unsigned num_addrs ) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - } - assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); - for(unsigned i=0; i<num_addrs; i++) - m_per_scalar_thread[n].memreqaddr[i] = addr[i]; + m_per_scalar_thread[n].memreqaddr[0] = addr; + } + void set_addr(unsigned n, new_addr_type *addr, unsigned num_addrs) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; } - void print_m_accessq(){ - - if(accessq_empty()) - return; - else{ - printf("Printing mem access generated\n"); - std::list<mem_access_t>::iterator it; - for (it = m_accessq.begin(); it != m_accessq.end(); ++it){ - printf("MEM_TXN_GEN:%s:%llx, Size:%d \n",mem_access_type_str(it->get_type()), it->get_addr(),it->get_size()); - } - } - } - struct transaction_info { - std::bitset<4> chunks; // bitmask: 32-byte chunks accessed - mem_access_byte_mask_t bytes; - active_mask_t active; // threads in this transaction - - bool test_bytes(unsigned start_bit, unsigned end_bit) { - for( unsigned i=start_bit; i<=end_bit; i++ ) - if(bytes.test(i)) - return true; - return false; - } - }; - - void generate_mem_accesses(); - void memory_coalescing_arch( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ); - void memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ); - - void add_callback( unsigned lane_id, - void (*function)(const class inst_t*, class ptx_thread_info*), - const inst_t *inst, - class ptx_thread_info *thread, - bool atomic) - { - if( !m_per_scalar_thread_valid ) { - m_per_scalar_thread.resize(m_config->warp_size); - m_per_scalar_thread_valid=true; - if(atomic) m_isatomic=true; - } - m_per_scalar_thread[lane_id].callback.function = function; - m_per_scalar_thread[lane_id].callback.instruction = inst; - m_per_scalar_thread[lane_id].callback.thread = thread; + assert(num_addrs <= MAX_ACCESSES_PER_INSN_PER_THREAD); + for (unsigned i = 0; i < num_addrs; i++) + m_per_scalar_thread[n].memreqaddr[i] = addr[i]; + } + void print_m_accessq() { + if (accessq_empty()) + return; + else { + printf("Printing mem access generated\n"); + std::list<mem_access_t>::iterator it; + for (it = m_accessq.begin(); it != m_accessq.end(); ++it) { + printf("MEM_TXN_GEN:%s:%llx, Size:%d \n", + mem_access_type_str(it->get_type()), it->get_addr(), + it->get_size()); + } } - void set_active( const active_mask_t &active ); - - void clear_active( const active_mask_t &inactive ); - void set_not_active( unsigned lane_id ); + } + struct transaction_info { + std::bitset<4> chunks; // bitmask: 32-byte chunks accessed + mem_access_byte_mask_t bytes; + active_mask_t active; // threads in this transaction - // accessors - virtual void print_insn(FILE *fp) const - { - fprintf(fp," [inst @ pc=0x%04x] ", pc ); - for (int i=(int)m_config->warp_size-1; i>=0; i--) - fprintf(fp, "%c", ((m_warp_active_mask[i])?'1':'0') ); - } - bool active( unsigned thread ) const { return m_warp_active_mask.test(thread); } - unsigned active_count() const { return m_warp_active_mask.count(); } - unsigned issued_count() const { assert(m_empty == false); return m_warp_issued_mask.count(); } // for instruction counting - bool empty() const { return m_empty; } - unsigned warp_id() const - { - assert( !m_empty ); - return m_warp_id; - } - unsigned warp_id_func() const // to be used in functional simulations only - { - return m_warp_id; - } - unsigned dynamic_warp_id() const - { - assert( !m_empty ); - return m_dynamic_warp_id; - } - bool has_callback( unsigned n ) const - { - return m_warp_active_mask[n] && m_per_scalar_thread_valid && - (m_per_scalar_thread[n].callback.function!=NULL); - } - new_addr_type get_addr( unsigned n ) const - { - assert( m_per_scalar_thread_valid ); - return m_per_scalar_thread[n].memreqaddr[0]; + bool test_bytes(unsigned start_bit, unsigned end_bit) { + for (unsigned i = start_bit; i <= end_bit; i++) + if (bytes.test(i)) return true; + return false; } + }; - bool isatomic() const { return m_isatomic; } - - unsigned warp_size() const { return m_config->warp_size; } - - bool accessq_empty() const { return m_accessq.empty(); } - unsigned accessq_count() const { return m_accessq.size(); } - const mem_access_t &accessq_back() { return m_accessq.back(); } - void accessq_pop_back() { m_accessq.pop_back(); } + void generate_mem_accesses(); + void memory_coalescing_arch(bool is_write, mem_access_type access_type); + void memory_coalescing_arch_atomic(bool is_write, + mem_access_type access_type); + void memory_coalescing_arch_reduce_and_send(bool is_write, + mem_access_type access_type, + const transaction_info &info, + new_addr_type addr, + unsigned segment_size); - bool dispatch_delay() - { - if( cycles > 0 ) - cycles--; - return cycles > 0; + void add_callback(unsigned lane_id, + void (*function)(const class inst_t *, + class ptx_thread_info *), + const inst_t *inst, class ptx_thread_info *thread, + bool atomic) { + if (!m_per_scalar_thread_valid) { + m_per_scalar_thread.resize(m_config->warp_size); + m_per_scalar_thread_valid = true; + if (atomic) m_isatomic = true; } + m_per_scalar_thread[lane_id].callback.function = function; + m_per_scalar_thread[lane_id].callback.instruction = inst; + m_per_scalar_thread[lane_id].callback.thread = thread; + } + void set_active(const active_mask_t &active); - bool has_dispatch_delay(){ - return cycles > 0; - } + void clear_active(const active_mask_t &inactive); + void set_not_active(unsigned lane_id); - void print( FILE *fout ) const; - unsigned get_uid() const { return m_uid; } - unsigned get_schd_id() const { return m_scheduler_id; } + // accessors + virtual void print_insn(FILE *fp) const { + fprintf(fp, " [inst @ pc=0x%04x] ", pc); + for (int i = (int)m_config->warp_size - 1; i >= 0; i--) + fprintf(fp, "%c", ((m_warp_active_mask[i]) ? '1' : '0')); + } + bool active(unsigned thread) const { return m_warp_active_mask.test(thread); } + unsigned active_count() const { return m_warp_active_mask.count(); } + unsigned issued_count() const { + assert(m_empty == false); + return m_warp_issued_mask.count(); + } // for instruction counting + bool empty() const { return m_empty; } + unsigned warp_id() const { + assert(!m_empty); + return m_warp_id; + } + unsigned warp_id_func() const // to be used in functional simulations only + { + return m_warp_id; + } + unsigned dynamic_warp_id() const { + assert(!m_empty); + return m_dynamic_warp_id; + } + bool has_callback(unsigned n) const { + return m_warp_active_mask[n] && m_per_scalar_thread_valid && + (m_per_scalar_thread[n].callback.function != NULL); + } + new_addr_type get_addr(unsigned n) const { + assert(m_per_scalar_thread_valid); + return m_per_scalar_thread[n].memreqaddr[0]; + } -protected: + bool isatomic() const { return m_isatomic; } - unsigned m_uid; - bool m_empty; - bool m_cache_hit; - unsigned long long issue_cycle; - unsigned cycles; // used for implementing initiation interval delay - bool m_isatomic; - bool m_is_printf; - unsigned m_warp_id; - unsigned m_dynamic_warp_id; - const core_config *m_config; - active_mask_t m_warp_active_mask; // dynamic active mask for timing model (after predication) - active_mask_t m_warp_issued_mask; // active mask at issue (prior to predication test) -- for instruction counting + unsigned warp_size() const { return m_config->warp_size; } - struct per_thread_info { - per_thread_info() { - for(unsigned i=0; i<MAX_ACCESSES_PER_INSN_PER_THREAD; i++) - memreqaddr[i] = 0; - } - dram_callback_t callback; - new_addr_type memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; // effective address, upto 8 different requests (to support 32B access in 8 chunks of 4B each) - }; - bool m_per_scalar_thread_valid; - std::vector<per_thread_info> m_per_scalar_thread; - bool m_mem_accesses_created; - std::list<mem_access_t> m_accessq; + bool accessq_empty() const { return m_accessq.empty(); } + unsigned accessq_count() const { return m_accessq.size(); } + const mem_access_t &accessq_back() { return m_accessq.back(); } + void accessq_pop_back() { m_accessq.pop_back(); } - unsigned m_scheduler_id; //the scheduler that issues this inst + bool dispatch_delay() { + if (cycles > 0) cycles--; + return cycles > 0; + } - //Jin: cdp support -public: - int m_is_cdp; - -}; + bool has_dispatch_delay() { return cycles > 0; } -void move_warp( warp_inst_t *&dst, warp_inst_t *&src ); + void print(FILE *fout) const; + unsigned get_uid() const { return m_uid; } + unsigned get_schd_id() const { return m_scheduler_id; } -size_t get_kernel_code_size( class function_info *entry ); -class checkpoint -{ -public: + protected: + unsigned m_uid; + bool m_empty; + bool m_cache_hit; + unsigned long long issue_cycle; + unsigned cycles; // used for implementing initiation interval delay + bool m_isatomic; + bool m_is_printf; + unsigned m_warp_id; + unsigned m_dynamic_warp_id; + const core_config *m_config; + active_mask_t m_warp_active_mask; // dynamic active mask for timing model + // (after predication) + active_mask_t + m_warp_issued_mask; // active mask at issue (prior to predication test) + // -- for instruction counting - checkpoint(); - ~checkpoint(){ - printf("clasfsfss destructed\n"); + struct per_thread_info { + per_thread_info() { + for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) + memreqaddr[i] = 0; } + dram_callback_t callback; + new_addr_type + memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; // effective address, + // upto 8 different + // requests (to support + // 32B access in 8 chunks + // of 4B each) + }; + bool m_per_scalar_thread_valid; + std::vector<per_thread_info> m_per_scalar_thread; + bool m_mem_accesses_created; + std::list<mem_access_t> m_accessq; - void load_global_mem(class memory_space *temp_mem, char * f1name); - void store_global_mem(class memory_space *mem, char * fname , char * format); - unsigned radnom; + unsigned m_scheduler_id; // the scheduler that issues this inst + // Jin: cdp support + public: + int m_is_cdp; +}; + +void move_warp(warp_inst_t *&dst, warp_inst_t *&src); + +size_t get_kernel_code_size(class function_info *entry); +class checkpoint { + public: + checkpoint(); + ~checkpoint() { printf("clasfsfss destructed\n"); } + void load_global_mem(class memory_space *temp_mem, char *f1name); + void store_global_mem(class memory_space *mem, char *fname, char *format); + unsigned radnom; }; /* - * This abstract class used as a base for functional and performance and simulation, it has basic functional simulation - * data structures and procedures. + * This abstract class used as a base for functional and performance and + * simulation, it has basic functional simulation data structures and + * procedures. */ class core_t { - public: - core_t( gpgpu_sim *gpu, - kernel_info_t *kernel, - unsigned warp_size, - unsigned threads_per_shader ) - : m_gpu( gpu ), - m_kernel( kernel ), - m_simt_stack( NULL ), - m_thread( NULL ), - m_warp_size( warp_size ) - { - m_warp_count = threads_per_shader/m_warp_size; - // Handle the case where the number of threads is not a - // multiple of the warp size - if ( threads_per_shader % m_warp_size != 0 ) { - m_warp_count += 1; - } - assert( m_warp_count * m_warp_size > 0 ); - m_thread = ( ptx_thread_info** ) - calloc( m_warp_count * m_warp_size, - sizeof( ptx_thread_info* ) ); - initilizeSIMTStack(m_warp_count,m_warp_size); + public: + core_t(gpgpu_sim *gpu, kernel_info_t *kernel, unsigned warp_size, + unsigned threads_per_shader) + : m_gpu(gpu), + m_kernel(kernel), + m_simt_stack(NULL), + m_thread(NULL), + m_warp_size(warp_size) { + m_warp_count = threads_per_shader / m_warp_size; + // Handle the case where the number of threads is not a + // multiple of the warp size + if (threads_per_shader % m_warp_size != 0) { + m_warp_count += 1; + } + assert(m_warp_count * m_warp_size > 0); + m_thread = (ptx_thread_info **)calloc(m_warp_count * m_warp_size, + sizeof(ptx_thread_info *)); + initilizeSIMTStack(m_warp_count, m_warp_size); - for(unsigned i=0; i<MAX_CTA_PER_SHADER; i++){ - for(unsigned j=0; j<MAX_BARRIERS_PER_CTA; j++){ - reduction_storage[i][j]=0; - } - } + for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) { + for (unsigned j = 0; j < MAX_BARRIERS_PER_CTA; j++) { + reduction_storage[i][j] = 0; + } + } + } + virtual ~core_t() { free(m_thread); } + virtual void warp_exit(unsigned warp_id) = 0; + virtual bool warp_waiting_at_barrier(unsigned warp_id) const = 0; + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) = 0; + class gpgpu_sim *get_gpu() { + return m_gpu; + } + void execute_warp_inst_t(warp_inst_t &inst, unsigned warpId = (unsigned)-1); + bool ptx_thread_done(unsigned hw_thread_id) const; + void updateSIMTStack(unsigned warpId, warp_inst_t *inst); + void initilizeSIMTStack(unsigned warp_count, unsigned warps_size); + void deleteSIMTStack(); + warp_inst_t getExecuteWarp(unsigned warpId); + void get_pdom_stack_top_info(unsigned warpId, unsigned *pc, + unsigned *rpc) const; + kernel_info_t *get_kernel_info() { return m_kernel; } + class ptx_thread_info **get_thread_info() { + return m_thread; + } + unsigned get_warp_size() const { return m_warp_size; } + void and_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] &= value; + } + void or_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] |= value; + } + void popc_reduction(unsigned ctaid, unsigned barid, bool value) { + reduction_storage[ctaid][barid] += value; + } + unsigned get_reduction_value(unsigned ctaid, unsigned barid) { + return reduction_storage[ctaid][barid]; + } - } - virtual ~core_t() { free(m_thread); } - virtual void warp_exit( unsigned warp_id ) = 0; - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const = 0; - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid)=0; - class gpgpu_sim * get_gpu() {return m_gpu;} - void execute_warp_inst_t(warp_inst_t &inst, unsigned warpId =(unsigned)-1); - bool ptx_thread_done( unsigned hw_thread_id ) const ; - void updateSIMTStack(unsigned warpId, warp_inst_t * inst); - void initilizeSIMTStack(unsigned warp_count, unsigned warps_size); - void deleteSIMTStack(); - warp_inst_t getExecuteWarp(unsigned warpId); - void get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const; - kernel_info_t * get_kernel_info(){ return m_kernel;} - class ptx_thread_info ** get_thread_info() { return m_thread; } - unsigned get_warp_size() const { return m_warp_size; } - void and_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] &= value; } - void or_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] |= value; } - void popc_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] += value;} - unsigned get_reduction_value(unsigned ctaid, unsigned barid) {return reduction_storage[ctaid][barid];} - protected: - class gpgpu_sim *m_gpu; - kernel_info_t *m_kernel; - simt_stack **m_simt_stack; // pdom based reconvergence context for each warp - class ptx_thread_info ** m_thread; - unsigned m_warp_size; - unsigned m_warp_count; - unsigned reduction_storage[MAX_CTA_PER_SHADER][MAX_BARRIERS_PER_CTA]; + protected: + class gpgpu_sim *m_gpu; + kernel_info_t *m_kernel; + simt_stack **m_simt_stack; // pdom based reconvergence context for each warp + class ptx_thread_info **m_thread; + unsigned m_warp_size; + unsigned m_warp_count; + unsigned reduction_storage[MAX_CTA_PER_SHADER][MAX_BARRIERS_PER_CTA]; }; - -//register that can hold multiple instructions. +// register that can hold multiple instructions. class register_set { -public: - register_set(unsigned num, const char* name){ - for( unsigned i = 0; i < num; i++ ) { - regs.push_back(new warp_inst_t()); - } - m_name = name; - } - bool has_free(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( regs[i]->empty() ) { - return true; - } - } - return false; - } - bool has_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return has_free(); + public: + register_set(unsigned num, const char *name) { + for (unsigned i = 0; i < num; i++) { + regs.push_back(new warp_inst_t()); + } + m_name = name; + } + bool has_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return true; + } + } + return false; + } + bool has_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return has_free(); - assert(reg_id < regs.size()); - return regs[reg_id]->empty(); - } - bool has_ready(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - return true; - } - } - return false; - } + assert(reg_id < regs.size()); + return regs[reg_id]->empty(); + } + bool has_ready() { + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + return true; + } + } + return false; + } - void move_in( warp_inst_t *&src ){ - warp_inst_t** free = get_free(); - move_warp(*free, src); - } - //void copy_in( warp_inst_t* src ){ - // src->copy_contents_to(*get_free()); - //} - void move_out_to( warp_inst_t *&dest ){ - warp_inst_t **ready=get_ready(); - move_warp(dest, *ready); - } + void move_in(warp_inst_t *&src) { + warp_inst_t **free = get_free(); + move_warp(*free, src); + } + // void copy_in( warp_inst_t* src ){ + // src->copy_contents_to(*get_free()); + //} + void move_out_to(warp_inst_t *&dest) { + warp_inst_t **ready = get_ready(); + move_warp(dest, *ready); + } - warp_inst_t** get_ready(){ - warp_inst_t** ready; - ready = NULL; - for( unsigned i = 0; i < regs.size(); i++ ) { - if( not regs[i]->empty() ) { - if( ready and (*ready)->get_uid() < regs[i]->get_uid() ) { - // ready is oldest - } else { - ready = ®s[i]; - } - } - } - return ready; - } + warp_inst_t **get_ready() { + warp_inst_t **ready; + ready = NULL; + for (unsigned i = 0; i < regs.size(); i++) { + if (not regs[i]->empty()) { + if (ready and (*ready)->get_uid() < regs[i]->get_uid()) { + // ready is oldest + } else { + ready = ®s[i]; + } + } + } + return ready; + } - void print(FILE* fp) const{ - fprintf(fp, "%s : @%p\n", m_name, this); - for( unsigned i = 0; i < regs.size(); i++ ) { - fprintf(fp, " "); - regs[i]->print(fp); - fprintf(fp, "\n"); - } - } + void print(FILE *fp) const { + fprintf(fp, "%s : @%p\n", m_name, this); + for (unsigned i = 0; i < regs.size(); i++) { + fprintf(fp, " "); + regs[i]->print(fp); + fprintf(fp, "\n"); + } + } - warp_inst_t ** get_free(){ - for( unsigned i = 0; i < regs.size(); i++ ) { - if( regs[i]->empty() ) { - return ®s[i]; - } - } - assert(0 && "No free registers found"); - return NULL; - } + warp_inst_t **get_free() { + for (unsigned i = 0; i < regs.size(); i++) { + if (regs[i]->empty()) { + return ®s[i]; + } + } + assert(0 && "No free registers found"); + return NULL; + } - warp_inst_t ** get_free(bool sub_core_model, unsigned reg_id){ - //in subcore model, each sched has a one specific reg to use (based on sched id) - if(!sub_core_model) - return get_free(); + warp_inst_t **get_free(bool sub_core_model, unsigned reg_id) { + // in subcore model, each sched has a one specific reg to use (based on + // sched id) + if (!sub_core_model) return get_free(); - assert(reg_id < regs.size()); - if( regs[reg_id]->empty() ) { - return ®s[reg_id]; - } - assert(0 && "No free register found"); - return NULL; - } + assert(reg_id < regs.size()); + if (regs[reg_id]->empty()) { + return ®s[reg_id]; + } + assert(0 && "No free register found"); + return NULL; + } - unsigned get_size(){ - return regs.size(); - } + unsigned get_size() { return regs.size(); } -private: - std::vector<warp_inst_t*> regs; - const char* m_name; + private: + std::vector<warp_inst_t *> regs; + const char *m_name; }; -#endif // #ifdef __cplusplus +#endif // #ifdef __cplusplus -#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED +#endif // #ifndef ABSTRACT_HARDWARE_MODEL_INCLUDED diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 9a5468c..67539d4 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,61 +10,60 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ - #ifndef CUDA_MATH #define CUDA_MATH @@ -74,32 +73,31 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN - struct int4 { - int x, y, z, w; - }; - struct uint4 { - unsigned int x, y, z, w; - }; - struct float4 { - float x, y, z, w; - }; - struct float2 { - float x, y; - }; - +struct int4 { + int x, y, z, w; +}; +struct uint4 { + unsigned int x, y, z, w; +}; +struct float4 { + float x, y, z, w; +}; +struct float2 { + float x, y; +}; // DEVICE_BUILTIN - typedef struct int4 int4; - typedef struct uint4 uint4; - typedef struct float4 float4; - typedef struct float2 float2; +typedef struct int4 int4; +typedef struct uint4 uint4; +typedef struct float4 float4; +typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include <device_types.h> @@ -108,38 +106,36 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -147,205 +143,211 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include <device_types.h> #include <fenv.h> - // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; } -int __internal_float2int(float a, enum cudaRoundMode mode) -{ - return float2int(a, mode); +int __internal_float2int(float a, enum cudaRoundMode mode) { + return float2int(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - unsigned int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + unsigned int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; } -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) -{ - return float2uint(a, mode); +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { + return float2uint(a, mode); } -// intrinsic for division -float fdividef(float a, float b) -{ - return (a / b); -} +// intrinsic for division +float fdividef(float a, float b) { return (a / b); } -float __internal_accurate_fdividef(float a, float b) -{ - return fdividef(a, b); -} +float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) -{ - float b; - if (std::isnan(a)) b = 0.0f; - else if (a >= 1.0f) b = 1.0f; - else if (a <= 0.0f) b = 0.0f; - else b = a; - return b; +float __saturatef(float a) { + float b; + if (std::isnan(a)) + b = 0.0f; + else if (a >= 1.0f) + b = 1.0f; + else if (a <= 0.0f) + b = 0.0f; + else + b = a; + return b; } -// intrinsic for power -float __powf(float a, float b) -{ - return powf(a, b); -} +// intrinsic for power +float __powf(float a, float b) { return powf(a, b); } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) -{ - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) { + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -355,14 +357,11 @@ int __signbitd(double d) #endif -} +} // namespace cuda_math // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) -{ - return (std::isnan(a)); -} -#endif +int isnanf(float a) { return (std::isnan(a)); } +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 7a130ea..39e2b7e 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,283 +8,340 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda-sim.h" #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "ptx_sim.h" +typedef void *yyscan_t; #include <stdio.h> -#include <sstream> -#include "opcodes.h" -#include "../statwrapper.h" -#include <set> #include <map> +#include <set> +#include <sstream> +#include "../../libcuda/gpgpu_context.h" #include "../abstract_hardware_model.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "../gpgpusim_entrypoint.h" +#include "../statwrapper.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "decuda_pred_table/decuda_pred_table.h" #include "memory.h" +#include "opcodes.h" #include "ptx-stats.h" +#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" -#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "../gpgpusim_entrypoint.h" -#include "decuda_pred_table/decuda_pred_table.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" int g_debug_execution = 0; // Output debug information to file options - -void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, - "Opcode latencies for integers <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, - "Opcode latencies for single precision floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, - "Opcode latencies for double precision floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, - "Opcode initiation intervals for single precision floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, - "Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>" - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency <cudaStreamCreateWithFlags, \ +void cuda_sim::ptx_opcocde_latency_options(option_parser_t opp) { + option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, + &opcode_latency_int, + "Opcode latencies for integers <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, + &opcode_latency_fp, + "Opcode latencies for single precision floating " + "points <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, + &opcode_latency_dp, + "Opcode latencies for double precision floating " + "points <ADD,MAX,MUL,MAD,DIV>" + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, + &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, + &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register( + opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, + &opcode_initiation_fp, + "Opcode initiation intervals for single precision " + "floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, + &opcode_initiation_dp, + "Opcode initiation intervals for double precision " + "floating points <ADD,MAX,MUL,MAD,DIV>" + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, + &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, + &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency <cudaStreamCreateWithFlags, \ cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \ cudaLaunchDeviceV2_init_perWarp, cudaLaunchDevicV2_perKernel>" - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); } -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) -{ - std::string texname(name); - if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ - m_NameToTextureRef[texname] = std::set<const struct textureReference*>(); - }else{ - const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); - assert(tr!=NULL); - //asserts that all texrefs in set have same fields - assert(tr->normalized==texref->normalized&& - tr->filterMode==texref->filterMode&& - tr->addressMode[0]==texref->addressMode[0]&& - tr->addressMode[1]==texref->addressMode[1]&& - tr->addressMode[2]==texref->addressMode[2]&& - tr->channelDesc.x==texref->channelDesc.x&& - tr->channelDesc.y==texref->channelDesc.y&& - tr->channelDesc.z==texref->channelDesc.z&& - tr->channelDesc.w==texref->channelDesc.w&& - tr->channelDesc.f==texref->channelDesc.f - ); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( + const char *name, const struct textureReference *texref, int dim, + int readmode, int ext) { + std::string texname(name); + if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { + m_NameToTextureRef[texname] = std::set<const struct textureReference *>(); + } else { + const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); + assert(tr != NULL); + // asserts that all texrefs in set have same fields + assert(tr->normalized == texref->normalized && + tr->filterMode == texref->filterMode && + tr->addressMode[0] == texref->addressMode[0] && + tr->addressMode[1] == texref->addressMode[1] && + tr->addressMode[2] == texref->addressMode[2] && + tr->channelDesc.x == texref->channelDesc.x && + tr->channelDesc.y == texref->channelDesc.y && + tr->channelDesc.z == texref->channelDesc.z && + tr->channelDesc.w == texref->channelDesc.w && + tr->channelDesc.f == texref->channelDesc.f); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr( + texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; } -const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) -{ - std::map<const struct textureReference*, std::string>::const_iterator t=m_TextureRefToName.find(texref); - assert( t != m_TextureRefToName.end() ); - return t->second.c_str(); +const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref) { + std::map<const struct textureReference *, std::string>::const_iterator t = + m_TextureRefToName.find(texref); + assert(t != m_TextureRefToName.end()); + return t->second.c_str(); } -unsigned int intLOGB2( unsigned int v ) { - unsigned int shift; - unsigned int r; +unsigned int intLOGB2(unsigned int v) { + unsigned int shift; + unsigned int r; - r = 0; + r = 0; - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; - return r; + return r; } -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) -{ - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( + const struct textureReference *texref, const struct cudaArray *array) { + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); - //check that there's nothing there first - if(t != m_NameToCudaArray.end()){ - printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits/8; - unsigned int Tx, Ty; - int r; + std::map<std::string, const struct cudaArray *>::const_iterator t = + m_NameToCudaArray.find(texname); + // check that there's nothing there first + if (t != m_NameToCudaArray.end()) { + printf( + "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " + "was previously bound.\nImplicitly unbinding texref associated to %s " + "first\n", + texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = + array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits / 8; + unsigned int Tx, Ty; + int r; - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); - //first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: Tx = 4; break; - case 32: Tx = 8; break; - case 64: Tx = 8; break; - case 128: Tx = 16; break; - case 256: Tx = 16; break; - default: - printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", + m_function_model_config.get_texcache_linesize()); + // first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: + Tx = 4; + break; + case 32: + Tx = 8; + break; + case 64: + Tx = 8; + break; + case 128: + Tx = 16; + break; + case 256: + Tx = 16; + break; + default: + printf( + "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", + m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - //modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - //by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); + } + r = texel_size >> 2; + // modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + // by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); - printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); - printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; + printf( + "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", + Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", + texel_size, intLOGB2(texel_size)); + printf( + "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", + array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo *texInfo = + (struct textureInfo *)malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; } -void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) -{ - //assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); +void gpgpu_t::gpgpu_ptx_sim_unbindTexture( + const struct textureReference *texref) { + // assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() -{ - if( m_assembled ) { - return; - } +void function_info::ptx_assemble() { + if (m_assembled) { + return; + } - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); - m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); + m_instr_mem = new ptx_instruction *[m_instr_mem_size]; - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); - fflush(stdout); - std::list<ptx_instruction*>::iterator i; + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", + m_name.c_str()); + fflush(stdout); + std::list<ptx_instruction *>::iterator i; - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) - // start function on an aligned address - for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - PC += PC%MAX_INST_SIZE; - m_start_PC = PC; + addr_t PC = + gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address + // (across functions) + // start function on an aligned address + for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + PC += PC % MAX_INST_SIZE; + m_start_PC = PC; - addr_t n=0; // offset in m_instr_mem - //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. - //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); - for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if ( pI->is_label() ) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert( pI->inst_size() <= MAX_INST_SIZE ); - for( unsigned i=1; i < pI->inst_size(); i++ ) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - m_instr_mem[n+i]=NULL; - } - n += pI->inst_size(); - PC += pI->inst_size(); + addr_t n = 0; // offset in m_instr_mem + // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? + // reserve is cumulative. s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + + // MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); + for (i = m_instructions.begin(); i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert(pI->inst_size() <= MAX_INST_SIZE); + for (unsigned i = 1; i < pI->inst_size(); i++) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + m_instr_mem[n + i] = NULL; } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; - for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); //get operand, e.g. target name - if ( labels.find(target.name()) == labels.end() ) { - printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", - pI->source_file(),pI->source_line(), target.name().c_str() ); - abort(); - } - unsigned index = labels[ target.name() ]; //determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address( target.get_symbol(), PC ); - target.set_type(label_t); + n += pI->inst_size(); + PC += pI->inst_size(); + } + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; + for (unsigned ii = 0; ii < n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || + pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); // get operand, e.g. target name + if (labels.find(target.name()) == labels.end()) { + printf( + "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " + "appear in assembly code.", + pI->source_file(), pI->source_line(), target.name().c_str()); + abort(); } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + unsigned index = labels[target.name()]; // determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address(target.get_symbol(), PC); + target.set_type(label_t); + } + } + m_n = n; + printf(" done.\n"); + fflush(stdout); - //disable pdom analysis here and do it at runtime + // disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -323,2247 +380,2445 @@ void function_info::ptx_assemble() #endif } -addr_t shared_to_generic( unsigned smid, addr_t addr ) -{ - assert( addr < SHARED_MEM_SIZE_MAX ); - return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; +addr_t shared_to_generic(unsigned smid, addr_t addr) { + assert(addr < SHARED_MEM_SIZE_MAX); + return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; } -addr_t global_to_generic( addr_t addr ) -{ - return addr; -} - -bool isspace_shared( unsigned smid, addr_t addr ) -{ - addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; - if( (addr >= end) || (addr < start) ) - return false; - return true; -} +addr_t global_to_generic(addr_t addr) { return addr; } -bool isspace_global( addr_t addr ) -{ - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +bool isspace_shared(unsigned smid, addr_t addr) { + addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; + if ((addr >= end) || (addr < start)) return false; + return true; } -memory_space_t whichspace( addr_t addr ) -{ - if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { - return global_space; - } else if( addr >= SHARED_GENERIC_START ) { - return shared_space; - } else { - return local_space; - } +bool isspace_global(addr_t addr) { + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); } -addr_t generic_to_shared( unsigned smid, addr_t addr ) -{ - assert(isspace_shared(smid,addr)); - return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); +memory_space_t whichspace(addr_t addr) { + if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { + return global_space; + } else if (addr >= SHARED_GENERIC_START) { + return shared_space; + } else { + return local_space; + } } -addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +addr_t generic_to_shared(unsigned smid, addr_t addr) { + assert(isspace_shared(smid, addr)); + return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); } -bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); - if( (addr >= end) || (addr < start) ) - return false; - return true; +addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; } -addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(isspace_local(smid,hwtid,addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); +bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); + if ((addr >= end) || (addr < start)) return false; + return true; } -addr_t generic_to_global( addr_t addr ) -{ - return addr; +addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { + assert(isspace_local(smid, hwtid, addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid)); } +addr_t generic_to_global(addr_t addr) { return addr; } -void* gpgpu_t::gpu_malloc( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; +void *gpgpu_t::gpu_malloc(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; } -void* gpgpu_t::gpu_mallocarray( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; +void *gpgpu_t::gpu_mallocarray(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; } +void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, + size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", + count, (unsigned long long)src, (unsigned long long)dst_start_addr); + fflush(stdout); + } + char *src_data = (char *)src; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); -void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - char *src_data = (char*)src; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } } -void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char*)dst; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->read(src_start_addr+n,1,dst_data+n); +void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", + count, (unsigned long long)src_start_addr, (unsigned long long)dst); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char *)dst; + for (unsigned n = 0; n < count; n++) + m_global_mem->read(src_start_addr + n, 1, dst_data + n); - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } } -void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, - (unsigned long long) src, (unsigned long long) dst ); - fflush(stdout); - } - for (unsigned n=0; n < count; n ++ ) { - unsigned char tmp; - m_global_mem->read(src+n,1,&tmp); - m_global_mem->write(dst+n,1, &tmp,NULL,NULL); - } - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } +void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", + count, (unsigned long long)src, (unsigned long long)dst); + fflush(stdout); + } + for (unsigned n = 0; n < count; n++) { + unsigned char tmp; + m_global_mem->read(src + n, 1, &tmp); + m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); + } + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } } -void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", - count, (unsigned char) c, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } +void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " + "0x%Lx... ", + count, (unsigned char)c, (unsigned long long)dst_start_addr); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } } -void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) -{ - std::map<unsigned,function_info*>::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - fprintf(fp,"<no instruction at address 0x%x>", pc ); - return; - } - function_info *finfo = f->second; - assert( finfo ); - finfo->print_insn(pc,fp); +void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { + std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { + fprintf(fp, "<no instruction at address 0x%x>", pc); + return; + } + function_info *finfo = f->second; + assert(finfo); + finfo->print_insn(pc, fp); } -std::string cuda_sim::ptx_get_insn_str( address_type pc ) -{ - std::map<unsigned,function_info*>::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - #define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE,"<no instruction at address 0x%x>", pc ); - return std::string(buff); - } - function_info *finfo = f->second; - assert( finfo ); - return finfo->get_insn_str(pc); +std::string cuda_sim::ptx_get_insn_str(address_type pc) { + std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { +#define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "<no instruction at address 0x%x>", pc); + return std::string(buff); + } + function_info *finfo = f->second; + assert(finfo); + return finfo->get_insn_str(pc); } -void ptx_instruction::set_fp_or_int_archop(){ - oprnd_type=UN_OP; - if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ - // do nothing - }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ - if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; +void ptx_instruction::set_fp_or_int_archop() { + oprnd_type = UN_OP; + if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || + (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || + (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || + (m_opcode == CALL_OP)) { + // do nothing + } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || + m_opcode == SLCT_OP)) { + if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || + get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; - }else{ - if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - } + } else { + if (get_type() == F16_TYPE || get_type() == F32_TYPE || + get_type() == F64_TYPE || get_type() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + } } -void ptx_instruction::set_mul_div_or_other_archop(){ - sp_op=OTHER_OP; - if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ - if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - switch(get_opcode()){ - case MUL_OP: - case MAD_OP: - sp_op=FP_MUL_OP; - break; - case DIV_OP: - sp_op=FP_DIV_OP; - break; - case LG2_OP: - sp_op=FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op=FP_SQRT_OP; - break; - case RCP_OP: - sp_op=FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op=FP_SIN_OP; - break; - case EX2_OP: - sp_op=FP_EXP_OP; - break; - default: - if((op==ALU_OP)||(op==TENSOR_CORE_OP)) - sp_op=FP__OP; - break; - - } - }else { - switch(get_opcode()){ - case MUL24_OP: - case MAD24_OP: - sp_op=INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) - sp_op=INT_MUL32_OP; - else - sp_op=INT_MUL_OP; - break; - case DIV_OP: - sp_op=INT_DIV_OP; - break; - default: - if((op==ALU_OP)) - sp_op=INT__OP; - break; - } - } - } - +void ptx_instruction::set_mul_div_or_other_archop() { + sp_op = OTHER_OP; + if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && + (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && + (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && + (m_opcode != CALL_OP)) { + if (get_type() == F32_TYPE || get_type() == F64_TYPE || + get_type() == FF64_TYPE) { + switch (get_opcode()) { + case MUL_OP: + case MAD_OP: + sp_op = FP_MUL_OP; + break; + case DIV_OP: + sp_op = FP_DIV_OP; + break; + case LG2_OP: + sp_op = FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op = FP_SQRT_OP; + break; + case RCP_OP: + sp_op = FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op = FP_SIN_OP; + break; + case EX2_OP: + sp_op = FP_EXP_OP; + break; + default: + if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; + break; + } + } else { + switch (get_opcode()) { + case MUL24_OP: + case MAD24_OP: + sp_op = INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if (get_type() == U32_TYPE || get_type() == S32_TYPE || + get_type() == B32_TYPE) + sp_op = INT_MUL32_OP; + else + sp_op = INT_MUL_OP; + break; + case DIV_OP: + sp_op = INT_DIV_OP; + break; + default: + if ((op == ALU_OP)) sp_op = INT__OP; + break; + } + } + } } - - -void ptx_instruction::set_bar_type() -{ - if(m_opcode==BAR_OP) { - switch(m_barrier_op){ - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch(m_atomic_spec){ - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } - else if(m_opcode==SST_OP) { - bar_type = SYNC; - } +void ptx_instruction::set_bar_type() { + if (m_opcode == BAR_OP) { + switch (m_barrier_op) { + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch (m_atomic_spec) { + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } else if (m_opcode == SST_OP) { + bar_type = SYNC; + } } +void ptx_instruction::set_opcode_and_latency() { + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], + &int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], + &fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], + &dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); -void ptx_instruction::set_opcode_and_latency() -{ - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0],&int_latency[1],&int_latency[2], - &int_latency[3],&int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0],&fp_latency[1],&fp_latency[2], - &fp_latency[3],&fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0],&dp_latency[1],&dp_latency[2], - &dp_latency[3],&dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", - &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", - &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0],&int_init[1],&int_init[2], - &int_init[3],&int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0],&fp_init[1],&fp_init[2], - &fp_init[3],&fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0],&dp_init[1],&dp_init[2], - &dp_init[3],&dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", - &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", - &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - - if(!m_operands.empty()){ - std::vector<operand_info>::iterator it; - for(it=++m_operands.begin();it!=m_operands.end();it++){ - num_operands++; - if((it->is_reg() || it->is_vector())){ - num_regs++; - } - } - } - op = ALU_OP; - mem_op= NOT_TEX; - initiation_interval = latency = 1; - switch( m_opcode ) { - case MOV_OP: - assert( !(has_memory_read() && has_memory_write()) ); - if ( has_memory_read() ) op = LOAD_OP; - if ( has_memory_write() ) op = STORE_OP; - break; - case LD_OP: op = LOAD_OP; break; - case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; - case LDU_OP: op = LOAD_OP; break; - case ST_OP: op = STORE_OP; break; - case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; - case BRA_OP: op = BRANCH_OP; break; - case BREAKADDR_OP: op = BRANCH_OP; break; - case TEX_OP: op = LOAD_OP; mem_op=TEX; break; - case ATOM_OP: op = LOAD_OP; break; - case BAR_OP: op = BARRIER_OP; break; - case SST_OP: op = BARRIER_OP; break; - case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; - case CALL_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case CALLP_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case RET_OP: case RETP_OP: op = RET_OPS;break; - case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: - //ADD,SUB latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: case MIN_OP: - //MAX,MIN latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - //MUL latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: case MADC_OP: case MADP_OP: - //MAD latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; + if (!m_operands.empty()) { + std::vector<operand_info>::iterator it; + for (it = ++m_operands.begin(); it != m_operands.end(); it++) { + num_operands++; + if ((it->is_reg() || it->is_vector())) { + num_regs++; + } + } + } + op = ALU_OP; + mem_op = NOT_TEX; + initiation_interval = latency = 1; + switch (m_opcode) { + case MOV_OP: + assert(!(has_memory_read() && has_memory_write())); + if (has_memory_read()) op = LOAD_OP; + if (has_memory_write()) op = STORE_OP; + break; + case LD_OP: + op = LOAD_OP; + break; + case MMA_LD_OP: + op = TENSOR_CORE_LOAD_OP; + break; + case LDU_OP: + op = LOAD_OP; + break; + case ST_OP: + op = STORE_OP; + break; + case MMA_ST_OP: + op = TENSOR_CORE_STORE_OP; + break; + case BRA_OP: + op = BRANCH_OP; + break; + case BREAKADDR_OP: + op = BRANCH_OP; + break; + case TEX_OP: + op = LOAD_OP; + mem_op = TEX; + break; + case ATOM_OP: + op = LOAD_OP; + break; + case BAR_OP: + op = BARRIER_OP; + break; + case SST_OP: + op = BARRIER_OP; + break; + case MEMBAR_OP: + op = MEMORY_BARRIER_OP; + break; + case CALL_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case CALLP_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case RET_OP: + case RETP_OP: + op = RET_OPS; + break; + case ADD_OP: + case ADDP_OP: + case ADDC_OP: + case SUB_OP: + case SUBC_OP: + // ADD,SUB latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: + case MIN_OP: + // MAX,MIN latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + // MUL latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: + case MADC_OP: + case MADP_OP: + // MAD latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only op = SFU_OP; + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op=TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); - + case SQRT_OP: + case SIN_OP: + case COS_OP: + case EX2_OP: + case LG2_OP: + case RSQRT_OP: + case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; + op = SFU_OP; + break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op = TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); } -void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const -{ - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t&)*pI; - assert( inst.valid() ); +void ptx_thread_info::ptx_fetch_inst(inst_t &inst) const { + addr_t pc = get_pc(); + const ptx_instruction *pI = m_func_info->get_instruction(pc); + inst = (const inst_t &)*pI; + assert(inst.valid()); } -static unsigned datatype2size( unsigned data_type ) -{ - unsigned data_size; - switch ( data_type ) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; break; - case BB128_TYPE: - data_size = 16; break; - default: assert(0); break; - } - return data_size; +static unsigned datatype2size(unsigned data_type) { + unsigned data_size; + switch (data_type) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; + break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; + break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; + break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; + break; + case BB128_TYPE: + data_size = 16; + break; + default: + assert(0); + break; + } + return data_size; } -void ptx_instruction::pre_decode() -{ - pc = m_PC; - isize = m_inst_size; - for(unsigned i=0; i<MAX_OUTPUT_VALUES; i++) { - out[i] = 0; - } - for(unsigned i=0; i<MAX_INPUT_VALUES; i++) { - in[i] = 0; - } - incount=0; - outcount=0; - is_vectorin = 0; - is_vectorout = 0; - std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); - std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); - pred = 0; - ar1 = 0; - ar2 = 0; - space = m_space_spec; - memory_op = no_memory_op; - data_size = 0; - if ( has_memory_read() || has_memory_write() ) { - unsigned to_type = get_type(); - data_size = datatype2size(to_type); - memory_op = has_memory_read() ? memory_load : memory_store; - } +void ptx_instruction::pre_decode() { + pc = m_PC; + isize = m_inst_size; + for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { + out[i] = 0; + } + for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { + in[i] = 0; + } + incount = 0; + outcount = 0; + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if (has_memory_read() || has_memory_write()) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } - bool has_dst = false ; + bool has_dst = false; - switch ( get_opcode() ) { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; + switch (get_opcode()) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF - default: - printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); + default: + printf("Execution error: Invalid opcode (0x%x)\n", get_opcode()); break; - } + } - switch( m_cache_option ) { - case CA_OPTION: cache_op = CACHE_ALL; break; - case NC_OPTION: cache_op = CACHE_L1; break; - case CG_OPTION: cache_op = CACHE_GLOBAL; break; - case CS_OPTION: cache_op = CACHE_STREAMING; break; - case LU_OPTION: cache_op = CACHE_LAST_USE; break; - case CV_OPTION: cache_op = CACHE_VOLATILE; break; - case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; - case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; - default: - //if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) - cache_op = CACHE_ALL; - //else if( m_opcode == ST_OP ) - else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) - cache_op = CACHE_WRITE_BACK; - else if( m_opcode == ATOM_OP ) - cache_op = CACHE_GLOBAL; + switch (m_cache_option) { + case CA_OPTION: + cache_op = CACHE_ALL; break; - } + case NC_OPTION: + cache_op = CACHE_L1; + break; + case CG_OPTION: + cache_op = CACHE_GLOBAL; + break; + case CS_OPTION: + cache_op = CACHE_STREAMING; + break; + case LU_OPTION: + cache_op = CACHE_LAST_USE; + break; + case CV_OPTION: + cache_op = CACHE_VOLATILE; + break; + case WB_OPTION: + cache_op = CACHE_WRITE_BACK; + break; + case WT_OPTION: + cache_op = CACHE_WRITE_THROUGH; + break; + default: + // if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if (m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP) + cache_op = CACHE_ALL; + // else if( m_opcode == ST_OP ) + else if (m_opcode == MMA_ST_OP || m_opcode == ST_OP) + cache_op = CACHE_WRITE_BACK; + else if (m_opcode == ATOM_OP) + cache_op = CACHE_GLOBAL; + break; + } - set_opcode_and_latency(); - set_bar_type(); - // Get register operands - int n=0,m=0; - ptx_instruction::const_iterator opr=op_iter_begin(); - for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands - const operand_info &o = *opr; - if ( has_dst && n==0 ) { - // Do not set the null register "_" as an architectural register - if ( o.is_reg() && !o.is_non_arch_reg() ) { - out[0] = o.reg_num(); - arch_reg.dst[0] = o.arch_reg_num(); - } else if ( o.is_vector() ) { - is_vectorin = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) out[0] = o.reg1_num(); - if( num_elem >= 2 ) out[1] = o.reg2_num(); - if( num_elem >= 3 ) out[2] = o.reg3_num(); - if( num_elem >= 4 ) out[3] = o.reg4_num(); - if( num_elem >= 5 ) out[4] = o.reg5_num(); - if( num_elem >= 6 ) out[5] = o.reg6_num(); - if( num_elem >= 7 ) out[6] = o.reg7_num(); - if( num_elem >= 8 ) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if ( o.is_reg() && !o.is_non_arch_reg() ) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch ( m ) { - case 0: in[0] = reg_num; break; - case 1: in[1] = reg_num; break; - case 2: in[2] = reg_num; break; - default: break; - } - m++; - } else if ( o.is_vector() ) { - //assert(m == 0); //only support 1 vector operand (for textures) right now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[m+0] = o.reg1_num(); - if( num_elem >= 2 ) in[m+1] = o.reg2_num(); - if( num_elem >= 3 ) in[m+2] = o.reg3_num(); - if( num_elem >= 4 ) in[m+3] = o.reg4_num(); - if( num_elem >= 5 ) in[m+4] = o.reg5_num(); - if( num_elem >= 6 ) in[m+5] = o.reg6_num(); - if( num_elem >= 7 ) in[m+6] = o.reg7_num(); - if( num_elem >= 8 ) in[m+7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m+i] = o.arch_reg_num(i); - m+=num_elem; - } + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n = 0, m = 0; + ptx_instruction::const_iterator opr = op_iter_begin(); + for (; opr != op_iter_end(); opr++, n++) { // process operands + const operand_info &o = *opr; + if (has_dst && n == 0) { + // Do not set the null register "_" as an architectural register + if (o.is_reg() && !o.is_non_arch_reg()) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if (o.is_vector()) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) out[0] = o.reg1_num(); + if (num_elem >= 2) out[1] = o.reg2_num(); + if (num_elem >= 3) out[2] = o.reg3_num(); + if (num_elem >= 4) out[3] = o.reg4_num(); + if (num_elem >= 5) out[4] = o.reg5_num(); + if (num_elem >= 6) out[5] = o.reg6_num(); + if (num_elem >= 7) out[6] = o.reg7_num(); + if (num_elem >= 8) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } - } - - //Setting number of input and output operands which is required for scoreboard check - for(int i=0;i<MAX_OUTPUT_VALUES;i++) - if(out[i]>0) - outcount++; - - for(int i=0;i<MAX_INPUT_VALUES;i++) - if(in[i]>0) - incount++; - - // Get predicate - if(has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } + } else { + if (o.is_reg() && !o.is_non_arch_reg()) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch (m) { + case 0: + in[0] = reg_num; + break; + case 1: + in[1] = reg_num; + break; + case 2: + in[2] = reg_num; + break; + default: + break; + } + m++; + } else if (o.is_vector()) { + // assert(m == 0); //only support 1 vector operand (for textures) right + // now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) in[m + 0] = o.reg1_num(); + if (num_elem >= 2) in[m + 1] = o.reg2_num(); + if (num_elem >= 3) in[m + 2] = o.reg3_num(); + if (num_elem >= 4) in[m + 3] = o.reg4_num(); + if (num_elem >= 5) in[m + 4] = o.reg5_num(); + if (num_elem >= 6) in[m + 5] = o.reg6_num(); + if (num_elem >= 7) in[m + 6] = o.reg7_num(); + if (num_elem >= 8) in[m + 7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m + i] = o.arch_reg_num(i); + m += num_elem; + } + } + } - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if( has_memory_read() || has_memory_write() ) { - ptx_instruction::const_iterator op=op_iter_begin(); - for ( ; op != op_iter_end(); op++, n++ ) { //process operands - const operand_info &o = *op; + // Setting number of input and output operands which is required for + // scoreboard check + for (int i = 0; i < MAX_OUTPUT_VALUES; i++) + if (out[i] > 0) outcount++; - if(o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert( !o.is_non_arch_reg() ); + for (int i = 0; i < MAX_INPUT_VALUES; i++) + if (in[i] > 0) incount++; - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if(o.is_memory_operand2()) { + // Get predicate + if (has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } - // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) - if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register as well - } - // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) - else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output register as well - } - } - else if(o.is_immediate_address()){ + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if (has_memory_read() || has_memory_write()) { + ptx_instruction::const_iterator op = op_iter_begin(); + for (; op != op_iter_end(); op++, n++) { // process operands + const operand_info &o = *op; - } - // Regular PTX operand - else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } + if (o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert(!o.is_non_arch_reg()); - } + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if (o.is_memory_operand2()) { + // memory operand with one address register (ex. g[$r1+0x4] or + // s[$r2+=0x4]) + if (o.get_double_operand_type() == 0 || + o.get_double_operand_type() == 3) { + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register + // as well + } + // memory operand with two address register (ex. s[$r1+$r1] or + // g[$r1+=$r2]) + else if (o.get_double_operand_type() == 1 || + o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output + // register as well + } + } else if (o.is_immediate_address()) { + } + // Regular PTX operand + else if (o.get_symbol() + ->type() + ->get_key() + .is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } } - } + } + } - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - m_decoded=true; + m_decoded = true; } -void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) -{ - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); - int ntokens = sscanf(name.c_str(),buffer,&parsed_index); - if( ntokens == 1 ) { - assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); - } else { - assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } +void function_info::add_param_name_type_size(unsigned index, std::string name, + int type, size_t size, bool ptr, + memory_space_t space) { + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); + int ntokens = sscanf(name.c_str(), buffer, &parsed_index); + if (ntokens == 1) { + assert(m_ptx_kernel_param_info.find(parsed_index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[parsed_index] = + param_info(name, type, size, ptr, space); + } else { + assert(m_ptx_kernel_param_info.find(index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } } -void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) -{ - const void *data = args->m_start; +void function_info::add_param_data(unsigned argn, + struct gpgpu_ptx_sim_arg *args) { + const void *data = args->m_start; - bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory + bool scratchpad_memory_param = + false; // Is this parameter in CUDA shared memory or OpenCL local memory - std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.find(argn); - if( i != m_ptx_kernel_param_info.end() ) { - if (i->second.is_ptr_shared()) { - assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned) args->m_offset); - } - } else { - scratchpad_memory_param = true; - } + std::map<unsigned, param_info>::iterator i = + m_ptx_kernel_param_info.find(argn); + if (i != m_ptx_kernel_param_info.end()) { + if (i->second.is_ptr_shared()) { + assert( + args->m_start == NULL && + "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned)args->m_offset); + } + } else { + scratchpad_memory_param = true; + } + + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. - - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); - - symbol *p = m_symtab->lookup(buffer); - if( p == NULL ) { - printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); - abort(); + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); + + symbol *p = m_symtab->lookup(buffer); + if (p == NULL) { + printf( + "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " + "bind buffer\n", + buffer); + abort(); + } + if (data) + p->set_address((addr_t) * (size_t *)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if (i != m_ptx_kernel_param_info.end()) { + is_ptr_shared = i->second.is_ptr_shared(); } - if( data ) - p->set_address((addr_t)*(size_t*)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if( i != m_ptx_kernel_param_info.end() ) { - is_ptr_shared = i->second.is_ptr_shared(); - } - if( !is_ptr_shared and !p->is_shared() ) { - printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); - abort(); - } - unsigned num_bits = 8*args->m_nbytes; - printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), - m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits/8 ); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; - p->set_address( addr+addr_pad ); - m_symtab->alloc_shared( num_bits/8 + addr_pad ); + if (!is_ptr_shared and !p->is_shared()) { + printf( + "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " + "shared memory\n"); + abort(); } - } + unsigned num_bits = 8 * args->m_nbytes; + printf( + "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " + "0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits / 8); + fflush(stdout); + assert((num_bits % 8) == 0); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = + num_bits + ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) + : 0; + p->set_address(addr + addr_pad); + m_symtab->alloc_shared(num_bits / 8 + addr_pad); + } + } } unsigned function_info::get_args_aligned_size() { - - if(m_args_aligned_size >= 0) - return m_args_aligned_size; - - unsigned param_address = 0; - unsigned int total_size = 0; - for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); + if (m_args_aligned_size >= 0) return m_args_aligned_size; - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } + unsigned param_address = 0; + unsigned int total_size = 0; + for (std::map<unsigned, param_info>::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); - m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } - return m_args_aligned_size; + m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word + return m_args_aligned_size; } +void function_info::finalize(memory_space *param_mem) { + unsigned param_address = 0; + for (std::map<unsigned, param_info>::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if (size != p.get_size() / 8) { + printf( + "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " + "formal size = %zu (using smaller of two)\n", + size, p.get_size() / 8); + size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); + } + // copy the parameter over word-by-word so that parameter that crosses a + // memory page can be copied over + // Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * + align_amount; // aligned -void function_info::finalize( memory_space *param_mem ) -{ - unsigned param_address = 0; - for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if( size != p.get_size() / 8) { - printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", - size, p.get_size()/8); - size = (size<(p.get_size()/8))?size:(p.get_size()/8); - } - // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over - //Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned - - const size_t word_size = 4; - //param_address = (param_address + size - 1) / size * size; //aligned with size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); - } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } + const size_t word_size = 4; + // param_address = (param_address + size - 1) / size * size; //aligned with + // size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast<const char *>(param_value.pdata) + + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } } -void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) -{ - // TODO: call this only for PTXPlus with GT200 models - //extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; +void function_info::param_to_shared(memory_space *shared_mem, + symbol_table *symtab) { + // TODO: call this only for PTXPlus with GT200 models + // extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) + return; - // copies parameters into simulated shared memory - for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); + // copies parameters into simulated shared memory + for (std::map<unsigned, param_info>::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype,size,tmp); + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype, size, tmp); - // Write to shared memory - offset + 0x10 - shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); - } + // Write to shared memory - offset + 0x10 + shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); + } } - -void function_info::list_param( FILE *fout ) const -{ - for( std::map<unsigned,param_info>::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); +void function_info::list_param(FILE *fout) const { + for (std::map<unsigned, param_info>::const_iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); } -void function_info::ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) -{ - static unsigned long long counter = 0; - std::vector< std::pair<size_t, unsigned char*> > param_data; - std::vector<unsigned> offsets; - std::vector<bool> paramIsPointer; +void function_info::ptx_jit_config( + std::map<unsigned long long, size_t> mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { + static unsigned long long counter = 0; + std::vector<std::pair<size_t, unsigned char *> > param_data; + std::vector<unsigned> offsets; + std::vector<bool> paramIsPointer; - char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path!=NULL); - char * wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path!=NULL); - std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); + char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path != NULL); + char *wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path != NULL); + std::string command = + std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/params.config" + + std::to_string(counter)); - //initialize paramList - char buff[1024]; - std::string filename_c(filename+"_c"); - snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2>pos1&&pos1>0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while(tok!=NULL){ - std::string param(tok); - if(param.find("<")!=std::string::npos){ - assert(param.find(">")==std::string::npos); - assert(param.find("*")==std::string::npos); - tmp = param; - } else { - if (tmp.length()>0){ - tmp = ""; - assert(param.find(">")!=std::string::npos); - assert(param.find("<")==std::string::npos); - assert(param.find("*")==std::string::npos); - } - printf("%s\n", param.c_str()); - if(param.find("*")!=std::string::npos){ - paramIsPointer.push_back(true); - }else{ - paramIsPointer.push_back(false); - } - } - tok = strtok(NULL, ","); + // initialize paramList + char buff[1024]; + std::string filename_c(filename + "_c"); + snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), + filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2 > pos1 && pos1 > 0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while (tok != NULL) { + std::string param(tok); + if (param.find("<") != std::string::npos) { + assert(param.find(">") == std::string::npos); + assert(param.find("*") == std::string::npos); + tmp = param; + } else { + if (tmp.length() > 0) { + tmp = ""; + assert(param.find(">") != std::string::npos); + assert(param.find("<") == std::string::npos); + assert(param.find("*") == std::string::npos); + } + printf("%s\n", param.c_str()); + if (param.find("*") != std::string::npos) { + paramIsPointer.push_back(true); + } else { + paramIsPointer.push_back(false); + } } + tok = strtok(NULL, ","); + } + for (std::map<unsigned, param_info>::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); - for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - - if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ - //is pointer - assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; - if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ - array_size = mallocPtr_Size[param_pointer]; - }else{ - for( std::map<unsigned long long, size_t>::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { - if(param_pointer>j->first&¶m_pointer<j->first + j->second){ - array_size = j->first + j->second - param_pointer; - break; - } - } - assert(array_size>0&&"pointer was not previously malloc'd"); - } - - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - unsigned char* array_val = (unsigned char*) malloc(array_size); - gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); - param_data.push_back(std::pair<size_t, unsigned char*>(array_size,array_val)); - paramIsPointer.push_back(true); - }else{ - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - param_data.push_back(std::pair<size_t, unsigned char*>(param_value.size,val)); - paramIsPointer.push_back(false); + if (paramIsPointer[i->first] && + (*(unsigned long long *)param_value.pdata != 0)) { + // is pointer + assert(param_value.size == sizeof(void *) && + "MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = + *(unsigned long long *)param_value.pdata; + if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { + array_size = mallocPtr_Size[param_pointer]; + } else { + for (std::map<unsigned long long, size_t>::iterator j = + mallocPtr_Size.begin(); + j != mallocPtr_Size.end(); j++) { + if (param_pointer > j->first && + param_pointer < j->first + j->second) { + array_size = j->first + j->second - param_pointer; + break; + } } - } + assert(array_size > 0 && "pointer was not previously malloc'd"); + } - FILE *fout = fopen (filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for( std::vector< std::pair<size_t,unsigned char*> >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { - if (paramIsPointer[index]){ - fprintf(fout, "*"); - } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; j<i->first; j++){ - fprintf(fout, " %u", i->second[j]); - } - fprintf(fout, " : %u", offsets[index]); - free (i->second); - fprintf(fout, "\n"); - index++; + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + unsigned char *array_val = (unsigned char *)malloc(array_size); + gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, + (void *)array_val); + param_data.push_back( + std::pair<size_t, unsigned char *>(array_size, array_val)); + paramIsPointer.push_back(true); + } else { + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + param_data.push_back( + std::pair<size_t, unsigned char *>(param_value.size, val)); + paramIsPointer.push_back(false); } - fflush(fout); - fclose(fout); - - //ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); - snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); + } + + FILE *fout = fopen(filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for (std::vector<std::pair<size_t, unsigned char *> >::const_iterator i = + param_data.begin(); + i != param_data.end(); i++) { + if (paramIsPointer[index]) { + fprintf(fout, "*"); } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j < i->first; j++) { + fprintf(fout, " %u", i->second[j]); } - fin = fopen(ptx_source, "r"); - assert(fin!=NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout!=NULL); - for (unsigned i = 0; i<line_number; i++){ - assert(fgets(buff, 1024, fin) != NULL); - assert(!feof(fin)); + fprintf(fout, " : %u", offsets[index]); + free(i->second); + fprintf(fout, "\n"); + index++; + } + fflush(fout); + fclose(fout); + + // ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/ptx.config" + + std::to_string(counter)); + snprintf(buff, 1024, + "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", + get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, + "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " + "\"NR>={}&&NR<={}+2\" %s > %s", + ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin != NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout != NULL); + for (unsigned i = 0; i < line_number; i++) { + assert(fgets(buff, 1024, fin) != NULL); + assert(!feof(fin)); + } + fprintf(fout, "\n\n"); + do { + fprintf(fout, "%s", buff); + assert(fgets(buff, 1024, fin) != NULL); + if (feof(fin)) { + break; } - fprintf(fout, "\n\n"); - do{ - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if(feof(fin)){ - break; - } - } while(strstr(buff, "entry")==NULL); + } while (strstr(buff, "entry") == NULL); - fclose(fin); - fflush(fout); - fclose(fout); - counter++; + fclose(fin); + fflush(fout); + fclose(fout); + counter++; } -template<int activate_level> -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) -{ - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { - return false; - } - if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { - return false; - } +template <int activate_level> +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ((g_debug_thread_uid != 0) && + (thd_uid != (unsigned)g_debug_thread_uid)) { + return false; + } + if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { + return false; + } + + return true; + } - return true; - } - - return false; + return false; } -void cuda_sim::init_inst_classification_stat() -{ - static std::set<unsigned> init; - if( init.find(g_ptx_kernel_count) != init.end() ) - return; - init.insert(g_ptx_kernel_count); +void cuda_sim::init_inst_classification_stat() { + static std::set<unsigned> init; + if (init.find(g_ptx_kernel_count) != init.end()) return; + init.insert(g_ptx_kernel_count); - #define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] =""; - if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); - assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! - g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); - if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); - g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); +#define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] = ""; + if (!g_inst_classification_stat) + g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", + g_ptx_kernel_count); + assert(g_ptx_kernel_count < + MAX_CLASS_KER); // a static limit on number of kernels increase it if + // it fails! + g_inst_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 20); + if (!g_inst_op_classification_stat) + g_inst_op_classification_stat = + (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", + g_ptx_kernel_count); + g_inst_op_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 100); } -static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &src1 = pI->src1(); //the name of the texture - std::string texname = src1.name(); +static unsigned get_tex_datasize(const ptx_instruction *pI, + ptx_thread_info *thread) { + const operand_info &src1 = pI->src1(); // the name of the texture + std::string texname = src1.name(); - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t& k = thread->get_kernel(); - const struct textureInfo* texInfo = k.get_texinfo(texname); + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t &k = thread->get_kernel(); + const struct textureInfo *texInfo = k.get_texinfo(texname); - unsigned data_size = texInfo->texel_size; - return data_size; + unsigned data_size = texInfo->texel_size; + return data_size; } -int tensorcore_op(int inst_opcode){ - - if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) - return 1; - else - return 0; +int tensorcore_op(int inst_opcode) { + if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || + (inst_opcode == MMA_ST_OP)) + return 1; + else + return 0; } -void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) -{ - - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert( pc == inst.pc ); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); - - set_npc( pc + pI->inst_size() ); - +void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert(pc == + inst.pc); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); - try { + set_npc(pc + pI->inst_size()); - clearRPC(); - m_last_set_operand_value.u64 = 0; + try { + clearRPC(); + m_last_set_operand_value.u64 = 0; - if(is_done()) - { - printf("attempted to execute instruction on a thread that is already done.\n"); + if (is_done()) { + printf( + "attempted to execute instruction on a thread that is already " + "done.\n"); assert(0); - } - - if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) - || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { - - clear_modifiedregs(); - enable_debug_trace(); + } + + if (g_debug_execution >= 6 || + m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || + (get_uid() == + (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { + clear_modifiedregs(); + enable_debug_trace(); } - } - - - if( pI->has_pred() ) { + } + + if (pI->has_pred()) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if(pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag + if (pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ + pI->get_pred_neg(); // ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode=pI->get_opcode(); - - if( skip ) { + } + int inst_opcode = pI->get_opcode(); + + if (skip) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if( pI->get_opcode() == VOTE_OP ) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t*)pJ) = inst; // copy active mask information - pI = pJ; + if (pI->get_opcode() == VOTE_OP) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t *)pJ) = inst; // copy active mask information + pI = pJ; } - - if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ - if(inst.active_count()!=MAX_WARP_SIZE) - { - printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); - assert(0); - } + + if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || + inst_opcode == MMA_ST_OP))) { + if (inst.active_count() != MAX_WARP_SIZE) { + printf( + "Tensor Core operation are warp synchronous operation. All the " + "threads needs to be active."); + assert(0); + } } - - //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation - if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ - switch ( inst_opcode ) { - #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; - #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; - #include "opcodes.def" - #undef OP_DEF - #undef OP_W_DEF - default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; - } + + // Tensorcore is warp synchronous operation. So these instructions needs + // to be executed only once. To make the simulation faster removing the + // redundant tensorcore operation + if (!tensorcore_op(inst_opcode) || + ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { + switch (inst_opcode) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, this); \ + op_classification = CLASSIFICATION; \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, get_core(), inst); \ + op_classification = CLASSIFICATION; \ + break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: + printf("Execution error: Invalid opcode (0x%x)\n", + pI->get_opcode()); + break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if(pI->is_exit()) - exit_impl(pI,this); - } - + if (pI->is_exit()) exit_impl(pI, this); + } + const gpgpu_functional_sim_config &config = m_gpu->get_config(); - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - - // Output instruction information to file and stdout - if( config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { - fprintf(m_gpu->get_ptx_inst_debug_file(), - "[thd=%u] : (%s:%u - %s)\n", - get_uid(), - pI->source_file(), pI->source_line(), pI->get_source() ); - //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + // Output instruction information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { + fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", + get_uid(), pI->source_file(), pI->source_line(), + pI->get_source()); + // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory + // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), + pc)) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, - get_uid(), - pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, - get_icount(), - pc, pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64 ); + printf( + "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " + "[pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), + ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, + pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ( (pI->has_memory_read() || pI->has_memory_write()) ) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ((pI->has_memory_read() || pI->has_memory_write())) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); - } + } + } + + if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, + false /*not atomic*/); + } - if ( pI->get_opcode() == ATOM_OP ) { + if (pI->get_opcode() == ATOM_OP) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr() ); - assert( inst.space == last_space() ); - insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr()); + assert(inst.space == last_space()); + insn_data_size = get_tex_datasize( + pI, + this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if( config.get_ptx_inst_debug_to_file()!=0 && - (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { + // Output register information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( g_debug_execution >= 6 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) - dump_modifiedregs(stdout); - } - if ( g_debug_execution >= 10 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - //not using it with functional simulation mode - if(!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); - - if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { + if (g_debug_execution >= 6) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), + pc)) + dump_modifiedregs(stdout); + } + if (g_debug_execution >= 10) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), + pc)) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + // not using it with functional simulation mode + if (!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type=0; - switch ( pI->get_space().get_type() ) { - case global_space: space_type = 10; break; - case local_space: space_type = 11; break; - case tex_space: space_type = 12; break; - case surf_space: space_type = 13; break; - case param_space_kernel: - case param_space_local: - space_type = 14; break; - case shared_space: space_type = 15; break; - case const_space: space_type = 16; break; - default: - space_type = 0 ; - break; + unsigned space_type = 0; + switch (pI->get_space().get_type()) { + case global_space: + space_type = 10; + break; + case local_space: + space_type = 11; + break; + case tex_space: + space_type = 12; + break; + case surf_space: + space_type = 13; + break; + case param_space_kernel: + case param_space_local: + space_type = 14; + break; + case shared_space: + space_type = 15; + break; + case const_space: + space_type = 16; + break; + default: + space_type = 0; + break; } - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); - if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); - } - if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + op_classification); + if (space_type) + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)space_type); + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)pI->get_opcode()); + } + if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); + DPRINTF(LIVENESS, + "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, + ctaid.z, tid.x, tid.y, tid.z); fflush(stdout); - } - - // "Return values" - if(!skip) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert( inst.memory_op == insn_memory_op ); - } - } - - } catch ( int x ) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); - abort(); - } - + } + + // "Return values" + if (!skip) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert(inst.memory_op == insn_memory_op); + } + } + + } catch (int x) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, + pI->source_file(), pI->source_line()); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); + abort(); + } } -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) -{ - gpgpu_param_num_shaders = num_shaders; +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { + gpgpu_param_num_shaders = num_shaders; } -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) -{ - return kernel->get_kernel_info(); +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const function_info *kernel) { + return kernel->get_kernel_info(); } -const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) -{ - return pc_to_instruction(pc); +const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { + return pc_to_instruction(pc); } -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool isInFunctionalSimulationMode) -{ - std::list<ptx_thread_info *> &active_threads = kernel.active_threads(); +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, bool isInFunctionalSimulationMode) { + std::list<ptx_thread_info *> &active_threads = kernel.active_threads(); - static std::map<unsigned,memory_space*> shared_memory_lookup; - static std::map<unsigned,memory_space*> sstarr_memory_lookup; - static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup; - static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup; - static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup; + static std::map<unsigned, memory_space *> shared_memory_lookup; + static std::map<unsigned, memory_space *> sstarr_memory_lookup; + static std::map<unsigned, ptx_cta_info *> ptx_cta_lookup; + static std::map<unsigned, ptx_warp_info *> ptx_warp_lookup; + static std::map<unsigned, std::map<unsigned, memory_space *> > + local_memory_lookup; - if ( *thread_info != NULL ) { - ptx_thread_info *thd = *thread_info; - assert( thd->is_done() ); - if ( g_debug_execution==-1 ) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", - ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } + if (*thread_info != NULL) { + ptx_thread_info *thd = *thread_info; + assert(thd->is_done()); + if (g_debug_execution == -1) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf( + "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) uid=%u\n", + ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } - if ( !active_threads.empty() ) { - assert( active_threads.size() <= threads_left ); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); - return 1; - } + if (!active_threads.empty()) { + assert(active_threads.size() <= threads_left); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + return 1; + } - if ( kernel.no_more_ctas_to_run() ) { - return 0; //finished! - } + if (kernel.no_more_ctas_to_run()) { + return 0; // finished! + } - if ( threads_left < kernel.threads_per_cta() ) { - return 0; - } + if (threads_left < kernel.threads_per_cta()) { + return 0; + } - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } - //initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; + // initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 - assert( max_cta_per_sm > 0 ); + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 + assert(max_cta_per_sm > 0); - //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; + // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = + hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { - if ( g_debug_execution >= 1 ) { - printf(" <CTA alloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - char buf[512]; - snprintf(buf,512,"shared_%u", sid); - shared_mem = new memory_space_impl<16*1024>(buf,4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf,512,"sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16*1024>(buf,4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if ( g_debug_execution >= 1 ) { - printf(" <CTA realloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } + if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { + if (g_debug_execution >= 1) { + printf(" <CTA alloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + char buf[512]; + snprintf(buf, 512, "shared_%u", sid); + shared_mem = new memory_space_impl<16 * 1024>(buf, 4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf, 512, "sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if (g_debug_execution >= 1) { + printf(" <CTA realloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } - std::map<unsigned,memory_space*> &local_mem_lookup = local_memory_lookup[sid]; - while( kernel.more_threads_in_cta() ) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + std::map<unsigned, memory_space *> &local_mem_lookup = + local_memory_lookup[sid]; + while (kernel.more_threads_in_cta()) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; - memory_space *local_mem = NULL; - std::map<unsigned,memory_space*>::iterator l = local_mem_lookup.find(new_tid); - if ( l != local_mem_lookup.end() ) { - local_mem = l->second; - } else { - char buf[512]; - snprintf(buf,512,"local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf,32); - local_mem_lookup[new_tid] = local_mem; - } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if( kernel.entry()->get_ptx_version().extensions() ) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem,st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); - fflush(stdout); - } - active_threads.push_back(thd); - } - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + memory_space *local_mem = NULL; + std::map<unsigned, memory_space *>::iterator l = + local_mem_lookup.find(new_tid); + if (l != local_mem_lookup.end()) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf, 512, "local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf, 32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if (kernel.entry()->get_ptx_version().extensions()) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem, st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if (g_debug_execution == -1) { + printf( + "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, + (unsigned long long)thd); fflush(stdout); - } + } + active_threads.push_back(thd); + } + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + fflush(stdout); + } - kernel.increment_cta_id(); + kernel.increment_cta_id(); - assert( active_threads.size() <= threads_left ); - *thread_info = active_threads.front(); - (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); - active_threads.pop_front(); - return 1; + assert(active_threads.size() <= threads_left); + *thread_info = active_threads.front(); + (*thread_info) + ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + active_threads.pop_front(); + return 1; } -size_t get_kernel_code_size( class function_info *entry ) -{ - return entry->get_function_size(); +size_t get_kernel_code_size(class function_info *entry) { + return entry->get_function_size(); } +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( + class function_info *entry, gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { + kernel_info_t *result = + new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), + gpu->getNameInfoMapping()); + unsigned argcount = args.size(); + unsigned argn = 1; + for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); + a++) { + entry->add_param_data(argcount - argn, &(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ) -{ - kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); - unsigned argcount=args.size(); - unsigned argn=1; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - entry->add_param_data(argcount-argn,&(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - - return result; + return result; } #include "../../version" #include "detailed_version" -void print_splash() -{ - static int splash_printed=0; - if ( !splash_printed ) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); - splash_printed=1; - } +void print_splash() { + static int splash_printed = 0; + if (!splash_printed) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", + g_gpgpusim_version_string, g_gpgpusim_build_string); + splash_printed = 1; + } } -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); - g_const_name_lookup[hostVar] = deviceName; +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", + deviceName, size); + g_const_name_lookup[hostVar] = deviceName; } -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); - g_global_name_lookup[hostVar] = deviceName; +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", + deviceName); + g_global_name_lookup[hostVar] = deviceName; } -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) -{ - printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; - - std::map<const void*,std::string>::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map<const void*,std::string>::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { - if ( found_sym ) { - printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar ); - abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if( g_globals.find(hostVar) != g_globals.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if( g_constants.find(hostVar) != g_constants.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu) { + printf( + "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", + hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; - if ( !found_sym ) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); + std::map<const void *, std::string>::iterator c = + gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map<const void *, std::string>::iterator g = + gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { + if (found_sym) { + printf( + "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " + "both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar); abort(); - } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); - const char *mem_name = NULL; - memory_space *mem = NULL; + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if (g_globals.find(hostVar) != g_globals.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if (g_constants.find(hostVar) != g_constants.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } - std::map<std::string,symbol_table*>::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); - symbol_table *symtab = st->second; + if (!found_sym) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", + (unsigned long long)hostVar); + abort(); + } else + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " + "hostVar=0x%Lx\n", + (unsigned long long)hostVar); + const char *mem_name = NULL; + memory_space *mem = NULL; - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + std::map<std::string, symbol_table *>::iterator st = + gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); - for ( unsigned n=0; n < count; n++ ) { - if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); - else mem->read(dst+n,1,((char*)src)+n); - } - fflush(stdout); + } + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " + "%s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); + for (unsigned n = 0; n < count; n++) { + if (to) + mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); + else + mem->read(dst + n, 1, ((char *)src) + n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() -{ - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; +void cuda_sim::read_sim_environment_variables() { + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if ( mode ) - sscanf(mode,"%u", &g_ptx_sim_mode); - printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); - printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if ( dbg_inter && strlen(dbg_inter) ) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); - fflush(stdout); - sscanf(dbg_level,"%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if ( dbg_thread && strlen(dbg_thread) ) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); - fflush(stdout); - sscanf(dbg_thread,"%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if ( dbg_pc && strlen(dbg_pc) ) { - printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); - fflush(stdout); - sscanf(dbg_pc,"%d", &g_debug_pc); - } + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); + printf( + "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " + "environment variable:\n", + g_ptx_sim_mode); + printf( + " 1=functional simulation only, 0=detailed performance " + "simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if (dbg_inter && strlen(dbg_inter)) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); + fflush(stdout); + sscanf(dbg_level, "%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if (dbg_thread && strlen(dbg_thread)) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", + dbg_thread); + fflush(stdout); + sscanf(dbg_thread, "%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if (dbg_pc && strlen(dbg_pc)) { + printf( + "GPGPU-Sim PTX: printing debug information for instruction with PC = " + "%s\n", + dbg_pc); + fflush(stdout); + sscanf(dbg_pc, "%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if( blocking && !strcmp(blocking,"1") ) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf( + "GPGPU-Sim PTX: overriding embedded ptx with ptx file " + "(PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if (blocking && !strcmp(blocking, "1")) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if ( g_debug_execution >= 40 ) { - ptx_debug = 1; - } + if (g_debug_execution >= 40) { + ptx_debug = 1; + } } -#define MAX(a,b) (((a)>(b))?(a):(b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) -{ - - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; +unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, + unsigned threads_per_cta, unsigned int warp_size, + unsigned int n_thread_per_shader, + unsigned int gpgpu_shmem_size, + unsigned int gpgpu_shader_registers, + unsigned int max_cta_per_core) { + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); - printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); - //Limit by CTA - unsigned int result_cta = max_cta_per_core; + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / + (padded_cta_size * ((kernel_info->regs + 3) & ~3)); + printf("padded cta size is %d and %d and %d", padded_cta_size, + kernel_info->regs, ((kernel_info->regs + 3) & ~3)); + // Limit by CTA + unsigned int result_cta = max_cta_per_core; - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); - printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf (" threads"); - if (result == result_shmem) printf (" shmem"); - if (result == result_regs) printf (" regs"); - if (result == result_cta) printf (" cta_limit"); - printf ("\n"); + printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf(" threads"); + if (result == result_shmem) printf(" shmem"); + if (result == result_regs) printf(" regs"); + if (result == result_cta) printf(" cta_limit"); + printf("\n"); - return result; + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t parameter -which holds the data for the CUDA kernel to be executed +This function simulates the CUDA code functionally, it takes a kernel_info_t +parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) -{ - printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); +void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, + bool openCL) { + printf( + "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", + kernel.name().c_str()); - //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here - //extern gpgpu_sim *g_the_gpu; - //before we execute, we should do PDOM analysis for functional simulation scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); + // using a shader core object for book keeping, it is not needed but as most + // function built for performance simulation need it we use it here + // extern gpgpu_sim *g_the_gpu; + // before we execute, we should do PDOM analysis for functional simulation + // scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = + ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); - } + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + kernel.name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + kernel.name().c_str()); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } - unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); - printf("Max CTA : %d\n",max_cta_tot); + unsigned max_cta_tot = max_cta( + kernel_info, kernel.threads_per_cta(), + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->n_thread_per_shader, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shmem_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shader_registers, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->max_cta_per_core); + printf("Max CTA : %d\n", max_cta_tot); - int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; + int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched = 0; - //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise - while(!kernel.no_more_ctas_to_run()){ - unsigned temp=kernel.get_next_cta_id_single(); - + // we excute the kernel one CTA (Block) at the time, as synchronization + // functions work block wise + while (!kernel.no_more_ctas_to_run()) { + unsigned temp = kernel.get_next_cta_id_single(); - if(cp_op==0 || (cp_op==1 && cta_launched<cp_cta_resume && kernel.get_uid()==cp_kernel) || kernel.get_uid()< cp_kernel) // just fro testing - { - functionalCoreSim cta( - &kernel, - gpgpu_ctx->the_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size - ); - cta.execute(cp_count,temp); + if (cp_op == 0 || + (cp_op == 1 && cta_launched < cp_cta_resume && + kernel.get_uid() == cp_kernel) || + kernel.get_uid() < cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); + cta.execute(cp_count, temp); - #if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); - #endif - } - else - { - kernel.increment_cta_id(); - } - cta_launched++; +#if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); +#endif + } else { + kernel.increment_cta_id(); } + cta_launched++; + } - - - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); - } + if (cp_op == 1) { + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", + kernel.get_uid()); + g_checkpoint->store_global_mem( + gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, + (char *)"%08x"); + } + // registering this kernel as done - - - //registering this kernel as done - - //openCL kernel simulation calls don't register the kernel so we don't register its exit - if(!openCL) { - //extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); - } + // openCL kernel simulation calls don't register the kernel so we don't + // register its exit + if (!openCL) { + // extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( + kernel.get_uid()); + } - //******PRINTING******* - printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); - if ( gpgpu_ptx_instruction_classification ) { - StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); - } + //******PRINTING******* + printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", + g_ptx_sim_num_insn); + if (gpgpu_ptx_instruction_classification) { + StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); + } - //time_t variables used to calculate the total simulation time - //the start time of simulation is hold by the global variable g_simulation_starttime - //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - + // time_t variables used to calculate the total simulation time + // the start time of simulation is hold by the global variable + // g_simulation_starttime g_simulation_starttime is initilized by + // gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = + MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - //calculating and printing simulation time in terms of days, hours, minutes and seconds - days = elapsed_time/(3600*24); - hrs = elapsed_time/3600 - 24*days; - minutes = elapsed_time/60 - 60*(hrs + 24*days); - sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); + // calculating and printing simulation time in terms of days, hours, minutes + // and seconds + days = elapsed_time / (3600 * 24); + hrs = elapsed_time / 3600 - 24 * days; + minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); + sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); - fflush(stdout); + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, + (unsigned)elapsed_time); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(g_ptx_sim_num_insn / elapsed_time)); + fflush(stdout); } -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) -{ - int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); - - for(int i=0; i< m_warp_count; i++){ - m_warpAtBarrier[i]=false; - m_liveThreadCount[i]=0; - } - for(int i=0; i< m_warp_count*m_warp_size;i++) - m_thread[i]=NULL; - - //get threads for a cta - for(unsigned i=0; i<m_kernel->threads_per_cta();i++) { - ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); - assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); - ctaLiveThreads++; - } +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { + int ctaLiveThreads = 0; + symbol_table *symtab = m_kernel->entry()->get_symtab(); + + for (int i = 0; i < m_warp_count; i++) { + m_warpAtBarrier[i] = false; + m_liveThreadCount[i] = 0; + } + for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; - for(int k=0;k<m_warp_count;k++) - createWarp(k); + // get threads for a cta + for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { + ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, + m_kernel->threads_per_cta() - i, + m_kernel->threads_per_cta(), this, 0, i / m_warp_size, + (gpgpu_t *)m_gpu, true); + assert(m_thread[i] != NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) + m_thread[i]->resume_reg_thread(fname, symtab); + ctaLiveThreads++; + } + + for (int k = 0; k < m_warp_count; k++) createWarp(k); } -void functionalCoreSim::createWarp(unsigned warpId) -{ - simt_mask_t initialMask; - unsigned liveThreadsCount=0; - initialMask.set(); - for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){ - if(m_thread[i]==NULL) initialMask.reset(i-warpId*m_warp_size); - else liveThreadsCount++; - } - - assert(m_thread[warpId*m_warp_size]!=NULL); - m_simt_stack[warpId]->launch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); +void functionalCoreSim::createWarp(unsigned warpId) { + simt_mask_t initialMask; + unsigned liveThreadsCount = 0; + initialMask.set(); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + if (m_thread[i] == NULL) + initialMask.reset(i - warpId * m_warp_size); + else + liveThreadsCount++; + } - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){ - m_thread[i]->set_npc(pc); - m_thread[i]->update_pc(); - } + assert(m_thread[warpId * m_warp_size] != NULL); + m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), + initialMask); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); - } - m_liveThreadCount[warpId]= liveThreadsCount; + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { + unsigned pc, rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } + } + m_liveThreadCount[warpId] = liveThreadsCount; } -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) - { - m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count=0; - while(true){ - bool someOneLive= false; - bool allAtBarrier = true; - for(unsigned i=0;i<m_warp_count;i++){ - executeWarp(i,allAtBarrier,someOneLive); - count++; - } - - if(inst_count>0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } - if(!someOneLive) break; - if(allAtBarrier){ - for(unsigned i=0;i<m_warp_count;i++) - m_warpAtBarrier[i]=false; - } +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { + m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count = 0; + while (true) { + bool someOneLive = false; + bool allAtBarrier = true; + for (unsigned i = 0; i < m_warp_count; i++) { + executeWarp(i, allAtBarrier, someOneLive); + count++; + } + + if (inst_count > 0 && count > inst_count && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { + someOneLive = false; + break; + } + if (!someOneLive) break; + if (allAtBarrier) { + for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; } + } - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - ptx_reg_t regval; - regval.u64= 123; + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;i<m_warp_count;i++) - { - - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i,ctaid-1 ); - FILE * fp = fopen(fname,"w"); - assert(fp!=NULL); - m_simt_stack[i]->print_checkpoint(fp); - fclose(fp); - } - } + ptx_reg_t regval; + regval.u64 = 123; -} + unsigned ctaid = m_kernel->get_next_cta_id_single(); + if (m_gpu->checkpoint_option == 1 && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t)) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, + (char *)"%08x"); + for (int i = 0; i < 32 * m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, + ctaid - 1); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", + i, ctaid - 1); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, + (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) -{ - if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ - warp_inst_t inst =getExecuteWarp(i); - execute_warp_inst_t(inst,i); - if(inst.isatomic()) inst.do_atomic(true); - if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; - updateSIMTStack( i, &inst ); + for (int i = 0; i < m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, + ctaid - 1); + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); } - if(m_liveThreadCount[i]>0) someOneLive=true; - if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; + } +} + +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, + bool &someOneLive) { + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { + warp_inst_t inst = getExecuteWarp(i); + execute_warp_inst_t(inst, i); + if (inst.isatomic()) inst.do_atomic(true); + if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) + m_warpAtBarrier[i] = true; + updateSIMTStack(i, &inst); + } + if (m_liveThreadCount[i] > 0) someOneLive = true; + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) -{ - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is + // the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map<unsigned,const char*> get_duplicate(); +extern std::map<unsigned, const char *> get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map<unsigned,const char*> g_duplicate; +static std::map<unsigned, const char *> g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() -{ - return g_ptxinfo_kname; -} +const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } -void print_ptxinfo() -{ - if(! get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if(get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - } +void print_ptxinfo() { + if (!get_ptxinfo_kname()) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if (get_ptxinfo_kname()) { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + } } - -struct gpgpu_ptx_sim_info get_ptxinfo() -{ - return g_ptxinfo; +struct gpgpu_ptx_sim_info get_ptxinfo() { + return g_ptxinfo; } -std::map<unsigned,const char*> get_duplicate() -{ - return g_duplicate; -} +std::map<unsigned, const char *> get_duplicate() { return g_duplicate; } -void ptxinfo_linenum( unsigned linenum ) -{ - g_duplicate[linenum] = g_last_dup_type; +void ptxinfo_linenum(unsigned linenum) { + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_dup_type( const char *dup_type ) -{ - g_last_dup_type = dup_type; -} +void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } -void ptxinfo_function(const char *fname ) -{ - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_function(const char *fname) { + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_regs( unsigned nregs ) -{ - g_ptxinfo.regs=nregs; -} +void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } -void ptxinfo_lmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.lmem=declared+system; +void ptxinfo_lmem(unsigned declared, unsigned system) { + g_ptxinfo.lmem = declared + system; } -void ptxinfo_gmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.gmem=declared+system; +void ptxinfo_gmem(unsigned declared, unsigned system) { + g_ptxinfo.gmem = declared + system; } -void ptxinfo_smem( unsigned declared, unsigned system ) -{ - g_ptxinfo.smem=declared+system; +void ptxinfo_smem(unsigned declared, unsigned system) { + g_ptxinfo.smem = declared + system; } -void ptxinfo_cmem( unsigned nbytes, unsigned bank ) -{ - g_ptxinfo.cmem+=nbytes; -} +void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } -void clear_ptxinfo() -{ - free(g_ptxinfo_kname); - g_ptxinfo_kname=NULL; - g_ptxinfo.regs=0; - g_ptxinfo.lmem=0; - g_ptxinfo.smem=0; - g_ptxinfo.cmem=0; - g_ptxinfo.gmem=0; - g_ptxinfo.ptx_version=0; - g_ptxinfo.sm_target=0; +void clear_ptxinfo() { + free(g_ptxinfo_kname); + g_ptxinfo_kname = NULL; + g_ptxinfo.regs = 0; + g_ptxinfo.lmem = 0; + g_ptxinfo.smem = 0; + g_ptxinfo.cmem = 0; + g_ptxinfo.gmem = 0; + g_ptxinfo.ptx_version = 0; + g_ptxinfo.sm_target = 0; } +void ptxinfo_opencl_addinfo(std::map<std::string, function_info *> &kernels) { + if (!g_ptxinfo_kname) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } -void ptxinfo_opencl_addinfo( std::map<std::string,function_info*> &kernels ) -{ + if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map<std::string, function_info *>::iterator k = + kernels.find(g_ptxinfo_kname); + if (k == kernels.end()) { + printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", + g_ptxinfo_kname); + abort(); + } else { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + function_info *finfo = k->second; + assert(finfo != NULL); + finfo->set_kernel_info(g_ptxinfo); + } + clear_ptxinfo(); +} - if(! g_ptxinfo_kname) { - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } +struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { + rec_pts tmp; + std::map<function_info *, rec_pts>::iterator r = g_rpts.find(finfo); - if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map<std::string,function_info*>::iterator k=kernels.find(g_ptxinfo_kname); - if( k==kernels.end() ) { - printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); - abort(); - } else { - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - function_info *finfo = k->second; - assert(finfo!=NULL); - finfo->set_kernel_info( g_ptxinfo ); - } - clear_ptxinfo(); -} + if (r == g_rpts.end()) { + int num_recon = finfo->get_num_reconvergence_pairs(); -struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) -{ - rec_pts tmp; - std::map<function_info*,rec_pts>::iterator r=g_rpts.find(finfo); - - if( r==g_rpts.end() ) { - int num_recon = finfo->get_num_reconvergence_pairs(); - - gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); - for (int i=0;i<num_recon;i++) { - printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i+1 ); - kernel_recon_points[i].source_inst->print_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ " ); - if( kernel_recon_points[i].target_inst ) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); + gpgpu_recon_t *kernel_recon_points = + (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", + finfo->get_name().c_str()); + for (int i = 0; i < num_recon; i++) { + printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); + kernel_recon_points[i].source_inst->print_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ "); + if (kernel_recon_points[i].target_inst) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", + finfo->get_name().c_str()); - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; } -address_type get_return_pc( void *thd ) -{ - // function call return - ptx_thread_info *the_thread = (ptx_thread_info*)thd; - assert( the_thread != NULL ); - return the_thread->get_return_PC(); +address_type get_return_pc(void *thd) { + // function call return + ptx_thread_info *the_thread = (ptx_thread_info *)thd; + assert(the_thread != NULL); + return the_thread->get_return_PC(); } -address_type cuda_sim::get_converge_point( address_type pc ) -{ - // the branch could encode the reconvergence point and/or a bit that indicates the - // reconvergence point is the return PC on the call stack in the case the branch has - // no immediate postdominator in the function (i.e., due to multiple return points). +address_type cuda_sim::get_converge_point(address_type pc) { + // the branch could encode the reconvergence point and/or a bit that indicates + // the reconvergence point is the return PC on the call stack in the case the + // branch has no immediate postdominator in the function (i.e., due to + // multiple return points). - std::map<unsigned,function_info*>::iterator f=g_pc_to_finfo.find(pc); - assert( f != g_pc_to_finfo.end() ); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); + std::map<unsigned, function_info *>::iterator f = g_pc_to_finfo.find(pc); + assert(f != g_pc_to_finfo.end()); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); - int i=0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; - } + int i = 0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; } - } - return NO_BRANCH_DIVERGENCE; + } + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit( unsigned warp_id ) -{ - for(int i=0;i<m_warp_count*m_warp_size;i++){ - if(m_thread[i]!=NULL){ - m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; - } +void functionalCoreSim::warp_exit(unsigned warp_id) { + for (int i = 0; i < m_warp_count * m_warp_size; i++) { + if (m_thread[i] != NULL) { + m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; } + } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 1be3d19..21e1ca0 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,34 +7,35 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED -#include "../abstract_hardware_model.h" -#include"../gpgpu-sim/shader.h" #include <stdlib.h> #include <map> -#include <vector> #include <string> -#include"ptx_sim.h" +#include <vector> +#include "../abstract_hardware_model.h" +#include "../gpgpu-sim/shader.h" +#include "ptx_sim.h" class gpgpu_context; class memory_space; @@ -44,69 +45,65 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( std::map<std::string,function_info*> &kernels ); -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - class ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - class core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( + std::map<std::string, function_info *> &kernels); +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + class ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, class core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures + * and procedures in core_t */ -class functionalCoreSim: public core_t -{ -public: - functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) - : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) - { - m_warpAtBarrier = new bool [m_warp_count]; - m_liveThreadCount = new unsigned [m_warp_count]; - } - virtual ~functionalCoreSim(){ - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit( unsigned warp_id ); - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const - { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); - } - -private: - void executeWarp(unsigned, bool &, bool &); - //initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) - { - if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ - m_liveThreadCount[tid/m_warp_size]--; - } +class functionalCoreSim : public core_t { + public: + functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) + : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { + m_warpAtBarrier = new bool[m_warp_count]; + m_liveThreadCount = new unsigned[m_warp_count]; + } + virtual ~functionalCoreSim() { + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit(unsigned warp_id); + virtual bool warp_waiting_at_barrier(unsigned warp_id) const { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); + } + + private: + void executeWarp(unsigned, bool &, bool &); + // initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) { + if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { + m_liveThreadCount[tid / m_warp_size]--; } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - //each warp live thread count and barrier indicator - unsigned * m_liveThreadCount; - bool* m_warpAtBarrier; + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + // each warp live thread count and barrier indicator + unsigned *m_liveThreadCount; + bool *m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc( void *thd ); +address_type get_return_pc(void *thd); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -114,89 +111,98 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; - class cuda_sim { - public: - cuda_sim( gpgpu_context* ctx ) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat= NULL; - g_assemble_code_next_pc=0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count=0; - g_ptx_thread_info_uid_next=1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - //global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map<const void*,std::string> g_global_name_lookup; // indexed by hostVar - std::map<const void*,std::string> g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map<function_info*,rec_pts> g_rpts; - bool g_cuda_launch_blocking; - void ** g_inst_classification_stat; - void ** g_inst_op_classification_stat; - std::set<std::string> g_globals; - std::set<std::string> g_constants; - std::map<unsigned,function_info*> g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set<unsigned long long> g_ptx_cta_info_sm_idx_used; - ptx_reg_t* ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context* gpgpu_ctx; - //global functions - void ptx_opcocde_latency_options (option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); - int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); - void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points( function_info *finfo ); - address_type get_converge_point( address_type pc ); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); - void ptx_print_insn( address_type pc, FILE *fp ); - std::string ptx_get_insn_str( address_type pc ); - template<int activate_level> bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim(gpgpu_context *ctx) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = + -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat = NULL; + g_assemble_code_next_pc = 0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count = 0; + g_ptx_thread_info_uid_next = 1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + // global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map<const void *, std::string> + g_global_name_lookup; // indexed by hostVar + std::map<const void *, std::string> + g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no + // notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map<function_info *, rec_pts> g_rpts; + bool g_cuda_launch_blocking; + void **g_inst_classification_stat; + void **g_inst_op_classification_stat; + std::set<std::string> g_globals; + std::set<std::string> g_constants; + std::map<unsigned, function_info *> g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set<unsigned long long> g_ptx_cta_info_sm_idx_used; + ptx_reg_t *ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // global functions + void ptx_opcocde_latency_options(option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); + int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size); + void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, + size_t size); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points(function_info *finfo); + address_type get_converge_point(address_type pc); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu); + void ptx_print_insn(address_type pc, FILE *fp); + std::string ptx_get_insn_str(address_type pc); + template <int activate_level> + bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 9ac0727..1da6b85 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,108 +7,112 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr,const char *arg_list) -{ - FILE *fp = stdout; - unsigned i=0,j=0; - unsigned arg_offset=0; - char buf[64]; - bool in_fmt=false; - while( fmtstr[i] ) { - char c = fmtstr[i++]; - if( !in_fmt ) { - if( c != '%' ) { - fprintf(fp,"%c",c); - } else { - in_fmt=true; - buf[0] = c; - j=1; - } +void my_cuda_printf(const char *fmtstr, const char *arg_list) { + FILE *fp = stdout; + unsigned i = 0, j = 0; + unsigned arg_offset = 0; + char buf[64]; + bool in_fmt = false; + while (fmtstr[i]) { + char c = fmtstr[i++]; + if (!in_fmt) { + if (c != '%') { + fprintf(fp, "%c", c); } else { - if(!( c == 'u' || c == 'f' || c == 'd' )) { - printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j+1] = 0; - void* ptr = (void*)&arg_list[arg_offset]; - //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if( c == 'u' || c == 'd' ) { - fprintf(fp,buf,*((unsigned long long*)ptr)); - } else if( c == 'f' ) { - double tmp = *((double*)ptr); - fprintf(fp,buf,tmp); - } - arg_offset++; - in_fmt=false; + in_fmt = true; + buf[0] = c; + j = 1; } - } + } else { + if (!(c == 'u' || c == 'f' || c == 'd')) { + printf( + "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " + "%%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j + 1] = 0; + void *ptr = (void *)&arg_list[arg_offset]; + // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if (c == 'u' || c == 'd') { + fprintf(fp, buf, *((unsigned long long *)ptr)); + } else if (c == 'f') { + double tmp = *((double *)ptr); + fprintf(fp, buf, tmp); + } + arg_offset++; + in_fmt = false; + } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) -{ - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size<1024*sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr,size,buffer); - addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem=NULL; - memory_space_t space = generic_space; - decode_space(space,thread,actual_param_op,mem,addr); // figure out which space - if( arg == 0 ) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr+len,1,&b); - len++; - } while(b); - fmtstr = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,fmtstr+i); - //mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,arg_list+i); - //mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr,arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size < 1024 * sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr, size, buffer); + addr_t addr = + (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem = NULL; + memory_space_t space = generic_space; + decode_space(space, thread, actual_param_op, mem, + addr); // figure out which space + if (arg == 0) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr + len, 1, &b); + len++; + } while (b); + fmtstr = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); + // mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); + // mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr, arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 4e9baaa..c4ee75d 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,27 +7,30 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); +void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, + class ptx_thread_info* thread, + const class function_info* target_func); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 4baced5..4a99c1c 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,297 +1,327 @@ -//Jin: cuda_device_runtime.cc -//Defines CUDA device runtime APIs for CDP support - +// Jin: cuda_device_runtime.cc +// Defines CUDA device runtime APIs for CDP support #include <iostream> #include <map> - - #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include <builtin_types.h> #include <driver_types.h> +#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "../stream_manager.h" #include "../gpgpusim_entrypoint.h" +#include "../stream_manager.h" +#include "cuda-sim.h" #include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" - -#define DEV_RUNTIME_REPORT(a) \ - if( g_debug_execution ) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } +#include "ptx_ir.h" +#define DEV_RUNTIME_REPORT(a) \ + if (g_debug_execution) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } +// Handling device runtime api: +// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 +// blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); -//Handling device runtime api: -//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) -{ - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 4 ); + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 4); - function_info * child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; + function_info *child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - if(arg == 0) {//function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); - } - else if(arg == 1) { //dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); - } - else if(arg == 2) { //dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); - } - else if(arg == 3) { //unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, & shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); - } + if (arg == 0) { // function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " + << child_kernel_entry->get_name()); + } else if (arg == 1) { // dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " + << grid_dim.z << ")"); + } else if (arg == 2) { // dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " + << block_dim.z << ")"); + } else if (arg == 3) { // unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); } + } - //get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); - if(g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; - - //store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + // get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " + << child_kernel_arg_size + << ", parameter buffer allocated at " << param_buffer); + if (g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; - //copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); + // store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, + child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == + g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + // copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, + NULL); } -//Handling device runtime api: -//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); +// Handling device runtime api: +// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); - kernel_info_t * device_grid = NULL; - function_info * device_kernel_entry = NULL; - void * parameter_buffer; - struct CUstream_st * child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; + kernel_info_t *device_grid = NULL; + function_info *device_kernel_entry = NULL; + void *parameter_buffer; + struct CUstream_st *child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - if(arg == 0) {//paramter buffer for child kernel (in global memory) - //get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); + if (arg == 0) { // paramter buffer for child kernel (in global memory) + // get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " + << parameter_buffer); - //get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - //device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); - - //PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() >0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } + // get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != + g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + // device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " + << device_kernel_entry->get_name()); - //copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; + // PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + device_kernel_entry->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + device_kernel_entry->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() > 0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } - //create child kernel_info_t and index it with parameter_buffer address - gpgpu_t* gpu=thread->get_gpu(); - device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t & parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << - thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << - "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << - ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); //kernel param - size_t param_start_address = 0; - //copy in word - for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); - } - } - else if(arg == 1) { //cudaStream for the child kernel + // copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = + device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); - - kernel_info_t & parent_kernel = thread->get_kernel(); - if(child_stream == 0) { //default stream on device for current CTA - child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); - } - else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to stream " << child_stream->get_uid() << ": " << child_stream); - } - - device_launch_op.stream = child_stream; - } - - } + // create child kernel_info_t and index it with parameter_buffer address + gpgpu_t *gpu = thread->get_gpu(); + device_grid = new kernel_info_t( + config.grid_dim, config.block_dim, device_kernel_entry, + gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t &parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT( + "child kernel launched by " + << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " + << thread->get_ctaid().y << ", " << thread->get_ctaid().z + << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y + << ", " << thread->get_tid().z << ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), + thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); // kernel + // param + size_t param_start_address = 0; + // copy in word + for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read( + (size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, + NULL, NULL); + } + } else if (arg == 1) { // cudaStream for the child kernel + + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); - - //launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); + kernel_info_t &parent_kernel = thread->get_kernel(); + if (child_stream == 0) { // default stream on device for current CTA + child_stream = + parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() + << " to default stream of the cta " + << child_stream->get_uid() << ": " << child_stream); + } else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() << " to stream " + << child_stream->get_uid() << ": " << child_stream); + } - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + device_launch_op.stream = child_stream; + } + } + + // launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } +// Handling device runtime api: +// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int +// flags) flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); -//Handling device runtime api: -//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) -//flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); + if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - - //pStream should be non-zero address in local memory - pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + // pStream should be non-zero address in local memory + pStream_addr = generic_to_local( + thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } - else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); - } + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } else if (arg == + 1) { // unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); } + } - //create stream and write back to param0 - CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); + // create stream and write back to param0 + CUstream_st *stream = + thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, + NULL); + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - void cuda_device_runtime::launch_one_device_kernel() { - if(!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); + if (!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); - stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + stream_operation stream_op = stream_operation( + op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while(!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while (!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 7f7a0ca..1d661b2 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,67 +1,66 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -//Jin: cuda_device_runtime.h -//Defines CUDA device runtime APIs for CDP support +// Jin: cuda_device_runtime.h +// Defines CUDA device runtime APIs for CDP support class device_launch_config_t { + public: + device_launch_config_t() {} -public: - device_launch_config_t() {} - - device_launch_config_t(dim3 _grid_dim, - dim3 _block_dim, - unsigned int _shared_mem, - function_info * _entry): - grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} - - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info * entry; + device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, + unsigned int _shared_mem, function_info* _entry) + : grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info* entry; }; class device_launch_operation_t { + public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) + : grid(_grid), stream(_stream) {} -public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t *_grid, - CUstream_st * _stream) : - grid(_grid), stream(_stream) {} - - kernel_info_t * grid; //a new child grid - - CUstream_st * stream; + kernel_info_t* grid; // a new child grid + CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime( gpgpu_context* ctx ) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map<void *, device_launch_config_t> g_cuda_device_launch_param_map; - std::list<device_launch_operation_t> g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime(gpgpu_context* ctx) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map<void*, device_launch_config_t> g_cuda_device_launch_param_map; + std::list<device_launch_operation_t> g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index 9f74bb7..54bedc2 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,17 +2,23 @@ //
// Copyright (c) 2012-2017 Christian Rau <[email protected]>
//
-// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
-// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
-// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
-// Software is furnished to do so, subject to the following conditions:
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
//
-// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
// Version 1.12.0
@@ -23,180 +29,182 @@ #define HALF_HALF_HPP
/// Combined gcc version number.
-#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
+#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-//check C++11 language features
-#if defined(__clang__) //clang
- #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
- #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
- #endif
- #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
- #define HALF_ENABLE_CPP11_CONSTEXPR 1
- #endif
- #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
- #define HALF_ENABLE_CPP11_NOEXCEPT 1
- #endif
- #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
- #define HALF_ENABLE_CPP11_USER_LITERALS 1
- #endif
- #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
- #define HALF_ENABLE_CPP11_LONG_LONG 1
- #endif
-/*#elif defined(__INTEL_COMPILER) //Intel C++
- #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
- #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
- #endif
- #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
- #define HALF_ENABLE_CPP11_CONSTEXPR 1
- #endif
- #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
- #define HALF_ENABLE_CPP11_NOEXCEPT 1
- #endif
- #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
- #define HALF_ENABLE_CPP11_LONG_LONG 1
- #endif*/
-#elif defined(__GNUC__) //gcc
- #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
- #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
- #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
- #endif
- #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
- #define HALF_ENABLE_CPP11_CONSTEXPR 1
- #endif
- #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
- #define HALF_ENABLE_CPP11_NOEXCEPT 1
- #endif
- #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
- #define HALF_ENABLE_CPP11_USER_LITERALS 1
- #endif
- #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
- #define HALF_ENABLE_CPP11_LONG_LONG 1
- #endif
- #endif
-#elif defined(_MSC_VER) //Visual C++
- #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
- #define HALF_ENABLE_CPP11_CONSTEXPR 1
- #endif
- #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
- #define HALF_ENABLE_CPP11_NOEXCEPT 1
- #endif
- #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
- #define HALF_ENABLE_CPP11_USER_LITERALS 1
- #endif
- #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
- #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
- #endif
- #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
- #define HALF_ENABLE_CPP11_LONG_LONG 1
- #endif
- #define HALF_POP_WARNINGS 1
- #pragma warning(push)
- #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
+// check C++11 language features
+#if defined(__clang__) // clang
+#if __has_feature(cxx_static_assert) && \
+ !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+#endif
+#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+#define HALF_ENABLE_CPP11_CONSTEXPR 1
+#endif
+#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+#define HALF_ENABLE_CPP11_NOEXCEPT 1
+#endif
+#if __has_feature(cxx_user_literals) && \
+ !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+#define HALF_ENABLE_CPP11_USER_LITERALS 1
+#endif
+#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \
+ !defined(HALF_ENABLE_CPP11_LONG_LONG)
+#define HALF_ENABLE_CPP11_LONG_LONG 1
+#endif
+/*#elif defined(__INTEL_COMPILER)
+ //Intel C++ #if __INTEL_COMPILER >= 1100 &&
+ !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? #define
+ HALF_ENABLE_CPP11_STATIC_ASSERT 1 #endif #if __INTEL_COMPILER >= 1300 &&
+ !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? #define
+ HALF_ENABLE_CPP11_CONSTEXPR 1 #endif #if __INTEL_COMPILER >= 1300 &&
+ !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? #define
+ HALF_ENABLE_CPP11_NOEXCEPT 1 #endif #if __INTEL_COMPILER >= 1100 &&
+ !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? #define
+ HALF_ENABLE_CPP11_LONG_LONG 1 #endif*/
+#elif defined(__GNUC__) // gcc
+#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+#endif
+#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+#define HALF_ENABLE_CPP11_CONSTEXPR 1
+#endif
+#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+#define HALF_ENABLE_CPP11_NOEXCEPT 1
+#endif
+#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+#define HALF_ENABLE_CPP11_USER_LITERALS 1
+#endif
+#if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+#define HALF_ENABLE_CPP11_LONG_LONG 1
+#endif
+#endif
+#elif defined(_MSC_VER) // Visual C++
+#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+#define HALF_ENABLE_CPP11_CONSTEXPR 1
+#endif
+#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+#define HALF_ENABLE_CPP11_NOEXCEPT 1
+#endif
+#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+#define HALF_ENABLE_CPP11_USER_LITERALS 1
+#endif
+#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+#define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+#endif
+#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+#define HALF_ENABLE_CPP11_LONG_LONG 1
+#endif
+#define HALF_POP_WARNINGS 1
+#pragma warning(push)
+#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if,
+ // negative unsigned
#endif
-//check C++11 library features
+// check C++11 library features
#include <utility>
-#if defined(_LIBCPP_VERSION) //libc++
- #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
- #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
- #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
- #endif
- #ifndef HALF_ENABLE_CPP11_CSTDINT
- #define HALF_ENABLE_CPP11_CSTDINT 1
- #endif
- #ifndef HALF_ENABLE_CPP11_CMATH
- #define HALF_ENABLE_CPP11_CMATH 1
- #endif
- #ifndef HALF_ENABLE_CPP11_HASH
- #define HALF_ENABLE_CPP11_HASH 1
- #endif
- #endif
-#elif defined(__GLIBCXX__) //libstdc++
- #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
- #ifdef __clang__
- #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
- #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
- #endif
- #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
- #define HALF_ENABLE_CPP11_CSTDINT 1
- #endif
- #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
- #define HALF_ENABLE_CPP11_CMATH 1
- #endif
- #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
- #define HALF_ENABLE_CPP11_HASH 1
- #endif
- #else
- #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
- #define HALF_ENABLE_CPP11_CSTDINT 1
- #endif
- #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
- #define HALF_ENABLE_CPP11_CMATH 1
- #endif
- #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
- #define HALF_ENABLE_CPP11_HASH 1
- #endif
- #endif
- #endif
-#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
- #if _CPPLIB_VER >= 520
- #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
- #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
- #endif
- #ifndef HALF_ENABLE_CPP11_CSTDINT
- #define HALF_ENABLE_CPP11_CSTDINT 1
- #endif
- #ifndef HALF_ENABLE_CPP11_HASH
- #define HALF_ENABLE_CPP11_HASH 1
- #endif
- #endif
- #if _CPPLIB_VER >= 610
- #ifndef HALF_ENABLE_CPP11_CMATH
- #define HALF_ENABLE_CPP11_CMATH 1
- #endif
- #endif
+#if defined(_LIBCPP_VERSION) // libc++
+#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+#endif
+#ifndef HALF_ENABLE_CPP11_CSTDINT
+#define HALF_ENABLE_CPP11_CSTDINT 1
+#endif
+#ifndef HALF_ENABLE_CPP11_CMATH
+#define HALF_ENABLE_CPP11_CMATH 1
+#endif
+#ifndef HALF_ENABLE_CPP11_HASH
+#define HALF_ENABLE_CPP11_HASH 1
+#endif
+#endif
+#elif defined(__GLIBCXX__) // libstdc++
+#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+#ifdef __clang__
+#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+#endif
+#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+#define HALF_ENABLE_CPP11_CSTDINT 1
+#endif
+#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+#define HALF_ENABLE_CPP11_CMATH 1
+#endif
+#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+#define HALF_ENABLE_CPP11_HASH 1
+#endif
+#else
+#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+#define HALF_ENABLE_CPP11_CSTDINT 1
+#endif
+#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+#define HALF_ENABLE_CPP11_CMATH 1
+#endif
+#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+#define HALF_ENABLE_CPP11_HASH 1
+#endif
+#endif
+#endif
+#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++
+#if _CPPLIB_VER >= 520
+#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+#define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+#endif
+#ifndef HALF_ENABLE_CPP11_CSTDINT
+#define HALF_ENABLE_CPP11_CSTDINT 1
+#endif
+#ifndef HALF_ENABLE_CPP11_HASH
+#define HALF_ENABLE_CPP11_HASH 1
+#endif
+#endif
+#if _CPPLIB_VER >= 610
+#ifndef HALF_ENABLE_CPP11_CMATH
+#define HALF_ENABLE_CPP11_CMATH 1
+#endif
+#endif
#endif
#undef HALF_GNUC_VERSION
-//support constexpr
+// support constexpr
#if HALF_ENABLE_CPP11_CONSTEXPR
- #define HALF_CONSTEXPR constexpr
- #define HALF_CONSTEXPR_CONST constexpr
+#define HALF_CONSTEXPR constexpr
+#define HALF_CONSTEXPR_CONST constexpr
#else
- #define HALF_CONSTEXPR
- #define HALF_CONSTEXPR_CONST const
+#define HALF_CONSTEXPR
+#define HALF_CONSTEXPR_CONST const
#endif
-//support noexcept
+// support noexcept
#if HALF_ENABLE_CPP11_NOEXCEPT
- #define HALF_NOEXCEPT noexcept
- #define HALF_NOTHROW noexcept
+#define HALF_NOEXCEPT noexcept
+#define HALF_NOTHROW noexcept
#else
- #define HALF_NOEXCEPT
- #define HALF_NOTHROW throw()
+#define HALF_NOEXCEPT
+#define HALF_NOTHROW throw()
#endif
#include <algorithm>
-#include <iostream>
-#include <limits>
#include <climits>
#include <cmath>
#include <cstring>
+#include <iostream>
+#include <limits>
#if HALF_ENABLE_CPP11_TYPE_TRAITS
- #include <type_traits>
+#include <type_traits>
#endif
#if HALF_ENABLE_CPP11_CSTDINT
- #include <cstdint>
+#include <cstdint>
#endif
#if HALF_ENABLE_CPP11_HASH
- #include <functional>
+#include <functional>
#endif
-
/// Default rounding mode.
-/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
-/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
-/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
+/// This specifies the rounding mode used for all conversions between
+/// [half](\ref half_float::half)s and `float`s as well as for the half_cast()
+/// if not specifying a rounding mode explicitly. It can be redefined (before
+/// including half.hpp) to one of the standard rounding modes using their
+/// respective constants or the equivalent values of `std::float_round_style`:
///
/// `std::float_round_style` | value | rounding
/// ---------------------------------|-------|-------------------------
@@ -206,2862 +214,3509 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity
/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
///
-/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
-/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
-/// to synchronize the rounding mode with that of the underlying single-precision implementation.
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses
+/// truncation (round toward zero, but with overflows set to infinity) and is
+/// the fastest rounding mode possible. It can even be set to
+/// `std::numeric_limits<float>::round_style` to synchronize the rounding mode
+/// with that of the underlying single-precision implementation.
#ifndef HALF_ROUND_STYLE
- #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+#define HALF_ROUND_STYLE -1 // = std::round_indeterminate
#endif
/// Tie-breaking behaviour for round to nearest.
-/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
-/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
-/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
+/// This specifies if ties in round to nearest should be resolved by rounding to
+/// the nearest even value. By default this is defined to `0` resulting in the
+/// faster but slightly more biased behaviour of rounding away from zero in
+/// half-way cases (and thus equal to the round() function), but can be
+/// redefined to `1` (before including half.hpp) if more IEEE-conformant
/// behaviour is needed.
#ifndef HALF_ROUND_TIES_TO_EVEN
- #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
#endif
/// Value signaling overflow.
-/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
-/// operation, in particular it just evaluates to positive infinity.
-#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to
+/// a positive value signaling the overflow of an operation, in particular it
+/// just evaluates to positive infinity.
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
/// Fast half-precision fma function.
-/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
-/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
+/// This symbol is only defined if the fma() function generally executes as fast
+/// as, or faster than, a separate half-precision multiplication followed by an
+/// addition. Due to the internal single-precision implementation of all
/// arithmetic operations, this is in fact always the case.
-#define FP_FAST_FMAH 1
+#define FP_FAST_FMAH 1
#ifndef FP_ILOGB0
- #define FP_ILOGB0 INT_MIN
+#define FP_ILOGB0 INT_MIN
#endif
#ifndef FP_ILOGBNAN
- #define FP_ILOGBNAN INT_MAX
+#define FP_ILOGBNAN INT_MAX
#endif
#ifndef FP_SUBNORMAL
- #define FP_SUBNORMAL 0
+#define FP_SUBNORMAL 0
#endif
#ifndef FP_ZERO
- #define FP_ZERO 1
+#define FP_ZERO 1
#endif
#ifndef FP_NAN
- #define FP_NAN 2
+#define FP_NAN 2
#endif
#ifndef FP_INFINITE
- #define FP_INFINITE 3
+#define FP_INFINITE 3
#endif
#ifndef FP_NORMAL
- #define FP_NORMAL 4
+#define FP_NORMAL 4
#endif
-
/// Main namespace for half precision functionality.
/// This namespace contains all the functionality provided by the library.
-namespace half_float
-{
- class half;
+namespace half_float {
+class half;
#if HALF_ENABLE_CPP11_USER_LITERALS
- /// Library-defined half-precision literals.
- /// Import this namespace to enable half-precision floating point literals:
- /// ~~~~{.cpp}
- /// using namespace half_float::literal;
- /// half_float::half = 4.2_h;
- /// ~~~~
- namespace literal
- {
- half operator"" _h(long double);
- }
+/// Library-defined half-precision literals.
+/// Import this namespace to enable half-precision floating point literals:
+/// ~~~~{.cpp}
+/// using namespace half_float::literal;
+/// half_float::half = 4.2_h;
+/// ~~~~
+namespace literal {
+half operator"" _h(long double);
+}
#endif
- /// \internal
- /// \brief Implementation details.
- namespace detail
- {
- #if HALF_ENABLE_CPP11_TYPE_TRAITS
- /// Conditional type.
- template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
+/// \internal
+/// \brief Implementation details.
+namespace detail {
+#if HALF_ENABLE_CPP11_TYPE_TRAITS
+/// Conditional type.
+template <bool B, typename T, typename F>
+struct conditional : std::conditional<B, T, F> {};
- /// Helper for tag dispatching.
- template<bool B> struct bool_type : std::integral_constant<bool,B> {};
- using std::true_type;
- using std::false_type;
+/// Helper for tag dispatching.
+template <bool B>
+struct bool_type : std::integral_constant<bool, B> {};
+using std::false_type;
+using std::true_type;
- /// Type traits for floating point types.
- template<typename T> struct is_float : std::is_floating_point<T> {};
- #else
- /// Conditional type.
- template<bool,typename T,typename> struct conditional { typedef T type; };
- template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
+/// Type traits for floating point types.
+template <typename T>
+struct is_float : std::is_floating_point<T> {};
+#else
+/// Conditional type.
+template <bool, typename T, typename>
+struct conditional {
+ typedef T type;
+};
+template <typename T, typename F>
+struct conditional<false, T, F> {
+ typedef F type;
+};
- /// Helper for tag dispatching.
- template<bool> struct bool_type {};
- typedef bool_type<true> true_type;
- typedef bool_type<false> false_type;
+/// Helper for tag dispatching.
+template <bool>
+struct bool_type {};
+typedef bool_type<true> true_type;
+typedef bool_type<false> false_type;
- /// Type traits for floating point types.
- template<typename> struct is_float : false_type {};
- template<typename T> struct is_float<const T> : is_float<T> {};
- template<typename T> struct is_float<volatile T> : is_float<T> {};
- template<typename T> struct is_float<const volatile T> : is_float<T> {};
- template<> struct is_float<float> : true_type {};
- template<> struct is_float<double> : true_type {};
- template<> struct is_float<long double> : true_type {};
- #endif
+/// Type traits for floating point types.
+template <typename>
+struct is_float : false_type {};
+template <typename T>
+struct is_float<const T> : is_float<T> {};
+template <typename T>
+struct is_float<volatile T> : is_float<T> {};
+template <typename T>
+struct is_float<const volatile T> : is_float<T> {};
+template <>
+struct is_float<float> : true_type {};
+template <>
+struct is_float<double> : true_type {};
+template <>
+struct is_float<long double> : true_type {};
+#endif
- /// Type traits for floating point bits.
- template<typename T> struct bits { typedef unsigned char type; };
- template<typename T> struct bits<const T> : bits<T> {};
- template<typename T> struct bits<volatile T> : bits<T> {};
- template<typename T> struct bits<const volatile T> : bits<T> {};
+/// Type traits for floating point bits.
+template <typename T>
+struct bits {
+ typedef unsigned char type;
+};
+template <typename T>
+struct bits<const T> : bits<T> {};
+template <typename T>
+struct bits<volatile T> : bits<T> {};
+template <typename T>
+struct bits<const volatile T> : bits<T> {};
- #if HALF_ENABLE_CPP11_CSTDINT
- /// Unsigned integer of (at least) 16 bits width.
- typedef std::uint_least16_t uint16;
+#if HALF_ENABLE_CPP11_CSTDINT
+/// Unsigned integer of (at least) 16 bits width.
+typedef std::uint_least16_t uint16;
- /// Unsigned integer of (at least) 32 bits width.
- template<> struct bits<float> { typedef std::uint_least32_t type; };
+/// Unsigned integer of (at least) 32 bits width.
+template <>
+struct bits<float> {
+ typedef std::uint_least32_t type;
+};
- /// Unsigned integer of (at least) 64 bits width.
- template<> struct bits<double> { typedef std::uint_least64_t type; };
- #else
- /// Unsigned integer of (at least) 16 bits width.
- typedef unsigned short uint16;
+/// Unsigned integer of (at least) 64 bits width.
+template <>
+struct bits<double> {
+ typedef std::uint_least64_t type;
+};
+#else
+/// Unsigned integer of (at least) 16 bits width.
+typedef unsigned short uint16;
+
+/// Unsigned integer of (at least) 32 bits width.
+template <>
+struct bits<float>
+ : conditional<std::numeric_limits<unsigned int>::digits >= 32, unsigned int,
+ unsigned long> {};
+
+#if HALF_ENABLE_CPP11_LONG_LONG
+/// Unsigned integer of (at least) 64 bits width.
+template <>
+struct bits<double>
+ : conditional<std::numeric_limits<unsigned long>::digits >= 64,
+ unsigned long, unsigned long long> {};
+#else
+/// Unsigned integer of (at least) 64 bits width.
+template <>
+struct bits<double> {
+ typedef unsigned long type;
+};
+#endif
+#endif
- /// Unsigned integer of (at least) 32 bits width.
- template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
+/// Tag type for binary construction.
+struct binary_t {};
- #if HALF_ENABLE_CPP11_LONG_LONG
- /// Unsigned integer of (at least) 64 bits width.
- template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
- #else
- /// Unsigned integer of (at least) 64 bits width.
- template<> struct bits<double> { typedef unsigned long type; };
- #endif
- #endif
+/// Tag for binary construction.
+HALF_CONSTEXPR_CONST binary_t binary = binary_t();
- /// Tag type for binary construction.
- struct binary_t {};
+/// Temporary half-precision expression.
+/// This class represents a half-precision expression which just stores a
+/// single-precision value internally.
+struct expr {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
- /// Tag for binary construction.
- HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
- /// Temporary half-precision expression.
- /// This class represents a half-precision expression which just stores a single-precision value internally.
- struct expr
- {
- /// Conversion constructor.
- /// \param f single-precision value to convert
- explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+};
- /// Conversion to single-precision.
- /// \return single precision value representing expression value
- HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+/// SFINAE helper for generic half-precision functions.
+/// This class template has to be specialized for each valid combination of
+/// argument types to provide a corresponding `type` member equivalent to \a T.
+/// \tparam T type to return
+template <typename T, typename, typename = void, typename = void>
+struct enable {};
+template <typename T>
+struct enable<T, half, void, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, void, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, half, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, expr, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, half, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, expr, void> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, half, half> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, half, expr> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, expr, half> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, half, expr, expr> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, half, half> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, half, expr> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, expr, half> {
+ typedef T type;
+};
+template <typename T>
+struct enable<T, expr, expr, expr> {
+ typedef T type;
+};
- private:
- /// Internal expression value stored in single-precision.
- float value_;
- };
+/// Return type for specialized generic 2-argument half-precision functions.
+/// This class template has to be specialized for each valid combination of
+/// argument types to provide a corresponding `type` member denoting the
+/// appropriate return type. \tparam T first argument type \tparam U first
+/// argument type
+template <typename T, typename U>
+struct result : enable<expr, T, U> {};
+template <>
+struct result<half, half> {
+ typedef half type;
+};
- /// SFINAE helper for generic half-precision functions.
- /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
- /// `type` member equivalent to \a T.
- /// \tparam T type to return
- template<typename T,typename,typename=void,typename=void> struct enable {};
- template<typename T> struct enable<T,half,void,void> { typedef T type; };
- template<typename T> struct enable<T,expr,void,void> { typedef T type; };
- template<typename T> struct enable<T,half,half,void> { typedef T type; };
- template<typename T> struct enable<T,half,expr,void> { typedef T type; };
- template<typename T> struct enable<T,expr,half,void> { typedef T type; };
- template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
- template<typename T> struct enable<T,half,half,half> { typedef T type; };
- template<typename T> struct enable<T,half,half,expr> { typedef T type; };
- template<typename T> struct enable<T,half,expr,half> { typedef T type; };
- template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
- template<typename T> struct enable<T,expr,half,half> { typedef T type; };
- template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
- template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
- template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
+/// \name Classification helpers
+/// \{
- /// Return type for specialized generic 2-argument half-precision functions.
- /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
- /// `type` member denoting the appropriate return type.
- /// \tparam T first argument type
- /// \tparam U first argument type
- template<typename T,typename U> struct result : enable<expr,T,U> {};
- template<> struct result<half,half> { typedef half type; };
+/// Check for infinity.
+/// \tparam T argument type (builtin floating point type)
+/// \param arg value to query
+/// \retval true if infinity
+/// \retval false else
+template <typename T>
+bool builtin_isinf(T arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+#elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) &&
+ !::_isnan(static_cast<double>(arg));
+#else
+ return arg == std::numeric_limits<T>::infinity() ||
+ arg == -std::numeric_limits<T>::infinity();
+#endif
+}
- /// \name Classification helpers
- /// \{
+/// Check for NaN.
+/// \tparam T argument type (builtin floating point type)
+/// \param arg value to query
+/// \retval true if not a number
+/// \retval false else
+template <typename T>
+bool builtin_isnan(T arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+#elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+#else
+ return arg != arg;
+#endif
+}
- /// Check for infinity.
- /// \tparam T argument type (builtin floating point type)
- /// \param arg value to query
- /// \retval true if infinity
- /// \retval false else
- template<typename T> bool builtin_isinf(T arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return std::isinf(arg);
- #elif defined(_MSC_VER)
- return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
- #else
- return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
- #endif
- }
+/// Check sign.
+/// \tparam T argument type (builtin floating point type)
+/// \param arg value to query
+/// \retval true if signbit set
+/// \retval false else
+template <typename T>
+bool builtin_signbit(T arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+#else
+ return arg < T() || (arg == T() && T(1) / arg < T());
+#endif
+}
- /// Check for NaN.
- /// \tparam T argument type (builtin floating point type)
- /// \param arg value to query
- /// \retval true if not a number
- /// \retval false else
- template<typename T> bool builtin_isnan(T arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return std::isnan(arg);
- #elif defined(_MSC_VER)
- return ::_isnan(static_cast<double>(arg)) != 0;
- #else
- return arg != arg;
- #endif
- }
+/// \}
+/// \name Conversion
+/// \{
- /// Check sign.
- /// \tparam T argument type (builtin floating point type)
- /// \param arg value to query
- /// \retval true if signbit set
- /// \retval false else
- template<typename T> bool builtin_signbit(T arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return std::signbit(arg);
- #else
- return arg < T() || (arg == T() && T(1)/arg < T());
- #endif
- }
+/// Convert IEEE single-precision to half-precision.
+/// Credit for this goes to [Jeroen van der
+/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \tparam R
+/// rounding mode to use, `std::round_indeterminate` for fastest rounding \param
+/// value single-precision value \return binary representation of half-precision
+/// value
+template <std::float_round_style R>
+uint16 float2half_impl(float value, true_type) {
+ typedef bits<float>::type uint32;
+ uint32 bits; // = *reinterpret_cast<uint32*>(&value); //violating
+ // strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+ /* uint16 hbits = (bits>>16) & 0x8000;
+ bits &= 0x7FFFFFFF;
+ int exp = bits >> 23;
+ if(exp == 255)
+ return hbits | 0x7C00 |
+ (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0)); if(exp > 142)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF +
+ (R!=std::round_toward_zero);
+ }
+ int g, s;
+ if(exp > 112)
+ {
+ g = (bits>>12) & 1;
+ s = (bits&0xFFF) != 0;
+ hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
+ }
+ else if(exp > 101)
+ {
+ int i = 125 - exp;
+ bits = (bits&0x7FFFFF) | 0x800000;
+ g = (bits>>i) & 1;
+ s = (bits&((1L<<i)-1)) != 0;
+ hbits |= bits >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s = bits != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ */
+ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010,
+ 0x0020, 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000,
+ 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400,
+ 0x3800, 0x3C00, 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800,
+ 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001,
+ 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, 0x8200,
+ 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400,
+ 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, 0xC000, 0xC400, 0xC800,
+ 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00,
+ 0xF000, 0xF400, 0xF800, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00};
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19,
+ 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23,
+ 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 13};
+ uint16 hbits =
+ base_table[bits >> 23] +
+ static_cast<uint16>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
+ if (R == std::round_to_nearest)
+ hbits +=
+ (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
+ (((bits >> 23) & 0xFF) == 102)) &
+ ((hbits & 0x7C00) != 0x7C00)
+#if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1) << (shift_table[bits >> 23] - 1)) - 1) &
+ bits) != 0) |
+ hbits)
+#endif
+ ;
+ else if (R == std::round_toward_zero)
+ hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23];
+ else if (R == std::round_toward_infinity)
+ hbits +=
+ ((((bits & 0x7FFFFF &
+ ((static_cast<uint32>(1) << (shift_table[bits >> 23])) - 1)) != 0) |
+ (((bits >> 23) <= 102) & ((bits >> 23) != 0))) &
+ (hbits < 0x7C00)) -
+ ((hbits == 0xFC00) & ((bits >> 23) != 511));
+ else if (R == std::round_toward_neg_infinity)
+ hbits +=
+ ((((bits & 0x7FFFFF &
+ ((static_cast<uint32>(1) << (shift_table[bits >> 23])) - 1)) != 0) |
+ (((bits >> 23) <= 358) & ((bits >> 23) != 256))) &
+ (hbits < 0xFC00) & (hbits >> 15)) -
+ ((hbits == 0x7C00) & ((bits >> 23) != 255));
+ return hbits;
+}
- /// \}
- /// \name Conversion
- /// \{
+/// Convert IEEE double-precision to half-precision.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \param value double-precision value \return binary representation
+/// of half-precision value
+template <std::float_round_style R>
+uint16 float2half_impl(double value, true_type) {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits; // = *reinterpret_cast<uint64*>(&value); //violating
+ // strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi >> 16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if (exp == 2047)
+ return hbits | 0x7C00 |
+ (0x3FF & -static_cast<unsigned>((bits & 0xFFFFFFFFFFFFF) != 0));
+ if (exp > 1038) {
+ if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15);
+ if (R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits >> 15);
+ return hbits | 0x7BFF + (R != std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if (exp > 1008) {
+ g = (hi >> 9) & 1;
+ s |= (hi & 0x1FF) != 0;
+ hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF);
+ } else if (exp > 997) {
+ int i = 1018 - exp;
+ hi = (hi & 0xFFFFF) | 0x100000;
+ g = (hi >> i) & 1;
+ s |= (hi & ((1L << i) - 1)) != 0;
+ hbits |= hi >> (i + 1);
+ } else {
+ g = 0;
+ s |= hi != 0;
+ }
+ if (R == std::round_to_nearest)
+#if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s | hbits);
+#else
+ hbits += g;
+#endif
+ else if (R == std::round_toward_infinity)
+ hbits += ~(hbits >> 15) & (s | g);
+ else if (R == std::round_toward_neg_infinity)
+ hbits += (hbits >> 15) & (g | s);
+ return hbits;
+}
- /// Convert IEEE single-precision to half-precision.
- /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \param value single-precision value
- /// \return binary representation of half-precision value
- template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
- {
- typedef bits<float>::type uint32;
- uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
- std::memcpy(&bits, &value, sizeof(float));
-/* uint16 hbits = (bits>>16) & 0x8000;
- bits &= 0x7FFFFFFF;
- int exp = bits >> 23;
- if(exp == 255)
- return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0));
- if(exp > 142)
- {
- if(R == std::round_toward_infinity)
- return hbits | 0x7C00 - (hbits>>15);
- if(R == std::round_toward_neg_infinity)
- return hbits | 0x7BFF + (hbits>>15);
- return hbits | 0x7BFF + (R!=std::round_toward_zero);
- }
- int g, s;
- if(exp > 112)
- {
- g = (bits>>12) & 1;
- s = (bits&0xFFF) != 0;
- hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
- }
- else if(exp > 101)
- {
- int i = 125 - exp;
- bits = (bits&0x7FFFFF) | 0x800000;
- g = (bits>>i) & 1;
- s = (bits&((1L<<i)-1)) != 0;
- hbits |= bits >> (i+1);
- }
- else
- {
- g = 0;
- s = bits != 0;
- }
- if(R == std::round_to_nearest)
- #if HALF_ROUND_TIES_TO_EVEN
- hbits += g & (s|hbits);
- #else
- hbits += g;
- #endif
- else if(R == std::round_toward_infinity)
- hbits += ~(hbits>>15) & (s|g);
- else if(R == std::round_toward_neg_infinity)
- hbits += (hbits>>15) & (g|s);
-*/ static const uint16 base_table[512] = {
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
- 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
- 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
- 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
- 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
- 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
- 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
- 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
- static const unsigned char shift_table[512] = {
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
- 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
- 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
- 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
- uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
- if(R == std::round_to_nearest)
- hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
- #if HALF_ROUND_TIES_TO_EVEN
- & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
- #endif
- ;
- else if(R == std::round_toward_zero)
- hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
- else if(R == std::round_toward_infinity)
- hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
- ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
- else if(R == std::round_toward_neg_infinity)
- hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
- ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
- return hbits;
- }
+/// Convert non-IEEE floating point to half-precision.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam T source type (builtin floating point type) \param value
+/// floating point value \return binary representation of half-precision value
+template <std::float_round_style R, typename T>
+uint16 float2half_impl(T value, ...) {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if (value == T()) return hbits;
+ if (builtin_isnan(value)) return hbits | 0x7FFF;
+ if (builtin_isinf(value)) return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if (exp > 16) {
+ if (R == std::round_toward_infinity)
+ return hbits | (0x7C00 - (hbits >> 15));
+ else if (R == std::round_toward_neg_infinity)
+ return hbits | (0x7BFF + (hbits >> 15));
+ return hbits | (0x7BFF + (R != std::round_toward_zero));
+ }
+ if (exp < -13)
+ value = std::ldexp(value, 24);
+ else {
+ value = std::ldexp(value, 11 - exp);
+ hbits |= ((exp + 13) << 10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if (R == std::round_to_nearest) {
+ frac = std::abs(frac);
+#if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits);
+#else
+ hbits += frac >= T(0.5);
+#endif
+ } else if (R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if (R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+}
- /// Convert IEEE double-precision to half-precision.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \param value double-precision value
- /// \return binary representation of half-precision value
- template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
- {
- typedef bits<float>::type uint32;
- typedef bits<double>::type uint64;
- uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
- std::memcpy(&bits, &value, sizeof(double));
- uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
- uint16 hbits = (hi>>16) & 0x8000;
- hi &= 0x7FFFFFFF;
- int exp = hi >> 20;
- if(exp == 2047)
- return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
- if(exp > 1038)
- {
- if(R == std::round_toward_infinity)
- return hbits | 0x7C00 - (hbits>>15);
- if(R == std::round_toward_neg_infinity)
- return hbits | 0x7BFF + (hbits>>15);
- return hbits | 0x7BFF + (R!=std::round_toward_zero);
- }
- int g, s = lo != 0;
- if(exp > 1008)
- {
- g = (hi>>9) & 1;
- s |= (hi&0x1FF) != 0;
- hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
- }
- else if(exp > 997)
- {
- int i = 1018 - exp;
- hi = (hi&0xFFFFF) | 0x100000;
- g = (hi>>i) & 1;
- s |= (hi&((1L<<i)-1)) != 0;
- hbits |= hi >> (i+1);
- }
- else
- {
- g = 0;
- s |= hi != 0;
- }
- if(R == std::round_to_nearest)
- #if HALF_ROUND_TIES_TO_EVEN
- hbits += g & (s|hbits);
- #else
- hbits += g;
- #endif
- else if(R == std::round_toward_infinity)
- hbits += ~(hbits>>15) & (s|g);
- else if(R == std::round_toward_neg_infinity)
- hbits += (hbits>>15) & (g|s);
- return hbits;
- }
+/// Convert floating point to half-precision.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam T source type (builtin floating point type) \param value
+/// floating point value \return binary representation of half-precision value
+template <std::float_round_style R, typename T>
+uint16 float2half(T value) {
+ return float2half_impl<R>(
+ value, bool_type < std::numeric_limits<T>::is_iec559 &&
+ sizeof(typename bits<T>::type) == sizeof(T) > ());
+}
- /// Convert non-IEEE floating point to half-precision.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam T source type (builtin floating point type)
- /// \param value floating point value
- /// \return binary representation of half-precision value
- template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
- {
- uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
- if(value == T())
- return hbits;
- if(builtin_isnan(value))
- return hbits | 0x7FFF;
- if(builtin_isinf(value))
- return hbits | 0x7C00;
- int exp;
- std::frexp(value, &exp);
- if(exp > 16)
- {
- if(R == std::round_toward_infinity)
- return hbits | (0x7C00 - (hbits>>15));
- else if(R == std::round_toward_neg_infinity)
- return hbits | (0x7BFF + (hbits>>15));
- return hbits | (0x7BFF + (R!=std::round_toward_zero));
- }
- if(exp < -13)
- value = std::ldexp(value, 24);
- else
- {
- value = std::ldexp(value, 11-exp);
- hbits |= ((exp+13)<<10);
- }
- T ival, frac = std::modf(value, &ival);
- hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
- if(R == std::round_to_nearest)
- {
- frac = std::abs(frac);
- #if HALF_ROUND_TIES_TO_EVEN
- hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
- #else
- hbits += frac >= T(0.5);
- #endif
- }
- else if(R == std::round_toward_infinity)
- hbits += frac > T();
- else if(R == std::round_toward_neg_infinity)
- hbits += frac < T();
- return hbits;
- }
+/// Convert integer to half-precision floating point.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam S `true` if value negative, `false` else \tparam T type to
+/// convert (builtin integer type) \param value non-negative integral value
+/// \return binary representation of half-precision value
+template <std::float_round_style R, bool S, typename T>
+uint16 int2half_impl(T value) {
+#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value,
+ "int to half conversion only supports builtin integer types");
+#endif
+ if (S) value = -value;
+ uint16 bits = S << 15;
+ if (value > 0xFFFF) {
+ if (R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if (R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R != std::round_toward_zero);
+ } else if (value) {
+ unsigned int m = value, exp = 24;
+ for (; m < 0x400; m <<= 1, --exp)
+ ;
+ for (; m > 0x7FF; m >>= 1, ++exp)
+ ;
+ bits |= (exp << 10) + m;
+ if (exp > 24) {
+ if (R == std::round_to_nearest)
+ bits += (value >> (exp - 25)) & 1
+#if HALF_ROUND_TIES_TO_EVEN
+ & (((((1 << (exp - 25)) - 1) & value) != 0) | bits)
+#endif
+ ;
+ else if (R == std::round_toward_infinity)
+ bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S;
+ else if (R == std::round_toward_neg_infinity)
+ bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S;
+ }
+ }
+ return bits;
+}
- /// Convert floating point to half-precision.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam T source type (builtin floating point type)
- /// \param value floating point value
- /// \return binary representation of half-precision value
- template<std::float_round_style R,typename T> uint16 float2half(T value)
- {
- return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
- }
+/// Convert integer to half-precision floating point.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam T type to convert (builtin integer type) \param value
+/// integral value \return binary representation of half-precision value
+template <std::float_round_style R, typename T>
+uint16 int2half(T value) {
+ return (value < 0) ? int2half_impl<R, true>(value)
+ : int2half_impl<R, false>(value);
+}
- /// Convert integer to half-precision floating point.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam S `true` if value negative, `false` else
- /// \tparam T type to convert (builtin integer type)
- /// \param value non-negative integral value
- /// \return binary representation of half-precision value
- template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
- {
- #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
- #endif
- if(S)
- value = -value;
- uint16 bits = S << 15;
- if(value > 0xFFFF)
- {
- if(R == std::round_toward_infinity)
- bits |= 0x7C00 - S;
- else if(R == std::round_toward_neg_infinity)
- bits |= 0x7BFF + S;
- else
- bits |= 0x7BFF + (R!=std::round_toward_zero);
- }
- else if(value)
- {
- unsigned int m = value, exp = 24;
- for(; m<0x400; m<<=1,--exp) ;
- for(; m>0x7FF; m>>=1,++exp) ;
- bits |= (exp<<10) + m;
- if(exp > 24)
- {
- if(R == std::round_to_nearest)
- bits += (value>>(exp-25)) & 1
- #if HALF_ROUND_TIES_TO_EVEN
- & (((((1<<(exp-25))-1)&value)!=0)|bits)
- #endif
- ;
- else if(R == std::round_toward_infinity)
- bits += ((value&((1<<(exp-24))-1))!=0) & !S;
- else if(R == std::round_toward_neg_infinity)
- bits += ((value&((1<<(exp-24))-1))!=0) & S;
- }
- }
- return bits;
- }
+/// Convert half-precision to IEEE single-precision.
+/// Credit for this goes to [Jeroen van der
+/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \param
+/// value binary representation of half-precision value \return single-precision
+/// value
+inline float half2float_impl(uint16 value, float, true_type) {
+ typedef bits<float>::type uint32;
+ /* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ bits |= 0x38000000 <<
+ static_cast<unsigned>(abs>=0x7C00); for(; abs<0x400;
+ abs<<=1,bits-=0x800000) ; bits += static_cast<uint32>(abs) << 13;
+ }
+ */
+ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000,
+ 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000,
+ 0x35400000, 0x35500000, 0x35600000, 0x35700000, 0x35800000, 0x35880000,
+ 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000,
+ 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000,
+ 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, 0x360C0000,
+ 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000,
+ 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000,
+ 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000,
+ 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000, 0x36820000,
+ 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000,
+ 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000,
+ 0x369C0000, 0x369E0000, 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000,
+ 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000,
+ 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000,
+ 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000,
+ 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000,
+ 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000,
+ 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000,
+ 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, 0x37020000, 0x37030000,
+ 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000,
+ 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000,
+ 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000,
+ 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, 0x37200000, 0x37210000,
+ 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000,
+ 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000,
+ 0x372E0000, 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000,
+ 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000,
+ 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000,
+ 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000,
+ 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000,
+ 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000,
+ 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000,
+ 0x375E0000, 0x375F0000, 0x37600000, 0x37610000, 0x37620000, 0x37630000,
+ 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000,
+ 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000,
+ 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000,
+ 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, 0x37800000, 0x37808000,
+ 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000,
+ 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000,
+ 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, 0x37898000,
+ 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000,
+ 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000,
+ 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000,
+ 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, 0x37988000,
+ 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000,
+ 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000,
+ 0x379F0000, 0x379F8000, 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000,
+ 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000,
+ 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000,
+ 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000,
+ 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000,
+ 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000,
+ 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000,
+ 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000,
+ 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000,
+ 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000,
+ 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000,
+ 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, 0x37C80000, 0x37C88000,
+ 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000,
+ 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000,
+ 0x37CF0000, 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000,
+ 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000,
+ 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000,
+ 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000,
+ 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000,
+ 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000,
+ 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000,
+ 0x37E70000, 0x37E78000, 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000,
+ 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000,
+ 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000,
+ 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000,
+ 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, 0x37F80000, 0x37F88000,
+ 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000,
+ 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000,
+ 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, 0x3800C000,
+ 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000,
+ 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000,
+ 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000,
+ 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, 0x38084000,
+ 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000,
+ 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000,
+ 0x380B8000, 0x380BC000, 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000,
+ 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000,
+ 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000,
+ 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000,
+ 0x38130000, 0x38134000, 0x38138000, 0x3813C000, 0x38140000, 0x38144000,
+ 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000,
+ 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000,
+ 0x38178000, 0x3817C000, 0x38180000, 0x38184000, 0x38188000, 0x3818C000,
+ 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000,
+ 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000,
+ 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000,
+ 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, 0x38200000, 0x38204000,
+ 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000,
+ 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000,
+ 0x38238000, 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000,
+ 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000,
+ 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000,
+ 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000,
+ 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000,
+ 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000,
+ 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000,
+ 0x382F8000, 0x382FC000, 0x38300000, 0x38304000, 0x38308000, 0x3830C000,
+ 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000,
+ 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000,
+ 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000,
+ 0x38370000, 0x38374000, 0x38378000, 0x3837C000, 0x38380000, 0x38384000,
+ 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000,
+ 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000,
+ 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000,
+ 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000,
+ 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000,
+ 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000,
+ 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000, 0x38444000,
+ 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000,
+ 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000,
+ 0x38478000, 0x3847C000, 0x38480000, 0x38484000, 0x38488000, 0x3848C000,
+ 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000,
+ 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000,
+ 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000,
+ 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, 0x38500000, 0x38504000,
+ 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000,
+ 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000,
+ 0x38538000, 0x3853C000, 0x38540000, 0x38544000, 0x38548000, 0x3854C000,
+ 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000,
+ 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000,
+ 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000,
+ 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, 0x385C0000, 0x385C4000,
+ 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000,
+ 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000,
+ 0x385F8000, 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000,
+ 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000,
+ 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000,
+ 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000,
+ 0x38670000, 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000,
+ 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000,
+ 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000,
+ 0x386B8000, 0x386BC000, 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000,
+ 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000,
+ 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000,
+ 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000,
+ 0x38730000, 0x38734000, 0x38738000, 0x3873C000, 0x38740000, 0x38744000,
+ 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000,
+ 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000,
+ 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000, 0x3878C000,
+ 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000,
+ 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000,
+ 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000,
+ 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000, 0x38002000,
+ 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000,
+ 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000,
+ 0x3801C000, 0x3801E000, 0x38020000, 0x38022000, 0x38024000, 0x38026000,
+ 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000,
+ 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000,
+ 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000,
+ 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, 0x38060000, 0x38062000,
+ 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000,
+ 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000,
+ 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, 0x38084000, 0x38086000,
+ 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000,
+ 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000,
+ 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000,
+ 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, 0x380C0000, 0x380C2000,
+ 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000,
+ 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000,
+ 0x380DC000, 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000,
+ 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000,
+ 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000,
+ 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000,
+ 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000,
+ 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000,
+ 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000,
+ 0x3813C000, 0x3813E000, 0x38140000, 0x38142000, 0x38144000, 0x38146000,
+ 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000,
+ 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000,
+ 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000,
+ 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, 0x38180000, 0x38182000,
+ 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000,
+ 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000,
+ 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000,
+ 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000,
+ 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000,
+ 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000,
+ 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, 0x381E2000,
+ 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000,
+ 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000,
+ 0x381FC000, 0x381FE000, 0x38200000, 0x38202000, 0x38204000, 0x38206000,
+ 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000,
+ 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000,
+ 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000,
+ 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, 0x38240000, 0x38242000,
+ 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000,
+ 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000,
+ 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, 0x38264000, 0x38266000,
+ 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000,
+ 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000,
+ 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000,
+ 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, 0x382A0000, 0x382A2000,
+ 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000,
+ 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000,
+ 0x382BC000, 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000,
+ 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000,
+ 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000,
+ 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000,
+ 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000,
+ 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000,
+ 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000,
+ 0x3831C000, 0x3831E000, 0x38320000, 0x38322000, 0x38324000, 0x38326000,
+ 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000,
+ 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000,
+ 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000,
+ 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, 0x38360000, 0x38362000,
+ 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000,
+ 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000,
+ 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, 0x38386000,
+ 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000,
+ 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000,
+ 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000,
+ 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, 0x383C2000,
+ 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000,
+ 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000,
+ 0x383DC000, 0x383DE000, 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000,
+ 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000,
+ 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000,
+ 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000,
+ 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, 0x38420000, 0x38422000,
+ 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000,
+ 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000,
+ 0x3843C000, 0x3843E000, 0x38440000, 0x38442000, 0x38444000, 0x38446000,
+ 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000,
+ 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000,
+ 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000,
+ 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, 0x38480000, 0x38482000,
+ 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000,
+ 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000,
+ 0x3849C000, 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000,
+ 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000,
+ 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000,
+ 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000,
+ 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000,
+ 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000,
+ 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000,
+ 0x384FC000, 0x384FE000, 0x38500000, 0x38502000, 0x38504000, 0x38506000,
+ 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000,
+ 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000,
+ 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000,
+ 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, 0x38540000, 0x38542000,
+ 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000,
+ 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000,
+ 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, 0x38566000,
+ 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000,
+ 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000,
+ 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000,
+ 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000, 0x385A2000,
+ 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000,
+ 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000,
+ 0x385BC000, 0x385BE000, 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000,
+ 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000,
+ 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000,
+ 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000,
+ 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, 0x38600000, 0x38602000,
+ 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000,
+ 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000,
+ 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, 0x38624000, 0x38626000,
+ 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000,
+ 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000,
+ 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000,
+ 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, 0x38660000, 0x38662000,
+ 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000,
+ 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000,
+ 0x3867C000, 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000,
+ 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000,
+ 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000,
+ 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000,
+ 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000,
+ 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000,
+ 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000,
+ 0x386DC000, 0x386DE000, 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000,
+ 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000,
+ 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000,
+ 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000,
+ 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, 0x38720000, 0x38722000,
+ 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000,
+ 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000,
+ 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000, 0x38746000,
+ 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000,
+ 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000,
+ 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000,
+ 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, 0x38782000,
+ 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000,
+ 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000,
+ 0x3879C000, 0x3879E000, 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000,
+ 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000,
+ 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000,
+ 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000,
+ 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000,
+ 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000,
+ 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000,
+ 0x387FC000, 0x387FE000};
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000,
+ 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000,
+ 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000,
+ 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000,
+ 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000,
+ 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000,
+ 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000,
+ 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000,
+ 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000,
+ 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000};
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0,
+ 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024};
+ uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] +
+ exponent_table[value >> 10];
+ // return *reinterpret_cast<float*>(&bits); //violating
+ //strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+}
- /// Convert integer to half-precision floating point.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam T type to convert (builtin integer type)
- /// \param value integral value
- /// \return binary representation of half-precision value
- template<std::float_round_style R,typename T> uint16 int2half(T value)
- {
- return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
- }
+/// Convert half-precision to IEEE double-precision.
+/// \param value binary representation of half-precision value
+/// \return double-precision value
+inline double half2float_impl(uint16 value, double, true_type) {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value & 0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if (abs) {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs >= 0x7C00);
+ for (; abs < 0x400; abs <<= 1, hi -= 0x100000)
+ ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+ // return *reinterpret_cast<double*>(&bits); //violating
+ //strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+}
- /// Convert half-precision to IEEE single-precision.
- /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
- /// \param value binary representation of half-precision value
- /// \return single-precision value
- inline float half2float_impl(uint16 value, float, true_type)
- {
- typedef bits<float>::type uint32;
-/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
- int abs = value & 0x7FFF;
- if(abs)
- {
- bits |= 0x38000000 << static_cast<unsigned>(abs>=0x7C00);
- for(; abs<0x400; abs<<=1,bits-=0x800000) ;
- bits += static_cast<uint32>(abs) << 13;
- }
-*/ static const uint32 mantissa_table[2048] = {
- 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
- 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
- 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
- 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
- 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
- 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
- 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
- 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
- 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
- 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
- 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
- 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
- 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
- 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
- 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
- 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
- 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
- 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
- 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
- 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
- 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
- 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
- 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
- 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
- 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
- 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
- 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
- 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
- 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
- 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
- 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
- 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
- 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
- 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
- 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
- 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
- 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
- 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
- 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
- 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
- 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
- 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
- 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
- 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
- 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
- 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
- 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
- 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
- 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
- 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
- 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
- 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
- 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
- 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
- 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
- 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
- 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
- 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
- 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
- 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
- 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
- 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
- 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
- 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
- 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
- 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
- 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
- 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
- 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
- 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
- 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
- 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
- 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
- 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
- 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
- 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
- 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
- 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
- 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
- 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
- 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
- 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
- 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
- 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
- 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
- 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
- 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
- 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
- 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
- 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
- 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
- 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
- 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
- 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
- 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
- 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
- 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
- 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
- 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
- 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
- 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
- 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
- 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
- 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
- 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
- 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
- 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
- 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
- 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
- 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
- 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
- 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
- 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
- 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
- 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
- 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
- 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
- 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
- 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
- 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
- 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
- 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
- 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
- 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
- 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
- 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
- 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
- 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
- static const uint32 exponent_table[64] = {
- 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
- 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
- 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
- 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
- static const unsigned short offset_table[64] = {
- 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
- 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
- uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
-// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
- float out;
- std::memcpy(&out, &bits, sizeof(float));
- return out;
- }
+/// Convert half-precision to non-IEEE floating point.
+/// \tparam T type to convert to (builtin integer type)
+/// \param value binary representation of half-precision value
+/// \return floating point value
+template <typename T>
+T half2float_impl(uint16 value, T, ...) {
+ T out;
+ int abs = value & 0x7FFF;
+ if (abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN
+ ? std::numeric_limits<T>::quiet_NaN()
+ : T();
+ else if (abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity
+ ? std::numeric_limits<T>::infinity()
+ : std::numeric_limits<T>::max();
+ else if (abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs & 0x3FF) | 0x400), (abs >> 10) - 25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value & 0x8000) ? -out : out;
+}
- /// Convert half-precision to IEEE double-precision.
- /// \param value binary representation of half-precision value
- /// \return double-precision value
- inline double half2float_impl(uint16 value, double, true_type)
- {
- typedef bits<float>::type uint32;
- typedef bits<double>::type uint64;
- uint32 hi = static_cast<uint32>(value&0x8000) << 16;
- int abs = value & 0x7FFF;
- if(abs)
- {
- hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
- for(; abs<0x400; abs<<=1,hi-=0x100000) ;
- hi += static_cast<uint32>(abs) << 10;
- }
- uint64 bits = static_cast<uint64>(hi) << 32;
-// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
- double out;
- std::memcpy(&out, &bits, sizeof(double));
- return out;
- }
+/// Convert half-precision to floating point.
+/// \tparam T type to convert to (builtin integer type)
+/// \param value binary representation of half-precision value
+/// \return floating point value
+template <typename T>
+T half2float(uint16 value) {
+ return half2float_impl(value, T(),
+ bool_type < std::numeric_limits<T>::is_iec559 &&
+ sizeof(typename bits<T>::type) == sizeof(T) > ());
+}
- /// Convert half-precision to non-IEEE floating point.
- /// \tparam T type to convert to (builtin integer type)
- /// \param value binary representation of half-precision value
- /// \return floating point value
- template<typename T> T half2float_impl(uint16 value, T, ...)
- {
- T out;
- int abs = value & 0x7FFF;
- if(abs > 0x7C00)
- out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
- else if(abs == 0x7C00)
- out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
- else if(abs > 0x3FF)
- out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
- else
- out = std::ldexp(static_cast<T>(abs), -24);
- return (value&0x8000) ? -out : out;
- }
+/// Convert half-precision floating point to integer.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam E `true` for round to even, `false` for round away from
+/// zero \tparam T type to convert to (buitlin integer type with at least 16
+/// bits precision, excluding any implicit sign bits) \param value binary
+/// representation of half-precision value \return integral value
+template <std::float_round_style R, bool E, typename T>
+T half2int_impl(uint16 value) {
+#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value,
+ "half to int conversion only supports builtin integer types");
+#endif
+ unsigned int e = value & 0x7FFF;
+ if (e >= 0x7C00)
+ return (value & 0x8000) ? std::numeric_limits<T>::min()
+ : std::numeric_limits<T>::max();
+ if (e < 0x3800) {
+ if (R == std::round_toward_infinity)
+ return T(~(value >> 15) & (e != 0));
+ else if (R == std::round_toward_neg_infinity)
+ return -T(value > 0x8000);
+ return T();
+ }
+ unsigned int m = (value & 0x3FF) | 0x400;
+ e >>= 10;
+ if (e < 25) {
+ if (R == std::round_to_nearest)
+ m += (1 << (24 - e)) - (~(m >> (25 - e)) & E);
+ else if (R == std::round_toward_infinity)
+ m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U);
+ else if (R == std::round_toward_neg_infinity)
+ m += -(value >> 15) & ((1 << (25 - e)) - 1U);
+ m >>= 25 - e;
+ } else
+ m <<= e - 25;
+ return (value & 0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+}
- /// Convert half-precision to floating point.
- /// \tparam T type to convert to (builtin integer type)
- /// \param value binary representation of half-precision value
- /// \return floating point value
- template<typename T> T half2float(uint16 value)
- {
- return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
- }
+/// Convert half-precision floating point to integer.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam T type to convert to (buitlin integer type with at least 16
+/// bits precision, excluding any implicit sign bits) \param value binary
+/// representation of half-precision value \return integral value
+template <std::float_round_style R, typename T>
+T half2int(uint16 value) {
+ return half2int_impl<R, HALF_ROUND_TIES_TO_EVEN, T>(value);
+}
- /// Convert half-precision floating point to integer.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam E `true` for round to even, `false` for round away from zero
- /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
- /// \param value binary representation of half-precision value
- /// \return integral value
- template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
- {
- #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
- #endif
- unsigned int e = value & 0x7FFF;
- if(e >= 0x7C00)
- return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
- if(e < 0x3800)
- {
- if(R == std::round_toward_infinity)
- return T(~(value>>15)&(e!=0));
- else if(R == std::round_toward_neg_infinity)
- return -T(value>0x8000);
- return T();
- }
- unsigned int m = (value&0x3FF) | 0x400;
- e >>= 10;
- if(e < 25)
- {
- if(R == std::round_to_nearest)
- m += (1<<(24-e)) - (~(m>>(25-e))&E);
- else if(R == std::round_toward_infinity)
- m += ((value>>15)-1) & ((1<<(25-e))-1U);
- else if(R == std::round_toward_neg_infinity)
- m += -(value>>15) & ((1<<(25-e))-1U);
- m >>= 25 - e;
- }
- else
- m <<= e - 25;
- return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
- }
+/// Convert half-precision floating point to integer using
+/// round-to-nearest-away-from-zero. \tparam T type to convert to (buitlin
+/// integer type with at least 16 bits precision, excluding any implicit sign
+/// bits) \param value binary representation of half-precision value \return
+/// integral value
+template <typename T>
+T half2int_up(uint16 value) {
+ return half2int_impl<std::round_to_nearest, 0, T>(value);
+}
- /// Convert half-precision floating point to integer.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
- /// \param value binary representation of half-precision value
- /// \return integral value
- template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
+/// Round half-precision number to nearest integer value.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \tparam E `true` for round to even, `false` for round away from
+/// zero \param value binary representation of half-precision value \return
+/// half-precision bits for nearest integral value
+template <std::float_round_style R, bool E>
+uint16 round_half_impl(uint16 value) {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if (e < 0x3C00) {
+ result &= 0x8000;
+ if (R == std::round_to_nearest)
+ result |= 0x3C00U & -(e >= (0x3800 + E));
+ else if (R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value >> 15) & (e != 0));
+ else if (R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value > 0x8000);
+ } else if (e < 0x6400) {
+ e = 25 - (e >> 10);
+ unsigned int mask = (1 << e) - 1;
+ if (R == std::round_to_nearest)
+ result += (1 << (e - 1)) - (~(result >> e) & E);
+ else if (R == std::round_toward_infinity)
+ result += mask & ((value >> 15) - 1);
+ else if (R == std::round_toward_neg_infinity)
+ result += mask & -(value >> 15);
+ result &= ~mask;
+ }
+ return result;
+}
- /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
- /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
- /// \param value binary representation of half-precision value
- /// \return integral value
- template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
+/// Round half-precision number to nearest integer value.
+/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest
+/// rounding \param value binary representation of half-precision value \return
+/// half-precision bits for nearest integral value
+template <std::float_round_style R>
+uint16 round_half(uint16 value) {
+ return round_half_impl<R, HALF_ROUND_TIES_TO_EVEN>(value);
+}
- /// Round half-precision number to nearest integer value.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \tparam E `true` for round to even, `false` for round away from zero
- /// \param value binary representation of half-precision value
- /// \return half-precision bits for nearest integral value
- template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
- {
- unsigned int e = value & 0x7FFF;
- uint16 result = value;
- if(e < 0x3C00)
- {
- result &= 0x8000;
- if(R == std::round_to_nearest)
- result |= 0x3C00U & -(e>=(0x3800+E));
- else if(R == std::round_toward_infinity)
- result |= 0x3C00U & -(~(value>>15)&(e!=0));
- else if(R == std::round_toward_neg_infinity)
- result |= 0x3C00U & -(value>0x8000);
- }
- else if(e < 0x6400)
- {
- e = 25 - (e>>10);
- unsigned int mask = (1<<e) - 1;
- if(R == std::round_to_nearest)
- result += (1<<(e-1)) - (~(result>>e)&E);
- else if(R == std::round_toward_infinity)
- result += mask & ((value>>15)-1);
- else if(R == std::round_toward_neg_infinity)
- result += mask & -(value>>15);
- result &= ~mask;
- }
- return result;
- }
+/// Round half-precision number to nearest integer value using
+/// round-to-nearest-away-from-zero. \param value binary representation of
+/// half-precision value \return half-precision bits for nearest integral value
+inline uint16 round_half_up(uint16 value) {
+ return round_half_impl<std::round_to_nearest, 0>(value);
+}
+/// \}
- /// Round half-precision number to nearest integer value.
- /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
- /// \param value binary representation of half-precision value
- /// \return half-precision bits for nearest integral value
- template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
+struct functions;
+template <typename>
+struct unary_specialized;
+template <typename, typename>
+struct binary_specialized;
+template <typename, typename, std::float_round_style>
+struct half_caster;
+} // namespace detail
- /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
- /// \param value binary representation of half-precision value
- /// \return half-precision bits for nearest integral value
- inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
- /// \}
+/// Half-precision floating point type.
+/// This class implements an IEEE-conformant half-precision floating point type
+/// with the usual arithmetic operators and conversions. It is implicitly
+/// convertible to single-precision floating point, which makes artihmetic
+/// expressions and functions with mixed-type operands to be of the most precise
+/// operand type. Additionally all arithmetic operations (and many mathematical
+/// functions) are carried out in single-precision internally. All conversions
+/// from single- to half-precision are done using the library's default rounding
+/// mode, but temporary results inside chained arithmetic expressions are kept
+/// in single-precision as long as possible (while of course still maintaining a
+/// strong half-precision type).
+///
+/// According to the C++98/03 definition, the half type is not a POD type. But
+/// according to C++11's less strict and extended definitions it is both a
+/// standard layout type and a trivially copyable type (even if not a POD type),
+/// which means it can be standard-conformantly copied using raw binary copies.
+/// But in this context some more words about the actual size of the type.
+/// Although the half is representing an IEEE 16-bit type, it does not
+/// neccessarily have to be of exactly 16-bits size. But on any reasonable
+/// implementation the actual binary representation of this type will most
+/// probably not ivolve any additional "magic" or padding beyond the simple
+/// binary representation of the underlying 16-bit IEEE number, even if not
+/// strictly guaranteed by the standard. But even then it only has an actual
+/// size of 16 bits if your C++ implementation supports an unsigned integer type
+/// of exactly 16 bits width. But this should be the case on nearly any
+/// reasonable platform.
+///
+/// So if your C++ implementation is not totally exotic or imposes special
+/// alignment requirements, it is a reasonable assumption that the data of a
+/// half is just comprised of the 2 bytes of the underlying IEEE representation.
+class half {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half, half>;
+ template <typename, typename, std::float_round_style>
+ friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+#if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+#endif
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+#endif
- struct functions;
- template<typename> struct unary_specialized;
- template<typename,typename> struct binary_specialized;
- template<typename,typename,std::float_round_style> struct half_caster;
- }
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin
+ /// types' default-initialization semantics and may be less efficient than no
+ /// initialization, it is needed to provide proper value-initialization
+ /// semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
- /// Half-precision floating point type.
- /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
- /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
- /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
- /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
- /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
- /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
- ///
- /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
- /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
- /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
- /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
- /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
- /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
- /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
- /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
- /// nearly any reasonable platform.
- ///
- /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
- /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
- class half
- {
- friend struct detail::functions;
- friend struct detail::unary_specialized<half>;
- friend struct detail::binary_specialized<half,half>;
- template<typename,typename,std::float_round_style> friend struct detail::half_caster;
- friend class std::numeric_limits<half>;
- #if HALF_ENABLE_CPP11_HASH
- friend struct std::hash<half>;
- #endif
- #if HALF_ENABLE_CPP11_USER_LITERALS
- friend half literal::operator"" _h(long double);
- #endif
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs)
+ : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
- public:
- /// Default constructor.
- /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
- /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
- HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
- /// Copy constructor.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to copy from
- half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
- /// Conversion constructor.
- /// \param rhs float to convert
- explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
-
- /// Conversion to single-precision.
- /// \return single precision value representing expression value
- operator float() const { return detail::half2float<float>(data_); }
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half &operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
- /// Assignment operator.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to copy from
- /// \return reference to this half
- half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template <typename T>
+ typename detail::enable<half &, T>::type operator+=(T rhs) {
+ return *this += static_cast<float>(rhs);
+ }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to add
- /// \return reference to this half
- template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template <typename T>
+ typename detail::enable<half &, T>::type operator-=(T rhs) {
+ return *this -= static_cast<float>(rhs);
+ }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to subtract
- /// \return reference to this half
- template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template <typename T>
+ typename detail::enable<half &, T>::type operator*=(T rhs) {
+ return *this *= static_cast<float>(rhs);
+ }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to multiply with
- /// \return reference to this half
- template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template <typename T>
+ typename detail::enable<half &, T>::type operator/=(T rhs) {
+ return *this /= static_cast<float>(rhs);
+ }
- /// Arithmetic assignment.
- /// \tparam T type of concrete half expression
- /// \param rhs half expression to divide by
- /// \return reference to this half
- template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half &operator=(float rhs) {
+ data_ = detail::float2half<round_style>(rhs);
+ return *this;
+ }
- /// Assignment operator.
- /// \param rhs single-precision value to copy from
- /// \return reference to this half
- half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half &operator+=(float rhs) {
+ data_ =
+ detail::float2half<round_style>(detail::half2float<float>(data_) + rhs);
+ return *this;
+ }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to add
- /// \return reference to this half
- half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half &operator-=(float rhs) {
+ data_ =
+ detail::float2half<round_style>(detail::half2float<float>(data_) - rhs);
+ return *this;
+ }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to subtract
- /// \return reference to this half
- half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half &operator*=(float rhs) {
+ data_ =
+ detail::float2half<round_style>(detail::half2float<float>(data_) * rhs);
+ return *this;
+ }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to multiply with
- /// \return reference to this half
- half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half &operator/=(float rhs) {
+ data_ =
+ detail::float2half<round_style>(detail::half2float<float>(data_) / rhs);
+ return *this;
+ }
- /// Arithmetic assignment.
- /// \param rhs single-precision value to divide by
- /// \return reference to this half
- half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
+ /// Prefix increment.
+ /// \return incremented half value
+ half &operator++() { return *this += 1.0f; }
- /// Prefix increment.
- /// \return incremented half value
- half& operator++() { return *this += 1.0f; }
+ /// Prefix decrement.
+ /// \return decremented half value
+ half &operator--() { return *this -= 1.0f; }
- /// Prefix decrement.
- /// \return decremented half value
- half& operator--() { return *this -= 1.0f; }
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) {
+ half out(*this);
+ ++*this;
+ return out;
+ }
- /// Postfix increment.
- /// \return non-incremented half value
- half operator++(int) { half out(*this); ++*this; return out; }
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) {
+ half out(*this);
+ --*this;
+ return out;
+ }
- /// Postfix decrement.
- /// \return non-decremented half value
- half operator--(int) { half out(*this); --*this; return out; }
-
- private:
- /// Rounding mode to use
- static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style =
+ (std::float_round_style)(HALF_ROUND_STYLE);
- /// Constructor.
- /// \param bits binary representation to set half to
- HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT
+ : data_(bits) {}
- /// Internal binary representation
- detail::uint16 data_;
- };
+ /// Internal binary representation
+ detail::uint16 data_;
+};
#if HALF_ENABLE_CPP11_USER_LITERALS
- namespace literal
- {
- /// Half literal.
- /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
- /// to rather involved conversions.
- /// \param value literal value
- /// \return half with given value (if representable)
- inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
- }
+namespace literal {
+/// Half literal.
+/// While this returns an actual half-precision value, half literals can
+/// unfortunately not be constant expressions due to rather involved
+/// conversions. \param value literal value \return half with given value (if
+/// representable)
+inline half operator"" _h(long double value) {
+ return half(detail::binary, detail::float2half<half::round_style>(value));
+}
+} // namespace literal
#endif
- namespace detail
- {
- /// Wrapper implementing unspecialized half-precision functions.
- struct functions
- {
- /// Addition implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision sum stored in single-precision
- static expr plus(float x, float y) { return expr(x+y); }
-
- /// Subtraction implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision difference stored in single-precision
- static expr minus(float x, float y) { return expr(x-y); }
+namespace detail {
+/// Wrapper implementing unspecialized half-precision functions.
+struct functions {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x + y); }
- /// Multiplication implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision product stored in single-precision
- static expr multiplies(float x, float y) { return expr(x*y); }
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x - y); }
- /// Division implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision quotient stored in single-precision
- static expr divides(float x, float y) { return expr(x/y); }
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x * y); }
- /// Output implementation.
- /// \param out stream to write to
- /// \param arg value to write
- /// \return reference to stream
- template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x / y); }
- /// Input implementation.
- /// \param in stream to read from
- /// \param arg half to read into
- /// \return reference to stream
- template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
- {
- float f;
- if(in >> f)
- arg = f;
- return in;
- }
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template <typename charT, typename traits>
+ static std::basic_ostream<charT, traits> &write(
+ std::basic_ostream<charT, traits> &out, float arg) {
+ return out << arg;
+ }
- /// Modulo implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision division remainder stored in single-precision
- static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template <typename charT, typename traits>
+ static std::basic_istream<charT, traits> &read(
+ std::basic_istream<charT, traits> &in, half &arg) {
+ float f;
+ if (in >> f) arg = f;
+ return in;
+ }
- /// Remainder implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Half-precision division remainder stored in single-precision
- static expr remainder(float x, float y)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::remainder(x, y));
- #else
- if(builtin_isnan(x) || builtin_isnan(y))
- return expr(std::numeric_limits<float>::quiet_NaN());
- float ax = std::fabs(x), ay = std::fabs(y);
- if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
- return expr(std::numeric_limits<float>::quiet_NaN());
- if(ay >= 65536.0f)
- return expr(x);
- if(ax == ay)
- return expr(builtin_signbit(x) ? -0.0f : 0.0f);
- ax = std::fmod(ax, ay+ay);
- float y2 = 0.5f * ay;
- if(ax > y2)
- {
- ax -= ay;
- if(ax >= y2)
- ax -= ay;
- }
- return expr(builtin_signbit(x) ? -ax : ax);
- #endif
- }
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
- /// Remainder implementation.
- /// \param x first operand
- /// \param y second operand
- /// \param quo address to store quotient bits at
- /// \return Half-precision division remainder stored in single-precision
- static expr remquo(float x, float y, int *quo)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::remquo(x, y, quo));
- #else
- if(builtin_isnan(x) || builtin_isnan(y))
- return expr(std::numeric_limits<float>::quiet_NaN());
- bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
- float ax = std::fabs(x), ay = std::fabs(y);
- if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
- return expr(std::numeric_limits<float>::quiet_NaN());
- if(ay >= 65536.0f)
- return expr(x);
- if(ax == ay)
- return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
- ax = std::fmod(ax, 8.0f*ay);
- int cquo = 0;
- if(ax >= 4.0f * ay)
- {
- ax -= 4.0f * ay;
- cquo += 4;
- }
- if(ax >= 2.0f * ay)
- {
- ax -= 2.0f * ay;
- cquo += 2;
- }
- float y2 = 0.5f * ay;
- if(ax > y2)
- {
- ax -= ay;
- ++cquo;
- if(ax >= y2)
- {
- ax -= ay;
- ++cquo;
- }
- }
- return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
- #endif
- }
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+#else
+ if (builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if (ay >= 65536.0f) return expr(x);
+ if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay + ay);
+ float y2 = 0.5f * ay;
+ if (ax > y2) {
+ ax -= ay;
+ if (ax >= y2) ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+#endif
+ }
- /// Positive difference implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return Positive difference stored in single-precision
- static expr fdim(float x, float y)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::fdim(x, y));
- #else
- return expr((x<=y) ? 0.0f : (x-y));
- #endif
- }
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+#else
+ if (builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x),
+ qsign = static_cast<bool>(sign ^ builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if (ay >= 65536.0f) return expr(x);
+ if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f * ay);
+ int cquo = 0;
+ if (ax >= 4.0f * ay) {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if (ax >= 2.0f * ay) {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if (ax > y2) {
+ ax -= ay;
+ ++cquo;
+ if (ax >= y2) {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+#endif
+ }
- /// Fused multiply-add implementation.
- /// \param x first operand
- /// \param y second operand
- /// \param z third operand
- /// \return \a x * \a y + \a z stored in single-precision
- static expr fma(float x, float y, float z)
- {
- #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
- return expr(std::fma(x, y, z));
- #else
- return expr(x*y+z);
- #endif
- }
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+#else
+ return expr((x <= y) ? 0.0f : (x - y));
+#endif
+ }
- /// Get NaN.
- /// \return Half-precision quiet NaN
- static half nanh() { return half(binary, 0x7FFF); }
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z) {
+#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+#else
+ return expr(x * y + z);
+#endif
+ }
- /// Exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr exp(float arg) { return expr(std::exp(arg)); }
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
- /// Exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr expm1(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::expm1(arg));
- #else
- return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
- #endif
- }
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
- /// Binary exponential implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr exp2(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::exp2(arg));
- #else
- return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
- #endif
- }
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+#else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg)) - 1.0));
+#endif
+ }
- /// Logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log(float arg) { return expr(std::log(arg)); }
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+#else
+ return expr(
+ static_cast<float>(std::exp(arg * 0.69314718055994530941723212145818)));
+#endif
+ }
- /// Common logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log10(float arg) { return expr(std::log10(arg)); }
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
- /// Logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log1p(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::log1p(arg));
- #else
- return expr(static_cast<float>(std::log(1.0+arg)));
- #endif
- }
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
- /// Binary logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr log2(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::log2(arg));
- #else
- return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
- #endif
- }
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+#else
+ return expr(static_cast<float>(std::log(1.0 + arg)));
+#endif
+ }
- /// Square root implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+#else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg)) *
+ 1.4426950408889634073599246810019));
+#endif
+ }
- /// Cubic root implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cbrt(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::cbrt(arg));
- #else
- if(builtin_isnan(arg) || builtin_isinf(arg))
- return expr(arg);
- return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
- static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
- #endif
- }
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
- /// Hypotenuse implementation.
- /// \param x first argument
- /// \param y second argument
- /// \return function value stored in single-preicision
- static expr hypot(float x, float y)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::hypot(x, y));
- #else
- return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
- static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
- #endif
- }
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+#else
+ if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg);
+ return expr(builtin_signbit(arg)
+ ? -static_cast<float>(
+ std::pow(-static_cast<double>(arg), 1.0 / 3.0))
+ : static_cast<float>(
+ std::pow(static_cast<double>(arg), 1.0 / 3.0)));
+#endif
+ }
- /// Power implementation.
- /// \param base value to exponentiate
- /// \param exp power to expontiate to
- /// \return function value stored in single-preicision
- static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+#else
+ return expr(
+ (builtin_isinf(x) || builtin_isinf(y))
+ ? std::numeric_limits<float>::infinity()
+ : static_cast<float>(std::sqrt(static_cast<double>(x) * x +
+ static_cast<double>(y) * y)));
+#endif
+ }
- /// Sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sin(float arg) { return expr(std::sin(arg)); }
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
- /// Cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cos(float arg) { return expr(std::cos(arg)); }
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
- /// Tan implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tan(float arg) { return expr(std::tan(arg)); }
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
- /// Arc sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr asin(float arg) { return expr(std::asin(arg)); }
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
- /// Arc cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr acos(float arg) { return expr(std::acos(arg)); }
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
- /// Arc tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr atan(float arg) { return expr(std::atan(arg)); }
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
- /// Arc tangent implementation.
- /// \param x first argument
- /// \param y second argument
- /// \return function value stored in single-preicision
- static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
- /// Hyperbolic sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr sinh(float arg) { return expr(std::sinh(arg)); }
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
- /// Hyperbolic cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr cosh(float arg) { return expr(std::cosh(arg)); }
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
- /// Hyperbolic tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tanh(float arg) { return expr(std::tanh(arg)); }
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
- /// Hyperbolic area sine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr asinh(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::asinh(arg));
- #else
- return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
- #endif
- }
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
- /// Hyperbolic area cosine implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr acosh(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::acosh(arg));
- #else
- return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
- #endif
- }
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+#else
+ return expr(
+ (arg == -std::numeric_limits<float>::infinity())
+ ? arg
+ : static_cast<float>(std::log(arg + std::sqrt(arg * arg + 1.0))));
+#endif
+ }
- /// Hyperbolic area tangent implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr atanh(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::atanh(arg));
- #else
- return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
- #endif
- }
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+#else
+ return expr((arg < -1.0f) ? std::numeric_limits<float>::quiet_NaN()
+ : static_cast<float>(std::log(
+ arg + std::sqrt(arg * arg - 1.0))));
+#endif
+ }
- /// Error function implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr erf(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::erf(arg));
- #else
- return expr(static_cast<float>(erf(static_cast<double>(arg))));
- #endif
- }
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+#else
+ return expr(static_cast<float>(0.5 * std::log((1.0 + arg) / (1.0 - arg))));
+#endif
+ }
- /// Complementary implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr erfc(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::erfc(arg));
- #else
- return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
- #endif
- }
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+#else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+#endif
+ }
- /// Gamma logarithm implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr lgamma(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::lgamma(arg));
- #else
- if(builtin_isinf(arg))
- return expr(std::numeric_limits<float>::infinity());
- if(arg < 0.0f)
- {
- float i, f = std::modf(-arg, &i);
- if(f == 0.0f)
- return expr(std::numeric_limits<float>::infinity());
- return expr(static_cast<float>(1.1447298858494001741434273513531-
- std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
- }
- return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
- #endif
- }
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+#else
+ return expr(static_cast<float>(1.0 - erf(static_cast<double>(arg))));
+#endif
+ }
- /// Gamma implementation.
- /// \param arg function argument
- /// \return function value stored in single-preicision
- static expr tgamma(float arg)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::tgamma(arg));
- #else
- if(arg == 0.0f)
- return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
- if(arg < 0.0f)
- {
- float i, f = std::modf(-arg, &i);
- if(f == 0.0f)
- return expr(std::numeric_limits<float>::quiet_NaN());
- double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
- return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
- }
- if(builtin_isinf(arg))
- return expr(arg);
- return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
- #endif
- }
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+#else
+ if (builtin_isinf(arg)) return expr(std::numeric_limits<float>::infinity());
+ if (arg < 0.0f) {
+ float i, f = std::modf(-arg, &i);
+ if (f == 0.0f) return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(
+ 1.1447298858494001741434273513531 -
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) -
+ lgamma(1.0 - arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+#endif
+ }
- /// Floor implementation.
- /// \param arg value to round
- /// \return rounded value
- static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+#else
+ if (arg == 0.0f)
+ return builtin_signbit(arg)
+ ? expr(-std::numeric_limits<float>::infinity())
+ : expr(std::numeric_limits<float>::infinity());
+ if (arg < 0.0f) {
+ float i, f = std::modf(-arg, &i);
+ if (f == 0.0f) return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 /
+ (std::sin(3.1415926535897932384626433832795 * f) *
+ std::exp(lgamma(1.0 - arg)));
+ return expr(
+ static_cast<float>((std::fmod(i, 2.0f) == 0.0f) ? -value : value));
+ }
+ if (builtin_isinf(arg)) return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+#endif
+ }
- /// Ceiling implementation.
- /// \param arg value to round
- /// \return rounded value
- static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) {
+ return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_));
+ }
- /// Truncation implementation.
- /// \param arg value to round
- /// \return rounded value
- static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) {
+ return half(binary, round_half<std::round_toward_infinity>(arg.data_));
+ }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) {
+ return half(binary, round_half<std::round_toward_zero>(arg.data_));
+ }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) {
+ return half(binary, round_half<half::round_style>(arg.data_));
+ }
- #if HALF_ENABLE_CPP11_LONG_LONG
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) {
+ return detail::half2int<half::round_style, long>(arg.data_);
+ }
- /// Nearest integer implementation.
- /// \param arg value to round
- /// \return rounded value
- static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
- #endif
+#if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) {
+ return detail::half2int_up<long long>(arg.data_);
+ }
- /// Decompression implementation.
- /// \param arg number to decompress
- /// \param exp address to store exponent at
- /// \return normalized significant
- static half frexp(half arg, int *exp)
- {
- int m = arg.data_ & 0x7FFF, e = -14;
- if(m >= 0x7C00 || !m)
- return *exp = 0, arg;
- for(; m<0x400; m<<=1,--e) ;
- return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
- }
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) {
+ return detail::half2int<half::round_style, long long>(arg.data_);
+ }
+#endif
- /// Decompression implementation.
- /// \param arg number to decompress
- /// \param iptr address to store integer part at
- /// \return fractional part
- static half modf(half arg, half *iptr)
- {
- unsigned int e = arg.data_ & 0x7FFF;
- if(e >= 0x6400)
- return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
- if(e < 0x3C00)
- return iptr->data_ = arg.data_ & 0x8000, arg;
- e >>= 10;
- unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
- iptr->data_ = arg.data_ & ~mask;
- if(!m)
- return half(binary, arg.data_&0x8000);
- for(; m<0x400; m<<=1,--e) ;
- return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
- }
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp) {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if (m >= 0x7C00 || !m) return *exp = 0, arg;
+ for (; m < 0x400; m <<= 1, --e)
+ ;
+ return *exp = e + (m >> 10),
+ half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF));
+ }
- /// Scaling implementation.
- /// \param arg number to scale
- /// \param exp power of two to scale by
- /// \return scaled number
- static half scalbln(half arg, long exp)
- {
- unsigned int m = arg.data_ & 0x7FFF;
- if(m >= 0x7C00 || !m)
- return arg;
- for(; m<0x400; m<<=1,--exp) ;
- exp += m >> 10;
- uint16 value = arg.data_ & 0x8000;
- if(exp > 30)
- {
- if(half::round_style == std::round_toward_zero)
- value |= 0x7BFF;
- else if(half::round_style == std::round_toward_infinity)
- value |= 0x7C00 - (value>>15);
- else if(half::round_style == std::round_toward_neg_infinity)
- value |= 0x7BFF + (value>>15);
- else
- value |= 0x7C00;
- }
- else if(exp > 0)
- value |= (exp<<10) | (m&0x3FF);
- else if(exp > -11)
- {
- m = (m&0x3FF) | 0x400;
- if(half::round_style == std::round_to_nearest)
- {
- m += 1 << -exp;
- #if HALF_ROUND_TIES_TO_EVEN
- m -= (m>>(1-exp)) & 1;
- #endif
- }
- else if(half::round_style == std::round_toward_infinity)
- m += ((value>>15)-1) & ((1<<(1-exp))-1U);
- else if(half::round_style == std::round_toward_neg_infinity)
- m += -(value>>15) & ((1<<(1-exp))-1U);
- value |= m >> (1-exp);
- }
- else if(half::round_style == std::round_toward_infinity)
- value -= (value>>15) - 1;
- else if(half::round_style == std::round_toward_neg_infinity)
- value += value >> 15;
- return half(binary, value);
- }
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr) {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if (e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00)));
+ if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if (!m) return half(binary, arg.data_ & 0x8000);
+ for (; m < 0x400; m <<= 1, --e)
+ ;
+ return half(binary, static_cast<uint16>((arg.data_ & 0x8000) | (e << 10) |
+ (m & 0x3FF)));
+ }
- /// Exponent implementation.
- /// \param arg number to query
- /// \return floating point exponent
- static int ilogb(half arg)
- {
- int abs = arg.data_ & 0x7FFF;
- if(!abs)
- return FP_ILOGB0;
- if(abs < 0x7C00)
- {
- int exp = (abs>>10) - 15;
- if(abs < 0x400)
- for(; abs<0x200; abs<<=1,--exp) ;
- return exp;
- }
- if(abs > 0x7C00)
- return FP_ILOGBNAN;
- return INT_MAX;
- }
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp) {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if (m >= 0x7C00 || !m) return arg;
+ for (; m < 0x400; m <<= 1, --exp)
+ ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if (exp > 30) {
+ if (half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if (half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value >> 15);
+ else if (half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value >> 15);
+ else
+ value |= 0x7C00;
+ } else if (exp > 0)
+ value |= (exp << 10) | (m & 0x3FF);
+ else if (exp > -11) {
+ m = (m & 0x3FF) | 0x400;
+ if (half::round_style == std::round_to_nearest) {
+ m += 1 << -exp;
+#if HALF_ROUND_TIES_TO_EVEN
+ m -= (m >> (1 - exp)) & 1;
+#endif
+ } else if (half::round_style == std::round_toward_infinity)
+ m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U);
+ else if (half::round_style == std::round_toward_neg_infinity)
+ m += -(value >> 15) & ((1 << (1 - exp)) - 1U);
+ value |= m >> (1 - exp);
+ } else if (half::round_style == std::round_toward_infinity)
+ value -= (value >> 15) - 1;
+ else if (half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
- /// Exponent implementation.
- /// \param arg number to query
- /// \return floating point exponent
- static half logb(half arg)
- {
- int abs = arg.data_ & 0x7FFF;
- if(!abs)
- return half(binary, 0xFC00);
- if(abs < 0x7C00)
- {
- int exp = (abs>>10) - 15;
- if(abs < 0x400)
- for(; abs<0x200; abs<<=1,--exp) ;
- uint16 bits = (exp<0) << 15;
- if(exp)
- {
- unsigned int m = std::abs(exp) << 6, e = 18;
- for(; m<0x400; m<<=1,--e) ;
- bits |= (e<<10) + m;
- }
- return half(binary, bits);
- }
- if(abs > 0x7C00)
- return arg;
- return half(binary, 0x7C00);
- }
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg) {
+ int abs = arg.data_ & 0x7FFF;
+ if (!abs) return FP_ILOGB0;
+ if (abs < 0x7C00) {
+ int exp = (abs >> 10) - 15;
+ if (abs < 0x400)
+ for (; abs < 0x200; abs <<= 1, --exp)
+ ;
+ return exp;
+ }
+ if (abs > 0x7C00) return FP_ILOGBNAN;
+ return INT_MAX;
+ }
- /// Enumeration implementation.
- /// \param from number to increase/decrease
- /// \param to direction to enumerate into
- /// \return next representable number
- static half nextafter(half from, half to)
- {
- uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
- if(fabs > 0x7C00)
- return from;
- if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
- return to;
- if(!fabs)
- return half(binary, (to.data_&0x8000)+1);
- bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
- ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
- return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
- }
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg) {
+ int abs = arg.data_ & 0x7FFF;
+ if (!abs) return half(binary, 0xFC00);
+ if (abs < 0x7C00) {
+ int exp = (abs >> 10) - 15;
+ if (abs < 0x400)
+ for (; abs < 0x200; abs <<= 1, --exp)
+ ;
+ uint16 bits = (exp < 0) << 15;
+ if (exp) {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for (; m < 0x400; m <<= 1, --e)
+ ;
+ bits |= (e << 10) + m;
+ }
+ return half(binary, bits);
+ }
+ if (abs > 0x7C00) return arg;
+ return half(binary, 0x7C00);
+ }
- /// Enumeration implementation.
- /// \param from number to increase/decrease
- /// \param to direction to enumerate into
- /// \return next representable number
- static half nexttoward(half from, long double to)
- {
- if(isnan(from))
- return from;
- long double lfrom = static_cast<long double>(from);
- if(builtin_isnan(to) || lfrom == to)
- return half(static_cast<float>(to));
- if(!(from.data_&0x7FFF))
- return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
- return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
- }
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to) {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if (fabs > 0x7C00) return from;
+ if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to;
+ if (!fabs) return half(binary, (to.data_ & 0x8000) + 1);
+ bool lt =
+ ((fabs == from.data_) ? static_cast<int>(fabs)
+ : -static_cast<int>(fabs)) <
+ ((tabs == to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary,
+ from.data_ +
+ (((from.data_ >> 15) ^ static_cast<unsigned>(lt)) << 1) -
+ 1);
+ }
- /// Sign implementation
- /// \param x first operand
- /// \param y second operand
- /// \return composed value
- static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to) {
+ if (isnan(from)) return from;
+ long double lfrom = static_cast<long double>(from);
+ if (builtin_isnan(to) || lfrom == to) return half(static_cast<float>(to));
+ if (!(from.data_ & 0x7FFF))
+ return half(binary,
+ (static_cast<detail::uint16>(builtin_signbit(to)) << 15) + 1);
+ return half(
+ binary,
+ from.data_ +
+ (((from.data_ >> 15) ^ static_cast<unsigned>(lfrom < to)) << 1) -
+ 1);
+ }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if infinite number
- /// \retval false else
- static int fpclassify(half arg)
- {
- unsigned int abs = arg.data_ & 0x7FFF;
- return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
- }
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) {
+ return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000));
+ }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if finite number
- /// \retval false else
- static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg) {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00)
+ ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE)
+ : FP_NORMAL)
+ : FP_SUBNORMAL)
+ : FP_ZERO;
+ }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if infinite number
- /// \retval false else
- static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if not a number
- /// \retval false else
- static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; }
- /// Classification implementation.
- /// \param arg value to classify
- /// \retval true if normal number
- /// \retval false else
- static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; }
- /// Sign bit implementation.
- /// \param arg value to check
- /// \retval true if signed
- /// \retval false if unsigned
- static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) {
+ return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00);
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands equal
- /// \retval false else
- static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands not equal
- /// \retval false else
- static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) {
+ return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x);
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x > \a y
- /// \retval false else
- static bool isgreater(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) {
+ return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x);
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x >= \a y
- /// \retval false else
- static bool isgreaterequal(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs <= 0x7C00 && yabs <= 0x7C00 &&
+ (((xabs == x.data_) ? xabs : -xabs) >
+ ((yabs == y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x < \a y
- /// \retval false else
- static bool isless(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs <= 0x7C00 && yabs <= 0x7C00 &&
+ (((xabs == x.data_) ? xabs : -xabs) >=
+ ((yabs == y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x <= \a y
- /// \retval false else
- static bool islessequal(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs <= 0x7C00 && yabs <= 0x7C00 &&
+ (((xabs == x.data_) ? xabs : -xabs) <
+ ((yabs == y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if either \a x > \a y nor \a x < \a y
- /// \retval false else
- static bool islessgreater(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if(xabs > 0x7C00 || yabs > 0x7C00)
- return false;
- int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
- return a < b || a > b;
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs <= 0x7C00 && yabs <= 0x7C00 &&
+ (((xabs == x.data_) ? xabs : -xabs) <=
+ ((yabs == y.data_) ? yabs : -yabs));
+ }
- /// Comparison implementation.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operand unordered
- /// \retval false else
- static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if (xabs > 0x7C00 || yabs > 0x7C00) return false;
+ int a = (xabs == x.data_) ? xabs : -xabs,
+ b = (yabs == y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
- private:
- static double erf(double arg)
- {
- if(builtin_isinf(arg))
- return (arg<0.0) ? -1.0 : 1.0;
- double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
- return builtin_signbit(arg) ? -value : value;
- }
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
- static double lgamma(double arg)
- {
- double v = 1.0;
- for(; arg<8.0; ++arg) v *= arg;
- double w = 1.0 / (arg*arg);
- return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
- -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
- -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
- -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
- 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
- }
- };
+ private:
+ static double erf(double arg) {
+ if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2,
+ value = std::sqrt(
+ 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) /
+ (1.0 + ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
- /// Wrapper for unary half-precision functions needing specialization for individual argument types.
- /// \tparam T argument type
- template<typename T> struct unary_specialized
- {
- /// Negation implementation.
- /// \param arg value to negate
- /// \return negated value
- static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
+ static double lgamma(double arg) {
+ double v = 1.0;
+ for (; arg < 8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg * arg);
+ return (((((((-0.02955065359477124183006535947712 * w +
+ 0.00641025641025641025641025641026) *
+ w +
+ -0.00191752691752691752691752691753) *
+ w +
+ 8.4175084175084175084175084175084e-4) *
+ w +
+ -5.952380952380952380952380952381e-4) *
+ w +
+ 7.9365079365079365079365079365079e-4) *
+ w +
+ -0.00277777777777777777777777777778) *
+ w +
+ 0.08333333333333333333333333333333) /
+ arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg +
+ (arg - 0.5) * std::log(arg);
+ }
+};
- /// Absolute value implementation.
- /// \param arg function argument
- /// \return absolute value
- static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
- };
- template<> struct unary_specialized<expr>
- {
- static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
- static expr fabs(float arg) { return expr(std::fabs(arg)); }
- };
+/// Wrapper for unary half-precision functions needing specialization for
+/// individual argument types. \tparam T argument type
+template <typename T>
+struct unary_specialized {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) {
+ return half(binary, arg.data_ ^ 0x8000);
+ }
- /// Wrapper for binary half-precision functions needing specialization for individual argument types.
- /// \tparam T first argument type
- /// \tparam U first argument type
- template<typename T,typename U> struct binary_specialized
- {
- /// Minimum implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return minimum value
- static expr fmin(float x, float y)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::fmin(x, y));
- #else
- if(builtin_isnan(x))
- return expr(y);
- if(builtin_isnan(y))
- return expr(x);
- return expr(std::min(x, y));
- #endif
- }
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); }
+};
+template <>
+struct unary_specialized<expr> {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+};
- /// Maximum implementation.
- /// \param x first operand
- /// \param y second operand
- /// \return maximum value
- static expr fmax(float x, float y)
- {
- #if HALF_ENABLE_CPP11_CMATH
- return expr(std::fmax(x, y));
- #else
- if(builtin_isnan(x))
- return expr(y);
- if(builtin_isnan(y))
- return expr(x);
- return expr(std::max(x, y));
- #endif
- }
- };
- template<> struct binary_specialized<half,half>
- {
- static half fmin(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if(xabs > 0x7C00)
- return y;
- if(yabs > 0x7C00)
- return x;
- return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
- }
- static half fmax(half x, half y)
- {
- int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
- if(xabs > 0x7C00)
- return y;
- if(yabs > 0x7C00)
- return x;
- return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
- }
- };
+/// Wrapper for binary half-precision functions needing specialization for
+/// individual argument types. \tparam T first argument type \tparam U first
+/// argument type
+template <typename T, typename U>
+struct binary_specialized {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+#else
+ if (builtin_isnan(x)) return expr(y);
+ if (builtin_isnan(y)) return expr(x);
+ return expr(std::min(x, y));
+#endif
+ }
- /// Helper class for half casts.
- /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
- /// function and a corresponding `type` member denoting its return type.
- /// \tparam T destination type
- /// \tparam U source type
- /// \tparam R rounding mode to use
- template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
- template<typename U,std::float_round_style R> struct half_caster<half,U,R>
- {
- #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
- #endif
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y) {
+#if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+#else
+ if (builtin_isnan(x)) return expr(y);
+ if (builtin_isnan(y)) return expr(x);
+ return expr(std::max(x, y));
+#endif
+ }
+};
+template <>
+struct binary_specialized<half, half> {
+ static half fmin(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if (xabs > 0x7C00) return y;
+ if (yabs > 0x7C00) return x;
+ return (((xabs == x.data_) ? xabs : -xabs) >
+ ((yabs == y.data_) ? yabs : -yabs))
+ ? y
+ : x;
+ }
+ static half fmax(half x, half y) {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if (xabs > 0x7C00) return y;
+ if (yabs > 0x7C00) return x;
+ return (((xabs == x.data_) ? xabs : -xabs) <
+ ((yabs == y.data_) ? yabs : -yabs))
+ ? y
+ : x;
+ }
+};
- static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+/// Helper class for half casts.
+/// This class template has to be specialized for all valid cast argument to
+/// define an appropriate static `cast` member function and a corresponding
+/// `type` member denoting its return type. \tparam T destination type \tparam U
+/// source type \tparam R rounding mode to use
+template <typename T, typename U,
+ std::float_round_style R = (std::float_round_style)(HALF_ROUND_STYLE)>
+struct half_caster {};
+template <typename U, std::float_round_style R>
+struct half_caster<half, U, R> {
+#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value,
+ "half_cast from non-arithmetic type unsupported");
+#endif
- private:
- static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
- static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
- };
- template<typename T,std::float_round_style R> struct half_caster<T,half,R>
- {
- #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
- #endif
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
- static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+ private:
+ static half cast_impl(U arg, true_type) {
+ return half(binary, float2half<R>(arg));
+ }
+ static half cast_impl(U arg, false_type) {
+ return half(binary, int2half<R>(arg));
+ }
+};
+template <typename T, std::float_round_style R>
+struct half_caster<T, half, R> {
+#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value,
+ "half_cast to non-arithmetic type unsupported");
+#endif
- private:
- static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
- static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
- };
- template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
- {
- #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
- static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
- #endif
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
- static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R, T>(arg.data_); }
+};
+template <typename T, std::float_round_style R>
+struct half_caster<T, expr, R> {
+#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value,
+ "half_cast to non-arithmetic type unsupported");
+#endif
- private:
- static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
- static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
- };
- template<std::float_round_style R> struct half_caster<half,half,R>
- {
- static half cast(half arg) { return arg; }
- };
- template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
- /// \name Comparison operators
- /// \{
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R, T>(arg.data_); }
+};
+template <std::float_round_style R>
+struct half_caster<half, half, R> {
+ static half cast(half arg) { return arg; }
+};
+template <std::float_round_style R>
+struct half_caster<half, expr, R> : half_caster<half, half, R> {};
- /// Comparison for equality.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands equal
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
+/// \name Comparison operators
+/// \{
- /// Comparison for inequality.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if operands not equal
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
+/// Comparison for equality.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if operands equal
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator==(T x, U y) {
+ return functions::isequal(x, y);
+}
- /// Comparison for less than.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x less than \a y
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
+/// Comparison for inequality.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if operands not equal
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator!=(T x, U y) {
+ return functions::isnotequal(x, y);
+}
- /// Comparison for greater than.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x greater than \a y
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
+/// Comparison for less than.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x less than \a y
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator<(T x, U y) {
+ return functions::isless(x, y);
+}
- /// Comparison for less equal.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x less equal \a y
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
+/// Comparison for greater than.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x greater than \a y
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator>(T x, U y) {
+ return functions::isgreater(x, y);
+}
- /// Comparison for greater equal.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x greater equal \a y
- /// \retval false else
- template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
+/// Comparison for less equal.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x less equal \a y
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator<=(T x, U y) {
+ return functions::islessequal(x, y);
+}
- /// \}
- /// \name Arithmetic operators
- /// \{
+/// Comparison for greater equal.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x greater equal \a y
+/// \retval false else
+template <typename T, typename U>
+typename enable<bool, T, U>::type operator>=(T x, U y) {
+ return functions::isgreaterequal(x, y);
+}
- /// Add halfs.
- /// \param x left operand
- /// \param y right operand
- /// \return sum of half expressions
- template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
+/// \}
+/// \name Arithmetic operators
+/// \{
- /// Subtract halfs.
- /// \param x left operand
- /// \param y right operand
- /// \return difference of half expressions
- template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
+/// Add halfs.
+/// \param x left operand
+/// \param y right operand
+/// \return sum of half expressions
+template <typename T, typename U>
+typename enable<expr, T, U>::type operator+(T x, U y) {
+ return functions::plus(x, y);
+}
- /// Multiply halfs.
- /// \param x left operand
- /// \param y right operand
- /// \return product of half expressions
- template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
+/// Subtract halfs.
+/// \param x left operand
+/// \param y right operand
+/// \return difference of half expressions
+template <typename T, typename U>
+typename enable<expr, T, U>::type operator-(T x, U y) {
+ return functions::minus(x, y);
+}
- /// Divide halfs.
- /// \param x left operand
- /// \param y right operand
- /// \return quotient of half expressions
- template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
+/// Multiply halfs.
+/// \param x left operand
+/// \param y right operand
+/// \return product of half expressions
+template <typename T, typename U>
+typename enable<expr, T, U>::type operator*(T x, U y) {
+ return functions::multiplies(x, y);
+}
- /// Identity.
- /// \param arg operand
- /// \return uncahnged operand
- template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
+/// Divide halfs.
+/// \param x left operand
+/// \param y right operand
+/// \return quotient of half expressions
+template <typename T, typename U>
+typename enable<expr, T, U>::type operator/(T x, U y) {
+ return functions::divides(x, y);
+}
- /// Negation.
- /// \param arg operand
- /// \return negated operand
- template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
+/// Identity.
+/// \param arg operand
+/// \return uncahnged operand
+template <typename T>
+HALF_CONSTEXPR typename enable<T, T>::type operator+(T arg) {
+ return arg;
+}
- /// \}
- /// \name Input and output
- /// \{
+/// Negation.
+/// \param arg operand
+/// \return negated operand
+template <typename T>
+HALF_CONSTEXPR typename enable<T, T>::type operator-(T arg) {
+ return unary_specialized<T>::negate(arg);
+}
- /// Output operator.
- /// \param out output stream to write into
- /// \param arg half expression to write
- /// \return reference to output stream
- template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
- operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
+/// \}
+/// \name Input and output
+/// \{
- /// Input operator.
- /// \param in input stream to read from
- /// \param arg half to read into
- /// \return reference to input stream
- template<typename charT,typename traits> std::basic_istream<charT,traits>&
- operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
+/// Output operator.
+/// \param out output stream to write into
+/// \param arg half expression to write
+/// \return reference to output stream
+template <typename T, typename charT, typename traits>
+typename enable<std::basic_ostream<charT, traits> &, T>::type operator<<(
+ std::basic_ostream<charT, traits> &out, T arg) {
+ return functions::write(out, arg);
+}
- /// \}
- /// \name Basic mathematical operations
- /// \{
+/// Input operator.
+/// \param in input stream to read from
+/// \param arg half to read into
+/// \return reference to input stream
+template <typename charT, typename traits>
+std::basic_istream<charT, traits> &operator>>(
+ std::basic_istream<charT, traits> &in, half &arg) {
+ return functions::read(in, arg);
+}
- /// Absolute value.
- /// \param arg operand
- /// \return absolute value of \a arg
-// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
- inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
- inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+/// \}
+/// \name Basic mathematical operations
+/// \{
- /// Absolute value.
- /// \param arg operand
- /// \return absolute value of \a arg
-// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
- inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
- inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+/// Absolute value.
+/// \param arg operand
+/// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) {
+//return unary_specialized<T>::fabs(arg); }
+inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
- /// Remainder of division.
- /// \param x first operand
- /// \param y second operand
- /// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
- inline expr fmod(half x, half y) { return functions::fmod(x, y); }
- inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
- inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
- inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+/// Absolute value.
+/// \param arg operand
+/// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) {
+//return unary_specialized<T>::fabs(arg); }
+inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
- /// Remainder of division.
- /// \param x first operand
- /// \param y second operand
- /// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
- inline expr remainder(half x, half y) { return functions::remainder(x, y); }
- inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
- inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
- inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+/// Remainder of division.
+/// \param x first operand
+/// \param y second operand
+/// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//fmod(T x, U y) { return functions::fmod(x, y); }
+inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
- /// Remainder of division.
- /// \param x first operand
- /// \param y second operand
- /// \param quo address to store some bits of quotient at
- /// \return remainder of floating point division.
-// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
- inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
- inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
- inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
- inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+/// Remainder of division.
+/// \param x first operand
+/// \param y second operand
+/// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//remainder(T x, U y) { return functions::remainder(x, y); }
+inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
- /// Fused multiply add.
- /// \param x first operand
- /// \param y second operand
- /// \param z third operand
- /// \return ( \a x * \a y ) + \a z rounded as one operation.
-// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
- inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
- inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
- inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
- inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
- inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
- inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
- inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
- inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+/// Remainder of division.
+/// \param x first operand
+/// \param y second operand
+/// \param quo address to store some bits of quotient at
+/// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+inline expr remquo(half x, half y, int *quo) {
+ return functions::remquo(x, y, quo);
+}
+inline expr remquo(half x, expr y, int *quo) {
+ return functions::remquo(x, y, quo);
+}
+inline expr remquo(expr x, half y, int *quo) {
+ return functions::remquo(x, y, quo);
+}
+inline expr remquo(expr x, expr y, int *quo) {
+ return functions::remquo(x, y, quo);
+}
- /// Maximum of half expressions.
- /// \param x first operand
- /// \param y second operand
- /// \return maximum of operands
-// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
- inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
- inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
- inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
- inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
+/// Fused multiply add.
+/// \param x first operand
+/// \param y second operand
+/// \param z third operand
+/// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename
+//enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z);
+//}
+inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
- /// Minimum of half expressions.
- /// \param x first operand
- /// \param y second operand
- /// \return minimum of operands
-// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
- inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
- inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
- inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
- inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
+/// Maximum of half expressions.
+/// \param x first operand
+/// \param y second operand
+/// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T
+//x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+inline half fmax(half x, half y) {
+ return binary_specialized<half, half>::fmax(x, y);
+}
+inline expr fmax(half x, expr y) {
+ return binary_specialized<half, expr>::fmax(x, y);
+}
+inline expr fmax(expr x, half y) {
+ return binary_specialized<expr, half>::fmax(x, y);
+}
+inline expr fmax(expr x, expr y) {
+ return binary_specialized<expr, expr>::fmax(x, y);
+}
- /// Positive difference.
- /// \param x first operand
- /// \param y second operand
- /// \return \a x - \a y or 0 if difference negative
-// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
- inline expr fdim(half x, half y) { return functions::fdim(x, y); }
- inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
- inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
- inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+/// Minimum of half expressions.
+/// \param x first operand
+/// \param y second operand
+/// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T
+//x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+inline half fmin(half x, half y) {
+ return binary_specialized<half, half>::fmin(x, y);
+}
+inline expr fmin(half x, expr y) {
+ return binary_specialized<half, expr>::fmin(x, y);
+}
+inline expr fmin(expr x, half y) {
+ return binary_specialized<expr, half>::fmin(x, y);
+}
+inline expr fmin(expr x, expr y) {
+ return binary_specialized<expr, expr>::fmin(x, y);
+}
- /// Get NaN value.
- /// \return quiet NaN
- inline half nanh(const char*) { return functions::nanh(); }
+/// Positive difference.
+/// \param x first operand
+/// \param y second operand
+/// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//fdim(T x, U y) { return functions::fdim(x, y); }
+inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
- /// \}
- /// \name Exponential functions
- /// \{
+/// Get NaN value.
+/// \return quiet NaN
+inline half nanh(const char *) { return functions::nanh(); }
- /// Exponential function.
- /// \param arg function argument
- /// \return e raised to \a arg
-// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
- inline expr exp(half arg) { return functions::exp(arg); }
- inline expr exp(expr arg) { return functions::exp(arg); }
+/// \}
+/// \name Exponential functions
+/// \{
- /// Exponential minus one.
- /// \param arg function argument
- /// \return e raised to \a arg subtracted by 1
-// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
- inline expr expm1(half arg) { return functions::expm1(arg); }
- inline expr expm1(expr arg) { return functions::expm1(arg); }
+/// Exponential function.
+/// \param arg function argument
+/// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) {
+//return functions::exp(arg); }
+inline expr exp(half arg) { return functions::exp(arg); }
+inline expr exp(expr arg) { return functions::exp(arg); }
- /// Binary exponential.
- /// \param arg function argument
- /// \return 2 raised to \a arg
-// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
- inline expr exp2(half arg) { return functions::exp2(arg); }
- inline expr exp2(expr arg) { return functions::exp2(arg); }
+/// Exponential minus one.
+/// \param arg function argument
+/// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) {
+//return functions::expm1(arg); }
+inline expr expm1(half arg) { return functions::expm1(arg); }
+inline expr expm1(expr arg) { return functions::expm1(arg); }
- /// Natural logorithm.
- /// \param arg function argument
- /// \return logarithm of \a arg to base e
-// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
- inline expr log(half arg) { return functions::log(arg); }
- inline expr log(expr arg) { return functions::log(arg); }
+/// Binary exponential.
+/// \param arg function argument
+/// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) {
+//return functions::exp2(arg); }
+inline expr exp2(half arg) { return functions::exp2(arg); }
+inline expr exp2(expr arg) { return functions::exp2(arg); }
- /// Common logorithm.
- /// \param arg function argument
- /// \return logarithm of \a arg to base 10
-// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
- inline expr log10(half arg) { return functions::log10(arg); }
- inline expr log10(expr arg) { return functions::log10(arg); }
+/// Natural logorithm.
+/// \param arg function argument
+/// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) {
+//return functions::log(arg); }
+inline expr log(half arg) { return functions::log(arg); }
+inline expr log(expr arg) { return functions::log(arg); }
- /// Natural logorithm.
- /// \param arg function argument
- /// \return logarithm of \a arg plus 1 to base e
-// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
- inline expr log1p(half arg) { return functions::log1p(arg); }
- inline expr log1p(expr arg) { return functions::log1p(arg); }
+/// Common logorithm.
+/// \param arg function argument
+/// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) {
+//return functions::log10(arg); }
+inline expr log10(half arg) { return functions::log10(arg); }
+inline expr log10(expr arg) { return functions::log10(arg); }
- /// Binary logorithm.
- /// \param arg function argument
- /// \return logarithm of \a arg to base 2
-// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
- inline expr log2(half arg) { return functions::log2(arg); }
- inline expr log2(expr arg) { return functions::log2(arg); }
+/// Natural logorithm.
+/// \param arg function argument
+/// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) {
+//return functions::log1p(arg); }
+inline expr log1p(half arg) { return functions::log1p(arg); }
+inline expr log1p(expr arg) { return functions::log1p(arg); }
- /// \}
- /// \name Power functions
- /// \{
+/// Binary logorithm.
+/// \param arg function argument
+/// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) {
+//return functions::log2(arg); }
+inline expr log2(half arg) { return functions::log2(arg); }
+inline expr log2(expr arg) { return functions::log2(arg); }
- /// Square root.
- /// \param arg function argument
- /// \return square root of \a arg
-// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
- inline expr sqrt(half arg) { return functions::sqrt(arg); }
- inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+/// \}
+/// \name Power functions
+/// \{
- /// Cubic root.
- /// \param arg function argument
- /// \return cubic root of \a arg
-// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
- inline expr cbrt(half arg) { return functions::cbrt(arg); }
- inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+/// Square root.
+/// \param arg function argument
+/// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) {
+//return functions::sqrt(arg); }
+inline expr sqrt(half arg) { return functions::sqrt(arg); }
+inline expr sqrt(expr arg) { return functions::sqrt(arg); }
- /// Hypotenuse function.
- /// \param x first argument
- /// \param y second argument
- /// \return square root of sum of squares without internal over- or underflows
-// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
- inline expr hypot(half x, half y) { return functions::hypot(x, y); }
- inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
- inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
- inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+/// Cubic root.
+/// \param arg function argument
+/// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) {
+//return functions::cbrt(arg); }
+inline expr cbrt(half arg) { return functions::cbrt(arg); }
+inline expr cbrt(expr arg) { return functions::cbrt(arg); }
- /// Power function.
- /// \param base first argument
- /// \param exp second argument
- /// \return \a base raised to \a exp
-// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
- inline expr pow(half base, half exp) { return functions::pow(base, exp); }
- inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
- inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
- inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+/// Hypotenuse function.
+/// \param x first argument
+/// \param y second argument
+/// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//hypot(T x, U y) { return functions::hypot(x, y); }
+inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
- /// \}
- /// \name Trigonometric functions
- /// \{
+/// Power function.
+/// \param base first argument
+/// \param exp second argument
+/// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//pow(T base, U exp) { return functions::pow(base, exp); }
+inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
- /// Sine function.
- /// \param arg function argument
- /// \return sine value of \a arg
-// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
- inline expr sin(half arg) { return functions::sin(arg); }
- inline expr sin(expr arg) { return functions::sin(arg); }
+/// \}
+/// \name Trigonometric functions
+/// \{
- /// Cosine function.
- /// \param arg function argument
- /// \return cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
- inline expr cos(half arg) { return functions::cos(arg); }
- inline expr cos(expr arg) { return functions::cos(arg); }
+/// Sine function.
+/// \param arg function argument
+/// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) {
+//return functions::sin(arg); }
+inline expr sin(half arg) { return functions::sin(arg); }
+inline expr sin(expr arg) { return functions::sin(arg); }
- /// Tangent function.
- /// \param arg function argument
- /// \return tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
- inline expr tan(half arg) { return functions::tan(arg); }
- inline expr tan(expr arg) { return functions::tan(arg); }
+/// Cosine function.
+/// \param arg function argument
+/// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) {
+//return functions::cos(arg); }
+inline expr cos(half arg) { return functions::cos(arg); }
+inline expr cos(expr arg) { return functions::cos(arg); }
- /// Arc sine.
- /// \param arg function argument
- /// \return arc sine value of \a arg
-// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
- inline expr asin(half arg) { return functions::asin(arg); }
- inline expr asin(expr arg) { return functions::asin(arg); }
+/// Tangent function.
+/// \param arg function argument
+/// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) {
+//return functions::tan(arg); }
+inline expr tan(half arg) { return functions::tan(arg); }
+inline expr tan(expr arg) { return functions::tan(arg); }
- /// Arc cosine function.
- /// \param arg function argument
- /// \return arc cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
- inline expr acos(half arg) { return functions::acos(arg); }
- inline expr acos(expr arg) { return functions::acos(arg); }
+/// Arc sine.
+/// \param arg function argument
+/// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) {
+//return functions::asin(arg); }
+inline expr asin(half arg) { return functions::asin(arg); }
+inline expr asin(expr arg) { return functions::asin(arg); }
- /// Arc tangent function.
- /// \param arg function argument
- /// \return arc tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
- inline expr atan(half arg) { return functions::atan(arg); }
- inline expr atan(expr arg) { return functions::atan(arg); }
+/// Arc cosine function.
+/// \param arg function argument
+/// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) {
+//return functions::acos(arg); }
+inline expr acos(half arg) { return functions::acos(arg); }
+inline expr acos(expr arg) { return functions::acos(arg); }
- /// Arc tangent function.
- /// \param x first argument
- /// \param y second argument
- /// \return arc tangent value
-// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
- inline expr atan2(half x, half y) { return functions::atan2(x, y); }
- inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
- inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
- inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+/// Arc tangent function.
+/// \param arg function argument
+/// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) {
+//return functions::atan(arg); }
+inline expr atan(half arg) { return functions::atan(arg); }
+inline expr atan(expr arg) { return functions::atan(arg); }
- /// \}
- /// \name Hyperbolic functions
- /// \{
+/// Arc tangent function.
+/// \param x first argument
+/// \param y second argument
+/// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type
+//atan2(T x, U y) { return functions::atan2(x, y); }
+inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
- /// Hyperbolic sine.
- /// \param arg function argument
- /// \return hyperbolic sine value of \a arg
-// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
- inline expr sinh(half arg) { return functions::sinh(arg); }
- inline expr sinh(expr arg) { return functions::sinh(arg); }
+/// \}
+/// \name Hyperbolic functions
+/// \{
- /// Hyperbolic cosine.
- /// \param arg function argument
- /// \return hyperbolic cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
- inline expr cosh(half arg) { return functions::cosh(arg); }
- inline expr cosh(expr arg) { return functions::cosh(arg); }
+/// Hyperbolic sine.
+/// \param arg function argument
+/// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) {
+//return functions::sinh(arg); }
+inline expr sinh(half arg) { return functions::sinh(arg); }
+inline expr sinh(expr arg) { return functions::sinh(arg); }
- /// Hyperbolic tangent.
- /// \param arg function argument
- /// \return hyperbolic tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
- inline expr tanh(half arg) { return functions::tanh(arg); }
- inline expr tanh(expr arg) { return functions::tanh(arg); }
+/// Hyperbolic cosine.
+/// \param arg function argument
+/// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) {
+//return functions::cosh(arg); }
+inline expr cosh(half arg) { return functions::cosh(arg); }
+inline expr cosh(expr arg) { return functions::cosh(arg); }
- /// Hyperbolic area sine.
- /// \param arg function argument
- /// \return area sine value of \a arg
-// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
- inline expr asinh(half arg) { return functions::asinh(arg); }
- inline expr asinh(expr arg) { return functions::asinh(arg); }
+/// Hyperbolic tangent.
+/// \param arg function argument
+/// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) {
+//return functions::tanh(arg); }
+inline expr tanh(half arg) { return functions::tanh(arg); }
+inline expr tanh(expr arg) { return functions::tanh(arg); }
- /// Hyperbolic area cosine.
- /// \param arg function argument
- /// \return area cosine value of \a arg
-// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
- inline expr acosh(half arg) { return functions::acosh(arg); }
- inline expr acosh(expr arg) { return functions::acosh(arg); }
+/// Hyperbolic area sine.
+/// \param arg function argument
+/// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) {
+//return functions::asinh(arg); }
+inline expr asinh(half arg) { return functions::asinh(arg); }
+inline expr asinh(expr arg) { return functions::asinh(arg); }
- /// Hyperbolic area tangent.
- /// \param arg function argument
- /// \return area tangent value of \a arg
-// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
- inline expr atanh(half arg) { return functions::atanh(arg); }
- inline expr atanh(expr arg) { return functions::atanh(arg); }
+/// Hyperbolic area cosine.
+/// \param arg function argument
+/// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) {
+//return functions::acosh(arg); }
+inline expr acosh(half arg) { return functions::acosh(arg); }
+inline expr acosh(expr arg) { return functions::acosh(arg); }
- /// \}
- /// \name Error and gamma functions
- /// \{
+/// Hyperbolic area tangent.
+/// \param arg function argument
+/// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) {
+//return functions::atanh(arg); }
+inline expr atanh(half arg) { return functions::atanh(arg); }
+inline expr atanh(expr arg) { return functions::atanh(arg); }
- /// Error function.
- /// \param arg function argument
- /// \return error function value of \a arg
-// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
- inline expr erf(half arg) { return functions::erf(arg); }
- inline expr erf(expr arg) { return functions::erf(arg); }
+/// \}
+/// \name Error and gamma functions
+/// \{
- /// Complementary error function.
- /// \param arg function argument
- /// \return 1 minus error function value of \a arg
-// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
- inline expr erfc(half arg) { return functions::erfc(arg); }
- inline expr erfc(expr arg) { return functions::erfc(arg); }
+/// Error function.
+/// \param arg function argument
+/// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) {
+//return functions::erf(arg); }
+inline expr erf(half arg) { return functions::erf(arg); }
+inline expr erf(expr arg) { return functions::erf(arg); }
- /// Natural logarithm of gamma function.
- /// \param arg function argument
- /// \return natural logarith of gamma function for \a arg
-// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
- inline expr lgamma(half arg) { return functions::lgamma(arg); }
- inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+/// Complementary error function.
+/// \param arg function argument
+/// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) {
+//return functions::erfc(arg); }
+inline expr erfc(half arg) { return functions::erfc(arg); }
+inline expr erfc(expr arg) { return functions::erfc(arg); }
- /// Gamma function.
- /// \param arg function argument
- /// \return gamma function value of \a arg
-// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
- inline expr tgamma(half arg) { return functions::tgamma(arg); }
- inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+/// Natural logarithm of gamma function.
+/// \param arg function argument
+/// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) {
+//return functions::lgamma(arg); }
+inline expr lgamma(half arg) { return functions::lgamma(arg); }
+inline expr lgamma(expr arg) { return functions::lgamma(arg); }
- /// \}
- /// \name Rounding
- /// \{
+/// Gamma function.
+/// \param arg function argument
+/// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) {
+//return functions::tgamma(arg); }
+inline expr tgamma(half arg) { return functions::tgamma(arg); }
+inline expr tgamma(expr arg) { return functions::tgamma(arg); }
- /// Nearest integer not less than half value.
- /// \param arg half to round
- /// \return nearest integer not less than \a arg
-// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
- inline half ceil(half arg) { return functions::ceil(arg); }
- inline half ceil(expr arg) { return functions::ceil(arg); }
+/// \}
+/// \name Rounding
+/// \{
- /// Nearest integer not greater than half value.
- /// \param arg half to round
- /// \return nearest integer not greater than \a arg
-// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
- inline half floor(half arg) { return functions::floor(arg); }
- inline half floor(expr arg) { return functions::floor(arg); }
+/// Nearest integer not less than half value.
+/// \param arg half to round
+/// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) {
+//return functions::ceil(arg); }
+inline half ceil(half arg) { return functions::ceil(arg); }
+inline half ceil(expr arg) { return functions::ceil(arg); }
- /// Nearest integer not greater in magnitude than half value.
- /// \param arg half to round
- /// \return nearest integer not greater in magnitude than \a arg
-// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
- inline half trunc(half arg) { return functions::trunc(arg); }
- inline half trunc(expr arg) { return functions::trunc(arg); }
+/// Nearest integer not greater than half value.
+/// \param arg half to round
+/// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) {
+//return functions::floor(arg); }
+inline half floor(half arg) { return functions::floor(arg); }
+inline half floor(expr arg) { return functions::floor(arg); }
- /// Nearest integer.
- /// \param arg half to round
- /// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
- inline half round(half arg) { return functions::round(arg); }
- inline half round(expr arg) { return functions::round(arg); }
+/// Nearest integer not greater in magnitude than half value.
+/// \param arg half to round
+/// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) {
+//return functions::trunc(arg); }
+inline half trunc(half arg) { return functions::trunc(arg); }
+inline half trunc(expr arg) { return functions::trunc(arg); }
- /// Nearest integer.
- /// \param arg half to round
- /// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
- inline long lround(half arg) { return functions::lround(arg); }
- inline long lround(expr arg) { return functions::lround(arg); }
+/// Nearest integer.
+/// \param arg half to round
+/// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) {
+//return functions::round(arg); }
+inline half round(half arg) { return functions::round(arg); }
+inline half round(expr arg) { return functions::round(arg); }
- /// Nearest integer using half's internal rounding mode.
- /// \param arg half expression to round
- /// \return nearest integer using default rounding mode
-// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
- inline half nearbyint(half arg) { return functions::rint(arg); }
- inline half nearbyint(expr arg) { return functions::rint(arg); }
+/// Nearest integer.
+/// \param arg half to round
+/// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) {
+//return functions::lround(arg); }
+inline long lround(half arg) { return functions::lround(arg); }
+inline long lround(expr arg) { return functions::lround(arg); }
- /// Nearest integer using half's internal rounding mode.
- /// \param arg half expression to round
- /// \return nearest integer using default rounding mode
-// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
- inline half rint(half arg) { return functions::rint(arg); }
- inline half rint(expr arg) { return functions::rint(arg); }
+/// Nearest integer using half's internal rounding mode.
+/// \param arg half expression to round
+/// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T
+//arg) { return functions::nearbyint(arg); }
+inline half nearbyint(half arg) { return functions::rint(arg); }
+inline half nearbyint(expr arg) { return functions::rint(arg); }
- /// Nearest integer using half's internal rounding mode.
- /// \param arg half expression to round
- /// \return nearest integer using default rounding mode
-// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
- inline long lrint(half arg) { return functions::lrint(arg); }
- inline long lrint(expr arg) { return functions::lrint(arg); }
- #if HALF_ENABLE_CPP11_LONG_LONG
- /// Nearest integer.
- /// \param arg half to round
- /// \return nearest integer, rounded away from zero in half-way cases
-// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
- inline long long llround(half arg) { return functions::llround(arg); }
- inline long long llround(expr arg) { return functions::llround(arg); }
+/// Nearest integer using half's internal rounding mode.
+/// \param arg half expression to round
+/// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) {
+//return functions::rint(arg); }
+inline half rint(half arg) { return functions::rint(arg); }
+inline half rint(expr arg) { return functions::rint(arg); }
- /// Nearest integer using half's internal rounding mode.
- /// \param arg half expression to round
- /// \return nearest integer using default rounding mode
-// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
- inline long long llrint(half arg) { return functions::llrint(arg); }
- inline long long llrint(expr arg) { return functions::llrint(arg); }
- #endif
+/// Nearest integer using half's internal rounding mode.
+/// \param arg half expression to round
+/// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) {
+//return functions::lrint(arg); }
+inline long lrint(half arg) { return functions::lrint(arg); }
+inline long lrint(expr arg) { return functions::lrint(arg); }
+#if HALF_ENABLE_CPP11_LONG_LONG
+/// Nearest integer.
+/// \param arg half to round
+/// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T
+//arg) { return functions::llround(arg); }
+inline long long llround(half arg) { return functions::llround(arg); }
+inline long long llround(expr arg) { return functions::llround(arg); }
- /// \}
- /// \name Floating point manipulation
- /// \{
+/// Nearest integer using half's internal rounding mode.
+/// \param arg half expression to round
+/// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T
+//arg) { return functions::llrint(arg); }
+inline long long llrint(half arg) { return functions::llrint(arg); }
+inline long long llrint(expr arg) { return functions::llrint(arg); }
+#endif
- /// Decompress floating point number.
- /// \param arg number to decompress
- /// \param exp address to store exponent at
- /// \return significant in range [0.5, 1)
-// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
- inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
- inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+/// \}
+/// \name Floating point manipulation
+/// \{
- /// Multiply by power of two.
- /// \param arg number to modify
- /// \param exp power of two to multiply with
- /// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
- inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
- inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+/// Decompress floating point number.
+/// \param arg number to decompress
+/// \param exp address to store exponent at
+/// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg,
+//int *exp) { return functions::frexp(arg, exp); }
+inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
- /// Extract integer and fractional parts.
- /// \param arg number to decompress
- /// \param iptr address to store integer part at
- /// \return fractional part
-// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
- inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
- inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+/// Multiply by power of two.
+/// \param arg number to modify
+/// \param exp power of two to multiply with
+/// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg,
+//int exp) { return functions::scalbln(arg, exp); }
+inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
- /// Multiply by power of two.
- /// \param arg number to modify
- /// \param exp power of two to multiply with
- /// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
- inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
- inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+/// Extract integer and fractional parts.
+/// \param arg number to decompress
+/// \param iptr address to store integer part at
+/// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg,
+//half *iptr) { return functions::modf(arg, iptr); }
+inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
- /// Multiply by power of two.
- /// \param arg number to modify
- /// \param exp power of two to multiply with
- /// \return \a arg multplied by 2 raised to \a exp
-// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
- inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
- inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+/// Multiply by power of two.
+/// \param arg number to modify
+/// \param exp power of two to multiply with
+/// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg,
+//int exp) { return functions::scalbln(arg, exp); }
+inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
- /// Extract exponent.
- /// \param arg number to query
- /// \return floating point exponent
- /// \retval FP_ILOGB0 for zero
- /// \retval FP_ILOGBNAN for NaN
- /// \retval MAX_INT for infinity
-// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
- inline int ilogb(half arg) { return functions::ilogb(arg); }
- inline int ilogb(expr arg) { return functions::ilogb(arg); }
+/// Multiply by power of two.
+/// \param arg number to modify
+/// \param exp power of two to multiply with
+/// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg,
+//long exp) { return functions::scalbln(arg, exp); }
+inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
- /// Extract exponent.
- /// \param arg number to query
- /// \return floating point exponent
-// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
- inline half logb(half arg) { return functions::logb(arg); }
- inline half logb(expr arg) { return functions::logb(arg); }
+/// Extract exponent.
+/// \param arg number to query
+/// \return floating point exponent
+/// \retval FP_ILOGB0 for zero
+/// \retval FP_ILOGBNAN for NaN
+/// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) {
+//return functions::ilogb(arg); }
+inline int ilogb(half arg) { return functions::ilogb(arg); }
+inline int ilogb(expr arg) { return functions::ilogb(arg); }
- /// Next representable value.
- /// \param from value to compute next representable value for
- /// \param to direction towards which to compute next value
- /// \return next representable value after \a from in direction towards \a to
-// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
- inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
- inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
- inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
- inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
+/// Extract exponent.
+/// \param arg number to query
+/// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) {
+//return functions::logb(arg); }
+inline half logb(half arg) { return functions::logb(arg); }
+inline half logb(expr arg) { return functions::logb(arg); }
- /// Next representable value.
- /// \param from value to compute next representable value for
- /// \param to direction towards which to compute next value
- /// \return next representable value after \a from in direction towards \a to
-// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
- inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
- inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
+/// Next representable value.
+/// \param from value to compute next representable value for
+/// \param to direction towards which to compute next value
+/// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type
+//nextafter(T from, U to) { return functions::nextafter(from, to); }
+inline half nextafter(half from, half to) {
+ return functions::nextafter(from, to);
+}
+inline half nextafter(half from, expr to) {
+ return functions::nextafter(from, to);
+}
+inline half nextafter(expr from, half to) {
+ return functions::nextafter(from, to);
+}
+inline half nextafter(expr from, expr to) {
+ return functions::nextafter(from, to);
+}
- /// Take sign.
- /// \param x value to change sign for
- /// \param y value to take sign from
- /// \return value equal to \a x in magnitude and to \a y in sign
-// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
- inline half copysign(half x, half y) { return functions::copysign(x, y); }
- inline half copysign(half x, expr y) { return functions::copysign(x, y); }
- inline half copysign(expr x, half y) { return functions::copysign(x, y); }
- inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+/// Next representable value.
+/// \param from value to compute next representable value for
+/// \param to direction towards which to compute next value
+/// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T
+//from, long double to) { return functions::nexttoward(from, to); }
+inline half nexttoward(half from, long double to) {
+ return functions::nexttoward(from, to);
+}
+inline half nexttoward(expr from, long double to) {
+ return functions::nexttoward(from, to);
+}
- /// \}
- /// \name Floating point classification
- /// \{
+/// Take sign.
+/// \param x value to change sign for
+/// \param y value to take sign from
+/// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type
+//copysign(T x, U y) { return functions::copysign(x, y); }
+inline half copysign(half x, half y) { return functions::copysign(x, y); }
+inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+/// \}
+/// \name Floating point classification
+/// \{
- /// Classify floating point value.
- /// \param arg number to classify
- /// \retval FP_ZERO for positive and negative zero
- /// \retval FP_SUBNORMAL for subnormal numbers
- /// \retval FP_INFINITY for positive and negative infinity
- /// \retval FP_NAN for NaNs
- /// \retval FP_NORMAL for all other (normal) values
-// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
- inline int fpclassify(half arg) { return functions::fpclassify(arg); }
- inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+/// Classify floating point value.
+/// \param arg number to classify
+/// \retval FP_ZERO for positive and negative zero
+/// \retval FP_SUBNORMAL for subnormal numbers
+/// \retval FP_INFINITY for positive and negative infinity
+/// \retval FP_NAN for NaNs
+/// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T
+//arg) { return functions::fpclassify(arg); }
+inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
- /// Check if finite number.
- /// \param arg number to check
- /// \retval true if neither infinity nor NaN
- /// \retval false else
-// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
- inline bool isfinite(half arg) { return functions::isfinite(arg); }
- inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+/// Check if finite number.
+/// \param arg number to check
+/// \retval true if neither infinity nor NaN
+/// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg)
+//{ return functions::isfinite(arg); }
+inline bool isfinite(half arg) { return functions::isfinite(arg); }
+inline bool isfinite(expr arg) { return functions::isfinite(arg); }
- /// Check for infinity.
- /// \param arg number to check
- /// \retval true for positive or negative infinity
- /// \retval false else
-// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
- inline bool isinf(half arg) { return functions::isinf(arg); }
- inline bool isinf(expr arg) { return functions::isinf(arg); }
+/// Check for infinity.
+/// \param arg number to check
+/// \retval true for positive or negative infinity
+/// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) {
+//return functions::isinf(arg); }
+inline bool isinf(half arg) { return functions::isinf(arg); }
+inline bool isinf(expr arg) { return functions::isinf(arg); }
- /// Check for NaN.
- /// \param arg number to check
- /// \retval true for NaNs
- /// \retval false else
-// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
- inline bool isnan(half arg) { return functions::isnan(arg); }
- inline bool isnan(expr arg) { return functions::isnan(arg); }
+/// Check for NaN.
+/// \param arg number to check
+/// \retval true for NaNs
+/// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) {
+//return functions::isnan(arg); }
+inline bool isnan(half arg) { return functions::isnan(arg); }
+inline bool isnan(expr arg) { return functions::isnan(arg); }
- /// Check if normal number.
- /// \param arg number to check
- /// \retval true if normal number
- /// \retval false if either subnormal, zero, infinity or NaN
-// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
- inline bool isnormal(half arg) { return functions::isnormal(arg); }
- inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+/// Check if normal number.
+/// \param arg number to check
+/// \retval true if normal number
+/// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg)
+//{ return functions::isnormal(arg); }
+inline bool isnormal(half arg) { return functions::isnormal(arg); }
+inline bool isnormal(expr arg) { return functions::isnormal(arg); }
- /// Check sign.
- /// \param arg number to check
- /// \retval true for negative number
- /// \retval false for positive number
-// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
- inline bool signbit(half arg) { return functions::signbit(arg); }
- inline bool signbit(expr arg) { return functions::signbit(arg); }
+/// Check sign.
+/// \param arg number to check
+/// \retval true for negative number
+/// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg)
+//{ return functions::signbit(arg); }
+inline bool signbit(half arg) { return functions::signbit(arg); }
+inline bool signbit(expr arg) { return functions::signbit(arg); }
- /// \}
- /// \name Comparison
- /// \{
+/// \}
+/// \name Comparison
+/// \{
- /// Comparison for greater than.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x greater than \a y
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
- inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
- inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
- inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
- inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+/// Comparison for greater than.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x greater than \a y
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//isgreater(T x, U y) { return functions::isgreater(x, y); }
+inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
- /// Comparison for greater equal.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x greater equal \a y
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
- inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
- inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
- inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
- inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
+/// Comparison for greater equal.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x greater equal \a y
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+inline bool isgreaterequal(half x, half y) {
+ return functions::isgreaterequal(x, y);
+}
+inline bool isgreaterequal(half x, expr y) {
+ return functions::isgreaterequal(x, y);
+}
+inline bool isgreaterequal(expr x, half y) {
+ return functions::isgreaterequal(x, y);
+}
+inline bool isgreaterequal(expr x, expr y) {
+ return functions::isgreaterequal(x, y);
+}
- /// Comparison for less than.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x less than \a y
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
- inline bool isless(half x, half y) { return functions::isless(x, y); }
- inline bool isless(half x, expr y) { return functions::isless(x, y); }
- inline bool isless(expr x, half y) { return functions::isless(x, y); }
- inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+/// Comparison for less than.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x less than \a y
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//isless(T x, U y) { return functions::isless(x, y); }
+inline bool isless(half x, half y) { return functions::isless(x, y); }
+inline bool isless(half x, expr y) { return functions::isless(x, y); }
+inline bool isless(expr x, half y) { return functions::isless(x, y); }
+inline bool isless(expr x, expr y) { return functions::isless(x, y); }
- /// Comparison for less equal.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if \a x less equal \a y
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
- inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
- inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
- inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
- inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+/// Comparison for less equal.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if \a x less equal \a y
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//islessequal(T x, U y) { return functions::islessequal(x, y); }
+inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
- /// Comarison for less or greater.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if either less or greater
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
- inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
- inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
- inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
- inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
+/// Comarison for less or greater.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if either less or greater
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+inline bool islessgreater(half x, half y) {
+ return functions::islessgreater(x, y);
+}
+inline bool islessgreater(half x, expr y) {
+ return functions::islessgreater(x, y);
+}
+inline bool islessgreater(expr x, half y) {
+ return functions::islessgreater(x, y);
+}
+inline bool islessgreater(expr x, expr y) {
+ return functions::islessgreater(x, y);
+}
- /// Check if unordered.
- /// \param x first operand
- /// \param y second operand
- /// \retval true if unordered (one or two NaN operands)
- /// \retval false else
-// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
- inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
- inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
- inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
- inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+/// Check if unordered.
+/// \param x first operand
+/// \param y second operand
+/// \retval true if unordered (one or two NaN operands)
+/// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type
+//isunordered(T x, U y) { return functions::isunordered(x, y); }
+inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
- /// \name Casting
- /// \{
+/// \name Casting
+/// \{
- /// Cast to or from half-precision floating point number.
- /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
- /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
- /// It uses the default rounding mode.
- ///
- /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
- /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
- /// error and casting between [half](\ref half_float::half)s is just a no-op.
- /// \tparam T destination type (half or built-in arithmetic type)
- /// \tparam U source type (half or built-in arithmetic type)
- /// \param arg value to cast
- /// \return \a arg converted to destination type
- template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
+/// Cast to or from half-precision floating point number.
+/// This casts between [half](\ref half_float::half) and any built-in arithmetic
+/// type. The values are converted directly using the given rounding mode,
+/// without any roundtrip over `float` that a `static_cast` would otherwise do.
+/// It uses the default rounding mode.
+///
+/// Using this cast with neither of the two types being a [half](\ref
+/// half_float::half) or with any of the two types not being a built-in
+/// arithmetic type (apart from [half](\ref half_float::half), of course)
+/// results in a compiler error and casting between [half](\ref
+/// half_float::half)s is just a no-op. \tparam T destination type (half or
+/// built-in arithmetic type) \tparam U source type (half or built-in arithmetic
+/// type) \param arg value to cast \return \a arg converted to destination type
+template <typename T, typename U>
+T half_cast(U arg) {
+ return half_caster<T, U>::cast(arg);
+}
- /// Cast to or from half-precision floating point number.
- /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
- /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
- ///
- /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
- /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
- /// error and casting between [half](\ref half_float::half)s is just a no-op.
- /// \tparam T destination type (half or built-in arithmetic type)
- /// \tparam R rounding mode to use.
- /// \tparam U source type (half or built-in arithmetic type)
- /// \param arg value to cast
- /// \return \a arg converted to destination type
- template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
- /// \}
- }
+/// Cast to or from half-precision floating point number.
+/// This casts between [half](\ref half_float::half) and any built-in arithmetic
+/// type. The values are converted directly using the given rounding mode,
+/// without any roundtrip over `float` that a `static_cast` would otherwise do.
+///
+/// Using this cast with neither of the two types being a [half](\ref
+/// half_float::half) or with any of the two types not being a built-in
+/// arithmetic type (apart from [half](\ref half_float::half), of course)
+/// results in a compiler error and casting between [half](\ref
+/// half_float::half)s is just a no-op. \tparam T destination type (half or
+/// built-in arithmetic type) \tparam R rounding mode to use. \tparam U source
+/// type (half or built-in arithmetic type) \param arg value to cast \return \a
+/// arg converted to destination type
+template <typename T, std::float_round_style R, typename U>
+T half_cast(U arg) {
+ return half_caster<T, U, R>::cast(arg);
+}
+/// \}
+} // namespace detail
- using detail::operator==;
- using detail::operator!=;
- using detail::operator<;
- using detail::operator>;
- using detail::operator<=;
- using detail::operator>=;
- using detail::operator+;
- using detail::operator-;
- using detail::operator*;
- using detail::operator/;
- using detail::operator<<;
- using detail::operator>>;
+using detail::operator==;
+using detail::operator!=;
+using detail::operator<;
+using detail::operator>;
+using detail::operator<=;
+using detail::operator>=;
+using detail::operator+;
+using detail::operator-;
+using detail::operator*;
+using detail::operator/;
+using detail::operator<<;
+using detail::operator>>;
- using detail::abs;
- using detail::fabs;
- using detail::fmod;
- using detail::remainder;
- using detail::remquo;
- using detail::fma;
- using detail::fmax;
- using detail::fmin;
- using detail::fdim;
- using detail::nanh;
- using detail::exp;
- using detail::expm1;
- using detail::exp2;
- using detail::log;
- using detail::log10;
- using detail::log1p;
- using detail::log2;
- using detail::sqrt;
- using detail::cbrt;
- using detail::hypot;
- using detail::pow;
- using detail::sin;
- using detail::cos;
- using detail::tan;
- using detail::asin;
- using detail::acos;
- using detail::atan;
- using detail::atan2;
- using detail::sinh;
- using detail::cosh;
- using detail::tanh;
- using detail::asinh;
- using detail::acosh;
- using detail::atanh;
- using detail::erf;
- using detail::erfc;
- using detail::lgamma;
- using detail::tgamma;
- using detail::ceil;
- using detail::floor;
- using detail::trunc;
- using detail::round;
- using detail::lround;
- using detail::nearbyint;
- using detail::rint;
- using detail::lrint;
+using detail::abs;
+using detail::acos;
+using detail::acosh;
+using detail::asin;
+using detail::asinh;
+using detail::atan;
+using detail::atan2;
+using detail::atanh;
+using detail::cbrt;
+using detail::ceil;
+using detail::cos;
+using detail::cosh;
+using detail::erf;
+using detail::erfc;
+using detail::exp;
+using detail::exp2;
+using detail::expm1;
+using detail::fabs;
+using detail::fdim;
+using detail::floor;
+using detail::fma;
+using detail::fmax;
+using detail::fmin;
+using detail::fmod;
+using detail::hypot;
+using detail::lgamma;
+using detail::log;
+using detail::log10;
+using detail::log1p;
+using detail::log2;
+using detail::lrint;
+using detail::lround;
+using detail::nanh;
+using detail::nearbyint;
+using detail::pow;
+using detail::remainder;
+using detail::remquo;
+using detail::rint;
+using detail::round;
+using detail::sin;
+using detail::sinh;
+using detail::sqrt;
+using detail::tan;
+using detail::tanh;
+using detail::tgamma;
+using detail::trunc;
#if HALF_ENABLE_CPP11_LONG_LONG
- using detail::llround;
- using detail::llrint;
+using detail::llrint;
+using detail::llround;
#endif
- using detail::frexp;
- using detail::ldexp;
- using detail::modf;
- using detail::scalbn;
- using detail::scalbln;
- using detail::ilogb;
- using detail::logb;
- using detail::nextafter;
- using detail::nexttoward;
- using detail::copysign;
- using detail::fpclassify;
- using detail::isfinite;
- using detail::isinf;
- using detail::isnan;
- using detail::isnormal;
- using detail::signbit;
- using detail::isgreater;
- using detail::isgreaterequal;
- using detail::isless;
- using detail::islessequal;
- using detail::islessgreater;
- using detail::isunordered;
-
- using detail::half_cast;
-}
+using detail::copysign;
+using detail::fpclassify;
+using detail::frexp;
+using detail::ilogb;
+using detail::isfinite;
+using detail::isgreater;
+using detail::isgreaterequal;
+using detail::isinf;
+using detail::isless;
+using detail::islessequal;
+using detail::islessgreater;
+using detail::isnan;
+using detail::isnormal;
+using detail::isunordered;
+using detail::ldexp;
+using detail::logb;
+using detail::modf;
+using detail::nextafter;
+using detail::nexttoward;
+using detail::scalbln;
+using detail::scalbn;
+using detail::signbit;
+using detail::half_cast;
+} // namespace half_float
/// Extensions to the C++ standard library.
-namespace std
-{
- /// Numeric limits for half-precision floats.
- /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
- /// `std::numeric_limits<float>`.
- template<> class numeric_limits<half_float::half> : public numeric_limits<float>
- {
- public:
- /// Supports signed values.
- static HALF_CONSTEXPR_CONST bool is_signed = true;
+namespace std {
+/// Numeric limits for half-precision floats.
+/// Because of the underlying single-precision implementation of many
+/// operations, it inherits some properties from `std::numeric_limits<float>`.
+template <>
+class numeric_limits<half_float::half> : public numeric_limits<float> {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
- /// Is not exact.
- static HALF_CONSTEXPR_CONST bool is_exact = false;
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
- /// Doesn't provide modulo arithmetic.
- static HALF_CONSTEXPR_CONST bool is_modulo = false;
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
- /// IEEE conformant.
- static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
- /// Supports infinity.
- static HALF_CONSTEXPR_CONST bool has_infinity = true;
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
- /// Supports quiet NaNs.
- static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
- /// Supports subnormal values.
- static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
- /// Rounding mode.
- /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
- /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
- /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
- /// single-precision rounding mode.
- static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
- half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the
+ /// rounding mode of the underlying single-precision implementation) with the
+ /// rounding mode of the single-to-half conversions, the actual rounding mode
+ /// might be `std::round_indeterminate` if the default half-precision rounding
+ /// mode doesn't match the single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style =
+ (std::numeric_limits<float>::round_style == half_float::half::round_style)
+ ? half_float::half::round_style
+ : round_indeterminate;
- /// Significant digits.
- static HALF_CONSTEXPR_CONST int digits = 11;
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
- /// Significant decimal digits.
- static HALF_CONSTEXPR_CONST int digits10 = 3;
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
- /// Required decimal digits to represent all possible values.
- static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
- /// Number base.
- static HALF_CONSTEXPR_CONST int radix = 2;
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
- /// One more than smallest exponent.
- static HALF_CONSTEXPR_CONST int min_exponent = -13;
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
- /// Smallest normalized representable power of 10.
- static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
- /// One more than largest exponent
- static HALF_CONSTEXPR_CONST int max_exponent = 16;
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
- /// Largest finitely representable power of 10.
- static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
- /// Smallest positive normal value.
- static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x0400);
+ }
- /// Smallest finite value.
- static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0xFBFF);
+ }
- /// Largest finite value.
- static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x7BFF);
+ }
- /// Difference between one and next representable value.
- static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x1400);
+ }
- /// Maximum rounding error.
- static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
- { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW {
+ return half_float::half(
+ half_float::detail::binary,
+ (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00);
+ }
- /// Positive infinity.
- static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x7C00);
+ }
- /// Quiet NaN.
- static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x7FFF);
+ }
- /// Signalling NaN.
- static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x7DFF);
+ }
- /// Smallest positive subnormal value.
- static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
- };
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW {
+ return half_float::half(half_float::detail::binary, 0x0001);
+ }
+};
#if HALF_ENABLE_CPP11_HASH
- /// Hash function for half-precision floats.
- /// This is only defined if C++11 `std::hash` is supported and enabled.
- template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
- {
- /// Type of function argument.
- typedef half_float::half argument_type;
+/// Hash function for half-precision floats.
+/// This is only defined if C++11 `std::hash` is supported and enabled.
+template <>
+struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+{
+ /// Type of function argument.
+ typedef half_float::half argument_type;
- /// Function return type.
- typedef size_t result_type;
+ /// Function return type.
+ typedef size_t result_type;
- /// Compute hash function.
- /// \param arg half to hash
- /// \return hash value
- result_type operator()(argument_type arg) const
- { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
- };
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const {
+ return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_) &
+ -(arg.data_ != 0x8000));
+ }
+};
#endif
-}
-
+} // namespace std
#undef HALF_CONSTEXPR
#undef HALF_CONSTEXPR_CONST
#undef HALF_NOEXCEPT
#undef HALF_NOTHROW
#ifdef HALF_POP_WARNINGS
- #pragma warning(pop)
- #undef HALF_POP_WARNINGS
+#pragma warning(pop)
+#undef HALF_POP_WARNINGS
#endif
#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 014e588..0d56558 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,51 +8,51 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include "instructions.h" #include "half.h" #include "half.hpp" -#include "instructions.h" -#include "ptx_ir.h" #include "opcodes.h" +#include "ptx_ir.h" #include "ptx_sim.h" -typedef void * yyscan_t; +typedef void *yyscan_t; class ptx_recognizer; -#include "ptx.tab.h" -#include <stdlib.h> +#include <assert.h> +#include <fenv.h> #include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> #include <cmath> -#include <fenv.h> -#include "cuda-math.h" +#include <map> +#include <sstream> +#include <string> #include "../abstract_hardware_model.h" -#include "ptx_loader.h" -#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include <assert.h> -#include <string.h> -#include <sstream> -#include <stdio.h> -#include <string> -#include <map> -#include <stdlib.h> +#include "cuda-math.h" +#include "cuda_device_printf.h" +#include "ptx.tab.h" +#include "ptx_loader.h" -//Jin: include device runtime for CDP +// Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include <stdarg.h> @@ -60,5629 +60,6420 @@ class ptx_recognizer; using half_float::half; - - const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -//Using profiled information::check the TensorCoreMatrixArrangement.xls for details -unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ - - unsigned offset; - unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; - unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; - unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; - unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; - unsigned thread_group = thread/4; - unsigned in_tg_index = thread%4; - - switch(wmma_type){ - case LOAD_A: - if(wmma_layout==ROW) - offset=load_a_row[thread_group]+16*in_tg_index; - else - offset=load_a_col[thread_group]+16*in_tg_index; - break; +// Using profiled information::check the TensorCoreMatrixArrangement.xls for +// details +unsigned thread_group_offset(int thread, unsigned wmma_type, + unsigned wmma_layout, unsigned type, int stride) { + unsigned offset; + unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned thread_group = thread / 4; + unsigned in_tg_index = thread % 4; + switch (wmma_type) { + case LOAD_A: + if (wmma_layout == ROW) + offset = load_a_row[thread_group] + 16 * in_tg_index; + else + offset = load_a_col[thread_group] + 16 * in_tg_index; + break; - case LOAD_B: - if(wmma_layout==ROW) - offset=load_b_row[thread_group]+16*in_tg_index; - else - offset=load_b_col[thread_group]+16*in_tg_index; - break; + case LOAD_B: + if (wmma_layout == ROW) + offset = load_b_row[thread_group] + 16 * in_tg_index; + else + offset = load_b_col[thread_group] + 16 * in_tg_index; + break; - case LOAD_C: - case STORE_D: - if(type==F16_TYPE){ - if(wmma_layout==ROW) - offset=load_c_half_row[thread_group]+16*in_tg_index; - else - offset=load_c_half_col[thread_group]+in_tg_index; - } - else{ - if(wmma_layout==ROW) - offset=load_c_float_row[thread_group]; - else - offset=load_c_float_col[thread_group]; + case LOAD_C: + case STORE_D: + if (type == F16_TYPE) { + if (wmma_layout == ROW) + offset = load_c_half_row[thread_group] + 16 * in_tg_index; + else + offset = load_c_half_col[thread_group] + in_tg_index; + } else { + if (wmma_layout == ROW) + offset = load_c_float_row[thread_group]; + else + offset = load_c_float_col[thread_group]; - switch(in_tg_index){ - case 0: - break; - case 1: - if(wmma_layout==ROW) - offset+=16; - else - offset+=1; - break; - case 2: - if(wmma_layout==ROW) - offset+=2; - else - offset+=32; - break; - case 3: - if(wmma_layout==ROW) - offset+=18; - else - offset+=33; - break; - default: - abort(); - } - } - break; + switch (in_tg_index) { + case 0: + break; + case 1: + if (wmma_layout == ROW) + offset += 16; + else + offset += 1; + break; + case 2: + if (wmma_layout == ROW) + offset += 2; + else + offset += 32; + break; + case 3: + if (wmma_layout == ROW) + offset += 18; + else + offset += 33; + break; + default: + abort(); + } + } + break; - default: - abort(); - - } - offset = (offset/16)*stride+offset%16; - return offset; + default: + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; } -int acc_float_offset(int index,int wmma_layout,int stride){ +int acc_float_offset(int index, int wmma_layout, int stride) { + int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; + int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; + int offset; - int c_row_offset[]={0,1,32,33,4,5,36,37}; - int c_col_offset[]={0,16,2,18,64,80,66,82}; - int offset; - - - if(wmma_layout==ROW) - offset=c_row_offset[index]; - else if(wmma_layout==COL) - offset=c_col_offset[index]; - else{ - printf("wrong layout"); - abort(); - } - offset = (offset/16)*stride+offset%16; - return offset; + if (wmma_layout == ROW) + offset = c_row_offset[index]; + else if (wmma_layout == COL) + offset = c_col_offset[index]; + else { + printf("wrong layout"); + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; } -void inst_not_implemented( const ptx_instruction * pI ) ; -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); +void inst_not_implemented(const ptx_instruction *pI); +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread); -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); -void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) -{ - assert( reg != NULL ); - if( reg->name() == "_" ) return; - assert( !m_regs.empty() ); - assert( reg->uid() > 0 ); - m_regs.back()[ reg ] = value; - if (m_enable_debug_trace ) - m_debug_trace_regs_modified.back()[ reg ] = value; - m_last_set_operand_value = value; +void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { + assert(reg != NULL); + if (reg->name() == "_") return; + assert(!m_regs.empty()); + assert(reg->uid() > 0); + m_regs.back()[reg] = value; + if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; + m_last_set_operand_value = value; } -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); +void ptx_thread_info::print_reg_thread(char *fname) { + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); - - } - //m_regs.pop_back(); + + if (size > 0) { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) { + const std::string &name = it->first->name(); + const std::string &dec = it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), + size); + } + // m_regs.pop_back(); } fclose(fp); +} +void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { + FILE *fp2 = fopen(fname, "r"); + assert(fp2 != NULL); + // m_regs.push_back( reg_map_t() ); + char line[200]; + while (fgets(line, sizeof line, fp2) != NULL) { + symbol *reg; + char *pch; + pch = strtok(line, " "); + char *name = pch; + reg = symtab->lookup(name); + ptx_reg_t data; + pch = strtok(NULL, " "); + data = atoi(pch); + pch = strtok(NULL, " "); + pch = strtok(NULL, " "); + m_regs.back()[reg] = data; } - -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - pch = strtok (NULL," "); - m_regs.back()[reg] = data; - } - fclose ( fp2 ); + fclose(fp2); } - -ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) -{ - static bool unfound_register_warned = false; - assert( reg != NULL ); - assert( !m_regs.empty() ); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert( reg->type()->get_key().is_reg() ); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway - std::string file_loc = get_location(); - if( !unfound_register_warned ) { - printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid ); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); - } - if (m_enable_debug_trace ) - m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; - return regs_iter->second; +ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { + static bool unfound_register_warned = false; + assert(reg != NULL); + assert(!m_regs.empty()); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert(reg->type()->get_key().is_reg()); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the + // user anyway + std::string file_loc = get_location(); + if (!unfound_register_warned) { + printf( + "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " + "(cuid:%u). Setting to 0X00000000. This is okay if you are " + "simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace) + m_debug_trace_regs_read.back()[reg] = regs_iter->second; + return regs_iter->second; } -ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) -{ - ptx_reg_t result, tmp; - +ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, + operand_info dstInfo, + unsigned opType, + ptx_thread_info *thread, + int derefFlag) { + ptx_reg_t result, tmp; - if(op.get_double_operand_type() == 0) { - if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { - if ( op.is_reg() ) { - result = get_reg( op.get_symbol() ); - } else if ( op.is_builtin()) { - result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); - } else if(op.is_immediate_address()){ - result.u64 = op.get_addr_offset(); - } else if ( op.is_memory_operand() ) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); + if (op.get_double_operand_type() == 0) { + if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && + (opType != FF64_TYPE)) || + (op.get_addr_space() != undefined_space)) { + if (op.is_reg()) { + result = get_reg(op.get_symbol()); + } else if (op.is_builtin()) { + result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); + } else if (op.is_immediate_address()) { + result.u64 = op.get_addr_offset(); + } else if (op.is_memory_operand()) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); - if ( info.is_reg() ) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if ( info.is_param_kernel() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_param_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_global() ) { - assert( op.get_addr_offset() == 0 ); - result.u64 = sym->get_address(); - } else if ( info.is_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_const() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); - abort(); - } - - } else if ( op.is_literal() ) { - result = op.get_literal_value(); - } else if ( op.is_label() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_const() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_global() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_local() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if ( op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); - }else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if ( info2.is_param_kernel() ) { - result.u64 = sym2->get_address()+ op.get_addr_offset(); - } - else{ - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); - assert(0); - } - } + if (info.is_reg()) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if (info.is_param_kernel()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_param_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_global()) { + assert(op.get_addr_offset() == 0); + result.u64 = sym->get_address(); + } else if (info.is_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_const()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " + "operand type for %s\n", + name); + abort(); + } - if(op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - result.u64 = (result.u64>>16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; - result.u128.low = get_reg( op.vec_symbol(1) ).u32; - result.u128.high = get_reg( op.vec_symbol(2) ).u32; - result.u128.highest = get_reg( op.vec_symbol(3) ).u32; + } else if (op.is_literal()) { + result = op.get_literal_value(); + } else if (op.is_label()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_const()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_global()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_local()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_function_address()) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if (op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { - // bb64 or ff64 - result.bits.ls = get_reg( op.vec_symbol(0) ).u32; - result.bits.ms = get_reg( op.vec_symbol(1) ).u32; + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if (info2.is_param_kernel()) { + result.u64 = sym2->get_address() + op.get_addr_offset(); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " + "type for %s\n", + name); + assert(0); + } } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; - secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0),firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(),firstHalf); - } - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size=0; - int t=0; - finalResult.u64=0; + if (op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + result.u64 = (result.u64 >> 16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg(op.vec_symbol(0)).u32; + result.u128.low = get_reg(op.vec_symbol(1)).u32; + result.u128.high = get_reg(op.vec_symbol(2)).u32; + result.u128.highest = get_reg(op.vec_symbol(3)).u32; + } else { + // bb64 or ff64 + result.bits.ls = get_reg(op.vec_symbol(0)).u32; + result.bits.ms = get_reg(op.vec_symbol(1)).u32; + } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is + // s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0), firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(), firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size = 0; + int t = 0; + finalResult.u64 = 0; - //complete other cases for reading from memory, such as reading from other const memory - if((op.get_addr_space() == global_space)&&(derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; + // complete other cases for reading from memory, such as reading from other + // const memory + if ((op.get_addr_space() == global_space) && (derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == const_space)&&(derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == local_space)&&(derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else { - finalResult = result; - } + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == const_space) && (derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read((result.u32 + op.get_const_mem_offset()), size / 8, + &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == local_space) && (derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else { + finalResult = result; + } - if((op.get_operand_neg() == true)&&(derefFlag)) { - switch( opType ) { + if ((op.get_operand_neg() == true) && (derefFlag)) { + switch (opType) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } - - } - - return finalResult; + assert(0); + } + } + return finalResult; } -unsigned get_operand_nbits( const operand_info &op ) -{ - if ( op.is_reg() ) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch( t.scalar_type() ) { - case PRED_TYPE: - return 1; - case B8_TYPE: case S8_TYPE: case U8_TYPE: - return 8; - case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: - return 16; - case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: - return 32; - case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: - return 64; +unsigned get_operand_nbits(const operand_info &op) { + if (op.is_reg()) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch (t.scalar_type()) { + case PRED_TYPE: + return 1; + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + return 8; + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + case B16_TYPE: + return 16; + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + case B32_TYPE: + return 32; + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf( + "ERROR: Need to implement get_operand_nbits() for currently " + "unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) -{ - assert( op.is_vector() ); - assert( num_elements <= 8 ); +void ptx_thread_info::get_vector_operand_values(const operand_info &op, + ptx_reg_t *ptx_regs, + unsigned num_elements) { + assert(op.is_vector()); + assert(num_elements <= 8); - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if( strcmp(sym->name().c_str(),"_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert( reg_iter != m_regs.back().end() ); - ptx_regs[idx] = reg_iter->second; - } - } + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if (strcmp(sym->name().c_str(), "_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert(reg_iter != m_regs.back().end()); + ptx_regs[idx] = reg_iter->second; + } + } } -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) -{ - if( !dst.is_reg() ) - return; - unsigned dst_size = get_operand_nbits( dst ); - if( src_size >= dst_size ) - return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size-1); - if( (mask & data.u64) == 0 ) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size-src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { + if (!dst.is_reg()) return; + unsigned dst_size = get_operand_nbits(dst); + if (src_size >= dst_size) return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size - 1); + if ((mask & data.u64) == 0) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size - src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) -{ - thread->set_operand_value( dst, data, type, thread, pI ); +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI, int overflow, + int carry) { + thread->set_operand_value(dst, data, type, thread, pI); - if (dst.get_double_operand_type() == -2) - { - ptx_reg_t predValue; - - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01)<<3); - predValue.u64 |= ((carry & 0x01)<<2); + if (dst.get_double_operand_type() == -2) { + ptx_reg_t predValue; - set_reg(sym,predValue); - } - else if (dst.get_double_operand_type() == 0) - { - //intentionally do nothing - } - else - { - printf("Unexpected double destination\n"); - assert(0); - } + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01) << 3); + predValue.u64 |= ((carry & 0x01) << 2); + set_reg(sym, predValue); + } else if (dst.get_double_operand_type() == 0) { + // intentionally do nothing + } else { + printf("Unexpected double destination\n"); + assert(0); + } } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) -{ - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI) { + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; - type_info_key::type_decode(type,size,t); + type_info_key::type_decode(type, size, t); - /*complete this section for other cases*/ - if(dst.get_addr_space() == undefined_space) - { - ptx_reg_t setValue; - setValue.u64 = data.u64; + /*complete this section for other cases*/ + if (dst.get_addr_space() == undefined_space) { + ptx_reg_t setValue; + setValue.u64 = data.u64; - // Double destination in set instruction ($p0|$p1) - second is negation of first - if (dst.get_double_operand_type() == -1) - { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); + // Double destination in set instruction ($p0|$p1) - second is negation of + // first + if (dst.get_double_operand_type() == -1) { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); - if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { - setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; - } else { - setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; - } - - set_reg(name1,setValue); - set_reg(name2,setValue2); + if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || + (type == FF64_TYPE)) { + setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; + } else { + setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; } - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) - { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; - - switch ( type ) { - case S8_TYPE: - if((setValue.s8 & 0x7F) == 0) - predValue.u64 |= 1; - break; - case S16_TYPE: - if((setValue.s16 & 0x7FFF) == 0) - predValue.u64 |= 1; - break; - case S32_TYPE: - if((setValue.s32 & 0x7FFFFFFF) == 0) - predValue.u64 |= 1; - break; - case S64_TYPE: - if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) - predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if(setValue.u8 == 0) - predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if(setValue.u16 == 0) - predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if(setValue.u32 == 0) - predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if(setValue.u64 == 0) - predValue.u64 |= 1; - break; - case F16_TYPE: - if(setValue.f16 == 0) - predValue.u64 |= 1; - break; - case F32_TYPE: - if(setValue.f32 == 0) - predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if(setValue.f64 == 0) - predValue.u64 |= 1; - break; - default: assert(0); break; - } + set_reg(name1, setValue); + set_reg(name2, setValue2); + } + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second + // register operand receives data, first predicate operand is set as + // $p0=($r4!=0) Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2) || + (dst.get_double_operand_type() == -3)) { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; - if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| - (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| - (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { - if((setValue.u32 & (1<<(size-1))) != 0) - predValue.u64 |= 1<<1; - } - if ( type==F32_TYPE ) { - if(setValue.f32 < 0) - predValue.u64 |= 1<<1; - } + switch (type) { + case S8_TYPE: + if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; + break; + case S16_TYPE: + if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; + break; + case S32_TYPE: + if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; + break; + case S64_TYPE: + if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if (setValue.u8 == 0) predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if (setValue.u16 == 0) predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if (setValue.u32 == 0) predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if (setValue.u64 == 0) predValue.u64 |= 1; + break; + case F16_TYPE: + if (setValue.f16 == 0) predValue.u64 |= 1; + break; + case F32_TYPE: + if (setValue.f32 == 0) predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if (setValue.f64 == 0) predValue.u64 |= 1; + break; + default: + assert(0); + break; + } - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } + if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || + (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || + (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || + (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { + if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; + } + if (type == F32_TYPE) { + if (setValue.f32 < 0) predValue.u64 |= 1 << 1; + } - set_reg(predName,predValue); - set_reg(regName,setValue); + if (dst.get_operand_lohi() == 1) { + setValue.u64 = + ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); } - else if (type == BB128_TYPE) - { - //b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; - const symbol *name1, *name2, *name3, *name4 = NULL; + set_reg(predName, predValue); + set_reg(regName, setValue); + } else if (type == BB128_TYPE) { + // b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); + const symbol *name1, *name2, *name3, *name4 = NULL; - set_reg(name1,setValue); - set_reg(name2,setValue2); - set_reg(name3,setValue3); - set_reg(name4,setValue4); - } - else if (type == BB64_TYPE || type == FF64_TYPE) - { - //ptxplus version of storing 64 bit values to registers stores to two adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; + set_reg(name1, setValue); + set_reg(name2, setValue2); + set_reg(name3, setValue3); + set_reg(name4, setValue4); + } else if (type == BB64_TYPE || type == FF64_TYPE) { + // ptxplus version of storing 64 bit values to registers stores to two + // adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; - const symbol *name1, *name2 = NULL; + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); + const symbol *name1, *name2 = NULL; - set_reg(name1,setValue); - set_reg(name2,setValue2); - } - else - { - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } - set_reg(dst.get_symbol(),setValue); - } - } + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); - // global memory - g[4], g[$r0] - else if(dst.get_addr_space() == global_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } + set_reg(name1, setValue); + set_reg(name2, setValue2); + } else { + if (dst.get_operand_lohi() == 1) { + setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = + ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(), setValue); + } + } - // shared memory - s[4], s[$r0] - else if(dst.get_addr_space() == shared_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); + // global memory - g[4], g[$r0] + else if (dst.get_addr_space() == global_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } - // local memory - l0[4], l0[$r0] - else if(dst.get_addr_space() == local_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type,size,t); + // shared memory - s[4], s[$r0] + else if (dst.get_addr_space() == shared_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } - else - { - printf("Destination stores to unknown location."); - assert(0); - } + // local memory - l0[4], l0[$r0] + else if (dst.get_addr_space() == local_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type, size, t); + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + else { + printf("Destination stores to unknown location."); + assert(0); + } } -void ptx_thread_info::set_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } +void ptx_thread_info::set_vector_operand_values(const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } + } + } + } - m_last_set_operand_value = data1; + m_last_set_operand_value = data1; } -void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4, - const ptx_reg_t &data5, - const ptx_reg_t &data6, - const ptx_reg_t &data7, - const ptx_reg_t &data8 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } - else{ - printf("error:set_wmma_vector_operands"); - } +void ptx_thread_info::set_wmma_vector_operand_values( + const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, + const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } else { + printf("error:set_wmma_vector_operands"); + } - m_last_set_operand_value = data8; + m_last_set_operand_value = data8; } -#define my_abs(a) (((a)<0)?(-a):(a)) - -#define MY_MAX_I(a,b) (a > b) ? a : b -#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b +#define my_abs(a) (((a) < 0) ? (-a) : (a)) -#define MY_MIN_I(a,b) (a < b) ? a : b -#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b +#define MY_MAX_I(a, b) (a > b) ? a : b +#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b -#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 -#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 +#define MY_MIN_I(a, b) (a < b) ? a : b +#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b -#define MY_CAS_I(a,b,c) (a == b) ? c : a +#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 +#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 -#define MY_EXCH(a,b) b +#define MY_CAS_I(a, b, c) (a == b) ? c : a -void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +#define MY_EXCH(a, b) b - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case S16_TYPE: d.s16 = my_abs(a.s16); break; - case S32_TYPE: d.s32 = my_abs(a.s32); break; - case S64_TYPE: d.s64 = my_abs(a.s64); break; - case U16_TYPE: d.s16 = my_abs(a.u16); break; - case U32_TYPE: d.s32 = my_abs(a.u32); break; - case U64_TYPE: d.s64 = my_abs(a.u64); break; - case F32_TYPE: d.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; - default: + switch (i_type) { + case S16_TYPE: + d.s16 = my_abs(a.s16); + break; + case S32_TYPE: + d.s32 = my_abs(a.s32); + break; + case S64_TYPE: + d.s64 = my_abs(a.s64); + break; + case U16_TYPE: + d.s16 = my_abs(a.u16); + break; + case U32_TYPE: + d.s32 = my_abs(a.u32); + break; + case U64_TYPE: + d.s64 = my_abs(a.u64); + break; + case F32_TYPE: + d.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = my_abs(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; +void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - //performs addition. Sets carry and overflow if needed. - //src3_data.pred&0x4 is the carry flag - switch ( i_type ) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; + // performs addition. Sets carry and overflow if needed. + // src3_data.pred&0x4 is the carry flag + switch (i_type) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100)>>8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000)>>16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000)>>32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; +void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - //performs addition. Sets carry and overflow if needed. - switch ( i_type ) { - case S8_TYPE: + // performs addition. Sets carry and overflow if needed. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000)>>32; + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; - case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; - case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; - default: + switch (i_type) { + case B16_TYPE: + src2_data.u16 = ~src2_data.u16; + break; + case B32_TYPE: + src2_data.u32 = ~src2_data.u32; + break; + case B64_TYPE: + src2_data.u64 = ~src2_data.u64; + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - data.u64 = src1_data.u64 & src2_data.u64; + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bar_callback( const inst_t* inst, ptx_thread_info* thread) -{ - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid,barid); - const ptx_instruction *pI = dynamic_cast<const ptx_instruction*>(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst,value, U32_TYPE, thread, pI); +void bar_callback(const inst_t *inst, ptx_thread_info *thread) { + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid, barid); + const ptx_instruction *pI = dynamic_cast<const ptx_instruction *>(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst, value, U32_TYPE, thread, pI); } -void atom_callback( const inst_t* inst, ptx_thread_info* thread) -{ - const ptx_instruction *pI = dynamic_cast<const ptx_instruction*>(inst); +void atom_callback(const inst_t *inst, ptx_thread_info *thread) { + const ptx_instruction *pI = dynamic_cast<const ptx_instruction *>(inst); - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result - bool data_ready = false; + bool data_ready = false; - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()){ - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - //This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } - - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if( whichspace(effective_address) == global_space ) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid,effective_address); - space = shared_space; - } else { - abort(); - } - } - assert( space == global_space || space == shared_space ); + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()) { + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + // This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } - memory_space *mem = NULL; - if(space == global_space) - mem = thread->get_global_memory(); - else if(space == shared_space) - mem = thread->m_shared_mem; - else - abort(); + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if (whichspace(effective_address) == global_space) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } + assert(space == global_space || space == shared_space); - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address,size/8,&data.s64); - if (dst.get_symbol()->type()){ - thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' - } + memory_space *mem = NULL; + if (space == global_space) + mem = thread->get_global_memory(); + else if (space == shared_space) + mem = thread->m_shared_mem; + else + abort(); - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address, size / 8, &data.s64); + if (dst.get_symbol()->type()) { + thread->set_operand_value(dst, data, to_type, thread, + pI); // Write value into register 'd' + } - switch ( m_atomic_spec ) { - // AND - case ATOMIC_AND: - { + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); - assert(0); - break; - } - - break; + switch (m_atomic_spec) { + // AND + case ATOMIC_AND: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.AND " + "only accepts b32\n", + to_type); + assert(0); + break; } - // OR - case ATOMIC_OR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); - assert(0); - break; - } - break; + break; + } + // OR + case ATOMIC_OR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.OR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // XOR - case ATOMIC_XOR: - { - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); - assert(0); - break; - } - - break; + break; + } + // XOR + case ATOMIC_XOR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.XOR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // CAS - case ATOMIC_CAS: - { - - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); - assert(0); - break; - } + break; + } + // CAS + case ATOMIC_CAS: { + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - break; + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.CAS " + "only accepts b32 and b64\n", + to_type); + assert(0); + break; } - // EXCH - case ATOMIC_EXCH: - { - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); - assert(0); - break; - } - break; + break; + } + // EXCH + case ATOMIC_EXCH: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.EXCH " + "only accepts b32\n", + to_type); + assert(0); + break; } - // ADD - case ATOMIC_ADD: - { - - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); - assert(0); - break; - } - break; + break; + } + // ADD + case ATOMIC_ADD: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.ADD only " + "accepts u32, s32, u64, and f32\n"); + assert(0); + break; } - // INC - case ATOMIC_INC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // INC + case ATOMIC_INC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.INC only " + "accepts u32 and s32\n"); + assert(0); + break; } - // DEC - case ATOMIC_DEC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // DEC + case ATOMIC_DEC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.DEC only " + "accepts u32 and s32\n"); + assert(0); + break; } - // MIN - case ATOMIC_MIN: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // MIN + case ATOMIC_MIN: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MIN only " + "accepts u32 and s32\n"); + assert(0); + break; } - // MAX - case ATOMIC_MAX: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); - assert(0); - break; - } - break; - } - // DEFAULT - default: - { - assert(0); - break; + break; + } + // MAX + case ATOMIC_MAX: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MAX only " + "accepts u32 and s32\n"); + assert(0); + break; } - } - // Write operation result into memory - // (i.e. copy src1_data to dst) - if ( data_ready ) { - mem->write(effective_address,size/8,&op_result.s64,thread,pI); - } else { - printf("Execution error: data_ready not set\n"); + break; + } + // DEFAULT + default: { assert(0); - } + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if (data_ready) { + mem->write(effective_address, size / 8, &op_result.s64, thread, pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } } -// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) -void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) +// atom_impl will now result in a callback being called in mem_ctrl_pop +// (gpu-sim.c) +void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) - // obtain memory space of the operation - memory_space_t space = pI->get_space(); + // obtain memory space of the operation + memory_space_t space = pI->get_space(); - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address + // calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; - addr_t effective_address_final; + addr_t effective_address_final; - // handle generic memory space by converting it to global - if ( space == undefined_space ) { - if( whichspace(effective_address) == global_space ) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid,effective_address); - space = shared_space; - } else { - abort(); - } - } else { - assert( space == global_space || space == shared_space ); - effective_address_final = effective_address; - } + // handle generic memory space by converting it to global + if (space == undefined_space) { + if (whichspace(effective_address) == global_space) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert(space == global_space || space == shared_space); + effective_address_final = effective_address; + } - // Check state space - assert( space == global_space || space == shared_space ); + // Check state space + assert(space == global_space || space == shared_space); - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; } -void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) -{ - ptx_instruction * pI = const_cast<ptx_instruction *>(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); +void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { + ptx_instruction *pI = const_cast<ptx_instruction *>(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); - switch(bar_op){ - case SYNC_OPTION: - { - if(pI->get_num_operands()>1){ - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - }else{ - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - } - break; - } - case ARRIVE_OPTION: - { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: - { - if(pI->get_num_operands()>3){ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32=!(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - default: - abort(); - break; - } - }else{ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32=!(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - default: - abort(); - break; - } - } - break; - } - default: - abort(); - break; - } + switch (bar_op) { + case SYNC_OPTION: { + if (pI->get_num_operands() > 1) { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + } else { + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: { + if (pI->get_num_operands() > 3) { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32 = !(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); + break; + default: + abort(); + break; + } + } else { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32 = !(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * + thread->get_ntid().z); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + default: + abort(); + break; + } + } + break; + } + default: + abort(); + break; + } - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; } -void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) - { - case U32_TYPE: - { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: - { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: - { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } - else - { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; - } - break; - } - case S64_TYPE: - { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; - } - else - { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; - } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); +void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) { + case U32_TYPE: { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } else { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; + } + break; + } + case S64_TYPE: { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; + } else { + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; + } + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - int i,max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; +void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + int i, max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - switch ( i_type ) { - case B32_TYPE: + switch (i_type) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data=src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for(i=0;i<len && pos+i<max;i++){ - data.u32=(~((0x00000001)<<(pos+i)))&data.u32; - data.u32=data.u32|((src1_data.u32&((0x00000001)<<(i)))<<(pos)); - } - thread->set_operand_value(dst, data, i_type, thread, pI); + } + data = src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for (i = 0; i < len && pos + i < max; i++) { + data.u32 = (~((0x00000001) << (pos + i))) & data.u32; + data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated +void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert( + pI->has_pred() == + false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); +void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - unsigned msb; - switch(i_type){ - case B32_TYPE: - msb = 31; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u32 & (1 << i))) - data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u64 & (1 << i))) - data.u64 |= 1 << (msb - i); - } - break; - default: assert(0); - } - thread->set_operand_value(dst,data, i_type, thread, pI); + unsigned msb; + switch (i_type) { + case B32_TYPE: + msb = 31; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); + } + break; + default: + assert(0); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 - //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); - //unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - //if (shifted_output == (pow(2,precision)-1)) round_up = 0; - //num = shifted_output + round_up; - num >>= (latest_one-precision+1); - } - return num; + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise + // it is 0 + // int round_up = (num & (1 << (latest_one-precision))) >> + // (latest_one-precision); unsigned shifted_output = num >> + // (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + // if (shifted_output == (pow(2,precision)-1)) round_up = 0; + // num = shifted_output + round_up; + num >>= (latest_one - precision + 1); + } + return num; } -void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ - int offset; - int c_row_offset[]={0,8,0,8,4,12,4,12}; - int c_col_offset[]={0,0,8,8,0,0,8,8}; - int c_tg_inside_row_offset[]={0,1,0,1}; - int c_tg_inside_col_offset[]={0,0,2,2}; - int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; - int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; +void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, + int stride, int &row, int &col, int &assg_offset) { + int offset; + int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; + int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; + int c_tg_inside_row_offset[] = {0, 1, 0, 1}; + int c_tg_inside_col_offset[] = {0, 0, 2, 2}; + int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; + int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; - offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); + offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); - if(wmma_type==LOAD_A){ - if(wmma_layout==ROW){ - offset+=index+8*((thread%16)/8); - } - else{ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if(wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - else{ - offset+=index+8*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if( wmma_type==LOAD_C){ - if(type==F16_TYPE){ - row=c_row_offset[thread/4]+thread%4; - col=c_col_offset[thread/4]+index; - } - else{ - row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; - col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; - } - assg_offset=index; - } + if (wmma_type == LOAD_A) { + if (wmma_layout == ROW) { + offset += index + 8 * ((thread % 16) / 8); + } else { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } else { + offset += index + 8 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_C) { + if (type == F16_TYPE) { + row = c_row_offset[thread / 4] + thread % 4; + col = c_col_offset[thread / 4] + index; + } else { + row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + + c_inside_row_offset[index]; + col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + + c_inside_col_offset[index]; + } + assg_offset = index; + } - if(wmma_type==LOAD_A||wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - row=offset/16; - col=offset%16; - } - else{ - col=offset/16; - row=offset%16; - } - } + if (wmma_type == LOAD_A || wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + row = offset / 16; + col = offset % 16; + } else { + col = offset / 16; + row = offset % 16; + } + } } -void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - int i,j,k,thrd; - int row,col,offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; +void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + int i, j, k, thrd; + int row, col, offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid ; - const operand_info &dst = pI->operand_lookup(0); - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - float temp; - half temp2; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:",thrd); - for(int operand_num=1;operand_num<=3;operand_num++){ - const operand_info &src_a= pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for(k = 0; k < nelem; k++){ - printf("%llx ",v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid; + const operand_info &dst = pI->operand_lookup(0); - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - if(k%2==1) - hex_val=(v[k/2].s64&0xffff); - else - hex_val=((v[k/2].s64&0xffff0000)>>16); - nw_v[k].f16 =*((half *)&hex_val); - } - } - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - temp=nw_v[k].f16; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ",temp); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - else{ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - for(k=0;k<8;k++){ - printf("%.2f ",v[k].f32); - } - printf("\n"); - } - } - switch(operand_num) { - case 1 ://operand 1 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_a[row][col]=nw_v[offset]; - } - break; - case 2 ://operand 2 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_b[row][col]=nw_v[offset]; - } - break; - case 3 ://operand 3 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - if(type2!=F16_TYPE){ - matrix_c[row][col]=v[offset]; - } - else { - matrix_c[row][col]=nw_v[offset]; - } - } - break; - default : - printf("Invalid Operand Index\n" ); - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_A\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_a[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_b[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type2==F16_TYPE){ - temp=matrix_c[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - matrix_d[i][j].f16=0; - } - } - - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - for(k=0;k<16;k++){ - matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; - } - if((type==F16_TYPE)&&(type2==F16_TYPE)) - matrix_d[i][j].f16+=matrix_c[i][j].f16; - else if((type==F32_TYPE)&&(type2==F16_TYPE)){ - temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; - temp=temp2; - matrix_d[i][j].f32=temp; - } - else if((type==F16_TYPE)&&(type2==F32_TYPE)){ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f16=half(temp); - } - else{ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f32=temp; - } - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_D\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type==F16_TYPE){ - temp=matrix_d[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - int row_t[8]; - int col_t[8]; - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); - } - thread = core->get_thread_info()[tid+thrd]; - - - if(type==F32_TYPE){ - thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - { - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - temp=matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ",temp); - } - printf("\n"); + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + float temp; + half temp2; - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); - nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); - nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); - nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); - thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); - - } - else{ - printf("wmma:mma:wrong type\n"); - abort(); - } - } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:", thrd); + for (int operand_num = 1; operand_num <= 3; operand_num++) { + const operand_info &src_a = pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values(src_a, v, nelem); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for (k = 0; k < nelem; k++) { + printf("%llx ", v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + if (k % 2 == 1) + hex_val = (v[k / 2].s64 & 0xffff); + else + hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); + nw_v[k].f16 = *((half *)&hex_val); + } + } + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + temp = nw_v[k].f16; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ", temp); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } else { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + for (k = 0; k < 8; k++) { + printf("%.2f ", v[k].f32); + } + printf("\n"); + } + } + switch (operand_num) { + case 1: // operand 1 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_a[row][col] = nw_v[offset]; + } + break; + case 2: // operand 2 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_b[row][col] = nw_v[offset]; + } + break; + case 3: // operand 3 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + if (type2 != F16_TYPE) { + matrix_c[row][col] = v[offset]; + } else { + matrix_c[row][col] = nw_v[offset]; + } + } + break; + default: + printf("Invalid Operand Index\n"); + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_A\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_a[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_b[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type2 == F16_TYPE) { + temp = matrix_c[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + matrix_d[i][j].f16 = 0; + } + } + + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + for (k = 0; k < 16; k++) { + matrix_d[i][j].f16 = + matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; + } + if ((type == F16_TYPE) && (type2 == F16_TYPE)) + matrix_d[i][j].f16 += matrix_c[i][j].f16; + else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { + temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; + temp = temp2; + matrix_d[i][j].f32 = temp; + } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f16 = half(temp); + } else { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f32 = temp; + } + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_D\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type == F16_TYPE) { + temp = matrix_d[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + int row_t[8]; + int col_t[8]; + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); + } + thread = core->get_thread_info()[tid + thrd]; + + if (type == F32_TYPE) { + thread->set_wmma_vector_operand_values( + dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], + matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], + matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], + matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + temp = matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ", temp); + } + printf("\n"); + + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | + ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); + nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | + ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); + nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | + ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); + nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | + ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); + thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, + nw_data4); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, + nw_data3.s64, nw_data4.s64); + + } else { + printf("wmma:mma:wrong type\n"); + abort(); + } + } } -void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static unsigned call_uid_next = 1; - - const operand_info &target = pI->func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (target_func->get_function_size() >0) - target_func->do_pdom(); - target_func->set_pdom(); - } +void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; - // check that number of args and return match function requirements - if( pI->has_return() ^ target_func->has_return() ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); + const operand_info &target = pI->func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + target_func->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + target_func->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() > 0) target_func->do_pdom(); + target_func->set_pdom(); + } - if( n_operands != (n_return+1+n_args) ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" - " call instruction and function declaration\n"); - abort(); - } + // check that number of args and return match function requirements + if (pI->has_return() ^ target_func->has_return()) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of return values " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); - // handle intrinsic functions - std::string fname = target_func->get_name(); - if( fname == "vprintf" ) { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } + if (n_operands != (n_return + 1 + n_args)) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + // handle intrinsic functions + std::string fname = target_func->get_name(); + if (fname == "vprintf") { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } #if (CUDART_VERSION >= 5000) - //Jin: handle device runtime apis for CDP - else if(fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); - return; - } - else if(fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); - return; - } - else if(fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); - return; - } + // Jin: handle device runtime apis for CDP + else if (fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( + pI, thread, target_func); + return; + } else if (fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( + pI, thread, target_func); + return; + } else if (fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( + pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if( target_func->has_return() ) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if (target_func->has_return()) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -//Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - - static unsigned call_uid_next = 1; +// Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch ( i_type ) { - case B32_TYPE: + switch (i_type) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64&mask) == 0) ) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64 & mask) == 0)) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst,d, B32_TYPE, thread, pI); + thread->set_operand_value(dst, d, B32_TYPE, thread, pI); } -void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; - case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; - case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; - case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; - default: + switch (i_type) { + case PRED_TYPE: + d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; + break; + case B16_TYPE: + d.u16 = (a.u16 == 0) ? 1 : 0; + break; + case B32_TYPE: + d.u32 = (a.u32 == 0) ? 1 : 0; + break; + case B64_TYPE: + d.u64 = (a.u64 == 0) ? 1 : 0; + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - switch ( to_width ) { - case 8: x.mask_and(0,0xFF); break; - case 16: x.mask_and(0,0xFFFF); break; - case 32: x.mask_and(0,0xFFFFFFFF); break; - case 64: break; - default: assert(0); - } - return x; +ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + switch (to_width) { + case 8: + x.mask_and(0, 0xFF); + break; + case 16: + x.mask_and(0, 0xFFFF); + break; + case 32: + x.mask_and(0, 0xFFFFFFFF); + break; + case 64: + break; + default: + assert(0); + } + return x; } -ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( from_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; +ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (from_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; } -// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 -ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( to_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; +// sign extend depending on the destination register size - hack to get +// SobelFilter working in CUDA 4.2 +ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (to_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; } -ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - return chop(x,0,from_width,0,rounding_mode,saturation_mode); +ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); } -int saturatei(int a, int max, int min) -{ - if (a > max) a = max; - else if (a < min) a = min; - return a; +int saturatei(int a, int max, int min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; } -unsigned int saturatei(unsigned int a, unsigned int max) -{ - if (a > max) a = max; - return a; +unsigned int saturatei(unsigned int a, unsigned int max) { + if (a > max) a = max; + return a; } -ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - half mytemp; - half_float::half tmp_h; - //assert( from_width == 32); +ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + half mytemp; + half_float::half tmp_h; + // assert( from_width == 32); - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: mode = cudaRoundZero; break; - case RNI_OPTION: mode = cudaRoundNearest; break; - case RMI_OPTION: mode = cudaRoundMinInf; break; - case RPI_OPTION: mode = cudaRoundPosInf; break; - default: break; - } + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: + mode = cudaRoundZero; + break; + case RNI_OPTION: + mode = cudaRoundNearest; + break; + case RMI_OPTION: + mode = cudaRoundMinInf; + break; + case RPI_OPTION: + mode = cudaRoundPosInf; + break; + default: + break; + } - ptx_reg_t y; - if ( to_sign == 1 ) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) - tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1<<to_width) - 1, -(1<<to_width)); - } - switch ( to_width ) { - case 8: y.s8 = (char)tmp; break; - case 16: y.s16 = (short)tmp; break; - case 32: y.s32 = (int)tmp; break; - case 64: y.s64 = (long long)tmp; break; - default: assert(0); break; - } - } else if ( to_sign == 0 ) { - unsigned int tmp = cuda_math::float2uint(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) - tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1<<to_width) - 1); - } - switch ( to_width ) { - case 8: y.u8 = (unsigned char)tmp; break; - case 16: y.u16 = (unsigned short)tmp; break; - case 32: y.u32 = (unsigned int)tmp; break; - case 64: y.u64 = (unsigned long long)tmp; break; - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: - y.f16 =half_float::half_cast<half,std::numeric_limits<float>::round_style>(x.f32);//mytemp; - break; - case 32: - y.f32=float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: assert(0); break; - } - } - return y; + ptx_reg_t y; + if (to_sign == 1) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); + } + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + unsigned int tmp = cuda_math::float2uint(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1); + } + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + y.f16 = half_float::half_cast<half, + std::numeric_limits<float>::round_style>( + x.f32); // mytemp; + break; + case 32: + y.f32 = float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: + assert(0); + break; + } + } + return y; } -double saturated2i (double a, double max, double min) { - if (a > max) a = max; - else if (a < min) a = min; - return a; +double saturated2i(double a, double max, double min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; } -ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - assert( from_width == 64); +ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + assert(from_width == 64); - double tmp; - switch (rounding_mode) { - case RZI_OPTION: tmp = trunc(x.f64); break; - case RNI_OPTION: tmp = nearbyint(x.f64); break; - case RMI_OPTION: tmp = floor(x.f64); break; - case RPI_OPTION: tmp = ceil(x.f64); break; - default: tmp = x.f64; break; - } + double tmp; + switch (rounding_mode) { + case RZI_OPTION: + tmp = trunc(x.f64); + break; + case RNI_OPTION: + tmp = nearbyint(x.f64); + break; + case RMI_OPTION: + tmp = floor(x.f64); + break; + case RPI_OPTION: + tmp = ceil(x.f64); + break; + default: + tmp = x.f64; + break; + } - ptx_reg_t y; - if ( to_sign == 1 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); - switch ( to_width ) { - case 8: y.s8 = (char)tmp; break; - case 16: y.s16 = (short)tmp; break; - case 32: y.s32 = (int)tmp; break; - case 64: y.s64 = (long long)tmp; break; - default: assert(0); break; - } - } else if ( to_sign == 0 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); - switch ( to_width ) { - case 8: y.u8 = (unsigned char)tmp; break; - case 16: y.u16 = (unsigned short)tmp; break; - case 32: y.u32 = (unsigned int)tmp; break; - case 64: y.u64 = (unsigned long long)tmp; break; - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; + ptx_reg_t y; + if (to_sign == 1) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: assert(0); break; - } - } - return y; + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: + assert(0); + break; + } + } + return y; } -ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; +ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; - if (from_width < 64) { // 32-bit conversion - y = sext(x,from_width,32,0,rounding_mode,saturation_mode); + if (from_width < 64) { // 32-bit conversion + y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; - case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; - case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; - case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; - default: break; - } - break; - case 64: y.f64 = y.s32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; - case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; - case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; - case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; - default: break; - } - break; - case 64: y.f64 = y.s64; break; // no internal implementation found - default: assert(0); break; - } - } + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__int2float_rz(y.s32); + break; + case RN_OPTION: + y.f32 = cuda_math::__int2float_rn(y.s32); + break; + case RM_OPTION: + y.f32 = cuda_math::__int2float_rd(y.s32); + break; + case RP_OPTION: + y.f32 = cuda_math::__int2float_ru(y.s32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ll2float_rz(y.s64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ll2float_rn(y.s64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ll2float_rd(y.s64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ll2float_ru(y.s64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s64; + break; // no internal implementation found + default: + assert(0); + break; + } + } - // saturating an integer to 1 or 0? - return y; + // saturating an integer to 1 or 0? + return y; } -ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; +ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; - if (from_width < 64) { // 32-bit conversion - y = zext(x,from_width,32,0,rounding_mode,saturation_mode); + if (from_width < 64) { // 32-bit conversion + y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; - case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; - case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; - case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; - default: break; - } - break; - case 64: y.f64 = y.u32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - default: break; - } - break; - case 64: y.f64 = y.u64; break; // no internal implementation found - default: assert(0); break; - } - } + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__uint2float_rz(y.u32); + break; + case RN_OPTION: + y.f32 = cuda_math::__uint2float_rn(y.u32); + break; + case RM_OPTION: + y.f32 = cuda_math::__uint2float_rd(y.u32); + break; + case RP_OPTION: + y.f32 = cuda_math::__uint2float_ru(y.u32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u64; + break; // no internal implementation found + default: + assert(0); + break; + } + } - // saturating an integer to 1 or 0? - return y; + // saturating an integer to 1 or 0? + return y; } -ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - if (from_width == 16){ - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float<float>(val); - }else{ - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: - #if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); - #else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); - #endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } - #if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) - #else - if (cuda_math::__cuda___isnanf(y.f32)) - #endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } +ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + if (from_width == 16) { + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float<float>(val); + } else { + switch (rounding_mode) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); +#else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); +#endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } +#if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) +#else + if (cuda_math::__cuda___isnanf(y.f32)) +#endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } - return y; + return y; } -ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: +ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + switch (rounding_mode) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: #if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); + y.f64 = nearbyint(x.f64); #else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); + y.f64 = cuda_math::__internal_nearbyintf(x.f64); #endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; } -ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode ) = { - { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, - { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} -}; +ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, + unsigned to_width, int to_sign, int rounding_mode, + int saturation_mode) = { + {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, + {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; -void ptx_round(ptx_reg_t& data, int rounding_mode, int type) -{ - if (rounding_mode == RN_OPTION) { - return; - } - switch ( rounding_mode ) { - case RZI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) data.f64 = ceil(data.f64); //negative - else data.f64 = floor(data.f64); //positive - break; - default: assert(0); break; +void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { + if (rounding_mode == RN_OPTION) { + return; + } + switch (rounding_mode) { + case RZI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = truncf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) + data.f64 = ceil(data.f64); // negative + else + data.f64 = floor(data.f64); // positive + break; + default: + assert(0); + break; } break; - case RNI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE:// assert(0); break; + case RNI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + data.f16 = nearbyintf(data.f16); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); #endif - break; - case F32_TYPE: + break; + case F32_TYPE: #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + data.f32 = nearbyintf(data.f32); #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); #endif - break; - case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; - default: assert(0); break; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = round(data.f64); + break; + default: + assert(0); + break; } break; - case RMI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; - default: assert(0); break; + case RMI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = floorf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = floor(data.f64); + break; + default: + assert(0); + break; } break; - case RPI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; - case F32_TYPE: data.f32 = ceilf(data.f32); break; - case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; - default: assert(0); break; + case RPI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = ceilf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = ceilf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = ceil(data.f64); + break; + default: + assert(0); + break; } break; - default: break; - } + default: + break; + } - if (type == F32_TYPE) { + if (type == F32_TYPE) { #if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) + if (isnanf(data.f32)) #else - if (cuda_math::__cuda___isnanf(data.f32)) + if (cuda_math::__cuda___isnanf(data.f32)) #endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE)||(type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE) || (type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) -{ - if (!saturation_mode) { - return; - } - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: //assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; //negative - if (data.f16 < 0.0f) data.f16 = 0.0f; //positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; //negative - if (data.f32 < 0.0f) data.f32 = 0.0f; //positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; //negative - if (data.f64 < 0.0f) data.f64 = 0.0f; //positive - break; - default: assert(0); break; - } - +void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { + if (!saturation_mode) { + return; + } + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; // negative + if (data.f16 < 0.0f) data.f16 = 0.0f; // positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; // negative + if (data.f32 < 0.0f) data.f32 = 0.0f; // positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; // negative + if (data.f64 < 0.0f) data.f64 = 0.0f; // positive + break; + default: + assert(0); + break; + } } -void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); +void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); -// if ( to_type == F16_TYPE || from_type == F16_TYPE ) -// abort(); + // if ( to_type == F16_TYPE || from_type == F16_TYPE ) + // abort(); - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = + type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - if(pI->is_neg()){ - - switch( from_type ) { + if (pI->is_neg()) { + switch (from_type) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - data.s8 = -data.s8; - break; + data.s8 = -data.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - data.s16 = -data.s16; - break; + data.s16 = -data.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - data.s32 = -data.s32; - break; + data.s32 = -data.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - data.s64 = -data.s64; - break; + data.s64 = -data.s64; + break; case F16_TYPE: - data.f16 = -data.f16; - break; + data.f16 = -data.f16; + break; case F32_TYPE: - data.f32 = -data.f32; - break; + data.f32 = -data.f32; + break; case F64_TYPE: case FF64_TYPE: - data.f64 = -data.f64; - break; + data.f64 = -data.f64; + break; default: - assert(0); - } - - } - + assert(0); + } + } - if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); - data = result; - } + if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( + data, from_width, to_width, to_sign, rounding_mode, saturation_mode); + data = result; + } - thread->set_operand_value(dst, data, to_type, thread, pI ); + thread->set_operand_value(dst, data, to_type, thread, pI); } -void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - if( to_non_generic ) { - switch( space.get_type() ) { - case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; - case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; - case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; - default: abort(); - } - } else { - switch( space.get_type() ) { - case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; - case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) - + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call - case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; - default: abort(); - } - } - - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(),to_addr); -} + if (to_non_generic) { + switch (space.get_type()) { + case shared_space: + to_addr_hw = generic_to_shared(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); + break; + case global_space: + to_addr_hw = generic_to_global(from_addr_hw); + break; + default: + abort(); + } + } else { + switch (space.get_type()) { + case shared_space: + to_addr_hw = shared_to_generic(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + + thread->get_local_mem_stack_pointer(); + break; // add stack ptr here so that it can be passed as a pointer at + // function call + case global_space: + to_addr_hw = global_to_generic(from_addr_hw); + break; + default: + abort(); + } + } -void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(), to_addr); +} - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - unsigned i_type = pI->get_type(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8 ; break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; - default: assert(0); break; - } - thread->set_operand_value(dst,data, i_type, thread,pI); + switch (i_type) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8; + break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; + break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; + break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 / src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 / src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 / src2_data.f64; + break; + default: + assert(0); + break; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - printf("DP4A instruction not implemented yet"); - assert(0); - +void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + printf("DP4A instruction not implemented yet"); + assert(0); } -void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); + unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: data.f32 = cuda_math::__powf(2.0, src1_data.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } - - thread->set_operand_value(dst,data, i_type, thread,pI); + } + + thread->set_operand_value(dst, data, i_type, thread, pI); } -void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - thread->set_done(); - thread->exitCore(); - thread->registerExit(); +void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry = false); -void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI,thread); +void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread); } -void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a; - bool t=false; +void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a; + bool t = false; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - switch( space.get_type() ) { - case shared_space: t = isspace_shared( smid, addr ); - case local_space: t = isspace_local( smid, hwtid, addr ); - case global_space: t = isspace_global( addr ); - default: abort(); - } + switch (space.get_type()) { + case shared_space: + t = isspace_shared(smid, addr); + case local_space: + t = isspace_local(smid, hwtid, addr); + case global_space: + t = isspace_global(addr); + default: + abort(); + } - ptx_reg_t p; - p.pred = t?1:0; + ptx_reg_t p; + p.pred = t ? 1 : 0; - thread->set_reg(dst.get_symbol(),p); + thread->set_reg(dst.get_symbol(), p); } -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) -{ - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr) { + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); - if( space == param_space_unclassified ) { - // need to op to determine whether it refers to a kernel param or local param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if( ti.is_param_kernel() ) - space = param_space_kernel; - else if( ti.is_param_local() ) { - space = param_space_local; - } - //mov r1, param-label - else if (ti.is_reg() ){ - space = param_space_kernel; - } - else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); - abort(); - } - } - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; + if (space == param_space_unclassified) { + // need to op to determine whether it refers to a kernel param or local + // param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if (ti.is_param_kernel()) + space = param_space_kernel; + else if (ti.is_param_local()) { + space = param_space_local; + } + // mov r1, param-label + else if (ti.is_reg()) { + space = param_space_kernel; + } else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", + s->name().c_str()); + abort(); + } + } + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: mem = thread->get_tex_memory(); break; - case surf_space: mem = thread->get_surf_memory(); break; - case param_space_kernel: mem = thread->get_param_memory(); break; - case shared_space: mem = thread->m_shared_mem; break; - case sstarr_space: mem = thread->m_sstarr_mem; break; - case const_space: mem = thread->get_global_memory(); break; - case generic_space: - if( thread->get_ptx_version().ver() >= 2.0 ) { - // convert generic address to memory space address - space = whichspace(addr); - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; - case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; - case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; - default: abort(); - } + break; + case tex_space: + mem = thread->get_tex_memory(); + break; + case surf_space: + mem = thread->get_surf_memory(); + break; + case param_space_kernel: + mem = thread->get_param_memory(); + break; + case shared_space: + mem = thread->m_shared_mem; + break; + case sstarr_space: + mem = thread->m_sstarr_mem; + break; + case const_space: + mem = thread->get_global_memory(); + break; + case generic_space: + if (thread->get_ptx_version().ver() >= 2.0) { + // convert generic address to memory space address + space = whichspace(addr); + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + addr = generic_to_global(addr); + break; + case local_space: + mem = thread->m_local_mem; + addr = generic_to_local(smid, hwtid, addr); + break; + case shared_space: + mem = thread->m_shared_mem; + addr = generic_to_shared(smid, addr); + break; + default: + abort(); + } } else { - abort(); + abort(); } break; - case param_space_unclassified: - case undefined_space: - default: + case param_space_unclassified: + case undefined_space: + default: abort(); - } + } } -void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned type = pI->get_type(); + unsigned type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); - memory_space *mem = NULL; - addr_t addr = src1_data.u32; + memory_space *mem = NULL; + addr_t addr = src1_data.u32; - decode_space(space,thread,src1,mem,addr); + decode_space(space, thread, src1, mem, addr); - size_t size; - int t; - data.u64=0; - type_info_key::type_decode(type,size,t); - if (!vector_spec) { - mem->read(addr,size/8,&data.s64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(data,size,dst); - thread->set_operand_value(dst,data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr,size/8,&data1.s64); - mem->read(addr+size/8,size/8,&data2.s64); - if (vector_spec != V2_TYPE) { //either V3 or V4 - mem->read(addr+2*size/8,size/8,&data3.s64); - if (vector_spec != V3_TYPE) { //v4 - mem->read(addr+3*size/8,size/8,&data4.s64); - thread->set_vector_operand_values(dst,data1,data2,data3,data4); - } else //v3 - thread->set_vector_operand_values(dst,data1,data2,data3,data3); - } else //v2 - thread->set_vector_operand_values(dst,data1,data2,data2,data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + size_t size; + int t; + data.u64 = 0; + type_info_key::type_decode(type, size, t); + if (!vector_spec) { + mem->read(addr, size / 8, &data.s64); + if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); + thread->set_operand_value(dst, data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr, size / 8, &data1.s64); + mem->read(addr + size / 8, size / 8, &data2.s64); + if (vector_spec != V2_TYPE) { // either V3 or V4 + mem->read(addr + 2 * size / 8, size / 8, &data3.s64); + if (vector_spec != V3_TYPE) { // v4 + mem->read(addr + 3 * size / 8, size / 8, &data4.s64); + thread->set_vector_operand_values(dst, data1, data2, data3, data4); + } else // v3 + thread->set_vector_operand_values(dst, data1, data2, data3, data3); + } else // v2 + thread->set_vector_operand_values(dst, data1, data2, data2, data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); +void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); } -void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); +void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); } -void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid ; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; +void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - for (thrd=0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid+thrd]; + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a= pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t* v= new ptx_reg_t[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - stride = src2_data.u32; - - memory_space_t space = pI->get_space(); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a = pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t *v = new ptx_reg_t[8]; + thread->get_vector_operand_values(src_a, v, nelem); + stride = src2_data.u32; - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - decode_space(space,thread,src1,mem,addr); + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; + + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + decode_space(space, thread, src1, mem, addr); - type_info_key::type_decode(type, size, t); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t push_addr; + type_info_key::type_decode(type, size, t); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, + addr_reg.u32, size, src2_data.u32); + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for (k = 0; k < 8; k++) { + if (k % 2 == 0) + nw_v[k].s64 = (v[k / 2].s64 & 0xffff); + else + nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); + } + + for (k = 0; k < 8; k++) { + if (type == F32_TYPE) { + // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); + mem->write(push_addr, size / 8, &v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, + v[6].s64, v[7].s64); + float temp; + int l; + printf("thread=%d:", thrd); + for (l = 0; l < 8; l++) { + temp = v[l].f32; + printf("%.2f", temp); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; + } else if (wmma_layout == COL) { + // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2 * stride; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + } - ptx_reg_t nw_v[8]; - for(k=0;k<8;k++){ - if(k%2==0) - nw_v[k].s64=(v[k/2].s64&0xffff); - else - nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); - } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, + nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); + } + } - for(k=0;k<8;k++){ - if(type==F32_TYPE){ - //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); - mem->write(push_addr,size/8,&v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); - float temp; - int l; - printf("thread=%d:",thrd); - for(l=0;l<8;l++){ - temp=v[l].f32; - printf("%.2f",temp); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - if(k%2==0) - mem_txn_addr[num_mem_txn++]=push_addr; - } - else if(wmma_layout==COL){ - //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2*stride; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - } - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); - } - } - - delete [] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + delete[] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction + if ((type == F16_TYPE) && + (wmma_layout == COL)) // check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction - assert( inst.memory_op == insn_memory_op ); - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } + assert(inst.memory_op == insn_memory_op); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - int t,i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + int t, i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd,stride; - ptx_thread_info *thread; - + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd, stride; + ptx_thread_info *thread; - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride=src2_data.u32; - memory_space_t space = pI->get_space(); + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t src1_data = + thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = + thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride = src2_data.u32; + memory_space_t space = pI->get_space(); - decode_space(space,thread,src1,mem,addr); - type_info_key::type_decode(type, size, t); - - ptx_reg_t data[16]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); - - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } - if(wmma_type==LOAD_A){ - for(i=0;i<16;i++){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_B){ - for(i=0;i<16;i++){ - if(wmma_layout==COL){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==ROW){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_C){ - for(i=0;i<8;i++){ - if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*stride*i; - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("mma_ld:wrong_type\n"); - abort(); - } - } - else if(type==F32_TYPE){ - //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("wrong type"); - abort(); - } - } - } - else{ - printf("wrong wmma type\n");; - abort(); - } - //generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + decode_space(space, thread, src1, mem, addr); + type_info_key::type_decode(type, size, t); - if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert( inst.memory_op == insn_memory_op ); + ptx_reg_t data[16]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, + src1_data.u32, size, src2_data.u32); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - if(type==F16_TYPE){ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<16;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - - printf("\nmma_ld:thread%d= ",thrd); - float temp; - for(i=0;i<16;i++){ - temp=data[i].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - else{ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%.2f ",data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - } - } + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; - if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ - thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); - } - else{ - ptx_reg_t nw_data[8]; - int num_reg; - - if(wmma_type==LOAD_C) - num_reg=4; - else - num_reg=8; + if (wmma_type == LOAD_A) { + for (i = 0; i < 16; i++) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_B) { + for (i = 0; i < 16; i++) { + if (wmma_layout == COL) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == ROW) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_C) { + for (i = 0; i < 8; i++) { + if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * stride * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("mma_ld:wrong_type\n"); + abort(); + } + } else if (type == F32_TYPE) { + // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("wrong type"); + abort(); + } + } + } else { + printf("wrong wmma type\n"); + ; + abort(); + } + // generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); - for(i=0;i<num_reg;i++){ - nw_data[i].s64= ((data[2*i].s64 & 0xffff)<<16)| ((data[2*i+1].s64 & 0xffff)); - } + if ((wmma_type == LOAD_C) && (type == F16_TYPE) && + (wmma_layout == COL)) // memory address is scattered, check the + // profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert(inst.memory_op == insn_memory_op); - if(wmma_type==LOAD_C) - thread->set_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); - else - thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); - printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); - printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); - printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); - if(wmma_type!=LOAD_C){ - printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); - printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); - printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); - printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); - } - } - } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + if (type == F16_TYPE) { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 16; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} + printf("\nmma_ld:thread%d= ", thrd); + float temp; + for (i = 0; i < 16; i++) { + temp = data[i].f16; + printf("%.2f ", temp); + } + printf("\n"); + } else { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%.2f ", data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + } + } -void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { + thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], + data[3], data[4], data[5], data[6], + data[7]); + } else { + ptx_reg_t nw_data[8]; + int num_reg; - unsigned i_type = pI->get_type(); + if (wmma_type == LOAD_C) + num_reg = 4; + else + num_reg = 8; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + for (i = 0; i < num_reg; i++) { + nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | + ((data[2 * i + 1].s64 & 0xffff)); + } + if (wmma_type == LOAD_C) + thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], + nw_data[2], nw_data[3]); + else + thread->set_wmma_vector_operand_values( + dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], + nw_data[5], nw_data[6], nw_data[7]); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", + data[0].u64, data[1].u64, nw_data[0].u64); + printf( + "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", + data[2].u64, data[3].u64, nw_data[1].u64); + printf( + "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", + data[4].u64, data[5].u64, nw_data[2].u64); + printf( + "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", + data[6].u64, data[7].u64, nw_data[3].u64); + if (wmma_type != LOAD_C) { + printf( + "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", + data[8].u64, data[9].u64, nw_data[4].s64); + printf( + "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" + "llx\n", + data[10].u64, data[11].u64, nw_data[5].u64); + printf( + "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" + "llx\n", + data[12].u64, data[13].u64, nw_data[6].u64); + printf( + "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" + "llx\n", + data[14].u64, data[15].u64, nw_data[3].u64); + } + } + } - switch ( i_type ) { - case F32_TYPE: - d.f32 = log(a.f32)/log(2); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } +} + +void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + d.f32 = log(a.f32) / log(2); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert( !pI->is_wide() ); + assert(!pI->is_wide()); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_hi() ) { - d.s64 = (t.s64>>16) + c.s32; - if ( sat_mode ) { - if ( d.s64 > (int)0x7FFFFFFF ) - d.s64 = (int)0x7FFFFFFF; - else if ( d.s64 < (int)0x80000000 ) - d.s64 = (int)0x80000000; - } - } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; - else assert(0); + if (pI->is_hi()) { + d.s64 = (t.s64 >> 16) + c.s32; + if (sat_mode) { + if (d.s64 > (int)0x7FFFFFFF) + d.s64 = (int)0x7FFFFFFF; + else if (d.s64 < (int)0x80000000) + d.s64 = (int)0x80000000; + } + } else if (pI->is_lo()) + d.s64 = t.s32 + c.s32; + else + assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; - else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; - else assert(0); + if (pI->is_hi()) + d.u64 = (t.u64 >> 16) + c.u32; + else if (pI->is_lo()) + d.u64 = t.u32 + c.u32; + else + assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, false); +void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, false); } -void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - int carry=0; - int overflow=0; + int carry = 0; + int overflow = 0; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>=2; - } + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>= 2; + } - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; - else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; - else assert(0); - carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; + else if (pI->is_lo()) + d.s16 = t.s16 + c.s16 + carry_bit.pred; + else + assert(0); + carry = + ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; - else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; + else if (pI->is_lo()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; - else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; - else assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if (pI->is_hi()) + d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; + else if (pI->is_lo()) + d.u16 = t.u16 + c.u16 + carry_bit.pred; + else + assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & + 0x100000000) >> + 32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; - else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if (pI->is_hi()) + d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; + else if (pI->is_lo()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else + assert(0); break; - case F16_TYPE:{ - // assert(0); - // break; - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64 + c.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; + case F16_TYPE: { + // assert(0); + // break; + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; } - default: + fesetround(orig_rm); + break; + } + case F32_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(float x) -{ - return std::isnan(x); -} +bool isNaN(float x) { return std::isnan(x); } -bool isNaN(double x) -{ - return std::isnan(x); -} +bool isNaN(double x) { return std::isnan(x); } -void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MAX_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MAX_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MAX_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MAX_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MAX_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MAX_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MAX_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MAX_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // handled by timing simulator +void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // handled by timing simulator } -void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MIN_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MIN_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MIN_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MIN_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MIN_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MIN_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MIN_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MIN_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert( src1.is_param_local() == 0 ); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert(src1.is_param_local() == 0); - if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; - - switch( pI->get_type() ) { - case B16_TYPE: nbits_to_move = 16; break; - case B32_TYPE: nbits_to_move = 32; break; - case B64_TYPE: nbits_to_move = 64; break; - default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; - } + if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && + (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - if( src1.is_vector() ) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem ); - - unsigned bits_per_src_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_src_elem) { - case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; - case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; - case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; - default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; - } - } - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + switch (pI->get_type()) { + case B16_TYPE: + nbits_to_move = 16; + break; + case B32_TYPE: + nbits_to_move = 32; + break; + case B64_TYPE: + nbits_to_move = 64; + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported type " + "qualifier\n"); + assert(0); + break; + } - switch( pI->get_type() ) { - case B16_TYPE: tmp_bits.u16 = data.u16; break; - case B32_TYPE: tmp_bits.u32 = data.u32; break; - case B64_TYPE: tmp_bits.u64 = data.u64; break; - default: assert(0); break; - } - } + if (src1.is_vector()) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem); - if( dst.is_vector() ) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_dst_elem) { - case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; - case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; - case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; - default: - printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); - assert(0); - break; - } - } - thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); - } else { - thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); + unsigned bits_per_src_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_src_elem) { + case 8: + tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); + break; + case 16: + tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); + break; + case 32: + tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (src)\n"); + assert(0); + break; + } } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in ptxplus + } else { data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { + switch (pI->get_type()) { + case B16_TYPE: + tmp_bits.u16 = data.u16; + break; + case B32_TYPE: + tmp_bits.u32 = data.u32; + break; + case B64_TYPE: + tmp_bits.u64 = data.u64; + break; + default: + assert(0); + break; + } + } - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + if (dst.is_vector()) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_dst_elem) { + case 8: + v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); + break; + case 16: + v[i].u16 = + (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); + break; + case 32: + v[i].u32 = + (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); + } else { + thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); + } + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in + // ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - thread->set_operand_value(dst, data, i_type, thread, pI); + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - } + thread->set_operand_value(dst, data, i_type, thread, pI); + } } -void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + src1_data.mask_and(0, 0x00FFFFFF); + src2_data.mask_and(0, 0x00FFFFFF); - src1_data.mask_and(0,0x00FFFFFF); - src2_data.mask_and(0,0x00FFFFFF); - - switch ( i_type ) { - case S32_TYPE: - if( src1_data.get_bit(23) ) - src1_data.mask_or(0xFFFFFFFF,0xFF000000); - if( src2_data.get_bit(23) ) - src2_data.mask_or(0xFFFFFFFF,0xFF000000); + switch (i_type) { + case S32_TYPE: + if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); + if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf( + "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if ( pI->is_hi() ) { - data.u64 = data.u64 >> 16; - data.mask_and(0,0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0,0xFFFFFFFF); - } + if (pI->is_hi()) { + data.u64 = data.u64 >> 16; + data.mask_and(0, 0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0, 0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if ( pI->is_wide() ) d.s32 = t.s32; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16); - else if ( pI->is_lo() ) d.s16 = t.s16; - else assert(0); + if (pI->is_wide()) + d.s32 = t.s32; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16); + else if (pI->is_lo()) + d.s16 = t.s16; + else + assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if ( pI->is_wide() ) d.s64 = t.s64; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32); - else if ( pI->is_lo() ) d.s32 = t.s32; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32); + else if (pI->is_lo()) + d.s32 = t.s32; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.s64 = t.s64; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.s64 = t.s64; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if ( pI->is_wide() ) d.u32 = t.u32; - else if ( pI->is_lo() ) d.u16 = t.u16; - else if ( pI->is_hi() ) d.u16 = (t.u32>>16); - else assert(0); + if (pI->is_wide()) + d.u32 = t.u32; + else if (pI->is_lo()) + d.u16 = t.u16; + else if (pI->is_hi()) + d.u16 = (t.u32 >> 16); + else + assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if ( pI->is_wide() ) d.u64 = t.u64; - else if ( pI->is_lo() ) d.u32 = t.u32; - else if ( pI->is_hi() ) d.u32 = (t.u64>>32); - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64; + else if (pI->is_lo()) + d.u32 = t.u32; + else if (pI->is_hi()) + d.u32 = (t.u64 >> 32); + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.u64 = t.u64; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.u64 = t.u64; + else + assert(0); break; - case F16_TYPE:{ - //assert(0); - //break; - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } + case F16_TYPE: { + // assert(0); + // break; + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - d.f16 = a.f16 * b.f16; + d.f16 = a.f16 * b.f16; - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - d.f32 = a.f32 * b.f32; + d.f32 = a.f32 * b.f32; - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE:{ - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; } - default: - assert(0); + fesetround(orig_rm); break; - } + } + case F64_TYPE: + case FF64_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: + assert(0); + break; + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); - switch ( to_type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); break; - case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; - default: assert(0); break; - } + switch (to_type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; + break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); + break; + case F16_TYPE: + data.f16 = 0.0f - src1_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = 0.0f - src1_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = 0.0f - src1_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, to_type, thread, pI); + thread->set_operand_value(dst, data, to_type, thread, pI); } -//nandn bitwise negates second operand then bitwise nands with the first operand -void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +// nandn bitwise negates second operand then bitwise nands with the first +// operand +void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); - - thread->set_operand_value(dst,data, i_type, thread, pI); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); + thread->set_operand_value(dst, data, i_type, thread, pI); } -//norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +// norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; + thread->set_operand_value(dst, data, i_type, thread, pI); } -void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; - case B16_TYPE: d.u16 = ~a.u16; break; - case B32_TYPE: d.u32 = ~a.u32; break; - case B64_TYPE: d.u64 = ~a.u64; break; - default: + switch (i_type) { + case PRED_TYPE: + d.pred = (~(a.pred) & 0x000F); + break; + case B16_TYPE: + d.u16 = ~a.u16; + break; + case B32_TYPE: + d.u32 = ~a.u32; + break; + case B64_TYPE: + d.u64 = ~a.u64; + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); - switch ( i_type ) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); + switch (i_type) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); data.u32 = mask.count(); - } break; - default: + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -int prmt_mode_present(int mode) -{ - int returnval=0; - switch(mode){ - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval=1; - break; - default: - break; - } - return returnval; +void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -int read_byte(int mode, int control, int d_sel_index, signed long long value){ - int returnval = 0; - int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; - int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; - int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; - int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; - int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; - int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; +int prmt_mode_present(int mode) { + int returnval = 0; + switch (mode) { + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval = 1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode, int control, int d_sel_index, signed long long value) { + int returnval = 0; + int prmt_f4e_mode[4][4] = { + {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; + int prmt_b4e_mode[4][4] = { + {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; + int prmt_rc8_mode[4][4] = { + {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; + int prmt_ecl_mode[4][4] = { + {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; + int prmt_ecr_mode[4][4] = { + {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; + int prmt_rc16_mode[4][4] = { + {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; - if(!prmt_mode_present(mode)){ - if(control&0x8){ - returnval=0xff; - } - else{ - returnval= (value>>(8*control)) & 0xff; - } - } - else{ - switch(mode){ - case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; - case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; - case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; - case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; - case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; - case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; - // Change the default from printing "ERROR" to just asserting - default: assert(false); - } - } - return (returnval << 8 * d_sel_index); + if (!prmt_mode_present(mode)) { + if (control & 0x8) { + returnval = 0xff; + } else { + returnval = (value >> (8 * control)) & 0xff; + } + } else { + switch (mode) { + case PRMT_F4E_MODE: + returnval = prmt_f4e_mode[control][d_sel_index]; + break; + case PRMT_B4E_MODE: + returnval = prmt_b4e_mode[control][d_sel_index]; + break; + case PRMT_RC8_MODE: + returnval = prmt_rc8_mode[control][d_sel_index]; + break; + case PRMT_ECL_MODE: + returnval = prmt_ecl_mode[control][d_sel_index]; + break; + case PRMT_ECR_MODE: + returnval = prmt_ecr_mode[control][d_sel_index]; + break; + case PRMT_RC16_MODE: + returnval = prmt_rc16_mode[control][d_sel_index]; + break; + // Change the default from printing "ERROR" to just asserting + default: + assert(false); + } + } + return (returnval << 8 * d_sel_index); } -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - - ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - tmpdata.s64=src1_data.s32|(src2_data.s64<<32); - int ctl[4]; + tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); + int ctl[4]; - if(!prmt_mode_present(mode)){ - ctl[0]=(src3_data.s32>>0)&0xf; - ctl[1]=(src3_data.s32>>4)&0xf; - ctl[2]=(src3_data.s32>>8)&0xf; - ctl[3]=(src3_data.s32>>12)&0xf; - } - else{ - ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; - } - - data.s32=0; - data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 - data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 - data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 - data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 - - thread->set_operand_value(dst,data, i_type, thread, pI); + if (!prmt_mode_present(mode)) { + ctl[0] = (src3_data.s32 >> 0) & 0xf; + ctl[1] = (src3_data.s32 >> 4) & 0xf; + ctl[2] = (src3_data.s32 >> 8) & 0xf; + ctl[3] = (src3_data.s32 >> 12) & 0xf; + } else { + ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; + } + data.s32 = 0; + data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First + // byte-0 + data.s32 = + data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 + data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third + // byte-2 + data.s32 = + data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 + thread->set_operand_value(dst, data, i_type, thread, pI); } -void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); +void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} -void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: assert(0); break; - } + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -//Ptxplus version of ret instruction. -void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop_plus(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +// Ptxplus version of ret instruction. +void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop_plus(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + case F64_TYPE: + case FF64_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -#define SAD(d,a,b,c) d = c + ((a<b) ? (b-a) : (a-b)) +#define SAD(d, a, b, c) d = c + ((a < b) ? (b - a) : (a - b)) -void sad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, c, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void sad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - switch ( i_type ) { - case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; - case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; - case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; - case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; - case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; - case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; - case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; - case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; - default: + switch (i_type) { + case U16_TYPE: + SAD(d.u16, a.u16, b.u16, c.u16); + break; + case U32_TYPE: + SAD(d.u32, a.u32, b.u32, c.u32); + break; + case U64_TYPE: + SAD(d.u64, a.u64, b.u64, c.u64); + break; + case S16_TYPE: + SAD(d.s16, a.s16, b.s16, c.s16); + break; + case S32_TYPE: + SAD(d.s32, a.s32, b.s32, c.s32); + break; + case S64_TYPE: + SAD(d.s64, a.s64, b.s64, c.s64); + break; + case F32_TYPE: + SAD(d.f32, a.f32, b.f32, c.f32); + break; + case F64_TYPE: + case FF64_TYPE: + SAD(d.f64, a.f64, b.f64, c.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001))?a:b; + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of c.pred must be inverted to get proper + // behavior + d = (!(c.pred & 0x0001)) ? a : b; - thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); } -bool isFloat(int type) -{ - switch ( type ) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) { + switch (type) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) -{ - bool t = false; +bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { + bool t = false; - switch ( type ) { - case B16_TYPE: + switch (type) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s16 == b.s16); break; - case NE_OPTION: t = (a.s16 != b.s16); break; - case LT_OPTION: t = (a.s16 < b.s16); break; - case LE_OPTION: t = (a.s16 <= b.s16); break; - case GT_OPTION: t = (a.s16 > b.s16); break; - case GE_OPTION: t = (a.s16 >= b.s16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s16 == b.s16); + break; + case NE_OPTION: + t = (a.s16 != b.s16); + break; + case LT_OPTION: + t = (a.s16 < b.s16); + break; + case LE_OPTION: + t = (a.s16 <= b.s16); + break; + case GT_OPTION: + t = (a.s16 > b.s16); + break; + case GE_OPTION: + t = (a.s16 >= b.s16); + break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s32 == b.s32); break; - case NE_OPTION: t = (a.s32 != b.s32); break; - case LT_OPTION: t = (a.s32 < b.s32); break; - case LE_OPTION: t = (a.s32 <= b.s32); break; - case GT_OPTION: t = (a.s32 > b.s32); break; - case GE_OPTION: t = (a.s32 >= b.s32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s32 == b.s32); + break; + case NE_OPTION: + t = (a.s32 != b.s32); + break; + case LT_OPTION: + t = (a.s32 < b.s32); + break; + case LE_OPTION: + t = (a.s32 <= b.s32); + break; + case GT_OPTION: + t = (a.s32 > b.s32); + break; + case GE_OPTION: + t = (a.s32 >= b.s32); + break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s64 == b.s64); break; - case NE_OPTION: t = (a.s64 != b.s64); break; - case LT_OPTION: t = (a.s64 < b.s64); break; - case LE_OPTION: t = (a.s64 <= b.s64); break; - case GT_OPTION: t = (a.s64 > b.s64); break; - case GE_OPTION: t = (a.s64 >= b.s64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s64 == b.s64); + break; + case NE_OPTION: + t = (a.s64 != b.s64); + break; + case LT_OPTION: + t = (a.s64 < b.s64); + break; + case LE_OPTION: + t = (a.s64 <= b.s64); + break; + case GT_OPTION: + t = (a.s64 > b.s64); + break; + case GE_OPTION: + t = (a.s64 >= b.s64); + break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - case LT_OPTION: t = (a.u16 < b.u16); break; - case LE_OPTION: t = (a.u16 <= b.u16); break; - case GT_OPTION: t = (a.u16 > b.u16); break; - case GE_OPTION: t = (a.u16 >= b.u16); break; - case LO_OPTION: t = (a.u16 < b.u16); break; - case LS_OPTION: t = (a.u16 <= b.u16); break; - case HI_OPTION: t = (a.u16 > b.u16); break; - case HS_OPTION: t = (a.u16 >= b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + case LT_OPTION: + t = (a.u16 < b.u16); + break; + case LE_OPTION: + t = (a.u16 <= b.u16); + break; + case GT_OPTION: + t = (a.u16 > b.u16); + break; + case GE_OPTION: + t = (a.u16 >= b.u16); + break; + case LO_OPTION: + t = (a.u16 < b.u16); + break; + case LS_OPTION: + t = (a.u16 <= b.u16); + break; + case HI_OPTION: + t = (a.u16 > b.u16); + break; + case HS_OPTION: + t = (a.u16 >= b.u16); + break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - case LT_OPTION: t = (a.u32 < b.u32); break; - case LE_OPTION: t = (a.u32 <= b.u32); break; - case GT_OPTION: t = (a.u32 > b.u32); break; - case GE_OPTION: t = (a.u32 >= b.u32); break; - case LO_OPTION: t = (a.u32 < b.u32); break; - case LS_OPTION: t = (a.u32 <= b.u32); break; - case HI_OPTION: t = (a.u32 > b.u32); break; - case HS_OPTION: t = (a.u32 >= b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + case LT_OPTION: + t = (a.u32 < b.u32); + break; + case LE_OPTION: + t = (a.u32 <= b.u32); + break; + case GT_OPTION: + t = (a.u32 > b.u32); + break; + case GE_OPTION: + t = (a.u32 >= b.u32); + break; + case LO_OPTION: + t = (a.u32 < b.u32); + break; + case LS_OPTION: + t = (a.u32 <= b.u32); + break; + case HI_OPTION: + t = (a.u32 > b.u32); + break; + case HS_OPTION: + t = (a.u32 >= b.u32); + break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - case LT_OPTION: t = (a.u64 < b.u64); break; - case LE_OPTION: t = (a.u64 <= b.u64); break; - case GT_OPTION: t = (a.u64 > b.u64); break; - case GE_OPTION: t = (a.u64 >= b.u64); break; - case LO_OPTION: t = (a.u64 < b.u64); break; - case LS_OPTION: t = (a.u64 <= b.u64); break; - case HI_OPTION: t = (a.u64 > b.u64); break; - case HS_OPTION: t = (a.u64 >= b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + case LT_OPTION: + t = (a.u64 < b.u64); + break; + case LE_OPTION: + t = (a.u64 <= b.u64); + break; + case GT_OPTION: + t = (a.u64 > b.u64); + break; + case GE_OPTION: + t = (a.u64 >= b.u64); + break; + case LO_OPTION: + t = (a.u64 < b.u64); + break; + case LS_OPTION: + t = (a.u64 <= b.u64); + break; + case HI_OPTION: + t = (a.u64 > b.u64); + break; + case HS_OPTION: + t = (a.u64 >= b.u64); + break; + default: + assert(0); } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: + case F16_TYPE: + assert(0); + break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; - case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case NE_OPTION: + t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LT_OPTION: + t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LE_OPTION: + t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GT_OPTION: + t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GE_OPTION: + t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case EQU_OPTION: + t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NEU_OPTION: + t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LTU_OPTION: + t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LEU_OPTION: + t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GTU_OPTION: + t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GEU_OPTION: + t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NUM_OPTION: + t = !isNaN(a.f32) && !isNaN(b.f32); + break; + case NAN_OPTION: + t = isNaN(a.f32) || isNaN(b.f32); + break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; - case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case NE_OPTION: + t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LT_OPTION: + t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LE_OPTION: + t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GT_OPTION: + t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GE_OPTION: + t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case EQU_OPTION: + t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NEU_OPTION: + t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LTU_OPTION: + t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LEU_OPTION: + t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GTU_OPTION: + t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GEU_OPTION: + t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NUM_OPTION: + t = !isNaN(a.f64) && !isNaN(b.f64); + break; + case NAN_OPTION: + t = isNaN(a.f64) || isNaN(b.f64); + break; + default: + assert(0); } break; - default: assert(0); break; - } + default: + assert(0); + break; + } - return t; + return t; } -void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type,a,b,cmpop); + t = CmpOp(type, a, b, cmpop); - ptx_reg_t data; + ptx_reg_t data; - //the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag + // the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = + (t == + 0); // inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); } -void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if(pI->is_abs()) { - switch ( src_type ) { - case S16_TYPE: a.s16 = my_abs(a.s16); break; - case S32_TYPE: a.s32 = my_abs(a.s32); break; - case S64_TYPE: a.s64 = my_abs(a.s64); break; - case U16_TYPE: a.u16 = a.u16; break; - case U32_TYPE: a.u32 = my_abs(a.u32); break; - case U64_TYPE: a.u64 = my_abs(a.u64); break; - case F32_TYPE: a.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; + // Take abs of first operand if needed + if (pI->is_abs()) { + switch (src_type) { + case S16_TYPE: + a.s16 = my_abs(a.s16); + break; + case S32_TYPE: + a.s32 = my_abs(a.s32); + break; + case S64_TYPE: + a.s64 = my_abs(a.s64); + break; + case U16_TYPE: + a.u16 = a.u16; + break; + case U32_TYPE: + a.u32 = my_abs(a.u32); + break; + case U64_TYPE: + a.u64 = my_abs(a.u64); + break; + case F32_TYPE: + a.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + a.f64 = my_abs(a.f64); + break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } - - t = CmpOp(src_type,a,b,cmpop); + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } - ptx_reg_t data; - if ( isFloat(pI->get_type()) ) { - data.f32 = (t!=0)?1.0f:0.0f; - } else { - data.u32 = (t!=0)?0xFFFFFFFF:0; - } + t = CmpOp(src_type, a, b, cmpop); - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); + ptx_reg_t data; + if (isFloat(pI->get_type())) { + data.f32 = (t != 0) ? 1.0f : 0.0f; + } else { + data.u32 = (t != 0) ? 0xFFFFFFFF : 0; + } + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); } -void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - unsigned i_type = pI->get_type(); - int tid; +void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + unsigned i_type = pI->get_type(); + int tid; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); - if(core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; - int src_idx; - unsigned p; - switch(pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; + int src_idx; + unsigned p; + switch (pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf( + "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " + "threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for + predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } } -void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case B16_TYPE: - case U16_TYPE: - if ( b.u16 >= 16 ) - d.u16 = 0; + switch (i_type) { + case B16_TYPE: + case U16_TYPE: + if (b.u16 >= 16) + d.u16 = 0; else - d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if ( b.u32 >= 32 ) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if (b.u32 >= 32) + d.u32 = 0; else - d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if ( b.u32 >= 64 ) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if (b.u32 >= 64) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: - case B16_TYPE: - if ( b.u16 < 16 ) - d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); + switch (i_type) { + case U16_TYPE: + case B16_TYPE: + if (b.u16 < 16) + d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if ( b.u32 < 32 ) - d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if (b.u32 < 32) + d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if ( b.u32 < 64 ) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if (b.u32 < 64) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if ( b.u16 < 16 ) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if (b.u16 < 16) + d.s64 = (a.s16 >> b.s16); else { - if ( a.s16 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s16 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if ( b.u32 < 32 ) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if (b.u32 < 32) + d.s64 = (a.s32 >> b.s32); else { - if ( a.s32 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s32 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if ( b.u64 < 64 ) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if (b.u64 < 64) + d.s64 = (a.s64 >> b.u64); else { - if ( a.s64 < 0 ) { - if ( b.s32 < 0 ) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if (a.s64 < 0) { + if (b.s32 < 0) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); - switch ( c_type ) { - case S32_TYPE: t = c.s32 >= 0; break; - case F32_TYPE: t = c.f32 >= 0; break; - default: assert(0); - } + switch (c_type) { + case S32_TYPE: + t = c.s32 >= 0; + break; + case F32_TYPE: + t = c.f32 >= 0; + break; + default: + assert(0); + } - switch ( i_type ) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: d.u16 = t?a.u16:b.u16; break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: d.u32 = t?a.u32:b.u32; break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: d.u64 = t?a.u64:b.u64; break; - default: assert(0); - } + switch (i_type) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + d.u16 = t ? a.u16 : b.u16; + break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + d.u32 = t ? a.u32 : b.u32; + break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: + d.u64 = t ? a.u64 : b.u64; + break; + default: + assert(0); + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) - d.f32 = nanf(""); + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f64 < 0 ) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f64 < 0) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); break; - default: + d.f64 = sqrt(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_instruction * cpI = const_cast<ptx_instruction *>(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; +void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_instruction *cpI = const_cast<ptx_instruction *>(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = + src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; - decode_space(space,thread,src1,mem,addr); + decode_space(space, thread, src1, mem, addr); - size_t size; - int t; - type_info_key::type_decode(type,size,t); + size_t size; + int t; + type_info_key::type_decode(type, size, t); - // store data in sstarr memory - mem->write(addr,size/8,&src3_data.s64,thread,pI); + // store data in sstarr memory + mem->write(addr, size / 8, &src3_data.s64, thread, pI); - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64=0; - mem->read(addr+(tid*4),size/8,&data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64 = 0; + mem->read(addr + (tid * 4), size / 8, &data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } - // squeeze the zeros out of the array and store data back into original array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space,thread,src1,mem,addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); - mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); - offset++; - } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4*(offset-1); - thread->set_operand_value(dst, data, type, thread, pI); + // squeeze the zeros out of the array and store data back into original + // array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space, thread, src1, mem, addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, + pI); + mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, + pI); + offset++; + } + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4 * (offset - 1); + thread->set_operand_value(dst, data, type, thread, pI); - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); - offset++; - } + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); + offset++; + } - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; - thread->m_last_memory_space = space; - } + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; + thread->m_last_memory_space = space; + } } -void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); - // TODO: add implementation +void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // printf("Execution Warning: unimplemented ssy instruction is treated as a + // nop\n"); + // TODO: add implementation } -void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); //may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); +void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); // may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; - decode_space(space,thread,dst,mem,addr); + decode_space(space, thread, dst, mem, addr); - size_t size; - int t; - type_info_key::type_decode(type,size,t); + size_t size; + int t; + type_info_key::type_decode(type, size, t); - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr,size/8,&data.s64,thread,pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); - delete [] ptx_regs; - } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr, size / 8, &data.s64, thread, pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); + delete[] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; } -void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //performs addition. Sets carry and overflow if needed. - //the constant is added in during subtraction so the carry bit is set properly. - switch ( i_type ) { - case S8_TYPE: + // performs addition. Sets carry and overflow if needed. + // the constant is added in during subtraction so the carry bit is set + // properly. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.s32 & 0x100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.s32 & 0x10000)>>16; + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x10000) >> 16; break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = ((data.u64)>>32) & 0x0001; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + + 0x100000000; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = ((data.u64) >> 32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; break; - case B8_TYPE: - case U8_TYPE: + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; + break; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + + 0x100000000; + carry = (data.u64 & 0x100000000) >> 32; + break; + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; - carry = (data.u64 & 0x100000000)>>32; + case F16_TYPE: + data.f16 = src1_data.f16 - src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 - src2_data.f32; break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; - default: assert(0); break; - } + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 - src2_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // Do nothing +void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // Do nothing } -void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - +void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision( float x, unsigned bits ) -{ - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1<<23)-1); - int mask = ((1<<bits)-1) << (23-bits); - int nv = (v & ((-1)-((1<<23)-1))) | (mask&man); - tmp.a = nv; - float result = tmp.b; - return result; +float reduce_precision(float x, unsigned bits) { + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1 << 23) - 1); + int mask = ((1 << bits) - 1) << (23 - bits); + int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); + tmp.a = nv; + float result = tmp.b; + return result; } -unsigned wrap( unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size ) -{ - unsigned nx = (mx+x)%mx; - unsigned ny = (my+y)%my; - return nx + mx*ny; +unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = (mx + x) % mx; + unsigned ny = (my + y) % my; + return nx + mx * ny; } -unsigned clamp( unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size ) -{ - unsigned nx = x; - while (nx >= mx) nx -= elem_size; - unsigned ny = (y >= my)? my - 1 : y; - return nx + mx*ny; +unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = x; + while (nx >= mx) nx -= elem_size; + unsigned ny = (y >= my) ? my - 1 : y; + return nx + mx * ny; } -typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); -float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, - int x, int y, unsigned int width, unsigned int height, size_t elem_size, - float alpha, float beta, texAddr_t b_lim) -{ - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; +typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size); +float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, + int y, unsigned int width, unsigned int height, + size_t elem_size, float alpha, float beta, + texAddr_t b_lim) { + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), + 4, &Ti1j); + mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, + &Tij1); + mem->read( + tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, + &Ti1j1); - mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); - mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); - mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); + float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + + (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; - float sample = (1-alpha)*(1-beta)*Tij + - alpha*(1-beta)*Ti1j + - (1-alpha)*beta*Tij1 + - alpha*beta*Ti1j1; - - return sample; + return sample; } -float textureNormalizeElementSigned(int element, int bits) -{ - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float) element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } +float textureNormalizeElementSigned(int element, int bits) { + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float)element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } } -float textureNormalizeElementUnsigned(unsigned int element, int bits) -{ - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } +float textureNormalizeElementUnsigned(unsigned int element, int bits) { + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } } -void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) -{ - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); - datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); - dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); - dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); - datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); - dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); - dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); - } else { - assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); - } +void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, + ptx_reg_t &datax, ptx_reg_t &datay, + ptx_reg_t &dataz, ptx_reg_t &dataw) { + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); + datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); + dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); + dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); + datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); + dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); + dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); + } else { + assert(0 && + "Undefined texture read mode: cudaReadModeNormalizedFloat expect " + "integer elements"); + } } -void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned dimension = pI->dimension(); - const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); //the name of the texture - const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched +void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned dimension = pI->dimension(); + const operand_info &dst = + pI->dst(); // the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); // the name of the texture + const operand_info &src2 = + pI->src2(); // the vector registers containing coordinates of the texel + // to be fetched - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = k.get_texarray(texname); - const struct textureInfo* texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values( + src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, + nelem); // ptx_reg should be 4 entry vector type...coordinates into + // texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference *texref = gpu->get_texref(texname); + const struct cudaArray *cuArray = k.get_texarray(texname); + const struct textureInfo *texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); - //assume always 2D f32 input - //access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha=0, beta=0; + // assume always 2D f32 input + // access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha = 0, beta = 0; - type_info_key::type_decode(to_type,size,t); - tex_array_base = cuArray->devPtr32; + type_info_key::type_decode(to_type, size, t); + tex_array_base = cuArray->devPtr32; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha,9); - beta = 0.0; + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha, 9); + beta = 0.0; - x = (int)floor(xb); - y = 0; - } else { - x = (int) floor(x_f32 * width); - y = 0; - } + x = (int)floor(xb); + y = 0; + } else { + x = (int)floor(x_f32 * width); + y = 0; + } } else { - switch ( c_type ) { - case S32_TYPE: + switch (c_type) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int) x_f32; - break; - default: assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1)? (width - 1) : x; - x = (x < 0)? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - + floor(x_f32); // offset into subtexel (for linear sampling) + x = (int)x_f32; + break; + default: + assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1) ? (width - 1) : x; + x = (x < 0) ? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); - y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); + x_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); + y_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); - if (texref->addressMode[0]) {//clamp - if (x_f32<0) x_f32 = 0; - if (x_f32>=1) x_f32 = 1 - 1/x_f32; - } else {//wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) {//clamp - if (y_f32<0) y_f32 = 0; - if (y_f32>=1) y_f32 = 1 - 1/y_f32; - } else {//wrap - y_f32 = y_f32 - floor(y_f32); - } + if (texref->addressMode[0]) { // clamp + if (x_f32 < 0) x_f32 = 0; + if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; + } else { // wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) { // clamp + if (y_f32 < 0) y_f32 = 0; + if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; + } else { // wrap + y_f32 = y_f32 - floor(y_f32); + } - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha,9); - beta = reduce_precision(beta,9); + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha, 9); + beta = reduce_precision(beta, 9); - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int) floor(x_f32 * width); - y = (int) floor(y_f32 * height); - } + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int)floor(x_f32 * width); + y = (int)floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); - x = (int) x_f32; - y = (int) y_f32; - if (texref->addressMode[0]) {//clamp - if (x<0) x = 0; - if (x>= (int)width) x = width-1; - } else {//wrap - x = x % width; - if (x < 0) x*= -1; - } - if (texref->addressMode[1]) {//clamp - if (y<0) y = 0; - if (y>= (int)height) y = height -1; - } else {//wrap - y = y % height; - if (y < 0) y *= -1; - } + x = (int)x_f32; + y = (int)y_f32; + if (texref->addressMode[0]) { // clamp + if (x < 0) x = 0; + if (x >= (int)width) x = width - 1; + } else { // wrap + x = x % width; + if (x < 0) x *= -1; + } + if (texref->addressMode[1]) { // clamp + if (y < 0) y = 0; + if (y >= (int)height) y = height - 1; + } else { // wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - tex_array_index = tex_array_base + (x + width*y); + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + tex_array_index = tex_array_base + (x + width * y); break; - default: - assert(0); break; - } - switch ( to_type ) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); - elementOffset += cuArray->desc.x/8; + default: + assert(0); + break; + } + switch (to_type) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); + elementOffset += cuArray->desc.x / 8; if (cuArray->desc.y) { - mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); - elementOffset += cuArray->desc.y/8; - if (cuArray->desc.z) { - mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); - elementOffset += cuArray->desc.z/8; - if (cuArray->desc.w) - mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); - } + mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, + &data2.u32); + elementOffset += cuArray->desc.y / 8; + if (cuArray->desc.z) { + mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, + &data3.u32); + elementOffset += cuArray->desc.z / 8; + if (cuArray->desc.w) + mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, + &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read( tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read(tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.u64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.u64); - } + mem->read(tex_array_index + 8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.u64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); + } } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: { - if( texref->filterMode == cudaFilterModeLinear ) { - texAddr_t b_lim = wrap; - if ( texref->addressMode[0] == cudaAddressModeClamp ) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; - size_t elem_ofst = 0; + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + if (texref->filterMode == cudaFilterModeLinear) { + texAddr_t b_lim = wrap; + if (texref->addressMode[0] == cudaAddressModeClamp) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z + cuArray->desc.w) / + 8; + size_t elem_ofst = 0; - data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - } - } + data1.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, + y, width, height, elem_size, alpha, + beta, b_lim); + } + } } else { - mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); - if (cuArray->desc.y) { - mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); - if (cuArray->desc.z) { - mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); - if (cuArray->desc.w) - mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); - } - } + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); + if (cuArray->desc.y) { + mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); + if (cuArray->desc.z) { + mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); + if (cuArray->desc.w) + mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read( tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read(tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.f64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.f64); - } + mem->read(tex_array_index + 8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.f64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); + } } break; - default: assert(0); break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: + assert(0); + break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; + memreqindex = + ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; - blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + + (y % (texInfo->Ty) + << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; + thread->m_last_effective_address = + tex_array_base + memreqindex; // tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; + } + thread->m_last_memory_space = tex_space; - // normalize output into floating point numbers according to the texture read mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } + // normalize output into floating point numbers according to the texture read + // mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } - thread->set_vector_operand_values(dst,data1,data2,data3,data4); + thread->set_vector_operand_values(dst, data1, data2, data3, data4); } -void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} -void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list<ptx_thread_info*> threads_in_warp; - static unsigned last_tid; +void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list<ptx_thread_info *> threads_in_warp; + static unsigned last_tid; - if( first_in_warp ) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset=31; - while( (offset>=0) && !pI->active(offset) ) - offset--; - assert( offset >= 0 ); - last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; - } + if (first_in_warp) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset = 31; + while ((offset >= 0) && !pI->active(offset)) offset--; + assert(offset >= 0); + last_tid = + (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + + offset; + } - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of src1_data.pred must be inverted to get proper behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of src1_data.pred must be inverted to get + // proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } - if( thread->get_hw_tid() == last_tid ) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for( std::list<ptx_thread_info*>::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; + if (thread->get_hw_tid() == last_tid) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for (std::list<ptx_thread_info *>::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); + } + } else { + bool pred_value = false; - switch( pI->vote_mode() ) { - case ptx_instruction::vote_any: pred_value = or_all; break; - case ptx_instruction::vote_all: pred_value = and_all; break; - case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true + switch (pI->vote_mode()) { + case ptx_instruction::vote_any: + pred_value = or_all; + break; + case ptx_instruction::vote_all: + pred_value = and_all; + break; + case ptx_instruction::vote_uni: + pred_value = (or_all ^ and_all); + break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, + // 1 = false and 0 = true - for( std::list<ptx_thread_info*>::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); - } + for (std::list<ptx_thread_info *>::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); } - first_in_warp = true; - } + } + first_in_warp = true; + } } -void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void inst_not_implemented( const ptx_instruction * pI ) -{ - printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), - pI->source_line(), - pI->get_opcode_cstr() ); - abort(); +void inst_not_implemented(const ptx_instruction *pI) { + printf( + "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); + abort(); } -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) -{ - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64=0; +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread) { + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64 = 0; - //complete other cases for reading from memory, such as reading from other const memory - if(opInfo.get_addr_space() == global_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else if(opInfo.get_addr_space() == shared_space) - { - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); + // complete other cases for reading from memory, such as reading from other + // const memory + if (opInfo.get_addr_space() == global_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else if (opInfo.get_addr_space() == shared_space) { + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); - } - else if(opInfo.get_addr_space() == const_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); + } else if (opInfo.get_addr_space() == const_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); - mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); + mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, + &result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else - { - result = opData; - } + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else { + result = opData; + } - if(opInfo.get_operand_lohi() == 1) - { - result.u64 = result.u64 & 0xFFFF; - } - else if(opInfo.get_operand_lohi() == 2) - { - result.u64 = (result.u64>>16) & 0xFFFF; - } + if (opInfo.get_operand_lohi() == 1) { + result.u64 = result.u64 & 0xFFFF; + } else if (opInfo.get_operand_lohi() == 2) { + result.u64 = (result.u64 >> 16) & 0xFFFF; + } - if(opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } + if (opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } - return result; + return result; } - diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 4b2acdf..1323837 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,210 +7,227 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "memory.h" #include <stdlib.h> -#include "../debug.h" #include "../../libcuda/gpgpu_context.h" +#include "../debug.h" -template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::string name, unsigned hash_size ) -{ - m_name = name; - MEM_MAP_RESIZE(hash_size); +template <unsigned BSIZE> +memory_space_impl<BSIZE>::memory_space_impl(std::string name, + unsigned hash_size) { + m_name = name; + MEM_MAP_RESIZE(hash_size); - m_log2_block_size = -1; - for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { - if( BSIZE & mask ) { - assert( m_log2_block_size == (unsigned)-1 ); - m_log2_block_size = n; - } - } - assert( m_log2_block_size != (unsigned)-1 ); + m_log2_block_size = -1; + for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { + if (BSIZE & mask) { + assert(m_log2_block_size == (unsigned)-1); + m_log2_block_size = n; + } + } + assert(m_log2_block_size != (unsigned)-1); } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) -{ - m_data[index].write(offset,length,(const unsigned char*)data); +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::write_only(mem_addr_t offset, mem_addr_t index, + size_t length, const void *data) { + m_data[index].write(offset, length, (const unsigned char *)data); } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) -{ +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::write(mem_addr_t addr, size_t length, + const void *data, + class ptx_thread_info *thd, + const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; - mem_addr_t index = addr >> m_log2_block_size; + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + m_data[index].write(offset, nbytes, (const unsigned char *)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; - if ( (addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - m_data[index].write(offset,nbytes,(const unsigned char*)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; + } - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, + &((const unsigned char *)data)[src_offset]); - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if( !m_watchpoints.empty() ) { - std::map<unsigned,mem_addr_t>::iterator i; - for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { - mem_addr_t wa = i->second; - if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); - } - } + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } + if (!m_watchpoints.empty()) { + std::map<unsigned, mem_addr_t>::iterator i; + for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { + mem_addr_t wa = i->second; + if (((addr <= wa) && ((addr + length) > wa)) || + ((addr > wa) && (addr < (wa + 4)))) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( + i->first, thd, pI); + } + } } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const -{ - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", - (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if( i == m_data.end() ) { - for( size_t n=0; n < length; n++ ) - ((unsigned char*)data)[n] = (unsigned char) 0; - //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - i->second.read(offset,nbytes,(unsigned char*)data); - } +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::read_single_block(mem_addr_t blk_idx, + mem_addr_t addr, size_t length, + void *data) const { + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf( + "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " + "addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf( + "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " + "index=0x%x, BSIZE=0x%x\n", + (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if (i == m_data.end()) { + for (size_t n = 0; n < length; n++) + ((unsigned char *)data)[n] = (unsigned char)0; + // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized + // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + i->second.read(offset, nbytes, (unsigned char *)data); + } } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, size_t length, void *data ) const -{ - mem_addr_t index = addr >> m_log2_block_size; - if ((addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::read(mem_addr_t addr, size_t length, + void *data) const { + mem_addr_t index = addr >> m_log2_block_size; + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); - - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - assert(nbytes_remain == 0); - } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, + &((unsigned char *)data)[dst_offset]); + + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const -{ - typename map_t::const_iterator i_page; +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::print(const char *format, FILE *fout) const { + typename map_t::const_iterator i_page; - for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template<unsigned BSIZE> void memory_space_impl<BSIZE>::set_watch( addr_t addr, unsigned watchpoint ) -{ - m_watchpoints[watchpoint]=addr; +template <unsigned BSIZE> +void memory_space_impl<BSIZE>::set_watch(addr_t addr, unsigned watchpoint) { + m_watchpoints[watchpoint] = addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16*1024>; +template class memory_space_impl<16 * 1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) -{ - mem->print(format,fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", + FILE *fout = stdout) { + mem->print(format, fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[] ) -{ - int errors_found=0; - memory_space *mem = new memory_space_impl<32>("test",4); - // write address to [address] - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) - mem->write(addr,4,&addr,NULL,NULL); +int main(int argc, char *argv[]) { + int errors_found = 0; + memory_space *mem = new memory_space_impl<32>("test", 4); + // write address to [address] + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) + mem->write(addr, 4, &addr, NULL, NULL); - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { - unsigned tmp=0; - mem->read(addr,4,&tmp); - if( tmp != addr ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); - } - } + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { + unsigned tmp = 0; + mem->read(addr, 4, &tmp); + if (tmp != addr) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); + } + } - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr,1,&val,NULL,NULL); - } + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr, 1, &val, NULL, NULL); + } - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned tmp=0; - mem->read(addr,1,&tmp); - unsigned char val = (addr + 128) % 256; - if( tmp != val ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); - } - } + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned tmp = 0; + mem->read(addr, 1, &tmp); + unsigned char val = (addr + 128) % 256; + if (tmp != val) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, + (unsigned)val); + } + } - if( errors_found ) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } + if (errors_found) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index ab588bc..5850aa1 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef memory_h_INCLUDED #define memory_h_INCLUDED @@ -33,101 +34,97 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 - #define MEM_MAP_RESIZE(hash_size) +#define MEM_MAP_RESIZE(hash_size) #else - #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) +#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include <assert.h> -#include <string.h> #include <stdio.h> -#include <string> -#include <map> #include <stdlib.h> +#include <string.h> +#include <map> +#include <string> typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4*1024) +#define MEM_BLOCK_SIZE (4 * 1024) -template<unsigned BSIZE> class mem_storage { -public: - mem_storage( const mem_storage &another ) - { - m_data = (unsigned char*)calloc(1,BSIZE); - memcpy(m_data,another.m_data,BSIZE); - } - mem_storage() - { - m_data = (unsigned char*)calloc(1,BSIZE); - } - ~mem_storage() - { - free(m_data); - } +template <unsigned BSIZE> +class mem_storage { + public: + mem_storage(const mem_storage &another) { + m_data = (unsigned char *)calloc(1, BSIZE); + memcpy(m_data, another.m_data, BSIZE); + } + mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } + ~mem_storage() { free(m_data); } - void write( unsigned offset, size_t length, const unsigned char *data ) - { - assert( offset + length <= BSIZE ); - memcpy(m_data+offset,data,length); - } + void write(unsigned offset, size_t length, const unsigned char *data) { + assert(offset + length <= BSIZE); + memcpy(m_data + offset, data, length); + } - void read( unsigned offset, size_t length, unsigned char *data ) const - { - assert( offset + length <= BSIZE ); - memcpy(data,m_data+offset,length); - } + void read(unsigned offset, size_t length, unsigned char *data) const { + assert(offset + length <= BSIZE); + memcpy(data, m_data + offset, length); + } - void print( const char *format, FILE *fout ) const - { - unsigned int *i_data = (unsigned int*)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); - } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); + void print(const char *format, FILE *fout) const { + unsigned int *i_data = (unsigned int *)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); } - fprintf(fout, "\n"); - fflush(fout); - } + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); + } + fprintf(fout, "\n"); + fflush(fout); + } -private: - unsigned m_nbytes; - unsigned char *m_data; + private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space -{ -public: - virtual ~memory_space() {} - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; - virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; - virtual void print( const char *format, FILE *fout ) const = 0; - virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; +class memory_space { + public: + virtual ~memory_space() {} + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI) = 0; + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data) = 0; + virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; + virtual void print(const char *format, FILE *fout) const = 0; + virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; }; -template<unsigned BSIZE> class memory_space_impl : public memory_space { -public: - memory_space_impl( std::string name, unsigned hash_size ); +template <unsigned BSIZE> +class memory_space_impl : public memory_space { + public: + memory_space_impl(std::string name, unsigned hash_size); + + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI); + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data); + virtual void read(mem_addr_t addr, size_t length, void *data) const; + virtual void print(const char *format, FILE *fout) const; - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); - virtual void read( mem_addr_t addr, size_t length, void *data ) const; - virtual void print( const char *format, FILE *fout ) const; - - virtual void set_watch( addr_t addr, unsigned watchpoint ); + virtual void set_watch(addr_t addr, unsigned watchpoint); -private: - void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map<mem_addr_t,mem_storage<BSIZE> > map_t; - map_t m_data; - std::map<unsigned,mem_addr_t> m_watchpoints; + private: + void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, + void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map<mem_addr_t, mem_storage<BSIZE> > map_t; + map_t m_data; + std::map<unsigned, mem_addr_t> m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 86d3b99..dc1e8c9 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,71 +7,71 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef opcodes_h_included #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type{ - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 - +enum wmma_type { + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 22517df..9f7e760 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,266 +7,285 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" -#include "ptx_sim.h" #include "ptx-stats.h" -#include "../option_parser.h" #include <stdio.h> #include <map> -#include "../tr1_hash_map.h" #include "../../libcuda/gpgpu_context.h" +#include "../option_parser.h" +#include "../tr1_hash_map.h" +#include "ptx_ir.h" +#include "ptx_sim.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) -{ - option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, - &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, - &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { + option_parser_register( + opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register( + opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line -{ -public: - ptx_file_line(const char* s, int l) { - if( s == NULL ) - st = "NULL_NAME"; - else - st = s; - line = l; - } +class ptx_file_line { + public: + ptx_file_line(const char *s, int l) { + if (s == NULL) + st = "NULL_NAME"; + else + st = s; + line = l; + } - bool operator<(const ptx_file_line &other) const { - if( st == other.st ) { - if( line < other.line ) - return true; - else - return false; - } else { - return st < other.st; - } + bool operator<(const ptx_file_line &other) const { + if (st == other.st) { + if (line < other.line) + return true; + else + return false; + } else { + return st < other.st; } + } - bool operator==(const ptx_file_line &other) const { - return (line==other.line) && (st==other.st); - } + bool operator==(const ptx_file_line &other) const { + return (line == other.line) && (st == other.st); + } - std::string st; - unsigned line; + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats -{ -public: - ptx_file_line_stats() - : exec_count(0), latency(0), dram_traffic(0), - smem_n_way_bank_conflict_total(0), smem_warp_count(0), - gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), - warp_divergence(0) - { } - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction - unsigned long gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) - unsigned long long warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats { + public: + ptx_file_line_stats() + : exec_count(0), + latency(0), + dram_traffic(0), + smem_n_way_bank_conflict_total(0), + smem_warp_count(0), + gmem_n_access_total(0), + gmem_warp_count(0), + exposed_latency(0), + warp_divergence(0) {} + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long + smem_n_way_bank_conflict_total; // total number of banks accessed by this + // instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in + // total from this instruction + unsigned long + gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles + // (attributed to this instruction) + unsigned long long + warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats> ptx_file_line_stats_map_t; +typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats> + ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line -{ - std::size_t operator()(const ptx_file_line & pfline) const { - std::hash<unsigned> hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line { + std::size_t operator()(const ptx_file_line &pfline) const { + std::hash<unsigned> hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats, hash_ptx_file_line> ptx_file_line_stats_map_t; +typedef tr1_hash_map<ptx_file_line, ptx_file_line_stats, hash_ptx_file_line> + ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() -{ - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; +void ptx_stats::ptx_file_line_stats_write_file() { + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; - ptx_file_line_stats_map_t::iterator it; - FILE * pfile; + ptx_file_line_stats_map_t::iterator it; + FILE *pfile; - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); - for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf( + pfile, + "kernel line : count latency dram_traffic smem_bk_conflicts smem_warp " + "gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); + for (it = ptx_file_line_stats_tracker.begin(); + it != ptx_file_line_stats_tracker.end(); it++) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) -{ - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent +// in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, + unsigned dram_traffic) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of +// cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( + unsigned pc, unsigned n_way_bkconflict) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory +// requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, + unsigned n_access) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker -{ -public: - typedef std::map<const ptx_instruction *, int> insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker { + public: + typedef std::map<const ptx_instruction *, int> insn_count_map; - void add_count(const ptx_instruction * pInsn, int count = 1) - { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction *pInsn, int count = 1) { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction * pInsn, int count = 1) - { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction *pInsn, int count = 1) { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); - } + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); } + } - void attribute_exposed_latency(int count = 1) - { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; + void attribute_exposed_latency(int count = 1) { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.exposed_latency += count; - } + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats &line_stats = + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())]; + line_stats.exposed_latency += count; } + } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) -{ - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) -{ - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory +// instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, + int exposed_latency) { + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; -} +void ptx_stats::ptx_file_line_stats_add_warp_divergence( + unsigned pc, unsigned n_way_divergence) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; +} diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 246b4ce..be77c39 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,33 +7,33 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" - #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); #endif // stat collection interface to gpgpu-sim @@ -41,28 +41,29 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); - class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char * ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char* ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, + unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, + unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index d8943d2..aa1c25a 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,1170 +8,1263 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_parser.h" #include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "opcodes.h" +#include "ptx_parser.h" +typedef void *yyscan_t; +#include <assert.h> #include <stdio.h> #include <stdlib.h> -#include <list> -#include <assert.h> #include <algorithm> +#include <list> #include "assert.h" +#include "opcodes.h" +#include "ptx.tab.h" -#include "cuda-sim.h" #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" #define STR_SIZE 1024 -const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) -{ - if( pc < s_g_pc_to_insn.size() ) - return s_g_pc_to_insn[pc]; - else - return NULL; +const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { + if (pc < s_g_pc_to_insn.size()) + return s_g_pc_to_insn[pc]; + else + return NULL; } -unsigned symbol::get_uid() -{ - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +unsigned symbol::get_uid() { + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -void symbol::add_initializer( const std::list<operand_info> &init ) -{ - m_initializer = init; +void symbol::add_initializer(const std::list<operand_info> &init) { + m_initializer = init; } -void symbol::print_info(FILE *fp) const -{ - fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); - if( m_address_valid ) - fprintf(fp,"<address valid>, "); - if( m_is_label ) - fprintf(fp," is_label "); - if( m_is_shared ) - fprintf(fp," is_shared "); - if( m_is_const ) - fprintf(fp," is_const "); - if( m_is_global ) - fprintf(fp," is_global "); - if( m_is_local ) - fprintf(fp," is_local "); - if( m_is_tex ) - fprintf(fp," is_tex "); - if( m_is_func_addr ) - fprintf(fp," is_func_addr "); - if( m_function ) - fprintf(fp," %p ", m_function ); +void symbol::print_info(FILE *fp) const { + fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), + m_type); + if (m_address_valid) fprintf(fp, "<address valid>, "); + if (m_is_label) fprintf(fp, " is_label "); + if (m_is_shared) fprintf(fp, " is_shared "); + if (m_is_const) fprintf(fp, " is_const "); + if (m_is_global) fprintf(fp, " is_global "); + if (m_is_local) fprintf(fp, " is_local "); + if (m_is_tex) fprintf(fp, " is_tex "); + if (m_is_func_addr) fprintf(fp, " is_func_addr "); + if (m_function) fprintf(fp, " %p ", m_function); } -symbol_table::symbol_table() -{ - assert(0); -} +symbol_table::symbol_table() { assert(0); } -symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator=0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; +symbol_table::symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator = 0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; - //Jin: handle instruction group for cdp - m_inst_group_id = 0; + // Jin: handle instruction group for cdp + m_inst_group_id = 0; - m_parent = parent; - if ( m_parent ) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } + m_parent = parent; + if (m_parent) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } } -void symbol_table::set_name( const char *name ) -{ - m_scope_name = std::string(name); +void symbol_table::set_name(const char *name) { + m_scope_name = std::string(name); } -const ptx_version &symbol_table::get_ptx_version() const -{ - if( m_parent == NULL ) return m_ptx_version; - else return m_parent->get_ptx_version(); +const ptx_version &symbol_table::get_ptx_version() const { + if (m_parent == NULL) + return m_ptx_version; + else + return m_parent->get_ptx_version(); } -unsigned symbol_table::get_sm_target() const -{ - if( m_parent == NULL ) - return m_ptx_version.target(); - else return m_parent->get_sm_target(); +unsigned symbol_table::get_sm_target() const { + if (m_parent == NULL) + return m_ptx_version.target(); + else + return m_parent->get_sm_target(); } -void symbol_table::set_ptx_version( float ver, unsigned ext ) -{ - m_ptx_version = ptx_version(ver,ext); +void symbol_table::set_ptx_version(float ver, unsigned ext) { + m_ptx_version = ptx_version(ver, ext); } -void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) -{ - m_ptx_version.set_target(target,ext,ext2); +void symbol_table::set_sm_target(const char *target, const char *ext, + const char *ext2) { + m_ptx_version.set_target(target, ext, ext2); } -symbol *symbol_table::lookup( const char *identifier ) -{ - std::string key(identifier); - std::map<std::string, symbol *>::iterator i = m_symbols.find(key); - if ( i != m_symbols.end() ) { - return i->second; - } - if ( m_parent ) { - return m_parent->lookup(identifier); - } - return NULL; +symbol *symbol_table::lookup(const char *identifier) { + std::string key(identifier); + std::map<std::string, symbol *>::iterator i = m_symbols.find(key); + if (i != m_symbols.end()) { + return i->second; + } + if (m_parent) { + return m_parent->lookup(identifier); + } + return NULL; } -symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) -{ - char buf[1024]; - std::string key(identifier); - assert( m_symbols.find(key) == m_symbols.end() ); - snprintf(buf,1024,"%s:%u",filename,line); - symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); - m_symbols[ key ] = s; +symbol *symbol_table::add_variable(const char *identifier, + const type_info *type, unsigned size, + const char *filename, unsigned line) { + char buf[1024]; + std::string key(identifier); + assert(m_symbols.find(key) == m_symbols.end()); + snprintf(buf, 1024, "%s:%u", filename, line); + symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); + m_symbols[key] = s; - if ( type != NULL && type->get_key().is_global() ) { - m_globals.push_back(s); - } - if ( type != NULL && type->get_key().is_const() ) { - m_consts.push_back(s); - } + if (type != NULL && type->get_key().is_global()) { + m_globals.push_back(s); + } + if (type != NULL && type->get_key().is_const()) { + m_consts.push_back(s); + } - return s; + return s; } -void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) -{ - std::map<std::string, symbol *>::iterator i = m_symbols.find( func->get_name() ); - if( i != m_symbols.end() ) - return; - char buf[1024]; - snprintf(buf,1024,"%s:%u",filename,linenumber); - type_info *type = add_type( func ); - symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); - s->set_function(func); - m_symbols[ func->get_name() ] = s; +void symbol_table::add_function(function_info *func, const char *filename, + unsigned linenumber) { + std::map<std::string, symbol *>::iterator i = + m_symbols.find(func->get_name()); + if (i != m_symbols.end()) return; + char buf[1024]; + snprintf(buf, 1024, "%s:%u", filename, linenumber); + type_info *type = add_type(func); + symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); + s->set_function(func); + m_symbols[func->get_name()] = s; } -//Jin: handle instruction group for cdp -symbol_table* symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); +// Jin: handle instruction group for cdp +symbol_table *symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), + m_inst_group_id); + + // previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == + m_inst_group_symtab.end()); + symbol_table *sym_table = + new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); - //previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); - symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - return sym_table; + return sym_table; } -symbol_table * symbol_table::end_inst_group() { - symbol_table * sym_table = m_parent; - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; +symbol_table *symbol_table::end_inst_group() { + symbol_table *sym_table = m_parent; - return sym_table; + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; } -void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl +void register_ptx_function(const char *name, + function_info *impl); // either libcuda or libopencl -bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) -{ - std::string key = std::string(name); - bool prior_decl = false; - if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } +bool symbol_table::add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **sym_table) { + std::string key = std::string(name); + bool prior_decl = false; + if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } - if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { - assert( prior_decl ); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert( !prior_decl ); - *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info* null_type_info = new type_info( NULL, null_key ); - symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); - null_reg->set_regno(0, 0); - - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert( (*func_info)->get_symtab() == *sym_table ); - register_ptx_function(name,*func_info); - } - return prior_decl; + if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { + assert(prior_decl); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert(!prior_decl); + *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key(reg_space, 0, 0, 0, 0, 0); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info *null_type_info = new type_info(NULL, null_key); + symbol *null_reg = + (*sym_table)->add_variable("_", null_type_info, 0, "", 0); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert((*func_info)->get_symtab() == *sym_table); + register_ptx_function(name, *func_info); + } + return prior_decl; } -function_info *symbol_table::lookup_function( std::string name ) -{ - std::string key = std::string(name); - std::map<std::string,function_info*>::iterator it = m_function_info_lookup.find(key); - assert ( it != m_function_info_lookup.end() ); - return it->second; +function_info *symbol_table::lookup_function(std::string name) { + std::string key = std::string(name); + std::map<std::string, function_info *>::iterator it = + m_function_info_lookup.find(key); + assert(it != m_function_info_lookup.end()); + return it->second; } -type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) -{ - if( space_spec == param_space_unclassified ) - space_spec = param_space_local; - type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); - type_info *pt; - pt = new type_info(this,t); - return pt; +type_info *symbol_table::add_type(memory_space_t space_spec, + int scalar_type_spec, int vector_spec, + int alignment_spec, int extern_spec) { + if (space_spec == param_space_unclassified) space_spec = param_space_local; + type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, + extern_spec, 0); + type_info *pt; + pt = new type_info(this, t); + return pt; } -type_info *symbol_table::add_type( function_info *func ) -{ - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this,t); - return pt; +type_info *symbol_table::add_type(function_info *func) { + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this, t); + return pt; } -type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) -{ - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this,t); - //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. - //TODO: find where m_types can be used in future and solve the seg fault. - //pt = m_types[t] = new type_info(this,t); - return pt; +type_info *symbol_table::get_array_type(type_info *base_type, + unsigned array_dim) { + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this, t); + // Where else is m_types being used? As of now, I dont find any use of it and + // causing seg fault. So disabling m_types. + // TODO: find where m_types can be used in future and solve the seg fault. + // pt = m_types[t] = new type_info(this,t); + return pt; } -void symbol_table::set_label_address( const symbol *label, unsigned addr ) -{ - std::map<std::string, symbol *>::iterator i=m_symbols.find(label->name()); - assert( i != m_symbols.end() ); - symbol *s = i->second; - s->set_label_address(addr); +void symbol_table::set_label_address(const symbol *label, unsigned addr) { + std::map<std::string, symbol *>::iterator i = m_symbols.find(label->name()); + assert(i != m_symbols.end()); + symbol *s = i->second; + s->set_label_address(addr); } -void symbol_table::dump() -{ - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); - std::map<std::string, symbol *>::iterator i; - for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { - printf("%30s : ", i->first.c_str() ); - if( i->second ) - i->second->print_info(stdout); - else - printf(" <no symbol object> "); - printf("\n"); - } - printf("\n"); +void symbol_table::dump() { + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); + std::map<std::string, symbol *>::iterator i; + for (i = m_symbols.begin(); i != m_symbols.end(); i++) { + printf("%30s : ", i->first.c_str()); + if (i->second) + i->second->print_info(stdout); + else + printf(" <no symbol object> "); + printf("\n"); + } + printf("\n"); } -unsigned operand_info::get_uid() -{ - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; +unsigned operand_info::get_uid() { + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; } -std::list<ptx_instruction*>::iterator function_info::find_next_real_instruction( std::list<ptx_instruction*>::iterator i) -{ - while( (i != m_instructions.end()) && (*i)->is_label() ) - i++; - return i; +std::list<ptx_instruction *>::iterator +function_info::find_next_real_instruction( + std::list<ptx_instruction *>::iterator i) { + while ((i != m_instructions.end()) && (*i)->is_label()) i++; + return i; } -void function_info::create_basic_blocks() -{ - std::list<ptx_instruction*> leaders; - std::list<ptx_instruction*>::iterator i, l; +void function_info::create_basic_blocks() { + std::list<ptx_instruction *> leaders; + std::list<ptx_instruction *>::iterator i, l; - // first instruction is a leader - i=m_instructions.begin(); - leaders.push_back(*i); - i++; - while( i!=m_instructions.end() ) { - ptx_instruction *pI = *i; - if( pI->is_label() ) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch( pI->get_opcode() ) { - case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: + // first instruction is a leader + i = m_instructions.begin(); + leaders.push_back(*i); + i++; + while (i != m_instructions.end()) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch (pI->get_opcode()) { + case BRA_OP: + case RET_OP: + case EXIT_OP: + case RETP_OP: + case BREAK_OP: + i++; + if (i != m_instructions.end()) leaders.push_back(*i); + i = find_next_real_instruction(i); + break; + case CALL_OP: + case CALLP_OP: + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); + if (i != m_instructions.end()) leaders.push_back(*i); i = find_next_real_instruction(i); - break; - case CALL_OP: case CALLP_OP: - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); - i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); - i = find_next_real_instruction(i); - } else i++; - break; - default: + } else i++; - } - } - } + break; + default: + i++; + } + } + } - if( leaders.empty() ) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); - return; - } + if (leaders.empty()) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", + m_name.c_str()); + return; + } - unsigned bb_id = 0; - l=leaders.begin(); - i=m_instructions.begin(); - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); - ptx_instruction *last_real_inst=*(l++); + unsigned bb_id = 0; + l = leaders.begin(); + i = m_instructions.begin(); + m_basic_blocks.push_back( + new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); + ptx_instruction *last_real_inst = *(l++); - for( ; i!=m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if( l != leaders.end() && *i == *l ) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); - last_real_inst = *find_next_real_instruction(i); - } - // start search for next leader - l++; + for (; i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (l != leaders.end() && *i == *l) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if (find_next_real_instruction(i) != + m_instructions.end()) { // if not bogus trailing label + m_basic_blocks.push_back(new basic_block_t( + bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); + last_real_inst = *find_next_real_instruction(i); } - pI->assign_bb( m_basic_blocks.back() ); - if( !pI->is_label() ) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); + // start search for next leader + l++; + } + pI->assign_bb(m_basic_blocks.back()); + if (!pI->is_label()) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( + /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); } -void function_info::print_basic_blocks() -{ - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); - std::list<ptx_instruction*>::iterator ptx_itr; - unsigned last_bb=0; - for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { - if( (*ptx_itr)->get_bb() ) { - if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; - } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); - printf("\n"); +void function_info::print_basic_blocks() { + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); + std::list<ptx_instruction *>::iterator ptx_itr; + unsigned last_bb = 0; + for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); + ptx_itr++) { + if ((*ptx_itr)->get_bb()) { + if ((*ptx_itr)->get_bb()->bb_id != last_bb) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); - std::vector<basic_block_t*>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else printf(" last: NULL\t"); + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); printf("\n"); - } - printf("\n"); + } + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); + std::vector<basic_block_t *>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else + printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else + printf(" last: NULL\t"); + printf("\n"); + } + printf("\n"); } -void function_info::print_basic_block_links() -{ - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); - std::vector<basic_block_t*>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if ( !(*bb_itr)->predecessor_ids.empty() ) { - printf("Predecessors:"); - std::set<int>::iterator p; - for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { - printf(" %d", *p); - } - printf("\t"); +void function_info::print_basic_block_links() { + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); + std::vector<basic_block_t *>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if (!(*bb_itr)->predecessor_ids.empty()) { + printf("Predecessors:"); + std::set<int>::iterator p; + for (p = (*bb_itr)->predecessor_ids.begin(); + p != (*bb_itr)->predecessor_ids.end(); p++) { + printf(" %d", *p); } - if ( !(*bb_itr)->successor_ids.empty() ) { - printf("Successors:"); - std::set<int>::iterator s; - for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - printf(" %d", *s); - } + printf("\t"); + } + if (!(*bb_itr)->successor_ids.empty()) { + printf("Successors:"); + std::set<int>::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + printf(" %d", *s); } - printf("\n"); - } + } + printf("\n"); + } } -operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction +operand_info *function_info::find_break_target( + ptx_instruction *p_break_insn) // find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for(const basic_block_t *p_bb = break_bb; - p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) - { - // reverse search through instructions in basic block for breakaddr instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { + // reverse search through instructions in basic block for breakaddr + // instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) + continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); } - } + } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); - insn_addr >= 0; insn_addr--) - { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; + insn_addr--) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together +void function_info::connect_basic_blocks() // iterate across m_basic_blocks of + // function, connecting basic blocks + // together { - std::vector<basic_block_t*>::iterator bb_itr; - std::vector<basic_block_t*>::iterator bb_target_itr; - basic_block_t* exit_bb = m_basic_blocks.back(); + std::vector<basic_block_t *>::iterator bb_itr; + std::vector<basic_block_t *>::iterator bb_target_itr; + basic_block_t *exit_bb = m_basic_blocks.back(); - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - } - continue; - } else if (pI->get_opcode() == BRA_OP) { - //find successor and link that basic_block to this one - operand_info &target = pI->dst(); //get operand, e.g. target name - unsigned addr = labels[ target.name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || + pI->get_opcode() == EXIT_OP) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } + } + continue; + } else if (pI->get_opcode() == BRA_OP) { + // find successor and link that basic_block to this one + operand_info &target = pI->dst(); // get operand, e.g. target name + unsigned addr = labels[target.name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } - if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } + if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } } -bool function_info::connect_break_targets() //connecting break instructions with proper targets +bool function_info::connect_break_targets() // connecting break instructions + // with proper targets { - std::vector<basic_block_t*>::iterator bb_itr; - std::vector<basic_block_t*>::iterator bb_target_itr; - bool modified = false; - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set<int> orig_successor_ids = p_bb->successor_ids; + std::vector<basic_block_t *>::iterator bb_itr; + std::vector<basic_block_t *>::iterator bb_target_itr; + bool modified = false; - // erase the previous linkage with old successors - for(std::set<int>::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); - } - p_bb->successor_ids.clear(); + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set<int> orig_successor_ids = p_bb->successor_ids; - //find successor and link that basic_block to this one - //successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[ target->name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); + // erase the previous linkage with old successors + for (std::set<int>::iterator succ_ids = p_bb->successor_ids.begin(); + succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); + } + p_bb->successor_ids.clear(); - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); - } + // find successor and link that basic_block to this one + // successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[target->name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); - modified = modified || (orig_successor_ids != p_bb->successor_ids); + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); } - } - return modified; + modified = modified || (orig_successor_ids != p_bb->successor_ids); + } + } + + return modified; } -void function_info::do_pdom() -{ - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); +void function_info::do_pdom() { + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); - if ( g_debug_execution>=50 ) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if ( g_debug_execution>=2 ) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if ( g_debug_execution>=50 ) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); - for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); - fflush(stdout); - m_assembled = true; + if (g_debug_execution >= 50) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if (g_debug_execution >= 2) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if (g_debug_execution >= 50) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", + m_name.c_str()); + for (unsigned ii = 0; ii < m_n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", + m_name.c_str()); + fflush(stdout); + m_assembled = true; } -void intersect( std::set<int> &A, const std::set<int> &B ) -{ - // return intersection of A and B in A - for( std::set<int>::iterator a=A.begin(); a!=A.end(); ) { - std::set<int>::iterator a_next = a; - a_next++; - if( B.find(*a) == B.end() ) { - A.erase(*a); - a = a_next; - } else - a++; - } +void intersect(std::set<int> &A, const std::set<int> &B) { + // return intersection of A and B in A + for (std::set<int>::iterator a = A.begin(); a != A.end();) { + std::set<int>::iterator a_next = a; + a_next++; + if (B.find(*a) == B.end()) { + A.erase(*a); + a = a_next; + } else + a++; + } } -bool is_equal( const std::set<int> &A, const std::set<int> &B ) -{ - if( A.size() != B.size() ) - return false; - for( std::set<int>::iterator b=B.begin(); b!=B.end(); b++ ) - if( A.find(*b) == A.end() ) - return false; - return true; +bool is_equal(const std::set<int> &A, const std::set<int> &B) { + if (A.size() != B.size()) return false; + for (std::set<int>::iterator b = B.begin(); b != B.end(); b++) + if (A.find(*b) == A.end()) return false; + return true; } -void print_set(const std::set<int> &A) -{ - std::set<int>::iterator a; - for (a= A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); +void print_set(const std::set<int> &A) { + std::set<int>::iterator a; + for (a = A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); } -void function_info::find_dominators( ) -{ - // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - std::vector<basic_block_t*>::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry - //copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set<int> T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set<int>::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; - } +void function_info::find_dominators() { + // find dominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + std::vector<basic_block_t *>::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert( + (*bb_itr)->bb_id); // the only dominator of the entry block is the entry + // copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set<int> T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set<int>::iterator s = + m_basic_blocks[h]->predecessor_ids.begin(); + s != m_basic_blocks[h]->predecessor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; } - } - //clean the basic block of dominators of it has no predecessors -- except for entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) - (*bb_itr)->dominator_ids.clear(); - } + } + } + // clean the basic block of dominators of it has no predecessors -- except for + // entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); + } } -void function_info::find_postdominators( ) -{ - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - std::vector<basic_block_t*>::reverse_iterator bb_itr = m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block - for (unsigned i=0; i<m_basic_blocks.size(); i++) - (*bb_itr)->postdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set<int> T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set<int>::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; - } +void function_info::find_postdominators() { + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + std::vector<basic_block_t *>::reverse_iterator bb_itr = + m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert( + (*bb_itr) + ->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr; bb_itr != m_basic_blocks.rend(); + bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT + // for the exit block + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->postdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set<int> T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set<int>::iterator s = m_basic_blocks[h]->successor_ids.begin(); + s != m_basic_blocks[h]->successor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; } - } + } + } } -void function_info::find_ipostdominators( ) -{ - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - for (unsigned i=0; i<m_basic_blocks.size(); i++) { //initialize Tmp(n) to all pdoms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = m_basic_blocks.size()-2; n >=0;--n) { - // point iterator to basic block before the exit - for( std::set<int>::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set<int>::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set<int>::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } - } - } - unsigned num_ipdoms=0; - for ( int n = m_basic_blocks.size()-1; n >=0;--n) { - assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; +void function_info::find_ipostdominators() { + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all pdoms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { + // point iterator to basic block before the exit + for (std::set<int>::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set<int>::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set<int>::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->postdominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - assert( num_ipdoms == m_basic_blocks.size()-1 ); - // the exit node does not have an immediate post dominator, but everyone else should + } + } + unsigned num_ipdoms = 0; + for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { + assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatepostdominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; + } + } + assert(num_ipdoms == m_basic_blocks.size() - 1); + // the exit node does not have an immediate post dominator, but everyone else + // should } -void function_info::find_idominators( ) -{ - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - for (unsigned i=0; i<m_basic_blocks.size(); i++) { //initialize Tmp(n) to all doms of n except for n - m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for( std::set<int>::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set<int>::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set<int>::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } - } - } - unsigned num_idoms=0; - unsigned num_nopred = 0; - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; +void function_info::find_idominators() { + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all doms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for (std::set<int>::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set<int>::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set<int>::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->dominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - assert( num_idoms == m_basic_blocks.size()-num_nopred ); - // the entry node does not have an immediate dominator, but everyone else should + } + } + unsigned num_idoms = 0; + unsigned num_nopred = 0; + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatedominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; + } + } + assert(num_idoms == m_basic_blocks.size() - num_nopred); + // the entry node does not have an immediate dominator, but everyone else + // should } -void function_info::print_dominators() -{ - printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set<int>::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } +void function_info::print_dominators() { + printf("Printing dominators for function \'%s\':\n", m_name.c_str()); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set<int>::iterator j = m_basic_blocks[i]->dominator_ids.begin(); + j != m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } } -void function_info::print_postdominators() -{ - printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set<int>::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } +void function_info::print_postdominators() { + printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set<int>::iterator j = + m_basic_blocks[i]->postdominator_ids.begin(); + j != m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } } -void function_info::print_ipostdominators() -{ - printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } +void function_info::print_ipostdominators() { + printf("Printing immediate postdominators for function \'%s\':\n", + m_name.c_str()); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } } -void function_info::print_idominators() -{ - printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); - std::vector<int>::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } +void function_info::print_idominators() { + printf("Printing immediate dominators for function \'%s\':\n", + m_name.c_str()); + std::vector<int>::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } } -unsigned function_info::get_num_reconvergence_pairs() -{ - if (!num_reconvergence_pairs) { - if( m_basic_blocks.size() == 0 ) - return 0; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; - } +unsigned function_info::get_num_reconvergence_pairs() { + if (!num_reconvergence_pairs) { + if (m_basic_blocks.size() == 0) return 0; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; } - } - return num_reconvergence_pairs; + } + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) -{ - unsigned idx=0; //array index - if( m_basic_blocks.size() == 0 ) - return; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { + unsigned idx = 0; // array index + if (m_basic_blocks.size() == 0) return; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); fflush(stdout); + printf("i=%d\n", i); + fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, + m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", + m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { - recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); - recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin) { + recon_points[idx].target_pc = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->get_PC(); + recon_points[idx].target_inst = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); + fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() -{ - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector<basic_block_t*>::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("\t"); - std::set<int>::iterator s; - for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); - } - printf("\n"); - } - printf("}\n"); +void function_info::print_basic_block_dot() { + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector<basic_block_t *>::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("\t"); + std::set<int>::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); + } + printf("\n"); + } + printf("}\n"); } -unsigned ptx_kernel_shmem_size( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; +unsigned ptx_kernel_shmem_size(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; } -unsigned ptx_kernel_nregs( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; +unsigned ptx_kernel_nregs(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; } -unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const -{ - int type = scalar_type(); - return type_decode(type,size,basic_type); +unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { + int type = scalar_type(); + return type_decode(type, size, basic_type); } -unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) -{ - switch ( type ) { - case S8_TYPE: size=8; basic_type=1; return 0; - case S16_TYPE: size=16; basic_type=1; return 1; - case S32_TYPE: size=32; basic_type=1; return 2; - case S64_TYPE: size=64; basic_type=1; return 3; - case U8_TYPE: size=8; basic_type=0; return 4; - case U16_TYPE: size=16; basic_type=0; return 5; - case U32_TYPE: size=32; basic_type=0; return 6; - case U64_TYPE: size=64; basic_type=0; return 7; - case F16_TYPE: size=16; basic_type=-1; return 8; - case F32_TYPE: size=32; basic_type=-1; return 9; - case F64_TYPE: size=64; basic_type=-1; return 10; - case FF64_TYPE: size=64; basic_type=-1; return 10; - case PRED_TYPE: size=1; basic_type=2; return 11; - case B8_TYPE: size=8; basic_type=0; return 12; - case B16_TYPE: size=16; basic_type=0; return 13; - case B32_TYPE: size=32; basic_type=0; return 14; - case B64_TYPE: size=64; basic_type=0; return 15; - case BB64_TYPE: size=64; basic_type=0; return 15; - case BB128_TYPE: size=128; basic_type=0; return 16; - case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: - size=32; basic_type=3; return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); - assert(0); +unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { + switch (type) { + case S8_TYPE: + size = 8; + basic_type = 1; + return 0; + case S16_TYPE: + size = 16; + basic_type = 1; + return 1; + case S32_TYPE: + size = 32; + basic_type = 1; + return 2; + case S64_TYPE: + size = 64; + basic_type = 1; + return 3; + case U8_TYPE: + size = 8; + basic_type = 0; + return 4; + case U16_TYPE: + size = 16; + basic_type = 0; + return 5; + case U32_TYPE: + size = 32; + basic_type = 0; + return 6; + case U64_TYPE: + size = 64; + basic_type = 0; + return 7; + case F16_TYPE: + size = 16; + basic_type = -1; + return 8; + case F32_TYPE: + size = 32; + basic_type = -1; + return 9; + case F64_TYPE: + size = 64; + basic_type = -1; + return 10; + case FF64_TYPE: + size = 64; + basic_type = -1; + return 10; + case PRED_TYPE: + size = 1; + basic_type = 2; + return 11; + case B8_TYPE: + size = 8; + basic_type = 0; + return 12; + case B16_TYPE: + size = 16; + basic_type = 0; + return 13; + case B32_TYPE: + size = 32; + basic_type = 0; + return 14; + case B64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB128_TYPE: + size = 128; + basic_type = 0; + return 16; + case TEXREF_TYPE: + case SAMPLERREF_TYPE: + case SURFREF_TYPE: + size = 32; + basic_type = 3; + return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", + decode_token(type)); + assert(0); return 0xDEADBEEF; - } + } } -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) -{ - if( actual_param_op.is_reg() ) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param,actual_param_op,value); - } else if ( actual_param_op.is_param_local() ) { - unsigned size=formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size<1024); - thread->m_local_mem->read(from_addr,size,buffer); - return arg_buffer_t(formal_param,actual_param_op,buffer,size); - } else { - printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); - abort(); - } +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param) { + if (actual_param_op.is_reg()) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param, actual_param_op, value); + } else if (actual_param_op.is_param_local()) { + unsigned size = formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size < 1024); + thread->m_local_mem->read(from_addr, size, buffer); + return arg_buffer_t(formal_param, actual_param_op, buffer, size); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " + "call/return\n"); + abort(); + } } -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ) -{ - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); - } +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values) { + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( + copy_arg_to_buffer(thread, actual_param_op, formal_param)); + } } -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) -{ - if( a.is_reg() ) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(),value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); - } +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { + if (a.is_reg()) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = + operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(), value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); + } } -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) -{ - arg_buffer_list_t::iterator a; - for( a=arg_values.begin(); a != arg_values.end(); a++ ) { - copy_buffer_to_frame(thread, *a); - } +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values) { + arg_buffer_list_t::iterator a; + for (a = arg_values.begin(); a != arg_values.end(); a++) { + copy_buffer_to_frame(thread, *a); + } } - - -static std::list<operand_info> check_operands( int opcode, - const std::list<int> &scalar_type, - const std::list<operand_info> &operands, - gpgpu_context* ctx) -{ - static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if( !g_warn_literal_operands_two_type_inst ) { - std::list<operand_info>::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } - } +static std::list<operand_info> check_operands( + int opcode, const std::list<int> &scalar_type, + const std::list<operand_info> &operands, gpgpu_context *ctx) { + static int g_warn_literal_operands_two_type_inst; + if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || + (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if (!g_warn_literal_operands_two_type_inst) { + std::list<operand_info>::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + printf( + "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " + "operand.\n"); + g_warn_literal_operands_two_type_inst = 1; } - } else { - assert( scalar_type.size() < 2 ); - if( scalar_type.size() == 1 ) { - std::list<operand_info> result; - int inst_type = scalar_type.front(); - std::list<operand_info>::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); - } - } - return result; - } + } + } + } else { + assert(scalar_type.size() < 2); + if (scalar_type.size() == 1) { + std::list<operand_info> result; + int inst_type = scalar_type.front(); + std::list<operand_info>::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; } - return operands; + } + return operands; } - -ptx_instruction::ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list<operand_info> &operands, - const operand_info &return_var, - const std::list<int> &options, - const std::list<int> &wmma_options, - const std::list<int> &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) -{ - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list<operand_info> checked_operands = check_operands(opcode,scalar_type,operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr=0; - std::list<int>::const_iterator i; - unsigned n=1; - for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type=last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++]=last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; - } - } - rr=0; - n=1; - for ( i=options.begin(); i!= options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { +ptx_instruction::ptx_instruction( + int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, + const std::list<operand_info> &operands, const operand_info &return_var, + const std::list<int> &options, const std::list<int> &wmma_options, + const std::list<int> &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx) + : warp_inst_t(config), m_return_var(ctx) { + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list<operand_info> checked_operands = + check_operands(opcode, scalar_type, operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), + checked_operands.end()); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr = 0; + std::list<int>::const_iterator i; + unsigned n = 1; + for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type = last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++] = last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr = 0; + n = 1; + for (i = options.begin(); i != options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1186,16 +1279,16 @@ ptx_instruction::ptx_instruction( int opcode, case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1204,38 +1297,39 @@ ptx_instruction::ptx_instruction( int opcode, case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert( !m_lo ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert(!m_lo); + assert(!m_wide); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert( !m_hi ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert(!m_hi); + assert(!m_wide); + break; case WIDE_OPTION: - m_wide = true; - assert( !m_lo ); - assert( !m_hi ); - break; + m_wide = true; + assert(!m_lo); + assert(!m_hi); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing + // flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1246,223 +1340,225 @@ ptx_instruction::ptx_instruction( int opcode, case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: + case CG_OPTION: + case CS_OPTION: + case LU_OPTION: + case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { - m_space_spec = generic_space; - } - for( std::vector<operand_info>::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { - const operand_info &op = *i; - if( op.get_addr_space() != undefined_space ) - m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst - } - if( opcode == TEX_OP ) - m_space_spec = tex_space; - - m_source_file = file?file:"<unknown>"; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && + (space_spec == undefined_space)) { + m_space_spec = generic_space; + } + for (std::vector<operand_info>::const_iterator i = m_operands.begin(); + i != m_operands.end(); ++i) { + const operand_info &op = *i; + if (op.get_addr_space() != undefined_space) + m_space_spec = + op.get_addr_space(); // TODO: can have more than one memory space for + // ptxplus (g8x) inst + } + if (opcode == TEX_OP) m_space_spec = tex_space; - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); + m_source_file = file ? file : "<unknown>"; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), + m_source.end()); - if (fname =="vprintf"){ - m_is_printf = true; - } - if(fname == "cudaStreamCreateWithFlags") - m_is_cdp = 1; - if(fname == "cudaGetParameterBufferV2") - m_is_cdp = 2; - if(fname == "cudaLaunchDeviceV2") - m_is_cdp = 4; + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); - } + if (fname == "vprintf") { + m_is_printf = true; + } + if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; + if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; + if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; + } } -void ptx_instruction::print_insn() const -{ - print_insn(stdout); - fflush(stdout); +void ptx_instruction::print_insn() const { + print_insn(stdout); + fflush(stdout); } -void ptx_instruction::print_insn( FILE *fp ) const -{ - fprintf( fp, "%s", to_string().c_str() ); +void ptx_instruction::print_insn(FILE *fp) const { + fprintf(fp, "%s", to_string().c_str()); } -std::string ptx_instruction::to_string() const -{ - char buf[ STR_SIZE ]; - unsigned used_bytes = 0; - if( !is_label() ) { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); - } else { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); - } - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, - "(%s:%d) %s", - m_source_file.c_str(), m_source_line, - m_source.c_str() ); - return std::string( buf ); +std::string ptx_instruction::to_string() const { + char buf[STR_SIZE]; + unsigned used_bytes = 0; + if (!is_label()) { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); + } else { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); + } + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", + m_source_file.c_str(), m_source_line, m_source.c_str()); + return std::string(buf); } -operand_info ptx_instruction::get_pred() const -{ - return operand_info( m_pred, gpgpu_ctx); +operand_info ptx_instruction::get_pred() const { + return operand_info(m_pred, gpgpu_ctx); } - -function_info::function_info(int entry_point, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point==1)?true:false; - m_extern = (entry_point==2)?true:false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; //initialize it to false +function_info::function_info(int entry_point, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point == 1) ? true : false; + m_extern = (entry_point == 2) ? true : false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; // initialize it to false } -unsigned function_info::print_insn( unsigned pc, FILE * fp ) const -{ - unsigned inst_size=1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command,1024,"c++filt -p %s",m_name.c_str()); - FILE *p = popen(command,"r"); - buffer[0]=0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp,"%s",buffer); - if ( index >= m_instr_mem_size ) { - fprintf(fp, "<past last instruction (max pc=%u)>", m_start_PC + m_instr_mem_size - 1 ); - } else { - if ( m_instr_mem[index] != NULL ) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "<no instruction at pc = %u>", pc ); - } - pclose(p); - return inst_size; +unsigned function_info::print_insn(unsigned pc, FILE *fp) const { + unsigned inst_size = 1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); + FILE *p = popen(command, "r"); + buffer[0] = 0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp, "%s", buffer); + if (index >= m_instr_mem_size) { + fprintf(fp, "<past last instruction (max pc=%u)>", + m_start_PC + m_instr_mem_size - 1); + } else { + if (m_instr_mem[index] != NULL) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "<no instruction at pc = %u>", pc); + } + pclose(p); + return inst_size; } -std::string function_info::get_insn_str( unsigned pc ) const -{ - unsigned index = pc - m_start_PC; - if ( index >= m_instr_mem_size ) { +std::string function_info::get_insn_str(unsigned pc) const { + unsigned index = pc - m_start_PC; + if (index >= m_instr_mem_size) { + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "<past last instruction (max pc=%u)>", + m_start_PC + m_instr_mem_size - 1); + return std::string(buff); + } else { + if (m_instr_mem[index] != NULL) { + return m_instr_mem[index]->to_string(); + } else { char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "<past last instruction (max pc=%u)>", m_start_PC + m_instr_mem_size - 1 ); + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "<no instruction at pc = %u>", pc); return std::string(buff); - } else { - if ( m_instr_mem[index] != NULL ) { - return m_instr_mem[index]->to_string(); - } else { - char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "<no instruction at pc = %u>", pc ); - return std::string(buff); - } - } + } + } } -void gpgpu_ptx_assemble( std::string kname, void *kinfo ) -{ - function_info *func_info = (function_info *)kinfo; - if((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); - return; - } - if( func_info->is_extern() ) { - printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); - return; - } - func_info->ptx_assemble(); +void gpgpu_ptx_assemble(std::string kname, void *kinfo) { + function_info *func_info = (function_info *)kinfo; + if ((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", + kname.c_str()); + return; + } + if (func_info->is_extern()) { + printf( + "GPGPU-Sim PTX: skipping assembly for extern declared function " + "\'%s\'\n", + func_info->get_name().c_str()); + return; + } + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4c5c37..6627847 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,36 +7,37 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ptx_ir_INCLUDED #define ptx_ir_INCLUDED #include "../abstract_hardware_model.h" +#include <assert.h> #include <cstdlib> #include <cstring> -#include <string> #include <list> #include <map> +#include <string> #include <vector> -#include <assert.h> //#include "ptx.tab.h" #include "ptx_sim.h" @@ -46,1537 +47,1517 @@ class gpgpu_context; class type_info_key { -public: - type_info_key() - { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) - { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() - { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } + public: + type_info_key() { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec, + int array_dim) { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } + + void set_array_dim(int array_dim) { m_array_dim = array_dim; } + int get_array_dim() const { + assert(m_init); + return m_array_dim; + } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - void set_array_dim( int array_dim ) { m_array_dim = array_dim; } - int get_array_dim() const { assert(m_init); return m_array_dim; } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space; } + bool is_param_kernel() const { return m_space_spec == param_space_kernel; } + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { + return m_space_spec == param_space_unclassified; + } + bool is_global() const { return m_space_spec == global_space; } + bool is_local() const { return m_space_spec == local_space; } + bool is_shared() const { return m_space_spec == shared_space; } + bool is_const() const { return m_space_spec.get_type() == const_space; } + bool is_tex() const { return m_space_spec == tex_space; } + bool is_func_addr() const { return m_is_function ? true : false; } + int scalar_type() const { return m_scalar_type_spec; } + int get_alignment_spec() const { return m_alignment_spec; } + unsigned type_decode(size_t &size, int &t) const; + static unsigned type_decode(int type, size_t &size, int &t); + memory_space_t get_memory_space() const { return m_space_spec; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space;} - bool is_param_kernel() const { return m_space_spec == param_space_kernel;} - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } - bool is_global() const { return m_space_spec == global_space;} - bool is_local() const { return m_space_spec == local_space;} - bool is_shared() const { return m_space_spec == shared_space;} - bool is_const() const { return m_space_spec.get_type() == const_space;} - bool is_tex() const { return m_space_spec == tex_space;} - bool is_func_addr() const { return m_is_function?true:false; } - int scalar_type() const { return m_scalar_type_spec;} - int get_alignment_spec() const { return m_alignment_spec;} - unsigned type_decode( size_t &size, int &t ) const; - static unsigned type_decode( int type, size_t &size, int &t ); - memory_space_t get_memory_space() const { return m_space_spec; } -private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; + private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; - friend struct type_info_key_compare; + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()( const type_info_key &a, const type_info_key &b ) const - { - assert( a.m_init && b.m_init ); - if ( a.m_space_spec < b.m_space_spec ) return true; - if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; - if ( a.m_vector_spec < b.m_vector_spec ) return true; - if ( a.m_alignment_spec < b.m_alignment_spec ) return true; - if ( a.m_extern_spec < b.m_extern_spec ) return true; - if ( a.m_array_dim < b.m_array_dim ) return true; - if ( a.m_is_function < b.m_is_function ) return true; + bool operator()(const type_info_key &a, const type_info_key &b) const { + assert(a.m_init && b.m_init); + if (a.m_space_spec < b.m_space_spec) return true; + if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; + if (a.m_vector_spec < b.m_vector_spec) return true; + if (a.m_alignment_spec < b.m_alignment_spec) return true; + if (a.m_extern_spec < b.m_extern_spec) return true; + if (a.m_array_dim < b.m_array_dim) return true; + if (a.m_is_function < b.m_is_function) return true; - return false; - } + return false; + } }; class type_info { -public: - type_info( symbol_table *scope, type_info_key t ) - { - m_type_info = t; - } - const type_info_key &get_key() const { return m_type_info;} + public: + type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } + const type_info_key &get_key() const { return m_type_info; } -private: - symbol_table *m_scope; - type_info_key m_type_info; + private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, - unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, - v_int_t, v_unsigned_t, undef_t + reg_t, + vector_t, + builtin_t, + address_t, + memory_t, + float_op_t, + double_op_t, + int_t, + unsigned_t, + symbolic_t, + label_t, + v_reg_t, + v_float_op_t, + v_double_op_t, + v_int_t, + v_unsigned_t, + undef_t }; class operand_info; class symbol { -public: - symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) - { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num=(unsigned)-1; - m_arch_reg_num=(unsigned)-1; - m_address=(unsigned)-1; - m_initializer.clear(); - if ( type ) m_is_shared = type->get_key().is_shared(); - if ( type ) m_is_const = type->get_key().is_const(); - if ( type ) m_is_global = type->get_key().is_global(); - if ( type ) m_is_local = type->get_key().is_local(); - if ( type ) m_is_param_local = type->get_key().is_param_local(); - if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); - if ( type ) m_is_tex = type->get_key().is_tex(); - if ( type ) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const - { - return m_size; - } - const std::string &name() const { return m_name;} - const std::string &decl_location() const { return m_decl_location;} - const type_info *type() const { return m_type;} - addr_t get_address() const - { - assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions - assert( m_address_valid ); - return m_address; - } - function_info *get_pc() const - { - return m_function; - } - void set_regno( unsigned regno, unsigned arch_regno ) - { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } + public: + symbol(const char *name, const type_info *type, const char *location, + unsigned size, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num = (unsigned)-1; + m_arch_reg_num = (unsigned)-1; + m_address = (unsigned)-1; + m_initializer.clear(); + if (type) m_is_shared = type->get_key().is_shared(); + if (type) m_is_const = type->get_key().is_const(); + if (type) m_is_global = type->get_key().is_global(); + if (type) m_is_local = type->get_key().is_local(); + if (type) m_is_param_local = type->get_key().is_param_local(); + if (type) m_is_param_kernel = type->get_key().is_param_kernel(); + if (type) m_is_tex = type->get_key().is_tex(); + if (type) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const { return m_size; } + const std::string &name() const { return m_name; } + const std::string &decl_location() const { return m_decl_location; } + const type_info *type() const { return m_type; } + addr_t get_address() const { + assert(m_is_label || + !m_type->get_key().is_reg()); // todo : other assertions + assert(m_address_valid); + return m_address; + } + function_info *get_pc() const { return m_function; } + void set_regno(unsigned regno, unsigned arch_regno) { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } - void set_address( addr_t addr ) - { - m_address_valid = true; - m_address = addr; - } - void set_label_address( addr_t addr) - { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function( function_info *func ) - { - m_function = func; - m_is_func_addr = true; - } + void set_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + } + void set_label_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function(function_info *func) { + m_function = func; + m_is_func_addr = true; + } - bool is_label() const { return m_is_label;} - bool is_shared() const { return m_is_shared;} - bool is_sstarr() const { return m_is_sstarr;} - bool is_const() const { return m_is_const;} - bool is_global() const { return m_is_global;} - bool is_local() const { return m_is_local;} - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex;} - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } + bool is_label() const { return m_is_label; } + bool is_shared() const { return m_is_shared; } + bool is_sstarr() const { return m_is_sstarr; } + bool is_const() const { return m_is_const; } + bool is_global() const { return m_is_global; } + bool is_local() const { return m_is_local; } + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex; } + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } - void add_initializer( const std::list<operand_info> &init ); - bool has_initializer() const - { - return m_initializer.size() > 0; - } - std::list<operand_info> get_initializer() const - { - return m_initializer; - } - unsigned reg_num() const - { - assert( m_reg_num_valid ); - return m_reg_num; - } - unsigned arch_reg_num() const - { - assert( m_reg_num_valid ); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } + void add_initializer(const std::list<operand_info> &init); + bool has_initializer() const { return m_initializer.size() > 0; } + std::list<operand_info> get_initializer() const { return m_initializer; } + unsigned reg_num() const { + assert(m_reg_num_valid); + return m_reg_num; + } + unsigned arch_reg_num() const { + assert(m_reg_num_valid); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } -private: - gpgpu_context* gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; + private: + gpgpu_context *gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; - unsigned m_address; - function_info *m_function; // used for function symbols + unsigned m_address; + function_info *m_function; // used for function symbols - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; - std::list<operand_info> m_initializer; + std::list<operand_info> m_initializer; }; class symbol_table { -public: - symbol_table(); - symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); - void set_name( const char *name ); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version( float ver, unsigned ext ); - void set_sm_target( const char *target, const char *ext, const char *ext2 ); - symbol* lookup( const char *identifier ); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); - void add_function( function_info *func, const char *filename, unsigned linenumber ); - bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); - function_info *lookup_function(std::string name); - type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); - type_info *add_type( function_info *func ); - type_info *get_array_type( type_info *base_type, unsigned array_dim ); - void set_label_address( const symbol *label, unsigned addr ); - unsigned next_reg_num() { return ++m_reg_allocator;} - addr_t get_shared_next() { return m_shared_next;} - addr_t get_sstarr_next() { return m_sstarr_next;} - addr_t get_global_next() { return m_global_next;} - addr_t get_local_next() { return m_local_next;} - addr_t get_tex_next() { return m_tex_next;} - void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} - void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} - void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} - void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} - void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} + public: + symbol_table(); + symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx); + void set_name(const char *name); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version(float ver, unsigned ext); + void set_sm_target(const char *target, const char *ext, const char *ext2); + symbol *lookup(const char *identifier); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable(const char *identifier, const type_info *type, + unsigned size, const char *filename, unsigned line); + void add_function(function_info *func, const char *filename, + unsigned linenumber); + bool add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **symbol_table); + function_info *lookup_function(std::string name); + type_info *add_type(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec); + type_info *add_type(function_info *func); + type_info *get_array_type(type_info *base_type, unsigned array_dim); + void set_label_address(const symbol *label, unsigned addr); + unsigned next_reg_num() { return ++m_reg_allocator; } + addr_t get_shared_next() { return m_shared_next; } + addr_t get_sstarr_next() { return m_sstarr_next; } + addr_t get_global_next() { return m_global_next; } + addr_t get_local_next() { return m_local_next; } + addr_t get_tex_next() { return m_tex_next; } + void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } + void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } + void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } + void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } + void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } + + typedef std::list<symbol *>::iterator iterator; + + iterator global_iterator_begin() { return m_globals.begin(); } + iterator global_iterator_end() { return m_globals.end(); } - typedef std::list<symbol*>::iterator iterator; + iterator const_iterator_begin() { return m_consts.begin(); } + iterator const_iterator_end() { return m_consts.end(); } - iterator global_iterator_begin() { return m_globals.begin();} - iterator global_iterator_end() { return m_globals.end();} + void dump(); - iterator const_iterator_begin() { return m_consts.begin();} - iterator const_iterator_end() { return m_consts.end();} + // Jin: handle instruction group for cdp + symbol_table *start_inst_group(); + symbol_table *end_inst_group(); - void dump(); - - //Jin: handle instruction group for cdp - symbol_table* start_inst_group(); - symbol_table* end_inst_group(); + // backward pointer + class gpgpu_context *gpgpu_ctx; - // backward pointer - class gpgpu_context* gpgpu_ctx; + private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; -private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map<std::string, symbol *> + m_symbols; // map from name of register to pointers to the registers + std::map<type_info_key, type_info *, type_info_key_compare> m_types; + std::list<symbol *> m_globals; + std::list<symbol *> m_consts; + std::map<std::string, function_info *> m_function_info_lookup; + std::map<std::string, symbol_table *> m_function_symtab_lookup; - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map<std::string, symbol *> m_symbols; //map from name of register to pointers to the registers - std::map<type_info_key,type_info*,type_info_key_compare> m_types; - std::list<symbol*> m_globals; - std::list<symbol*> m_consts; - std::map<std::string,function_info*> m_function_info_lookup; - std::map<std::string,symbol_table*> m_function_symtab_lookup; - - //Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map<std::string,symbol_table*> m_inst_group_symtab; + // Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map<std::string, symbol_table *> m_inst_group_symtab; }; class operand_info { -public: - operand_info(gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address=false; - m_addr_offset = 0; - m_value.m_symbolic=NULL; - } - operand_info( const symbol *addr, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if ( addr->is_label() ) { - m_type = label_t; - } else if ( addr->is_shared() ) { - m_type = symbolic_t; - } else if ( addr->is_const() ) { - m_type = symbolic_t; - } else if ( addr->is_global() ) { - m_type = symbolic_t; - } else if ( addr->is_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_kernel() ) { - m_type = symbolic_t; - } else if ( addr->is_tex() ) { - m_type = symbolic_t; - } else if ( addr->is_func_addr() ) { - m_type = symbolic_t; - } else if ( !addr->is_reg() ) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } - - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( unsigned x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=true; - } - operand_info( int x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( float x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( double x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } + public: + operand_info(gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address = false; + m_addr_offset = 0; + m_value.m_symbolic = NULL; + } + operand_info(const symbol *addr, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if (addr->is_label()) { + m_type = label_t; + } else if (addr->is_shared()) { + m_type = symbolic_t; + } else if (addr->is_const()) { + m_type = symbolic_t; + } else if (addr->is_global()) { + m_type = symbolic_t; + } else if (addr->is_local()) { + m_type = symbolic_t; + } else if (addr->is_param_local()) { + m_type = symbolic_t; + } else if (addr->is_param_kernel()) { + m_type = symbolic_t; + } else if (addr->is_tex()) { + m_type = symbolic_t; + } else if (addr->is_func_addr()) { + m_type = symbolic_t; + } else if (!addr->is_reg()) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } - void init(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_uid=(unsigned)-1; - m_valid=false; - m_vector=false; - m_type=undef_t; - m_immediate_address=false; - m_addr_space=undefined_space; - m_operand_lohi=0; - m_double_operand_type=0; - m_operand_neg=false; - m_const_mem_offset=(unsigned)-1; - m_value.m_int=0; - m_value.m_unsigned=(unsigned)-1; - m_value.m_float=0; - m_value.m_double=0; - for(unsigned i=0; i<4; i++){ - m_value.m_vint[i]=0; - m_value.m_vunsigned[i]=0; - m_value.m_vfloat[i]=0; - m_value.m_vdouble[i]=0; - } - m_value.m_symbolic=NULL; - m_value.m_vector_symbolic=NULL; - m_addr_offset=0; - m_neg_pred=0; - m_is_return_var=0; - m_is_non_arch_reg=0; + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(unsigned x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = true; + } + operand_info(int x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(float x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(double x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, const symbol *s5, const symbol *s6, + const symbol *s7, const symbol *s8, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } - } - void make_memory_operand() { m_type = memory_t;} - void set_return() { m_is_return_var = true; } - void set_immediate_addr() {m_immediate_address=true;} - const std::string &name() const - { - assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } + void init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (unsigned)-1; + m_valid = false; + m_vector = false; + m_type = undef_t; + m_immediate_address = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = (unsigned)-1; + m_value.m_int = 0; + m_value.m_unsigned = (unsigned)-1; + m_value.m_float = 0; + m_value.m_double = 0; + for (unsigned i = 0; i < 4; i++) { + m_value.m_vint[i] = 0; + m_value.m_vunsigned[i] = 0; + m_value.m_vfloat[i] = 0; + m_value.m_vdouble[i] = 0; + } + m_value.m_symbolic = NULL; + m_value.m_vector_symbolic = NULL; + m_addr_offset = 0; + m_neg_pred = 0; + m_is_return_var = 0; + m_is_non_arch_reg = 0; + } + void make_memory_operand() { m_type = memory_t; } + void set_return() { m_is_return_var = true; } + void set_immediate_addr() { m_immediate_address = true; } + const std::string &name() const { + assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || + m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } - unsigned get_vect_nelem() const - { - assert( is_vector() ); - if( !m_value.m_vector_symbolic[0] ) return 0; - if( !m_value.m_vector_symbolic[1] ) return 1; - if( !m_value.m_vector_symbolic[2] ) return 2; - if( !m_value.m_vector_symbolic[3] ) return 3; - if( !m_value.m_vector_symbolic[4] ) return 4; - if( !m_value.m_vector_symbolic[5] ) return 5; - if( !m_value.m_vector_symbolic[6] ) return 6; - if( !m_value.m_vector_symbolic[7] ) return 7; - return 8; - } + unsigned get_vect_nelem() const { + assert(is_vector()); + if (!m_value.m_vector_symbolic[0]) return 0; + if (!m_value.m_vector_symbolic[1]) return 1; + if (!m_value.m_vector_symbolic[2]) return 2; + if (!m_value.m_vector_symbolic[3]) return 3; + if (!m_value.m_vector_symbolic[4]) return 4; + if (!m_value.m_vector_symbolic[5]) return 5; + if (!m_value.m_vector_symbolic[6]) return 6; + if (!m_value.m_vector_symbolic[7]) return 7; + return 8; + } - const symbol* vec_symbol(int idx) const - { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert( result != NULL ); - return result; - } + const symbol *vec_symbol(int idx) const { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert(result != NULL); + return result; + } - const std::string &vec_name1() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } + const std::string &vec_name1() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } - const std::string &vec_name2() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } + const std::string &vec_name2() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } - const std::string &vec_name3() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } + const std::string &vec_name3() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } - const std::string &vec_name4() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } + const std::string &vec_name4() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } - bool is_reg() const - { - if ( m_type == reg_t ) { - return true; - } - if ( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } + bool is_reg() const { + if (m_type == reg_t) { + return true; + } + if (m_type != symbolic_t) { + return false; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } - bool is_param_kernel() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } + bool is_param_kernel() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } - bool is_vector() const - { - if ( m_vector) return true; - return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num();} - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} - int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } - int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } - int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } - int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } - int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } - int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } - bool is_label() const { return m_type == label_t;} - bool is_builtin() const { return m_type == builtin_t;} + bool is_vector() const { + if (m_vector) return true; + return false; + } + int reg_num() const { return m_value.m_symbolic->reg_num(); } + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } + int reg3_num() const { + return m_value.m_vector_symbolic[2] + ? m_value.m_vector_symbolic[2]->reg_num() + : 0; + } + int reg4_num() const { + return m_value.m_vector_symbolic[3] + ? m_value.m_vector_symbolic[3]->reg_num() + : 0; + } + int reg5_num() const { + return m_value.m_vector_symbolic[4] + ? m_value.m_vector_symbolic[4]->reg_num() + : 0; + } + int reg6_num() const { + return m_value.m_vector_symbolic[5] + ? m_value.m_vector_symbolic[5]->reg_num() + : 0; + } + int reg7_num() const { + return m_value.m_vector_symbolic[6] + ? m_value.m_vector_symbolic[6]->reg_num() + : 0; + } + int reg8_num() const { + return m_value.m_vector_symbolic[7] + ? m_value.m_vector_symbolic[7]->reg_num() + : 0; + } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { + return (m_value.m_vector_symbolic[n]) + ? m_value.m_vector_symbolic[n]->arch_reg_num() + : -1; + } + bool is_label() const { return m_type == label_t; } + bool is_builtin() const { return m_type == builtin_t; } - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t;} + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t; } - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. - bool is_memory_operand2() const { - return (m_addr_space!=undefined_space); - } + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address + // space during parsing. + bool is_memory_operand2() const { return (m_addr_space != undefined_space); } - bool is_immediate_address() const { - return m_immediate_address; - } + bool is_immediate_address() const { return m_immediate_address; } - bool is_literal() const { return m_type == int_t || - m_type == float_op_t || - m_type == double_op_t || - m_type == unsigned_t;} - bool is_shared() const { - if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} - bool is_const() const { return m_value.m_symbolic->is_const();} - bool is_global() const { return m_value.m_symbolic->is_global();} - bool is_local() const { return m_value.m_symbolic->is_local();} - bool is_tex() const { return m_value.m_symbolic->is_tex();} - bool is_return_var() const { return m_is_return_var; } + bool is_literal() const { + return m_type == int_t || m_type == float_op_t || m_type == double_op_t || + m_type == unsigned_t; + } + bool is_shared() const { + if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } + bool is_const() const { return m_value.m_symbolic->is_const(); } + bool is_global() const { return m_value.m_symbolic->is_global(); } + bool is_local() const { return m_value.m_symbolic->is_local(); } + bool is_tex() const { return m_value.m_symbolic->is_tex(); } + bool is_return_var() const { return m_is_return_var; } - bool is_function_address() const - { - if( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->is_func_addr(); - } + bool is_function_address() const { + if (m_type != symbolic_t) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } - ptx_reg_t get_literal_value() const - { - ptx_reg_t result; - switch ( m_type ) { - case int_t: result.s64 = m_value.m_int; break; - case float_op_t: result.f32 = m_value.m_float; break; - case double_op_t: result.f64 = m_value.m_double; break; - case unsigned_t: result.u32 = m_value.m_unsigned; break; + ptx_reg_t get_literal_value() const { + ptx_reg_t result; + switch (m_type) { + case int_t: + result.s64 = m_value.m_int; + break; + case float_op_t: + result.f32 = m_value.m_float; + break; + case double_op_t: + result.f64 = m_value.m_double; + break; + case unsigned_t: + result.u32 = m_value.m_unsigned; + break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int;} - int get_addr_offset() const { return m_addr_offset;} - const symbol *get_symbol() const { return m_value.m_symbolic;} - void set_type( enum operand_type type ) - { - m_type = type; - } - enum operand_type get_type() const { - return m_type; - } - void set_neg_pred() - { - assert( m_valid ); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int; } + int get_addr_offset() const { return m_addr_offset; } + const symbol *get_symbol() const { return m_value.m_symbolic; } + void set_type(enum operand_type type) { m_type = type; } + enum operand_type get_type() const { return m_type; } + void set_neg_pred() { + assert(m_valid); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } - void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } + void set_addr_space(enum _memory_space_t set_value) { + m_addr_space = set_value; + } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { + m_double_operand_type = set_value; + } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { + m_const_mem_offset = set_value; + } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } -private: - gpgpu_context* gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol* m_symbolic; - const symbol** m_vector_symbolic; - } m_value; + private: + gpgpu_context *gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol *m_symbolic; + const symbol **m_vector_symbolic; + } m_value; - int m_addr_offset; + int m_addr_offset; - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; - unsigned get_uid(); + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) - { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry=entry; - is_exit=ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } + basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, + bool entry, bool ex) { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry = entry; + is_exit = ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } - ptx_instruction* ptx_begin; - ptx_instruction* ptx_end; - std::set<int> predecessor_ids; //indices of other basic blocks in m_basic_blocks array - std::set<int> successor_ids; - std::set<int> postdominator_ids; - std::set<int> dominator_ids; - std::set<int> Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; + ptx_instruction *ptx_begin; + ptx_instruction *ptx_end; + std::set<int> + predecessor_ids; // indices of other basic blocks in m_basic_blocks array + std::set<int> successor_ids; + std::set<int> postdominator_ids; + std::set<int> dominator_ids; + std::set<int> Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); - } + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != + B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction* source_inst; - class ptx_instruction* target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction *source_inst; + class ptx_instruction *target_inst; }; class ptx_instruction : public warp_inst_t { -public: - ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list<operand_info> &operands, - const operand_info &return_var, - const std::list<int> &options, - const std::list<int> &wmma_options, - const std::list<int> &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx); + public: + ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, + symbol *label, const std::list<operand_info> &operands, + const operand_info &return_var, const std::list<int> &options, + const std::list<int> &wmma_options, + const std::list<int> &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx); - void print_insn() const; - virtual void print_insn( FILE *fp ) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid;} - int get_opcode() const { return m_opcode;} - const char *get_opcode_cstr() const - { - if ( m_opcode != -1 ) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str();} - unsigned source_line() const { return m_source_line;} - unsigned get_num_operands() const { return m_operands.size();} - bool has_pred() const { return m_pred != NULL;} - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred;} - int get_pred_mod() const { return m_pred_mod;} - const char *get_source() const { return m_source.c_str();} + void print_insn() const; + virtual void print_insn(FILE *fp) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid; } + int get_opcode() const { return m_opcode; } + const char *get_opcode_cstr() const { + if (m_opcode != -1) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str(); } + unsigned source_line() const { return m_source_line; } + unsigned get_num_operands() const { return m_operands.size(); } + bool has_pred() const { return m_pred != NULL; } + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred; } + int get_pred_mod() const { return m_pred_mod; } + const char *get_source() const { return m_source.c_str(); } - typedef std::vector<operand_info>::const_iterator const_iterator; + typedef std::vector<operand_info>::const_iterator const_iterator; - const_iterator op_iter_begin() const - { - return m_operands.begin(); - } + const_iterator op_iter_begin() const { return m_operands.begin(); } - const_iterator op_iter_end() const - { - return m_operands.end(); - } + const_iterator op_iter_end() const { return m_operands.end(); } - const operand_info &dst() const - { - assert( !m_operands.empty() ); - return m_operands[0]; - } + const operand_info &dst() const { + assert(!m_operands.empty()); + return m_operands[0]; + } - const operand_info &func_addr() const - { - assert( !m_operands.empty() ); - if( !m_operands[0].is_return_var() ) { - return m_operands[0]; - } else { - assert( m_operands.size() >= 2 ); - return m_operands[1]; - } - } - - operand_info &dst() - { - assert( !m_operands.empty() ); + const operand_info &func_addr() const { + assert(!m_operands.empty()); + if (!m_operands[0].is_return_var()) { return m_operands[0]; - } - - const operand_info &src1() const - { - assert( m_operands.size() > 1 ); + } else { + assert(m_operands.size() >= 2); return m_operands[1]; - } + } + } - const operand_info &src2() const - { - assert( m_operands.size() > 2 ); - return m_operands[2]; - } + operand_info &dst() { + assert(!m_operands.empty()); + return m_operands[0]; + } - const operand_info &src3() const - { - assert( m_operands.size() > 3 ); - return m_operands[3]; - } - const operand_info &src4() const - { - assert( m_operands.size() > 4 ); - return m_operands[4]; - } - const operand_info &src5() const - { - assert( m_operands.size() > 5 ); - return m_operands[5]; - } - const operand_info &src6() const - { - assert( m_operands.size() > 6 ); - return m_operands[6]; - } - const operand_info &src7() const - { - assert( m_operands.size() > 7 ); - return m_operands[7]; - } - const operand_info &src8() const - { - assert( m_operands.size() > 8 ); - return m_operands[8]; - } + const operand_info &src1() const { + assert(m_operands.size() > 1); + return m_operands[1]; + } - const operand_info &operand_lookup( unsigned n ) const - { - assert( n < m_operands.size() ); - return m_operands[n]; - } - bool has_return() const - { - return m_return_var.is_valid(); - } + const operand_info &src2() const { + assert(m_operands.size() > 2); + return m_operands[2]; + } - memory_space_t get_space() const { return m_space_spec;} - unsigned get_vector() const { return m_vector_spec;} - unsigned get_atomic() const { return m_atomic_spec;} + const operand_info &src3() const { + assert(m_operands.size() > 3); + return m_operands[3]; + } + const operand_info &src4() const { + assert(m_operands.size() > 4); + return m_operands[4]; + } + const operand_info &src5() const { + assert(m_operands.size() > 5); + return m_operands[5]; + } + const operand_info &src6() const { + assert(m_operands.size() > 6); + return m_operands[6]; + } + const operand_info &src7() const { + assert(m_operands.size() > 7); + return m_operands[7]; + } + const operand_info &src8() const { + assert(m_operands.size() > 8); + return m_operands[8]; + } - int get_wmma_type() const { - return m_wmma_type; - } - int get_wmma_layout(int index) const { - return m_wmma_layout[index];//0->Matrix D,1->Matrix C - } - int get_type() const - { - assert( !m_scalar_type.empty() ); - return m_scalar_type.front(); - } + const operand_info &operand_lookup(unsigned n) const { + assert(n < m_operands.size()); + return m_operands[n]; + } + bool has_return() const { return m_return_var.is_valid(); } - int get_type2() const - { - assert( m_scalar_type.size()==2 ); - return m_scalar_type.back(); - } + memory_space_t get_space() const { return m_space_spec; } + unsigned get_vector() const { return m_vector_spec; } + unsigned get_atomic() const { return m_atomic_spec; } - void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t* get_bb() { return m_basic_block;} - void set_m_instr_mem_index(unsigned index) { - m_instr_mem_index = index; - } - void set_PC( addr_t PC ) - { - m_PC = PC; - } - addr_t get_PC() const - { - return m_PC; - } + int get_wmma_type() const { return m_wmma_type; } + int get_wmma_layout(int index) const { + return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C + } + int get_type() const { + assert(!m_scalar_type.empty()); + return m_scalar_type.front(); + } - unsigned get_m_instr_mem_index() { return m_instr_mem_index;} - unsigned get_cmpop() const { return m_compare_op;} - const symbol *get_label() const { return m_label;} - bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} - bool is_hi() const { return m_hi;} - bool is_lo() const { return m_lo;} - bool is_wide() const { return m_wide;} - bool is_uni() const { return m_uni;} - bool is_exit() const { return m_exit;} - bool is_abs() const { return m_abs;} - bool is_neg() const { return m_neg;} - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode;} - unsigned saturation_mode() const { return m_saturation_mode;} - unsigned dimension() const { return m_geom_spec;} - unsigned barrier_op() const {return m_barrier_op;} - unsigned shfl_op() const {return m_shfl_op;} - unsigned prmt_op() const {return m_prmt_op;} - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } + int get_type2() const { + assert(m_scalar_type.size() == 2); + return m_scalar_type.back(); + } - int membar_level() const { return m_membar_level; } + void assign_bb( + basic_block_t *basic_block) // assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t *get_bb() { return m_basic_block; } + void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } + void set_PC(addr_t PC) { m_PC = PC; } + addr_t get_PC() const { return m_PC; } - bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands - if( n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands - if( n==0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } + unsigned get_m_instr_mem_index() { return m_instr_mem_index; } + unsigned get_cmpop() const { return m_compare_op; } + const symbol *get_label() const { return m_label; } + bool is_label() const { + if (m_label) { + assert(m_opcode == -1); + return true; + } + return false; + } + bool is_hi() const { return m_hi; } + bool is_lo() const { return m_lo; } + bool is_wide() const { return m_wide; } + bool is_uni() const { return m_uni; } + bool is_exit() const { return m_exit; } + bool is_abs() const { return m_abs; } + bool is_neg() const { return m_neg; } + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode; } + unsigned saturation_mode() const { return m_saturation_mode; } + unsigned dimension() const { return m_geom_spec; } + unsigned barrier_op() const { return m_barrier_op; } + unsigned shfl_op() const { return m_shfl_op; } + unsigned prmt_op() const { return m_prmt_op; } + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } + int membar_level() const { return m_membar_level; } -private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); + bool has_memory_read() const { + if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || + m_opcode == MMA_LD_OP) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; op != op_iter_end(); op++, n++) { // process operands + if (n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; (op != op_iter_end() && n < 1); + op++, n++) { // process operands + if (n == 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; + private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector<operand_info> m_operands; - operand_info m_return_var; + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; - std::list<int> m_options; - std::list<int> m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector<operand_info> m_operands; + operand_info m_return_var; - std::list<int> m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; //index into m_instr_mem array - unsigned m_inst_size; // bytes + std::list<int> m_options; + std::list<int> m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; // if branch instruction, this evaluates to true for uniform + // branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context* gpgpu_ctx; + std::list<int> m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; // index into m_instr_mem array + unsigned m_inst_size; // bytes + + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context *gpgpu_ctx; }; class param_info { -public: - param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } - param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) - { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data( param_t v ) { - assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset( unsigned offset ) { m_offset = offset; } - unsigned get_offset() { assert(m_valid); return m_offset; } - std::string get_name() const { assert(m_valid); return m_name; } - int get_type() const { assert(m_valid); return m_type; } - param_t get_value() const { assert(m_value_set); return m_value; } - size_t get_size() const { assert(m_valid); return m_size; } - bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } -private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; + public: + param_info() { + m_valid = false; + m_value_set = false; + m_size = 0; + m_is_ptr = false; + } + param_info(std::string name, int type, size_t size, bool is_ptr, + memory_space_t ptr_space) { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data(param_t v) { + assert((!m_value_set) || + (m_value.size == v.size)); // if this fails concurrent kernel + // launches might execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset(unsigned offset) { m_offset = offset; } + unsigned get_offset() { + assert(m_valid); + return m_offset; + } + std::string get_name() const { + assert(m_valid); + return m_name; + } + int get_type() const { + assert(m_valid); + return m_type; + } + param_t get_value() const { + assert(m_value_set); + return m_value; + } + size_t get_size() const { + assert(m_valid); + return m_size; + } + bool is_ptr_shared() const { + assert(m_valid); + return (m_is_ptr and m_ptr_space == shared_space); + } + + private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { -public: - function_info(int entry_point, gpgpu_context* ctx ); - const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) - { - m_name = name; - } - void set_symtab(symbol_table *symtab ) - { - m_symtab = symtab; - } - std::string get_name() const - { - return m_name; - } - unsigned print_insn( unsigned pc, FILE * fp ) const; - std::string get_insn_str( unsigned pc ) const; - void add_inst( const std::list<ptx_instruction*> &instructions ) - { - m_instructions = instructions; - } - std::list<ptx_instruction*>::iterator find_next_real_instruction( std::list<ptx_instruction*>::iterator i ); - void create_basic_blocks( ); + public: + function_info(int entry_point, gpgpu_context *ctx); + const ptx_version &get_ptx_version() const { + return m_symtab->get_ptx_version(); + } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) { m_name = name; } + void set_symtab(symbol_table *symtab) { m_symtab = symtab; } + std::string get_name() const { return m_name; } + unsigned print_insn(unsigned pc, FILE *fp) const; + std::string get_insn_str(unsigned pc) const; + void add_inst(const std::list<ptx_instruction *> &instructions) { + m_instructions = instructions; + } + std::list<ptx_instruction *>::iterator find_next_real_instruction( + std::list<ptx_instruction *>::iterator i); + void create_basic_blocks(); - void print_basic_blocks(); + void print_basic_blocks(); - void print_basic_block_links(); - void print_basic_block_dot(); + void print_basic_block_links(); + void print_basic_block_dot(); - operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction - void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together - bool connect_break_targets(); //connecting break instructions with proper targets + operand_info *find_break_target( + ptx_instruction *p_break_insn); // find the target of a break instruction + void connect_basic_blocks(); // iterate across m_basic_blocks of function, + // connecting basic blocks together + bool + connect_break_targets(); // connecting break instructions with proper targets - //iterate across m_basic_blocks of function, - //finding dominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators( ); - void print_dominators(); - void find_idominators(); - void print_idominators(); + // iterate across m_basic_blocks of function, + // finding dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators(); + void print_dominators(); + void find_idominators(); + void print_idominators(); - //iterate across m_basic_blocks of function, - //finding postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators( ); - void print_postdominators(); + // iterate across m_basic_blocks of function, + // finding postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators(); + void print_postdominators(); - //iterate across m_basic_blocks of function, - //finding immediate postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators( ); - void print_ipostdominators(); - void do_pdom(); //function to call pdom analysis + // iterate across m_basic_blocks of function, + // finding immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators(); + void print_ipostdominators(); + void do_pdom(); // function to call pdom analysis - unsigned get_num_reconvergence_pairs(); + unsigned get_num_reconvergence_pairs(); - void get_reconvergence_pairs(gpgpu_recon_t* recon_points); + void get_reconvergence_pairs(gpgpu_recon_t *recon_points); - unsigned get_function_size() { return m_instructions.size();} + unsigned get_function_size() { return m_instructions.size(); } - void ptx_assemble(); - - unsigned ptx_get_inst_op( ptx_thread_info *thread ); - void add_param( const char *name, struct param_t value ) - { - m_kernel_params[ name ] = value; - } - void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); - void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); - void add_return_var( const symbol *rv ) - { - m_return_var_sym = rv; - } - void add_arg( const symbol *arg ) - { - assert( arg != NULL ); - m_args.push_back(arg); - } - void remove_args() - { - m_args.clear(); - } - unsigned num_args() const - { - return m_args.size(); - } - unsigned get_args_aligned_size(); + void ptx_assemble(); - const symbol* get_arg( unsigned n ) const - { - assert( n < m_args.size() ); - return m_args[n]; - } - bool has_return() const - { - return m_return_var_sym != NULL; - } - const symbol *get_return_var() const - { - return m_return_var_sym; - } - const ptx_instruction *get_instruction( unsigned PC ) const - { - unsigned index = PC - m_start_PC; - if( index < m_instr_mem_size ) - return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const - { - return m_start_PC; - } + unsigned ptx_get_inst_op(ptx_thread_info *thread); + void add_param(const char *name, struct param_t value) { + m_kernel_params[name] = value; + } + void add_param_name_type_size(unsigned index, std::string name, int type, + size_t size, bool ptr, memory_space_t space); + void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); + void add_return_var(const symbol *rv) { m_return_var_sym = rv; } + void add_arg(const symbol *arg) { + assert(arg != NULL); + m_args.push_back(arg); + } + void remove_args() { m_args.clear(); } + unsigned num_args() const { return m_args.size(); } + unsigned get_args_aligned_size(); - void finalize( memory_space *param_mem ); - void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); - void list_param( FILE *fout ) const; - void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; + const symbol *get_arg(unsigned n) const { + assert(n < m_args.size()); + return m_args[n]; + } + bool has_return() const { return m_return_var_sym != NULL; } + const symbol *get_return_var() const { return m_return_var_sym; } + const ptx_instruction *get_instruction(unsigned PC) const { + unsigned index = PC - m_start_PC; + if (index < m_instr_mem_size) return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const { return m_start_PC; } - const struct gpgpu_ptx_sim_info* get_kernel_info () const - { - assert (m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } + void finalize(memory_space *param_mem); + void param_to_shared(memory_space *shared_mem, symbol_table *symtab); + void list_param(FILE *fout) const; + void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, + dim3 blockDim); - const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10*get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() - { - return m_symtab; - } + const struct gpgpu_ptx_sim_info *get_kernel_info() const { + assert(m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } - unsigned local_mem_framesize() const - { - return m_local_mem_framesize; - } - void set_framesize( unsigned sz ) - { - m_local_mem_framesize = sz; - } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } //return pdom flag - void set_pdom() { pdom_done = true; } //set pdom flag + const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() { return m_symtab; } - void add_config_param( size_t size, unsigned alignment ){ - unsigned offset = 0; - if (m_param_configs.size()>0){ - unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; - //ensure offset matches alignment requirements - offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; - } - m_param_configs.push_back(std::pair<size_t,unsigned>(size, offset)); - } + unsigned local_mem_framesize() const { return m_local_mem_framesize; } + void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } // return pdom flag + void set_pdom() { pdom_done = true; } // set pdom flag + + void add_config_param(size_t size, unsigned alignment) { + unsigned offset = 0; + if (m_param_configs.size() > 0) { + unsigned offset_nom = + m_param_configs.back().first + m_param_configs.back().second; + // ensure offset matches alignment requirements + offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment + : offset_nom; + } + m_param_configs.push_back(std::pair<size_t, unsigned>(size, offset)); + } + + std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { + return m_param_configs[param_num]; + } - std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } + unsigned get_maxnt_id() { return maxnt_id; } + // backward pointer + class gpgpu_context *gpgpu_ctx; - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} - unsigned get_maxnt_id() { return maxnt_id;} - // backward pointer - class gpgpu_context* gpgpu_ctx; + private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; // flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map<std::string, param_t> m_kernel_params; + std::map<unsigned, param_info> m_ptx_kernel_param_info; + std::vector<std::pair<size_t, unsigned> > m_param_configs; + const symbol *m_return_var_sym; + std::vector<const symbol *> m_args; + std::list<ptx_instruction *> m_instructions; + std::vector<basic_block_t *> m_basic_blocks; + std::list<std::pair<unsigned, unsigned> > m_back_edges; + std::map<std::string, unsigned> labels; + unsigned num_reconvergence_pairs; -private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; //flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map<std::string,param_t> m_kernel_params; - std::map<unsigned,param_info> m_ptx_kernel_param_info; - std::vector< std::pair<size_t, unsigned> > m_param_configs; - const symbol *m_return_var_sym; - std::vector<const symbol*> m_args; - std::list<ptx_instruction*> m_instructions; - std::vector<basic_block_t*> m_basic_blocks; - std::list<std::pair<unsigned, unsigned> > m_back_edges; - std::map<std::string,unsigned> labels; - unsigned num_reconvergence_pairs; + // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along + // with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; - //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; + symbol_table *m_symtab; - symbol_table *m_symtab; + // parameter size for device kernels + int m_args_aligned_size; - //parameter size for device kernels - int m_args_aligned_size; - - addr_t m_n; // offset in m_instr_mem (used in do_pdom) + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { -public: - arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) - { - m_is_reg=false; - m_is_param=false; - m_param_value=NULL; - m_reg_value=ptx_reg_t(); - } - arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) - { - make_copy(another); - } - void make_copy( const arg_buffer_t &another ) - { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if( m_is_param ) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value,another.m_param_value,m_param_bytes); + public: + arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { + m_is_reg = false; + m_is_param = false; + m_param_value = NULL; + m_reg_value = ptx_reg_t(); + } + arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) + : m_src_op(ctx) { + make_copy(another); + } + void make_copy(const arg_buffer_t &another) { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if (m_is_param) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value, another.m_param_value, m_param_bytes); + } + } + void operator=(const arg_buffer_t &another) { make_copy(another); } + ~arg_buffer_t() { + if (m_is_param) free(m_param_value); + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + ptx_reg_t source_value) + : m_src_op(src_op) { + m_dst = dst_sym; + m_reg_value = ptx_reg_t(); + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_reg()); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t), 1); + // new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); + } + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + void *source_param_value_array, unsigned array_size) + : m_src_op(src_op) { + m_dst = dst_sym; + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_param_local()); + assert(dst_sym->get_size_in_bytes() == array_size); + switch (array_size) { + case 1: + m_reg_value.u8 = *(unsigned char *)source_param_value_array; + break; + case 2: + m_reg_value.u16 = *(unsigned short *)source_param_value_array; + break; + case 4: + m_reg_value.u32 = *(unsigned int *)source_param_value_array; + break; + case 8: + m_reg_value.u64 = *(unsigned long long *)source_param_value_array; + break; + default: + printf( + "GPGPU-Sim PTX: ERROR ** source param size does not match known " + "register sizes\n"); + break; } - } - void operator=( const arg_buffer_t &another ) - { - make_copy(another); - } - ~arg_buffer_t() - { - if( m_is_param ) - free(m_param_value); - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) - { - m_dst = dst_sym; - m_reg_value=ptx_reg_t(); - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_reg() ); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t),1); - //new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); - } - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) - { - m_dst = dst_sym; - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_param_local() ); - assert( dst_sym->get_size_in_bytes() == array_size ); - switch( array_size ) { - case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; - case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; - case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; - case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; - default: - printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); - break; - } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size,1); - m_param_bytes = array_size; - memcpy(m_param_value,source_param_value_array,array_size); - } - } + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size, 1); + m_param_bytes = array_size; + memcpy(m_param_value, source_param_value_array, array_size); + } + } - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const - { - assert(m_is_reg); - return m_reg_value; - } + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const { + assert(m_is_reg); + return m_reg_value; + } - const void *get_param_buffer() const - { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const - { - assert(m_is_param); - return m_param_bytes; - } + const void *get_param_buffer() const { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const { + assert(m_is_param); + return m_param_bytes; + } - const symbol *get_dst() const { return m_dst; } + const symbol *get_dst() const { return m_dst; } -private: - // destination of copy - const symbol *m_dst; + private: + // destination of copy + const symbol *m_dst; - // source operand - operand_info m_src_op; + // source operand + operand_info m_src_op; - // source information - bool m_is_reg; - bool m_is_param; + // source information + bool m_is_reg; + bool m_is_param; - // source is register - ptx_reg_t m_reg_value; + // source is register + ptx_reg_t m_reg_value; - // source is param - void *m_param_value; - unsigned m_param_bytes; + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list< arg_buffer_t > arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ); -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); - +typedef std::list<arg_buffer_t> arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param); +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values); +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); struct textureInfo { - unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits,Ty_numbits; //log2(T) - unsigned int texel_size_numbits; //log2(texel_size) + unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx, + Ty; // tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits, Ty_numbits; // log2(T) + unsigned int texel_size_numbits; // log2(texel_size) }; -extern std::map<std::string,symbol_table*> g_sym_name_to_symbol_table; - +extern std::map<std::string, symbol_table *> g_sym_name_to_symbol_table; -void gpgpu_ptx_assemble( std::string kname, void *kinfo ); +void gpgpu_ptx_assemble(std::string kname, void *kinfo); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size( void *kernel_impl ); -unsigned ptx_kernel_nregs( void *kernel_impl ); +unsigned ptx_kernel_shmem_size(void *kernel_impl); +unsigned ptx_kernel_nregs(void *kernel_impl); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 33bcf45..372bda4 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,522 +7,576 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include "ptx_ir.h" -#include "cuda-sim.h" -#include "ptx_parser.h" -#include <unistd.h> #include <dirent.h> +#include <unistd.h> #include <fstream> #include <sstream> #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" +#include "ptx_ir.h" +#include "ptx_parser.h" /// extern prototypes -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char*, yyscan_t scanner); +extern int ptx__scan_string(const char *, yyscan_t scanner); -extern std::map<unsigned,const char*> get_duplicate(); +extern std::map<unsigned, const char *> get_duplicate(); -typedef void * yyscan_t; -extern int ptxinfo_lex_init(yyscan_t* scanner); -extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); +typedef void *yyscan_t; +extern int ptxinfo_lex_init(yyscan_t *scanner); +extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} - -void gpgpu_context::ptx_reg_options(option_parser_t opp) -{ - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, - "saves ptx files embedded in binary as <n>.ptx", - "0"); - option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", - "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +bool ptxinfo_data::keep_intermediate_files() { + return g_keep_intermediate_files; } -void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) -{ - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); - char *s = strdup(p); - char *t = s; - unsigned n=1; - while ( *t != '\0' ) { - char *u = t; - while ( (*u != '\n') && (*u != '\0') ) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); - char pc[64]; - if( pI && pI->get_PC() ) - snprintf(pc,64,"%4u", pI->get_PC() ); - else - snprintf(pc,64," "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); - if ( last ) break; - t = u+1; - n++; - } - free(s); - fflush(stdout); +void gpgpu_context::ptx_reg_options(option_parser_t opp) { + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, + &g_save_embedded_ptx, + "saves ptx files embedded in binary as <n>.ptx", "0"); + option_parser_register(opp, "-keep", OPT_BOOL, + &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when " + "interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, + &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register " + "usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) -{ - - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); +void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, + const char *filename) { + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); + char *s = strdup(p); + char *t = s; + unsigned n = 1; + while (*t != '\0') { + char *u = t; + while ((*u != '\n') && (*u != '\0')) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); + char pc[64]; + if (pI && pI->get_PC()) + snprintf(pc, 64, "%4u", pI->get_PC()); + else + snprintf(pc, 64, " "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); + if (last) break; + t = u + 1; + n++; + } + free(s); + fflush(stdout); +} - char fname_ptxplus[1024]; - snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); - int fd4=mkstemp(fname_ptxplus); - close(fd4); +char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptxfilename, const std::string elffilename, + const std::string sassfilename) { + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), - sassfilename.c_str(), - elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); - result = system(commandline); - if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} + char fname_ptxplus[1024]; + snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); + int fd4 = mkstemp(fname_ptxplus); + close(fd4); + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, + "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" + "cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", + commandline); + result = system(commandline); + if (result) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", + commandline); + exit(1); + } - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while(getline(fileStream,line)) { - text += (line + "\n"); - } - fileStream.close(); + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while (getline(fileStream, line)) { + text += (line + "\n"); + } + fileStream.close(); - char* ptxplus_str = new char [strlen(text.c_str())+1]; - strcpy(ptxplus_str, text.c_str()); + char *ptxplus_str = new char[strlen(text.c_str()) + 1]; + strcpy(ptxplus_str, text.c_str()); - if (!m_ptx_save_converted_ptxplus){ - char rm_commandline[1024]; + if (!m_ptx_save_converted_ptxplus) { + char rm_commandline[1024]; - snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); + snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); - int rm_result = system(rm_commandline); - if( rm_result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); - exit(1); - } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", + rm_commandline); + int rm_result = system(rm_commandline); + if (rm_result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", + rm_result); + exit(1); + } + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - return ptxplus_str; + return ptxplus_str; } +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( + const char *p, unsigned source_num) { + char buf[1024]; + snprintf(buf, 1024, "_%u.ptx", source_num); + if (g_save_embedded_ptx) { + FILE *fp = fopen(buf, "w"); + fprintf(fp, "%s", p); + fclose(fp); + } + symbol_table *symtab = init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse(ptx_parser->scanner, ptx_parser); + if (errors) { + char fname[1024]; + snprintf(fname, 1024, "_ptx_errors_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + printf( + "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " + ".ptx to \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) -{ - char buf[1024]; - snprintf(buf,1024,"_%u.ptx", source_num ); - if( g_save_embedded_ptx ) { - FILE *fp = fopen(buf,"w"); - fprintf(fp,"%s",p); - fclose(fp); - } - symbol_table *symtab=init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse (ptx_parser->scanner, ptx_parser); - if ( errors ) { - char fname[1024]; - snprintf(fname,1024,"_ptx_errors_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p ); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); - - if ( g_debug_execution >= 100 ) - print_ptx_file(p,source_num,buf); + if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); - return symtab; + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) -{ - symbol_table *symtab=init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( + const char *filename) { + symbol_table *symtab = init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; - // change the name of the ptx file to _temp_ptx - snprintf(commandline,1024,"mv %s %s",fname2,tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); - exit(1); - } + // change the name of the ptx file to _temp_ptx + snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", + fname2, tempfile); + exit(1); + } - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile,"r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile, "r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); - FILE *ptxdest = fopen(fname2,"w"); - std::map<unsigned,const char*> duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; + FILE *ptxdest = fopen(fname2, "w"); + std::map<unsigned, const char *> duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; - // recreate the ptx file without duplications - for ( std::map<unsigned,const char*>::iterator iter = duplicate.begin(); - iter != duplicate.end(); - iter++){ - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } - - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } + // recreate the ptx file without duplications + for (std::map<unsigned, const char *>::iterator iter = duplicate.begin(); + iter != duplicate.end(); iter++) { + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; - //find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') bracket++; - else if (*(lineptr + offset) == '}') bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - - //find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); - } + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + // find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') + bracket++; + else if (*(lineptr + offset) == '}') + bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline,1024,"rm -f %s",tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } -} + // find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", + iter->second); + } + + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline, 1024, "rm -f %s", tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } +} -//we need the application name here too. -char* get_app_binary_name(){ - char exe_path[1025]; - char *self_exe_path; +// we need the application name here too. +char *get_app_binary_name() { + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - //AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + // AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; - char *token = strtok(exe_path, "/"); - while(token !=NULL){ - self_exe_path = token; - token = strtok(NULL,"/"); - } + char *token = strtok(exe_path, "/"); + while (token != NULL) { + self_exe_path = token; + token = strtok(NULL, "/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) -{ - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0]=0; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } +void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, + unsigned sm_version) { + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0] = 0; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); + else + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); + snprintf( + buff, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) -{ - //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name=get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index=1; index <= no_of_ptx; index++){ - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); +void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, + unsigned source_num, + unsigned sm_version, + int no_of_ptx) { + // do ptxas for individual files instead of one big embedded ptx. This + // prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name = get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], + final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index = 1; index <= no_of_ptx; index++) { + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); - if (system(commandline) !=0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); + if (system(commandline) != 0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; #if CUDART_VERSION >= 3000 - if ( g_occupancy_sm_number == 0 ) { - fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences." ); - exit(1); + if (g_occupancy_sm_number == 0) { + fprintf( + stderr, + "gpgpusim.config must specify the sm version for the GPU that you " + "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used " + "to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences."); + exit(1); } - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + g_occupancy_sm_number); #endif - snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + snprintf(commandline, 1024, + "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if( result != 0 ) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo,"r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + if (result != 0) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo, "r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); - fix_duplicate_errors(fname2); - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - } + fix_duplicate_errors(fname2); + snprintf(commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", + commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } + } - //TODO: duplicate code! move it into a function so that it can be reused! - if(no_of_ptx==0) { - //For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); + // TODO: duplicate code! move it into a function so that it can be reused! + if (no_of_ptx == 0) { + // For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p_for_info); - fclose(ptxfile); + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p_for_info); + fclose(ptxfile); - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - //char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + // char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; - #if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - #endif +#if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); + else + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + sm_version); +#endif - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + snprintf( + commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); } + } - //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. - if(no_of_ptx>0){ - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); - snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3)!=0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n",commandline3); - exit(1); - } - } + // Now that we got resource usage per kernel in a ptx file, we dump all into + // one file and pass it to rest of the code as usual. + if (no_of_ptx > 0) { + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); + snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3) != 0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n", commandline3); + exit(1); + } + } - if(no_of_ptx>0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; - else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); + if (no_of_ptx > 0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; + else + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); - snprintf(commandline,1024,"rm -f *info"); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); - } - if( ! g_save_embedded_ptx ) { - if(no_of_ptx>0) - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); - else - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); - } + snprintf(commandline, 1024, "rm -f *info"); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if (!g_save_embedded_ptx) { + if (no_of_ptx > 0) + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + final_tempfile_ptxinfo); + else + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); } + } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index d8f1cbc..ff5d7ed 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef PTX_LOADER_H_INCLUDED #define PTX_LOADER_H_INCLUDED @@ -31,22 +32,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void * yyscan_t; -class ptxinfo_data{ - public: - ptxinfo_data(gpgpu_context* ctx) { - gpgpu_ctx = ctx; - } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char *g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); +typedef void* yyscan_t; +class ptxinfo_data { + public: + ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char* g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptx_str, const std::string sass_str, + const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a4f4a0c..3ae8de3 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,1017 +7,991 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "ptx_ir.h" #include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" +typedef void *yyscan_t; #include <stdarg.h> +#include "ptx.tab.h" -extern int ptx_get_lineno (yyscan_t yyscanner ); -extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern FILE *ptx_get_in (yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_get_lineno(yyscan_t yyscanner); +extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern FILE *ptx_get_in(yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) -{ - g_shader_core_config=warp_size; +void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { + g_shader_core_config = warp_size; } +#define PTX_PARSE_DPRINTF(...) \ + if (g_debug_ir_generation) { \ + printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } -#define PTX_PARSE_DPRINTF(...) \ - if( g_debug_ir_generation ) { \ - printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } - -static std::map<unsigned,std::string> g_ptx_token_decode; +static std::map<unsigned, std::string> g_ptx_token_decode; -const char *decode_token( int type ) -{ - return g_ptx_token_decode[type].c_str(); -} +const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } -void ptx_recognizer::read_parser_environment_variables() -{ - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - int debug_execution=0; - sscanf(dbg_level,"%d", &debug_execution); - if ( debug_execution >= 30 ) - g_debug_ir_generation=true; - } +void ptx_recognizer::read_parser_environment_variables() { + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + int debug_execution = 0; + sscanf(dbg_level, "%d", &debug_execution); + if (debug_execution >= 30) g_debug_ir_generation = true; + } } -void ptx_recognizer::init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; +void ptx_recognizer::init_directive_state() { + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_opcode = -1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; } -void ptx_recognizer::init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); +void ptx_recognizer::init_instruction_state() { + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); } -symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) -{ - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; - } - /*else { - g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - }*/ +symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = + new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = + g_global_allfiles_symbol_table; + } + /*else { + g_global_symbol_table = g_current_symbol_table = new + symbol_table("global",0,g_global_allfiles_symbol_table); + }*/ -#define DEF(X,Y) g_ptx_token_decode[X] = Y; +#define DEF(X, Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; } - -void ptx_recognizer::start_function( int entry_point ) -{ - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index=0; +void ptx_recognizer::start_function(int entry_point) { + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index = 0; } -void ptx_recognizer::add_function_name( const char *name ) -{ - PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); - bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); - if( g_add_identifier_cached__identifier ) { - add_identifier( g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident ); - free( g_add_identifier_cached__identifier ); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var( g_last_symbol ); - init_directive_state(); - } - if( prior_decl ) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); +void ptx_recognizer::add_function_name(const char *name) { + PTX_PARSE_DPRINTF( + "add_function_name %s %s", name, + ((g_entry_point == 1) ? "(entrypoint)" + : ((g_entry_point == 2) ? "(extern)" : ""))); + bool prior_decl = g_global_symbol_table->add_function_decl( + name, g_entry_point, &g_func_info, &g_current_symbol_table); + if (g_add_identifier_cached__identifier) { + add_identifier(g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident); + free(g_add_identifier_cached__identifier); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var(g_last_symbol); + init_directive_state(); + } + if (prior_decl) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, + ptx_get_lineno(scanner)); } -//Jin: handle instruction group for cdp +// Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); + PTX_PARSE_DPRINTF("end_instruction_group"); + g_current_symbol_table = g_current_symbol_table->end_inst_group(); } -void ptx_recognizer::add_directive() -{ - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); +void ptx_recognizer::add_directive() { + PTX_PARSE_DPRINTF("add_directive"); + init_directive_state(); } -#define mymax(a,b) ((a)>(b)?(a):(b)) +#define mymax(a, b) ((a) > (b) ? (a) : (b)) -void ptx_recognizer::end_function() -{ - PTX_PARSE_DPRINTF("end_function"); +void ptx_recognizer::end_function() { + PTX_PARSE_DPRINTF("end_function"); - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax( g_max_regs_per_thread, (g_current_symbol_table->next_reg_num()-1)); - g_func_info->add_inst( g_instructions ); - g_instructions.clear(); - gpgpu_ptx_assemble( g_func_info->get_name(), g_func_info ); - g_current_symbol_table = g_global_symbol_table; + init_directive_state(); + init_instruction_state(); + g_max_regs_per_thread = mymax(g_max_regs_per_thread, + (g_current_symbol_table->next_reg_num() - 1)); + g_func_info->add_inst(g_instructions); + g_instructions.clear(); + gpgpu_ptx_assemble(g_func_info->get_name(), g_func_info); + g_current_symbol_table = g_global_symbol_table; - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), g_func_info->get_start_PC()); + PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), + g_func_info->get_start_PC()); } -#define parse_error(msg, ...) parse_error_impl(__FILE__,__LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond,msg, ...) parse_assert_impl((cond),__FILE__,__LINE__, msg, ##__VA_ARGS__) +#define parse_error(msg, ...) \ + parse_error_impl(__FILE__, __LINE__, msg, ##__VA_ARGS__) +#define parse_assert(cond, msg, ...) \ + parse_assert_impl((cond), __FILE__, __LINE__, msg, ##__VA_ARGS__) -void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); +void ptx_recognizer::parse_error_impl(const char *file, unsigned line, + const char *msg, ...) { + va_list ap; + char buf[1024]; + va_start(ap, msg); + vsnprintf(buf, 1024, msg, ap); + va_end(ap); - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); + g_error_detected = 1; + printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, + ptx_get_lineno(scanner), buf, file, line); + ptx_error(scanner, NULL); + abort(); + exit(1); } -void ptx_recognizer::parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); +void ptx_recognizer::parse_assert_impl(int test_value, const char *file, + unsigned line, const char *msg, ...) { + va_list ap; + char buf[1024]; + va_start(ap, msg); + vsnprintf(buf, 1024, msg, ap); + va_end(ap); - if ( test_value == 0 ) - parse_error_impl(file,line, msg); + if (test_value == 0) parse_error_impl(file, line, msg); } - - -void ptx_recognizer::set_return() -{ - parse_assert( (g_opcode == CALL_OP || g_opcode == CALLP_OP), "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); +void ptx_recognizer::set_return() { + parse_assert((g_opcode == CALL_OP || g_opcode == CALLP_OP), + "only call can have return value"); + g_operands.front().set_return(); + g_return_var = g_operands.front(); } - -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( const char *filename, unsigned linenumber ) -{ - std::map<std::string,std::map<unsigned,const ptx_instruction*> >::iterator f=g_inst_lookup.find(filename); - if( f == g_inst_lookup.end() ) - return NULL; - std::map<unsigned,const ptx_instruction*>::iterator l=f->second.find(linenumber); - if( l == f->second.end() ) - return NULL; - return l->second; +const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( + const char *filename, unsigned linenumber) { + std::map<std::string, std::map<unsigned, const ptx_instruction *> >::iterator + f = g_inst_lookup.find(filename); + if (f == g_inst_lookup.end()) return NULL; + std::map<unsigned, const ptx_instruction *>::iterator l = + f->second.find(linenumber); + if (l == f->second.end()) return NULL; + return l->second; } -void ptx_recognizer::add_instruction() -{ - PTX_PARSE_DPRINTF("add_instruction: %s", ((g_opcode>0)?g_opcode_string[g_opcode]:"<label>") ); - assert( g_shader_core_config != 0 ); - ptx_instruction *i = new ptx_instruction( g_opcode, - g_pred, - g_neg_pred, - g_pred_mod, - g_label, - g_operands, - g_return_var, - g_options, - g_wmma_options, - g_scalar_type, - g_space_spec, - gpgpu_ctx->g_filename, - ptx_get_lineno(scanner), - linebuf, - g_shader_core_config, - gpgpu_ctx ); - g_instructions.push_back(i); - g_inst_lookup[gpgpu_ctx->g_filename][ptx_get_lineno(scanner)] = i; - init_instruction_state(); +void ptx_recognizer::add_instruction() { + PTX_PARSE_DPRINTF("add_instruction: %s", + ((g_opcode > 0) ? g_opcode_string[g_opcode] : "<label>")); + assert(g_shader_core_config != 0); + ptx_instruction *i = new ptx_instruction( + g_opcode, g_pred, g_neg_pred, g_pred_mod, g_label, g_operands, + g_return_var, g_options, g_wmma_options, g_scalar_type, g_space_spec, + gpgpu_ctx->g_filename, ptx_get_lineno(scanner), linebuf, + g_shader_core_config, gpgpu_ctx); + g_instructions.push_back(i); + g_inst_lookup[gpgpu_ctx->g_filename][ptx_get_lineno(scanner)] = i; + init_instruction_state(); } -void ptx_recognizer::add_variables() -{ - PTX_PARSE_DPRINTF("add_variables"); - if ( !g_operands.empty() ) { - assert( g_last_symbol != NULL ); - g_last_symbol->add_initializer(g_operands); - } - init_directive_state(); +void ptx_recognizer::add_variables() { + PTX_PARSE_DPRINTF("add_variables"); + if (!g_operands.empty()) { + assert(g_last_symbol != NULL); + g_last_symbol->add_initializer(g_operands); + } + init_directive_state(); } -void ptx_recognizer::set_variable_type() -{ - PTX_PARSE_DPRINTF("set_variable_type space_spec=%s scalar_type_spec=%s", - g_ptx_token_decode[g_space_spec.get_type()].c_str(), - g_ptx_token_decode[g_scalar_type_spec].c_str() ); - parse_assert( g_space_spec != undefined_space, "variable has no space specification" ); - parse_assert( g_scalar_type_spec != -1, "variable has no type information" ); // need to extend for structs? - g_var_type = g_current_symbol_table->add_type( g_space_spec, - g_scalar_type_spec, - g_vector_spec, - g_alignment_spec, - g_extern_spec ); +void ptx_recognizer::set_variable_type() { + PTX_PARSE_DPRINTF("set_variable_type space_spec=%s scalar_type_spec=%s", + g_ptx_token_decode[g_space_spec.get_type()].c_str(), + g_ptx_token_decode[g_scalar_type_spec].c_str()); + parse_assert(g_space_spec != undefined_space, + "variable has no space specification"); + parse_assert( + g_scalar_type_spec != -1, + "variable has no type information"); // need to extend for structs? + g_var_type = g_current_symbol_table->add_type( + g_space_spec, g_scalar_type_spec, g_vector_spec, g_alignment_spec, + g_extern_spec); } -bool ptx_recognizer::check_for_duplicates( const char *identifier ) -{ - const symbol *s = g_current_symbol_table->lookup(identifier); - return ( s != NULL ); +bool ptx_recognizer::check_for_duplicates(const char *identifier) { + const symbol *s = g_current_symbol_table->lookup(identifier); + return (s != NULL); } -// Returns padding that needs to be inserted ahead of address to make it aligned to min(size, maxalign) +// Returns padding that needs to be inserted ahead of address to make it aligned +// to min(size, maxalign) /* * @param address the address in bytes * @param size the size of the memory to be allocated in bytes - * @param maximum alignment in bytes. i.e. if size is too big then align to this instead + * @param maximum alignment in bytes. i.e. if size is too big then align to this + * instead */ -int pad_address (new_addr_type address, unsigned size, unsigned maxalign) { - assert(size >= 0); - assert(maxalign > 0); - int alignto = maxalign; - if (size < maxalign && - (size & (size-1)) == 0) { //size is a power of 2 - alignto = size; - } - return alignto ? ((alignto - (address % alignto)) % alignto) : 0; +int pad_address(new_addr_type address, unsigned size, unsigned maxalign) { + assert(size >= 0); + assert(maxalign > 0); + int alignto = maxalign; + if (size < maxalign && (size & (size - 1)) == 0) { // size is a power of 2 + alignto = size; + } + return alignto ? ((alignto - (address % alignto)) % alignto) : 0; } -void ptx_recognizer::add_identifier( const char *identifier, int array_dim, unsigned array_ident ) -{ - if(array_ident==ARRAY_IDENTIFIER){ - g_size *= array_dim; - } - if( g_func_decl && (g_func_info == NULL) ) { - // return variable decl... - assert( g_add_identifier_cached__identifier == NULL ); - g_add_identifier_cached__identifier = strdup(identifier); - g_add_identifier_cached__array_dim = array_dim; - g_add_identifier_cached__array_ident = array_ident; - return; - } - PTX_PARSE_DPRINTF("add_identifier \"%s\" (%u)", identifier, g_ident_add_uid); - g_ident_add_uid++; - type_info *type = g_var_type; - type_info_key ti = type->get_key(); - int basic_type; - int regnum; - size_t num_bits; - unsigned addr_pad; - new_addr_type addr; - ti.type_decode(num_bits,basic_type); +void ptx_recognizer::add_identifier(const char *identifier, int array_dim, + unsigned array_ident) { + if (array_ident == ARRAY_IDENTIFIER) { + g_size *= array_dim; + } + if (g_func_decl && (g_func_info == NULL)) { + // return variable decl... + assert(g_add_identifier_cached__identifier == NULL); + g_add_identifier_cached__identifier = strdup(identifier); + g_add_identifier_cached__array_dim = array_dim; + g_add_identifier_cached__array_ident = array_ident; + return; + } + PTX_PARSE_DPRINTF("add_identifier \"%s\" (%u)", identifier, g_ident_add_uid); + g_ident_add_uid++; + type_info *type = g_var_type; + type_info_key ti = type->get_key(); + int basic_type; + int regnum; + size_t num_bits; + unsigned addr_pad; + new_addr_type addr; + ti.type_decode(num_bits, basic_type); - bool duplicates = check_for_duplicates( identifier ); - if( duplicates ) { - symbol *s = g_current_symbol_table->lookup(identifier); - g_last_symbol = s; - if( g_func_decl ) - return; - std::string msg = std::string(identifier) + " was declared previous at " + s->decl_location() + " skipping new declaration"; - printf("GPGPU-Sim PTX: Warning %s\n", msg.c_str()); - return; - } + bool duplicates = check_for_duplicates(identifier); + if (duplicates) { + symbol *s = g_current_symbol_table->lookup(identifier); + g_last_symbol = s; + if (g_func_decl) return; + std::string msg = std::string(identifier) + " was declared previous at " + + s->decl_location() + " skipping new declaration"; + printf("GPGPU-Sim PTX: Warning %s\n", msg.c_str()); + return; + } - assert( g_var_type != NULL ); - switch ( array_ident ) { - case ARRAY_IDENTIFIER: - type = g_current_symbol_table->get_array_type(type,array_dim); + assert(g_var_type != NULL); + switch (array_ident) { + case ARRAY_IDENTIFIER: + type = g_current_symbol_table->get_array_type(type, array_dim); num_bits = array_dim * num_bits; break; - case ARRAY_IDENTIFIER_NO_DIM: - type = g_current_symbol_table->get_array_type(type,(unsigned)-1); + case ARRAY_IDENTIFIER_NO_DIM: + type = g_current_symbol_table->get_array_type(type, (unsigned)-1); num_bits = 0; break; - default: + default: break; - } - g_last_symbol = g_current_symbol_table->add_variable(identifier,type,num_bits/8,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); - switch ( ti.get_memory_space().get_type() ) { - case reg_space: { + } + g_last_symbol = g_current_symbol_table->add_variable( + identifier, type, num_bits / 8, gpgpu_ctx->g_filename, + ptx_get_lineno(scanner)); + switch (ti.get_memory_space().get_type()) { + case reg_space: { regnum = g_current_symbol_table->next_reg_num(); int arch_regnum = -1; for (int d = 0; d < strlen(identifier); d++) { - if (isdigit(identifier[d])) { - sscanf(identifier + d, "%d", &arch_regnum); - break; - } + if (isdigit(identifier[d])) { + sscanf(identifier + d, "%d", &arch_regnum); + break; + } } if (strcmp(identifier, "%sp") == 0) { - arch_regnum = 0; + arch_regnum = 0; } g_last_symbol->set_regno(regnum, arch_regnum); - } break; - case shared_space: - printf("GPGPU-Sim PTX: allocating shared region for \"%s\" ", - identifier); + } break; + case shared_space: + printf("GPGPU-Sim PTX: allocating shared region for \"%s\" ", identifier); fflush(stdout); - assert( (num_bits%8) == 0 ); + assert((num_bits % 8) == 0); addr = g_current_symbol_table->get_shared_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx (shared memory space)\n", - addr+addr_pad, - addr+addr_pad + num_bits/8); - fflush(stdout); - g_last_symbol->set_address( addr+addr_pad ); - g_current_symbol_table->alloc_shared( num_bits/8 + addr_pad ); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx (shared memory space)\n", addr + addr_pad, + addr + addr_pad + num_bits / 8); + fflush(stdout); + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_shared(num_bits / 8 + addr_pad); + break; + case sstarr_space: + printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", identifier); + fflush(stdout); + assert((num_bits % 8) == 0); + addr = g_current_symbol_table->get_sstarr_next(); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx (sstarr memory space)\n", addr + addr_pad, + addr + addr_pad + num_bits / 8); + fflush(stdout); + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_sstarr(num_bits / 8 + addr_pad); break; - case sstarr_space: - printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", - identifier); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr = g_current_symbol_table->get_sstarr_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx (sstarr memory space)\n", - addr+addr_pad, - addr+addr_pad + num_bits/8); - fflush(stdout); - g_last_symbol->set_address( addr+addr_pad ); - g_current_symbol_table->alloc_sstarr( num_bits/8 + addr_pad ); - break; - case const_space: - if( array_ident == ARRAY_IDENTIFIER_NO_DIM ) { - printf("GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" (need size information)\n", identifier ); + case const_space: + if (array_ident == ARRAY_IDENTIFIER_NO_DIM) { + printf( + "GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" " + "(need size information)\n", + identifier); } else { - printf("GPGPU-Sim PTX: allocating constant region for \"%s\" ", - identifier); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr = g_current_symbol_table->get_global_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx (global memory space) %u\n", - addr+addr_pad, - addr+addr_pad + num_bits/8, - g_const_alloc++); - fflush(stdout); - g_last_symbol->set_address( addr + addr_pad ); - g_current_symbol_table->alloc_global( num_bits/8 + addr_pad ); + printf("GPGPU-Sim PTX: allocating constant region for \"%s\" ", + identifier); + fflush(stdout); + assert((num_bits % 8) == 0); + addr = g_current_symbol_table->get_global_next(); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx (global memory space) %u\n", + addr + addr_pad, addr + addr_pad + num_bits / 8, + g_const_alloc++); + fflush(stdout); + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_global(num_bits / 8 + addr_pad); } - if( g_current_symbol_table == g_global_symbol_table ) { - gpgpu_ctx->func_sim->g_constants.insert( identifier ); + if (g_current_symbol_table == g_global_symbol_table) { + gpgpu_ctx->func_sim->g_constants.insert(identifier); } - assert( g_current_symbol_table != NULL ); - g_sym_name_to_symbol_table[ identifier ] = g_current_symbol_table; + assert(g_current_symbol_table != NULL); + g_sym_name_to_symbol_table[identifier] = g_current_symbol_table; break; - case global_space: - printf("GPGPU-Sim PTX: allocating global region for \"%s\" ", - identifier); + case global_space: + printf("GPGPU-Sim PTX: allocating global region for \"%s\" ", identifier); fflush(stdout); - assert( (num_bits%8) == 0 ); + assert((num_bits % 8) == 0); addr = g_current_symbol_table->get_global_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx (global memory space)\n", - addr+addr_pad, - addr+addr_pad + num_bits/8); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx (global memory space)\n", addr + addr_pad, + addr + addr_pad + num_bits / 8); fflush(stdout); - g_last_symbol->set_address( addr+addr_pad ); - g_current_symbol_table->alloc_global( num_bits/8 + addr_pad ); - gpgpu_ctx->func_sim->g_globals.insert( identifier ); - assert( g_current_symbol_table != NULL ); - g_sym_name_to_symbol_table[ identifier ] = g_current_symbol_table; + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_global(num_bits / 8 + addr_pad); + gpgpu_ctx->func_sim->g_globals.insert(identifier); + assert(g_current_symbol_table != NULL); + g_sym_name_to_symbol_table[identifier] = g_current_symbol_table; break; - case local_space: - if( g_func_info == NULL ) { - printf("GPGPU-Sim PTX: allocating local region for \"%s\" ", identifier); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr = g_current_symbol_table->get_local_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx (local memory space)\n", - addr+addr_pad, - addr+addr_pad + num_bits/8); - fflush(stdout); - g_last_symbol->set_address( addr+addr_pad); - g_current_symbol_table->alloc_local( num_bits/8 + addr_pad); - } else { - printf("GPGPU-Sim PTX: allocating stack frame region for .local \"%s\" ", + case local_space: + if (g_func_info == NULL) { + printf("GPGPU-Sim PTX: allocating local region for \"%s\" ", identifier); fflush(stdout); - assert( (num_bits%8) == 0 ); + assert((num_bits % 8) == 0); addr = g_current_symbol_table->get_local_next(); - addr_pad = pad_address(addr, num_bits/8, 128); - printf("from 0x%llx to 0x%llx\n", - addr+addr_pad, - addr+addr_pad + num_bits/8); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx (local memory space)\n", addr + addr_pad, + addr + addr_pad + num_bits / 8); fflush(stdout); - g_last_symbol->set_address( addr+addr_pad ); - g_current_symbol_table->alloc_local( num_bits/8 + addr_pad); - g_func_info->set_framesize( g_current_symbol_table->get_local_next() ); + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_local(num_bits / 8 + addr_pad); + } else { + printf( + "GPGPU-Sim PTX: allocating stack frame region for .local \"%s\" ", + identifier); + fflush(stdout); + assert((num_bits % 8) == 0); + addr = g_current_symbol_table->get_local_next(); + addr_pad = pad_address(addr, num_bits / 8, 128); + printf("from 0x%llx to 0x%llx\n", addr + addr_pad, + addr + addr_pad + num_bits / 8); + fflush(stdout); + g_last_symbol->set_address(addr + addr_pad); + g_current_symbol_table->alloc_local(num_bits / 8 + addr_pad); + g_func_info->set_framesize(g_current_symbol_table->get_local_next()); } break; - case tex_space: + case tex_space: printf("GPGPU-Sim PTX: encountered texture directive %s.\n", identifier); break; - case param_space_local: - printf("GPGPU-Sim PTX: allocating stack frame region for .param \"%s\" from 0x%x to 0x%lx\n", - identifier, - g_current_symbol_table->get_local_next(), - g_current_symbol_table->get_local_next() + num_bits/8 ); + case param_space_local: + printf( + "GPGPU-Sim PTX: allocating stack frame region for .param \"%s\" from " + "0x%x to 0x%lx\n", + identifier, g_current_symbol_table->get_local_next(), + g_current_symbol_table->get_local_next() + num_bits / 8); fflush(stdout); - assert( (num_bits%8) == 0 ); - g_last_symbol->set_address( g_current_symbol_table->get_local_next() ); - g_current_symbol_table->alloc_local( num_bits/8 ); - g_func_info->set_framesize( g_current_symbol_table->get_local_next() ); + assert((num_bits % 8) == 0); + g_last_symbol->set_address(g_current_symbol_table->get_local_next()); + g_current_symbol_table->alloc_local(num_bits / 8); + g_func_info->set_framesize(g_current_symbol_table->get_local_next()); break; - case param_space_kernel: + case param_space_kernel: break; - default: + default: abort(); break; - } + } - assert( !ti.is_param_unclassified() ); - if ( ti.is_param_kernel() ) { - bool is_ptr = (g_ptr_spec != undefined_space); - g_func_info->add_param_name_type_size(g_entry_func_param_index,identifier, ti.scalar_type(), num_bits, is_ptr, g_ptr_spec); - g_entry_func_param_index++; - } + assert(!ti.is_param_unclassified()); + if (ti.is_param_kernel()) { + bool is_ptr = (g_ptr_spec != undefined_space); + g_func_info->add_param_name_type_size(g_entry_func_param_index, identifier, + ti.scalar_type(), num_bits, is_ptr, + g_ptr_spec); + g_entry_func_param_index++; + } } -void ptx_recognizer::add_constptr(const char* identifier1, const char* identifier2, int offset) -{ - symbol *s1 = g_current_symbol_table->lookup(identifier1); - const symbol *s2 = g_current_symbol_table->lookup(identifier2); - parse_assert( s1 != NULL, "'from' constant identifier does not exist."); - parse_assert( s1 != NULL, "'to' constant identifier does not exist."); +void ptx_recognizer::add_constptr(const char *identifier1, + const char *identifier2, int offset) { + symbol *s1 = g_current_symbol_table->lookup(identifier1); + const symbol *s2 = g_current_symbol_table->lookup(identifier2); + parse_assert(s1 != NULL, "'from' constant identifier does not exist."); + parse_assert(s1 != NULL, "'to' constant identifier does not exist."); - unsigned addr = s2->get_address(); + unsigned addr = s2->get_address(); - printf("GPGPU-Sim PTX: moving \"%s\" from 0x%x to 0x%x (%s+%x)\n", - identifier1, s1->get_address(), addr+offset, identifier2, offset); + printf("GPGPU-Sim PTX: moving \"%s\" from 0x%x to 0x%x (%s+%x)\n", + identifier1, s1->get_address(), addr + offset, identifier2, offset); - s1->set_address( addr + offset ); + s1->set_address(addr + offset); } -void ptx_recognizer::add_function_arg() -{ - assert(g_size>0); - if( g_func_info ) { - PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); - g_func_info->add_arg(g_last_symbol); - unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec; - assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values - g_func_info->add_config_param( g_size, alignment); - } - +void ptx_recognizer::add_function_arg() { + assert(g_size > 0); + if (g_func_info) { + PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str()); + g_func_info->add_arg(g_last_symbol); + unsigned alignment = (g_alignment_spec == -1) ? g_size : g_alignment_spec; + assert(alignment == 1 || alignment == 2 || alignment == 4 || + alignment == 8 || alignment == 16); // known valid alignment values + g_func_info->add_config_param(g_size, alignment); + } } -void ptx_recognizer::add_extern_spec() -{ - PTX_PARSE_DPRINTF("add_extern_spec"); - g_extern_spec = 1; +void ptx_recognizer::add_extern_spec() { + PTX_PARSE_DPRINTF("add_extern_spec"); + g_extern_spec = 1; } -void ptx_recognizer::add_alignment_spec( int spec ) -{ - PTX_PARSE_DPRINTF("add_alignment_spec"); - parse_assert( g_alignment_spec == -1, "multiple .align specifiers per variable declaration not allowed." ); - g_alignment_spec = spec; +void ptx_recognizer::add_alignment_spec(int spec) { + PTX_PARSE_DPRINTF("add_alignment_spec"); + parse_assert( + g_alignment_spec == -1, + "multiple .align specifiers per variable declaration not allowed."); + g_alignment_spec = spec; } -void ptx_recognizer::add_ptr_spec( enum _memory_space_t spec ) -{ - PTX_PARSE_DPRINTF("add_ptr_spec \"%s\"", g_ptx_token_decode[spec].c_str() ); - parse_assert( g_ptr_spec == undefined_space, "multiple ptr space specifiers not allowed." ); - parse_assert( spec == global_space or spec == local_space or spec == shared_space, "invalid space for ptr directive." ); - g_ptr_spec = spec; +void ptx_recognizer::add_ptr_spec(enum _memory_space_t spec) { + PTX_PARSE_DPRINTF("add_ptr_spec \"%s\"", g_ptx_token_decode[spec].c_str()); + parse_assert(g_ptr_spec == undefined_space, + "multiple ptr space specifiers not allowed."); + parse_assert( + spec == global_space or spec == local_space or spec == shared_space, + "invalid space for ptr directive."); + g_ptr_spec = spec; } -void ptx_recognizer::add_space_spec( enum _memory_space_t spec, int value ) -{ - PTX_PARSE_DPRINTF("add_space_spec \"%s\"", g_ptx_token_decode[spec].c_str() ); - parse_assert( g_space_spec == undefined_space, "multiple space specifiers not allowed." ); - if( spec == param_space_unclassified ) { - if( g_func_decl ) { - if( g_entry_point == 1) - g_space_spec = param_space_kernel; - else - g_space_spec = param_space_local; - } else - g_space_spec = param_space_unclassified; - } else { - g_space_spec = spec; - if( g_space_spec == const_space ) - g_space_spec.set_bank((unsigned)value); - } +void ptx_recognizer::add_space_spec(enum _memory_space_t spec, int value) { + PTX_PARSE_DPRINTF("add_space_spec \"%s\"", g_ptx_token_decode[spec].c_str()); + parse_assert(g_space_spec == undefined_space, + "multiple space specifiers not allowed."); + if (spec == param_space_unclassified) { + if (g_func_decl) { + if (g_entry_point == 1) + g_space_spec = param_space_kernel; + else + g_space_spec = param_space_local; + } else + g_space_spec = param_space_unclassified; + } else { + g_space_spec = spec; + if (g_space_spec == const_space) g_space_spec.set_bank((unsigned)value); + } } -void ptx_recognizer::add_vector_spec(int spec ) -{ - PTX_PARSE_DPRINTF("add_vector_spec"); - parse_assert( g_vector_spec == -1, "multiple vector specifiers not allowed." ); - g_vector_spec = spec; +void ptx_recognizer::add_vector_spec(int spec) { + PTX_PARSE_DPRINTF("add_vector_spec"); + parse_assert(g_vector_spec == -1, "multiple vector specifiers not allowed."); + g_vector_spec = spec; } -void ptx_recognizer::add_scalar_type_spec( int type_spec ) -{ - //save size of parameter - switch ( type_spec ) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - g_size = 1; break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - g_size = 2; break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - g_size = 4; break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - g_size = 8; break; - case BB128_TYPE: - g_size = 16; break; - } - PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str()); - g_scalar_type.push_back( type_spec ); - if ( g_scalar_type.size() > 1 ) { - parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) - || (g_opcode == TEX_OP)|| (g_opcode==MMA_OP)|| (g_opcode == DP4A_OP), - "only cvt, set, slct, tex and dp4a can have more than one type specifier."); - } - g_scalar_type_spec = type_spec; +void ptx_recognizer::add_scalar_type_spec(int type_spec) { + // save size of parameter + switch (type_spec) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + g_size = 1; + break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + g_size = 2; + break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + g_size = 4; + break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + g_size = 8; + break; + case BB128_TYPE: + g_size = 16; + break; + } + PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", + g_ptx_token_decode[type_spec].c_str()); + g_scalar_type.push_back(type_spec); + if (g_scalar_type.size() > 1) { + parse_assert((g_opcode == -1) || (g_opcode == CVT_OP) || + (g_opcode == SET_OP) || (g_opcode == SLCT_OP) || + (g_opcode == TEX_OP) || (g_opcode == MMA_OP) || + (g_opcode == DP4A_OP), + "only cvt, set, slct, tex and dp4a can have more than one " + "type specifier."); + } + g_scalar_type_spec = type_spec; } -void ptx_recognizer::add_label( const char *identifier ) -{ - PTX_PARSE_DPRINTF("add_label"); - symbol *s = g_current_symbol_table->lookup(identifier); - if ( s != NULL ) { - g_label = s; - } else { - g_label = g_current_symbol_table->add_variable(identifier,NULL,0,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); - } +void ptx_recognizer::add_label(const char *identifier) { + PTX_PARSE_DPRINTF("add_label"); + symbol *s = g_current_symbol_table->lookup(identifier); + if (s != NULL) { + g_label = s; + } else { + g_label = g_current_symbol_table->add_variable( + identifier, NULL, 0, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); + } } -void ptx_recognizer::add_opcode( int opcode ) -{ - g_opcode = opcode; -} +void ptx_recognizer::add_opcode(int opcode) { g_opcode = opcode; } -void ptx_recognizer::add_pred( const char *identifier, int neg, int predModifier ) -{ - PTX_PARSE_DPRINTF("add_pred"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if ( s == NULL ) { - std::string msg = std::string("predicate \"") + identifier + "\" has no declaration."; - parse_error( msg.c_str() ); - } - g_pred = s; - g_neg_pred = neg; - g_pred_mod = predModifier; +void ptx_recognizer::add_pred(const char *identifier, int neg, + int predModifier) { + PTX_PARSE_DPRINTF("add_pred"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if (s == NULL) { + std::string msg = + std::string("predicate \"") + identifier + "\" has no declaration."; + parse_error(msg.c_str()); + } + g_pred = s; + g_neg_pred = neg; + g_pred_mod = predModifier; } -void ptx_recognizer::add_option( int option ) -{ - PTX_PARSE_DPRINTF("add_option"); - g_options.push_back( option ); +void ptx_recognizer::add_option(int option) { + PTX_PARSE_DPRINTF("add_option"); + g_options.push_back(option); } -void ptx_recognizer::add_wmma_option( int option ) -{ - PTX_PARSE_DPRINTF("add_option"); - g_wmma_options.push_back( option ); +void ptx_recognizer::add_wmma_option(int option) { + PTX_PARSE_DPRINTF("add_option"); + g_wmma_options.push_back(option); } -void ptx_recognizer::add_double_operand( const char *d1, const char *d2 ) -{ - //operands that access two variables. - //eg. s[$ofs1+$r0], g[$ofs1+=$r0] - //TODO: Not sure if I'm going to use this for storing to two destinations or not. +void ptx_recognizer::add_double_operand(const char *d1, const char *d2) { + // operands that access two variables. + // eg. s[$ofs1+$r0], g[$ofs1+=$r0] + // TODO: Not sure if I'm going to use this for storing to two destinations or + // not. - PTX_PARSE_DPRINTF("add_double_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - parse_assert( s1 != NULL && s2 != NULL, "component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2,gpgpu_ctx) ); + PTX_PARSE_DPRINTF("add_double_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + parse_assert(s1 != NULL && s2 != NULL, "component(s) missing declarations."); + g_operands.push_back(operand_info(s1, s2, gpgpu_ctx)); } -void ptx_recognizer::add_1vector_operand( const char *d1 ) -{ - // handles the single element vector operand ({%v1}) found in tex.1d instructions - PTX_PARSE_DPRINTF("add_1vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - parse_assert( s1 != NULL, "component(s) missing declarations."); - g_operands.push_back( operand_info(s1,NULL,NULL,NULL,gpgpu_ctx) ); +void ptx_recognizer::add_1vector_operand(const char *d1) { + // handles the single element vector operand ({%v1}) found in tex.1d + // instructions + PTX_PARSE_DPRINTF("add_1vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + parse_assert(s1 != NULL, "component(s) missing declarations."); + g_operands.push_back(operand_info(s1, NULL, NULL, NULL, gpgpu_ctx)); } -void ptx_recognizer::add_2vector_operand( const char *d1, const char *d2 ) -{ - PTX_PARSE_DPRINTF("add_2vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - parse_assert( s1 != NULL && s2 != NULL, "v2 component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2,NULL,NULL,gpgpu_ctx) ); +void ptx_recognizer::add_2vector_operand(const char *d1, const char *d2) { + PTX_PARSE_DPRINTF("add_2vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + parse_assert(s1 != NULL && s2 != NULL, + "v2 component(s) missing declarations."); + g_operands.push_back(operand_info(s1, s2, NULL, NULL, gpgpu_ctx)); } -void ptx_recognizer::add_3vector_operand( const char *d1, const char *d2, const char *d3 ) -{ - PTX_PARSE_DPRINTF("add_3vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - parse_assert( s1 != NULL && s2 != NULL && s3 != NULL, "v3 component(s) missing declarations."); - g_operands.push_back( operand_info(s1,s2,s3,NULL,gpgpu_ctx) ); +void ptx_recognizer::add_3vector_operand(const char *d1, const char *d2, + const char *d3) { + PTX_PARSE_DPRINTF("add_3vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + parse_assert(s1 != NULL && s2 != NULL && s3 != NULL, + "v3 component(s) missing declarations."); + g_operands.push_back(operand_info(s1, s2, s3, NULL, gpgpu_ctx)); } -void ptx_recognizer::add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ) -{ - PTX_PARSE_DPRINTF("add_4vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - const symbol *s4 = g_current_symbol_table->lookup(d4); - parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL, "v4 component(s) missing declarations."); - const symbol *null_op = g_current_symbol_table->lookup("_"); - if ( s2 == null_op ) s2 = NULL; - if ( s3 == null_op ) s3 = NULL; - if ( s4 == null_op ) s4 = NULL; - g_operands.push_back( operand_info(s1,s2,s3,s4,gpgpu_ctx) ); +void ptx_recognizer::add_4vector_operand(const char *d1, const char *d2, + const char *d3, const char *d4) { + PTX_PARSE_DPRINTF("add_4vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + parse_assert(s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL, + "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if (s2 == null_op) s2 = NULL; + if (s3 == null_op) s3 = NULL; + if (s4 == null_op) s4 = NULL; + g_operands.push_back(operand_info(s1, s2, s3, s4, gpgpu_ctx)); } -void ptx_recognizer::add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 ) -{ - PTX_PARSE_DPRINTF("add_8vector_operand"); - const symbol *s1 = g_current_symbol_table->lookup(d1); - const symbol *s2 = g_current_symbol_table->lookup(d2); - const symbol *s3 = g_current_symbol_table->lookup(d3); - const symbol *s4 = g_current_symbol_table->lookup(d4); - const symbol *s5 = g_current_symbol_table->lookup(d5); - const symbol *s6 = g_current_symbol_table->lookup(d6); - const symbol *s7 = g_current_symbol_table->lookup(d7); - const symbol *s8 = g_current_symbol_table->lookup(d8); - parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && s5 !=NULL && s6 !=NULL && s7 !=NULL && s8 !=NULL, "v4 component(s) missing declarations."); - const symbol *null_op = g_current_symbol_table->lookup("_"); - if ( s2 == null_op ) s2 = NULL; - if ( s3 == null_op ) s3 = NULL; - if ( s4 == null_op ) s4 = NULL; - if ( s5 == null_op ) s5 = NULL; - if ( s6 == null_op ) s6 = NULL; - if ( s7 == null_op ) s7 = NULL; - if ( s8 == null_op ) s8 = NULL; - g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8,gpgpu_ctx) ); +void ptx_recognizer::add_8vector_operand(const char *d1, const char *d2, + const char *d3, const char *d4, + const char *d5, const char *d6, + const char *d7, const char *d8) { + PTX_PARSE_DPRINTF("add_8vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + const symbol *s5 = g_current_symbol_table->lookup(d5); + const symbol *s6 = g_current_symbol_table->lookup(d6); + const symbol *s7 = g_current_symbol_table->lookup(d7); + const symbol *s8 = g_current_symbol_table->lookup(d8); + parse_assert(s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && + s5 != NULL && s6 != NULL && s7 != NULL && s8 != NULL, + "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if (s2 == null_op) s2 = NULL; + if (s3 == null_op) s3 = NULL; + if (s4 == null_op) s4 = NULL; + if (s5 == null_op) s5 = NULL; + if (s6 == null_op) s6 = NULL; + if (s7 == null_op) s7 = NULL; + if (s8 == null_op) s8 = NULL; + g_operands.push_back(operand_info(s1, s2, s3, s4, s5, s6, s7, s8, gpgpu_ctx)); } -void ptx_recognizer::add_builtin_operand( int builtin, int dim_modifier ) -{ - PTX_PARSE_DPRINTF("add_builtin_operand"); - g_operands.push_back( operand_info(builtin,dim_modifier,gpgpu_ctx) ); +void ptx_recognizer::add_builtin_operand(int builtin, int dim_modifier) { + PTX_PARSE_DPRINTF("add_builtin_operand"); + g_operands.push_back(operand_info(builtin, dim_modifier, gpgpu_ctx)); } -void ptx_recognizer::add_memory_operand() -{ - PTX_PARSE_DPRINTF("add_memory_operand"); - assert( !g_operands.empty() ); - g_operands.back().make_memory_operand(); +void ptx_recognizer::add_memory_operand() { + PTX_PARSE_DPRINTF("add_memory_operand"); + assert(!g_operands.empty()); + g_operands.back().make_memory_operand(); } /*TODO: add other memory locations*/ -void ptx_recognizer::change_memory_addr_space(const char *identifier) -{ - /*0 = N/A, not reading from memory - *1 = global memory - *2 = shared memory - *3 = const memory segment - *4 = local memory segment - */ +void ptx_recognizer::change_memory_addr_space(const char *identifier) { + /*0 = N/A, not reading from memory + *1 = global memory + *2 = shared memory + *3 = const memory segment + *4 = local memory segment + */ - bool recognizedType = false; + bool recognizedType = false; - PTX_PARSE_DPRINTF("change_memory_addr_space"); - assert( !g_operands.empty() ); - if(!strcmp(identifier, "g")) - { - g_operands.back().set_addr_space(global_space); - recognizedType = true; - } - if(!strcmp(identifier, "s")) - { - g_operands.back().set_addr_space(shared_space); - recognizedType = true; - } - // For constants, check if the first character is 'c' - char c[2]; - strncpy(c, identifier, 1); c[1] = '\0'; - if(!strcmp(c, "c")) - { - g_operands.back().set_addr_space(const_space); - parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Constant was not defined."); - g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); - recognizedType = true; - } - // For local memory, check if the first character is 'l' - char l[2]; - strncpy(l, identifier, 1); l[1] = '\0'; - if(!strcmp(l, "l")) - { - g_operands.back().set_addr_space(local_space); - //parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Local memory segment was not defined."); - //g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); - recognizedType = true; - } + PTX_PARSE_DPRINTF("change_memory_addr_space"); + assert(!g_operands.empty()); + if (!strcmp(identifier, "g")) { + g_operands.back().set_addr_space(global_space); + recognizedType = true; + } + if (!strcmp(identifier, "s")) { + g_operands.back().set_addr_space(shared_space); + recognizedType = true; + } + // For constants, check if the first character is 'c' + char c[2]; + strncpy(c, identifier, 1); + c[1] = '\0'; + if (!strcmp(c, "c")) { + g_operands.back().set_addr_space(const_space); + parse_assert(g_current_symbol_table->lookup(identifier) != NULL, + "Constant was not defined."); + g_operands.back().set_const_mem_offset( + g_current_symbol_table->lookup(identifier)->get_address()); + recognizedType = true; + } + // For local memory, check if the first character is 'l' + char l[2]; + strncpy(l, identifier, 1); + l[1] = '\0'; + if (!strcmp(l, "l")) { + g_operands.back().set_addr_space(local_space); + // parse_assert(g_current_symbol_table->lookup(identifier) != NULL, "Local + // memory segment was not defined."); + // g_operands.back().set_const_mem_offset(g_current_symbol_table->lookup(identifier)->get_address()); + recognizedType = true; + } - parse_assert(recognizedType, "Error: unrecognized memory type."); + parse_assert(recognizedType, "Error: unrecognized memory type."); } -void ptx_recognizer::change_operand_lohi( int lohi ) -{ - /*0 = N/A, read entire operand - *1 = lo, reading from lowest bits - *2 = hi, reading from highest bits - */ +void ptx_recognizer::change_operand_lohi(int lohi) { + /*0 = N/A, read entire operand + *1 = lo, reading from lowest bits + *2 = hi, reading from highest bits + */ - PTX_PARSE_DPRINTF("change_operand_lohi"); - assert( !g_operands.empty() ); - - g_operands.back().set_operand_lohi(lohi); + PTX_PARSE_DPRINTF("change_operand_lohi"); + assert(!g_operands.empty()); + g_operands.back().set_operand_lohi(lohi); } -void ptx_recognizer::set_immediate_operand_type () -{ - PTX_PARSE_DPRINTF("set_immediate_operand_type"); - assert( !g_operands.empty() ); - g_operands.back().set_immediate_addr(); +void ptx_recognizer::set_immediate_operand_type() { + PTX_PARSE_DPRINTF("set_immediate_operand_type"); + assert(!g_operands.empty()); + g_operands.back().set_immediate_addr(); } -void ptx_recognizer::change_double_operand_type( int operand_type ) -{ - /* - *-3 = reg / reg (set instruction, but both get same value) - *-2 = reg | reg (cvt instruction) - *-1 = reg | reg (set instruction) - *0 = N/A, default - *1 = reg + reg - *2 = reg += reg - *3 = reg += immediate - */ +void ptx_recognizer::change_double_operand_type(int operand_type) { + /* + *-3 = reg / reg (set instruction, but both get same value) + *-2 = reg | reg (cvt instruction) + *-1 = reg | reg (set instruction) + *0 = N/A, default + *1 = reg + reg + *2 = reg += reg + *3 = reg += immediate + */ - PTX_PARSE_DPRINTF("change_double_operand_type"); - assert( !g_operands.empty() ); + PTX_PARSE_DPRINTF("change_double_operand_type"); + assert(!g_operands.empty()); - // For double destination operands, ensure valid instruction - if( operand_type == -1 || operand_type == -2 ) { - if((g_opcode == SET_OP)||(g_opcode == SETP_OP)) - g_operands.back().set_double_operand_type(-1); - else - g_operands.back().set_double_operand_type(-2); - } else if( operand_type == -3 ) { - if(g_opcode == SET_OP || g_opcode == MAD_OP) - g_operands.back().set_double_operand_type(operand_type); - else - parse_assert(0, "Error: Unsupported use of double destination operand."); - } else { + // For double destination operands, ensure valid instruction + if (operand_type == -1 || operand_type == -2) { + if ((g_opcode == SET_OP) || (g_opcode == SETP_OP)) + g_operands.back().set_double_operand_type(-1); + else + g_operands.back().set_double_operand_type(-2); + } else if (operand_type == -3) { + if (g_opcode == SET_OP || g_opcode == MAD_OP) g_operands.back().set_double_operand_type(operand_type); - } - + else + parse_assert(0, "Error: Unsupported use of double destination operand."); + } else { + g_operands.back().set_double_operand_type(operand_type); + } } -void ptx_recognizer::change_operand_neg( ) -{ - PTX_PARSE_DPRINTF("change_operand_neg"); - assert( !g_operands.empty() ); - - g_operands.back().set_operand_neg(); +void ptx_recognizer::change_operand_neg() { + PTX_PARSE_DPRINTF("change_operand_neg"); + assert(!g_operands.empty()); + g_operands.back().set_operand_neg(); } -void ptx_recognizer::add_literal_int( int value ) -{ - PTX_PARSE_DPRINTF("add_literal_int"); - g_operands.push_back( operand_info(value,gpgpu_ctx) ); +void ptx_recognizer::add_literal_int(int value) { + PTX_PARSE_DPRINTF("add_literal_int"); + g_operands.push_back(operand_info(value, gpgpu_ctx)); } -void ptx_recognizer::add_literal_float( float value ) -{ - PTX_PARSE_DPRINTF("add_literal_float"); - g_operands.push_back( operand_info(value,gpgpu_ctx) ); +void ptx_recognizer::add_literal_float(float value) { + PTX_PARSE_DPRINTF("add_literal_float"); + g_operands.push_back(operand_info(value, gpgpu_ctx)); } -void ptx_recognizer::add_literal_double( double value ) -{ - PTX_PARSE_DPRINTF("add_literal_double"); - g_operands.push_back( operand_info(value,gpgpu_ctx) ); +void ptx_recognizer::add_literal_double(double value) { + PTX_PARSE_DPRINTF("add_literal_double"); + g_operands.push_back(operand_info(value, gpgpu_ctx)); } -void ptx_recognizer::add_scalar_operand( const char *identifier ) -{ - PTX_PARSE_DPRINTF("add_scalar_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if ( s == NULL ) { - if ( g_opcode == BRA_OP || g_opcode == CALLP_OP) { - // forward branch target... - s = g_current_symbol_table->add_variable(identifier,NULL,0,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); - } else { - std::string msg = std::string("operand \"") + identifier + "\" has no declaration."; - parse_error( msg.c_str() ); - } - } - g_operands.push_back( operand_info(s,gpgpu_ctx) ); +void ptx_recognizer::add_scalar_operand(const char *identifier) { + PTX_PARSE_DPRINTF("add_scalar_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if (s == NULL) { + if (g_opcode == BRA_OP || g_opcode == CALLP_OP) { + // forward branch target... + s = g_current_symbol_table->add_variable( + identifier, NULL, 0, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); + } else { + std::string msg = + std::string("operand \"") + identifier + "\" has no declaration."; + parse_error(msg.c_str()); + } + } + g_operands.push_back(operand_info(s, gpgpu_ctx)); } -void ptx_recognizer::add_neg_pred_operand( const char *identifier ) -{ - PTX_PARSE_DPRINTF("add_neg_pred_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if ( s == NULL ) { - s = g_current_symbol_table->add_variable(identifier,NULL,1,gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); - } - operand_info op(s, gpgpu_ctx); - op.set_neg_pred(); - g_operands.push_back( op ); +void ptx_recognizer::add_neg_pred_operand(const char *identifier) { + PTX_PARSE_DPRINTF("add_neg_pred_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if (s == NULL) { + s = g_current_symbol_table->add_variable( + identifier, NULL, 1, gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); + } + operand_info op(s, gpgpu_ctx); + op.set_neg_pred(); + g_operands.push_back(op); } -void ptx_recognizer::add_address_operand( const char *identifier, int offset ) -{ - PTX_PARSE_DPRINTF("add_address_operand"); - const symbol *s = g_current_symbol_table->lookup(identifier); - if ( s == NULL ) { - std::string msg = std::string("operand \"") + identifier + "\" has no declaration."; - parse_error( msg.c_str() ); - } - g_operands.push_back( operand_info(s,offset,gpgpu_ctx) ); +void ptx_recognizer::add_address_operand(const char *identifier, int offset) { + PTX_PARSE_DPRINTF("add_address_operand"); + const symbol *s = g_current_symbol_table->lookup(identifier); + if (s == NULL) { + std::string msg = + std::string("operand \"") + identifier + "\" has no declaration."; + parse_error(msg.c_str()); + } + g_operands.push_back(operand_info(s, offset, gpgpu_ctx)); } -void ptx_recognizer::add_address_operand2( int offset ) -{ - PTX_PARSE_DPRINTF("add_address_operand"); - g_operands.push_back( operand_info((unsigned)offset,gpgpu_ctx) ); +void ptx_recognizer::add_address_operand2(int offset) { + PTX_PARSE_DPRINTF("add_address_operand"); + g_operands.push_back(operand_info((unsigned)offset, gpgpu_ctx)); } -void ptx_recognizer::add_array_initializer() -{ - g_last_symbol->add_initializer(g_operands); +void ptx_recognizer::add_array_initializer() { + g_last_symbol->add_initializer(g_operands); } -void ptx_recognizer::add_version_info( float ver, unsigned ext ) -{ - g_global_symbol_table->set_ptx_version(ver,ext); +void ptx_recognizer::add_version_info(float ver, unsigned ext) { + g_global_symbol_table->set_ptx_version(ver, ext); } -void ptx_recognizer::add_file( unsigned num, const char *filename ) -{ - if( gpgpu_ctx->g_filename == NULL ) { - char *b = strdup(filename); - char *l=b; - char *n=b; - while( *n != '\0' ) { - if( *n == '/' ) - l = n+1; - n++; - } +void ptx_recognizer::add_file(unsigned num, const char *filename) { + if (gpgpu_ctx->g_filename == NULL) { + char *b = strdup(filename); + char *l = b; + char *n = b; + while (*n != '\0') { + if (*n == '/') l = n + 1; + n++; + } - char *p = strtok(l,"."); - char buf[1024]; - snprintf(buf,1024,"%s.ptx",p); + char *p = strtok(l, "."); + char buf[1024]; + snprintf(buf, 1024, "%s.ptx", p); - char *q = strtok(NULL,"."); - if( q && !strcmp(q,"cu") ) { - gpgpu_ctx->g_filename = strdup(buf); - } + char *q = strtok(NULL, "."); + if (q && !strcmp(q, "cu")) { + gpgpu_ctx->g_filename = strdup(buf); + } - free( b ); - } + free(b); + } - g_current_symbol_table = g_global_symbol_table; + g_current_symbol_table = g_global_symbol_table; } -void * ptx_recognizer::reset_symtab() -{ - void *result = g_current_symbol_table; - g_current_symbol_table = g_global_symbol_table; - return result; +void *ptx_recognizer::reset_symtab() { + void *result = g_current_symbol_table; + g_current_symbol_table = g_global_symbol_table; + return result; } -void ptx_recognizer::set_symtab(void*symtab) -{ - g_current_symbol_table = (symbol_table*)symtab; +void ptx_recognizer::set_symtab(void *symtab) { + g_current_symbol_table = (symbol_table *)symtab; } -void ptx_recognizer::add_pragma( const char *str ) -{ - printf("GPGPU-Sim PTX: Warning -- ignoring pragma '%s'\n", str ); +void ptx_recognizer::add_pragma(const char *str) { + printf("GPGPU-Sim PTX: Warning -- ignoring pragma '%s'\n", str); } -void ptx_recognizer::version_header(double a) {} //intentional dummy function +void ptx_recognizer::version_header(double a) {} // intentional dummy function -void ptx_recognizer::target_header(char* a) -{ - g_global_symbol_table->set_sm_target(a,NULL,NULL); +void ptx_recognizer::target_header(char *a) { + g_global_symbol_table->set_sm_target(a, NULL, NULL); } -void ptx_recognizer::target_header2(char* a, char* b) -{ - g_global_symbol_table->set_sm_target(a,b,NULL); +void ptx_recognizer::target_header2(char *a, char *b) { + g_global_symbol_table->set_sm_target(a, b, NULL); } -void ptx_recognizer::target_header3(char* a, char* b, char* c) -{ - g_global_symbol_table->set_sm_target(a,b,c); +void ptx_recognizer::target_header3(char *a, char *b, char *c) { + g_global_symbol_table->set_sm_target(a, b, c); } void ptx_recognizer::maxnt_id(int x, int y, int z) { g_func_info->set_maxnt_id(x * y * z); } -void ptx_recognizer::func_header(const char* a) {} //intentional dummy function -void ptx_recognizer::func_header_info(const char* a) {} //intentional dummy function -void ptx_recognizer::func_header_info_int(const char* a, int b) {} //intentional dummy function +void ptx_recognizer::func_header(const char *a) {} // intentional dummy + // function +void ptx_recognizer::func_header_info(const char *a) { +} // intentional dummy function +void ptx_recognizer::func_header_info_int(const char *a, int b) { +} // intentional dummy function diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 11a3d20..96eadf6 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ptx_parser_INCLUDED #define ptx_parser_INCLUDED @@ -32,155 +33,161 @@ #include "ptx_ir.h" class gpgpu_context; -typedef void * yyscan_t; +typedef void *yyscan_t; class ptx_recognizer { - public: - ptx_recognizer( gpgpu_context* ctx ) : g_return_var(ctx) { - scanner = NULL; - g_size = -1; - g_add_identifier_cached__identifier = NULL; - g_alignment_spec = -1; - g_var_type = NULL; - g_opcode = -1; - g_space_spec = undefined_space; - g_ptr_spec = undefined_space; - g_scalar_type_spec = -1; - g_vector_spec = -1; - g_extern_spec = 0; - g_func_decl = 0; - g_ident_add_uid = 0; - g_const_alloc = 1; - g_max_regs_per_thread = 0; - g_global_symbol_table = NULL; - g_current_symbol_table = NULL; - g_last_symbol = NULL; - g_error_detected = 0; - g_entry_func_param_index=0; - g_func_info = NULL; - g_debug_ir_generation=false; - gpgpu_ctx = ctx; - } - // global list - yyscan_t scanner; -#define PTX_LINEBUF_SIZE (4*1024) - char linebuf[PTX_LINEBUF_SIZE]; - unsigned col; - int g_size; - char *g_add_identifier_cached__identifier; - int g_add_identifier_cached__array_dim; - int g_add_identifier_cached__array_ident; - int g_alignment_spec; - // variable declaration stuff: - type_info *g_var_type; - // instruction definition stuff: - const symbol *g_pred; - int g_neg_pred; - int g_pred_mod; - symbol *g_label; - int g_opcode; - std::list<operand_info> g_operands; - std::list<int> g_options; - std::list<int> g_wmma_options; - std::list<int> g_scalar_type; - // type specifier stuff: - memory_space_t g_space_spec; - memory_space_t g_ptr_spec; - int g_scalar_type_spec; - int g_vector_spec; - int g_extern_spec; - int g_func_decl; - int g_ident_add_uid; - unsigned g_const_alloc; - unsigned g_max_regs_per_thread; - symbol_table *g_global_symbol_table; - symbol_table *g_current_symbol_table; - symbol *g_last_symbol; - std::list<ptx_instruction*> g_instructions; - int g_error_detected; - unsigned g_entry_func_param_index; - function_info *g_func_info; - operand_info g_return_var; - bool g_debug_ir_generation; - int g_entry_point; - const struct core_config *g_shader_core_config; - std::map<std::string,std::map<unsigned,const ptx_instruction*> > g_inst_lookup; - // the program intermediate representation... - std::map<std::string,symbol_table*> g_sym_name_to_symbol_table; - // backward pointer - class gpgpu_context* gpgpu_ctx; - - // member function list - void init_directive_state(); - void init_instruction_state(); - void start_function( int entry_point ); - void add_function_name( const char *fname ); - void add_directive(); - void end_function(); - void add_identifier( const char *s, int array_dim, unsigned array_ident ); - void add_function_arg(); - void add_scalar_type_spec( int type_spec ); - void add_scalar_operand( const char *identifier ); - void add_neg_pred_operand( const char *identifier ); - void add_variables(); - void set_variable_type(); - void add_opcode( int opcode ); - void add_pred( const char *identifier, int negate, int predModifier ); - void add_1vector_operand( const char *d1 ); - void add_2vector_operand( const char *d1, const char *d2 ); - void add_3vector_operand( const char *d1, const char *d2, const char *d3 ); - void add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ); - void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ,const char *d5,const char *d6,const char *d7,const char *d8); - void add_option(int option ); - void add_wmma_option(int option ); - void add_builtin_operand( int builtin, int dim_modifier ); - void add_memory_operand( ); - void add_literal_int( int value ); - void add_literal_float( float value ); - void add_literal_double( double value ); - void add_address_operand( const char *identifier, int offset ); - void add_address_operand2( int offset ); - void add_label( const char *idenfiier ); - void add_vector_spec(int spec ); - void add_space_spec( enum _memory_space_t spec, int value ); - void add_ptr_spec( enum _memory_space_t spec ); - void add_extern_spec(); - void add_instruction(); - void set_return(); - void add_alignment_spec( int spec ); - void add_array_initializer(); - void add_file( unsigned num, const char *filename ); - void add_version_info( float ver, unsigned ext); - void *reset_symtab(); - void set_symtab(void*); - void add_pragma( const char *str ); - void func_header(const char* a); - void func_header_info(const char* a); - void func_header_info_int(const char* a, int b); - void add_constptr(const char* identifier1, const char* identifier2, int offset); - void target_header(char* a); - void target_header2(char* a, char* b); - void target_header3(char* a, char* b, char* c); - void add_double_operand( const char *d1, const char *d2 ); - void change_memory_addr_space( const char *identifier ); - void change_operand_lohi( int lohi ); - void change_double_operand_type( int addr_type ); - void change_operand_neg( ); - void set_immediate_operand_type( ); - void version_header(double a); - void maxnt_id(int x, int y, int z); - void parse_error_impl( const char *file, unsigned line, const char *msg, ... ); - void parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ); - //Jin: handle instructino group for cdp - void start_inst_group(); - void end_inst_group(); - bool check_for_duplicates( const char *identifier ); - void read_parser_environment_variables(); - void set_ptx_warp_size(const struct core_config * warp_size); - const class ptx_instruction *ptx_instruction_lookup( const char *filename, unsigned linenumber ); + public: + ptx_recognizer(gpgpu_context *ctx) : g_return_var(ctx) { + scanner = NULL; + g_size = -1; + g_add_identifier_cached__identifier = NULL; + g_alignment_spec = -1; + g_var_type = NULL; + g_opcode = -1; + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_extern_spec = 0; + g_func_decl = 0; + g_ident_add_uid = 0; + g_const_alloc = 1; + g_max_regs_per_thread = 0; + g_global_symbol_table = NULL; + g_current_symbol_table = NULL; + g_last_symbol = NULL; + g_error_detected = 0; + g_entry_func_param_index = 0; + g_func_info = NULL; + g_debug_ir_generation = false; + gpgpu_ctx = ctx; + } + // global list + yyscan_t scanner; +#define PTX_LINEBUF_SIZE (4 * 1024) + char linebuf[PTX_LINEBUF_SIZE]; + unsigned col; + int g_size; + char *g_add_identifier_cached__identifier; + int g_add_identifier_cached__array_dim; + int g_add_identifier_cached__array_ident; + int g_alignment_spec; + // variable declaration stuff: + type_info *g_var_type; + // instruction definition stuff: + const symbol *g_pred; + int g_neg_pred; + int g_pred_mod; + symbol *g_label; + int g_opcode; + std::list<operand_info> g_operands; + std::list<int> g_options; + std::list<int> g_wmma_options; + std::list<int> g_scalar_type; + // type specifier stuff: + memory_space_t g_space_spec; + memory_space_t g_ptr_spec; + int g_scalar_type_spec; + int g_vector_spec; + int g_extern_spec; + int g_func_decl; + int g_ident_add_uid; + unsigned g_const_alloc; + unsigned g_max_regs_per_thread; + symbol_table *g_global_symbol_table; + symbol_table *g_current_symbol_table; + symbol *g_last_symbol; + std::list<ptx_instruction *> g_instructions; + int g_error_detected; + unsigned g_entry_func_param_index; + function_info *g_func_info; + operand_info g_return_var; + bool g_debug_ir_generation; + int g_entry_point; + const struct core_config *g_shader_core_config; + std::map<std::string, std::map<unsigned, const ptx_instruction *> > + g_inst_lookup; + // the program intermediate representation... + std::map<std::string, symbol_table *> g_sym_name_to_symbol_table; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // member function list + void init_directive_state(); + void init_instruction_state(); + void start_function(int entry_point); + void add_function_name(const char *fname); + void add_directive(); + void end_function(); + void add_identifier(const char *s, int array_dim, unsigned array_ident); + void add_function_arg(); + void add_scalar_type_spec(int type_spec); + void add_scalar_operand(const char *identifier); + void add_neg_pred_operand(const char *identifier); + void add_variables(); + void set_variable_type(); + void add_opcode(int opcode); + void add_pred(const char *identifier, int negate, int predModifier); + void add_1vector_operand(const char *d1); + void add_2vector_operand(const char *d1, const char *d2); + void add_3vector_operand(const char *d1, const char *d2, const char *d3); + void add_4vector_operand(const char *d1, const char *d2, const char *d3, + const char *d4); + void add_8vector_operand(const char *d1, const char *d2, const char *d3, + const char *d4, const char *d5, const char *d6, + const char *d7, const char *d8); + void add_option(int option); + void add_wmma_option(int option); + void add_builtin_operand(int builtin, int dim_modifier); + void add_memory_operand(); + void add_literal_int(int value); + void add_literal_float(float value); + void add_literal_double(double value); + void add_address_operand(const char *identifier, int offset); + void add_address_operand2(int offset); + void add_label(const char *idenfiier); + void add_vector_spec(int spec); + void add_space_spec(enum _memory_space_t spec, int value); + void add_ptr_spec(enum _memory_space_t spec); + void add_extern_spec(); + void add_instruction(); + void set_return(); + void add_alignment_spec(int spec); + void add_array_initializer(); + void add_file(unsigned num, const char *filename); + void add_version_info(float ver, unsigned ext); + void *reset_symtab(); + void set_symtab(void *); + void add_pragma(const char *str); + void func_header(const char *a); + void func_header_info(const char *a); + void func_header_info_int(const char *a, int b); + void add_constptr(const char *identifier1, const char *identifier2, + int offset); + void target_header(char *a); + void target_header2(char *a, char *b); + void target_header3(char *a, char *b, char *c); + void add_double_operand(const char *d1, const char *d2); + void change_memory_addr_space(const char *identifier); + void change_operand_lohi(int lohi); + void change_double_operand_type(int addr_type); + void change_operand_neg(); + void set_immediate_operand_type(); + void version_header(double a); + void maxnt_id(int x, int y, int z); + void parse_error_impl(const char *file, unsigned line, const char *msg, ...); + void parse_assert_impl(int test_value, const char *file, unsigned line, + const char *msg, ...); + // Jin: handle instructino group for cdp + void start_inst_group(); + void end_inst_group(); + bool check_for_duplicates(const char *identifier); + void read_parser_environment_variables(); + void set_ptx_warp_size(const struct core_config *warp_size); + const class ptx_instruction *ptx_instruction_lookup(const char *filename, + unsigned linenumber); }; -const char *decode_token( int type ); +const char *decode_token(int type); void read_parser_environment_variables(); #define NON_ARRAY_IDENTIFIER 1 diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index 949ee66..dc801f8 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -7,595 +7,611 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_sim.h" #include <string> #include "ptx_ir.h" class ptx_recognizer; -typedef void * yyscan_t; -#include "ptx.tab.h" +typedef void *yyscan_t; +#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include "../../libcuda/gpgpu_context.h" - -void feature_not_implemented( const char *f ); +#include "ptx.tab.h" +void feature_not_implemented(const char *f); -ptx_cta_info::ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ) -{ - assert( ctx->func_sim->g_ptx_cta_info_sm_idx_used.find(sm_idx) == ctx->func_sim->g_ptx_cta_info_sm_idx_used.end() ); - ctx->func_sim->g_ptx_cta_info_sm_idx_used.insert(sm_idx); +ptx_cta_info::ptx_cta_info(unsigned sm_idx, gpgpu_context *ctx) { + assert(ctx->func_sim->g_ptx_cta_info_sm_idx_used.find(sm_idx) == + ctx->func_sim->g_ptx_cta_info_sm_idx_used.end()); + ctx->func_sim->g_ptx_cta_info_sm_idx_used.insert(sm_idx); - m_sm_idx = sm_idx; - m_uid = (ctx->g_ptx_cta_info_uid)++; - m_bar_threads = 0; - gpgpu_ctx = ctx; + m_sm_idx = sm_idx; + m_uid = (ctx->g_ptx_cta_info_uid)++; + m_bar_threads = 0; + gpgpu_ctx = ctx; } -void ptx_cta_info::add_thread( ptx_thread_info *thd ) -{ - m_threads_in_cta.insert(thd); +void ptx_cta_info::add_thread(ptx_thread_info *thd) { + m_threads_in_cta.insert(thd); } -unsigned ptx_cta_info::num_threads() const -{ - return m_threads_in_cta.size(); -} +unsigned ptx_cta_info::num_threads() const { return m_threads_in_cta.size(); } -void ptx_cta_info::check_cta_thread_status_and_reset() -{ - bool fail = false; - if ( m_threads_that_have_exited.size() != m_threads_in_cta.size() ) { - printf("\n\n"); - printf("Execution error: Some threads still running in CTA during CTA reallocation! (1)\n"); - printf(" CTA uid = %Lu (sm_idx = %u) : %lu running out of %zu total\n", - m_uid, - m_sm_idx, - (m_threads_in_cta.size() - m_threads_that_have_exited.size()), m_threads_in_cta.size() ); - printf(" These are the threads that are still running:\n"); - std::set<ptx_thread_info*>::iterator t_iter; - for ( t_iter=m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); ++t_iter ) { - ptx_thread_info *t = *t_iter; - if ( m_threads_that_have_exited.find(t) == m_threads_that_have_exited.end() ) { - if ( m_dangling_pointers.find(t) != m_dangling_pointers.end() ) { - printf(" <thread deleted>\n"); - } else { - printf(" [done=%c] : ", (t->is_done()?'Y':'N') ); - t->print_insn( t->get_pc(), stdout ); - printf("\n"); - } - } +void ptx_cta_info::check_cta_thread_status_and_reset() { + bool fail = false; + if (m_threads_that_have_exited.size() != m_threads_in_cta.size()) { + printf("\n\n"); + printf( + "Execution error: Some threads still running in CTA during CTA " + "reallocation! (1)\n"); + printf(" CTA uid = %Lu (sm_idx = %u) : %lu running out of %zu total\n", + m_uid, m_sm_idx, + (m_threads_in_cta.size() - m_threads_that_have_exited.size()), + m_threads_in_cta.size()); + printf(" These are the threads that are still running:\n"); + std::set<ptx_thread_info *>::iterator t_iter; + for (t_iter = m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); + ++t_iter) { + ptx_thread_info *t = *t_iter; + if (m_threads_that_have_exited.find(t) == + m_threads_that_have_exited.end()) { + if (m_dangling_pointers.find(t) != m_dangling_pointers.end()) { + printf(" <thread deleted>\n"); + } else { + printf(" [done=%c] : ", (t->is_done() ? 'Y' : 'N')); + t->print_insn(t->get_pc(), stdout); + printf("\n"); + } } - printf("\n\n"); - fail = true; - } - if ( fail ) { - abort(); - } + } + printf("\n\n"); + fail = true; + } + if (fail) { + abort(); + } - bool fail2 = false; - std::set<ptx_thread_info*>::iterator t_iter; - for ( t_iter=m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); ++t_iter ) { - ptx_thread_info *t = *t_iter; - if ( m_dangling_pointers.find(t) == m_dangling_pointers.end() ) { - if ( !t->is_done() ) { - if ( !fail2 ) { - printf("Execution error: Some threads still running in CTA during CTA reallocation! (2)\n"); - printf(" CTA uid = %Lu (sm_idx = %u) :\n", m_uid, m_sm_idx ); - fail2 = true; - } - printf(" "); - t->print_insn( t->get_pc(), stdout ); - printf("\n"); - } + bool fail2 = false; + std::set<ptx_thread_info *>::iterator t_iter; + for (t_iter = m_threads_in_cta.begin(); t_iter != m_threads_in_cta.end(); + ++t_iter) { + ptx_thread_info *t = *t_iter; + if (m_dangling_pointers.find(t) == m_dangling_pointers.end()) { + if (!t->is_done()) { + if (!fail2) { + printf( + "Execution error: Some threads still running in CTA during CTA " + "reallocation! (2)\n"); + printf(" CTA uid = %Lu (sm_idx = %u) :\n", m_uid, m_sm_idx); + fail2 = true; + } + printf(" "); + t->print_insn(t->get_pc(), stdout); + printf("\n"); } - } - if ( fail2 ) { - abort(); - } - m_threads_in_cta.clear(); - m_threads_that_have_exited.clear(); - m_dangling_pointers.clear(); + } + } + if (fail2) { + abort(); + } + m_threads_in_cta.clear(); + m_threads_that_have_exited.clear(); + m_dangling_pointers.clear(); } -void ptx_cta_info::register_thread_exit( ptx_thread_info *thd ) -{ - assert( m_threads_that_have_exited.find(thd) == m_threads_that_have_exited.end() ); - m_threads_that_have_exited.insert(thd); +void ptx_cta_info::register_thread_exit(ptx_thread_info *thd) { + assert(m_threads_that_have_exited.find(thd) == + m_threads_that_have_exited.end()); + m_threads_that_have_exited.insert(thd); } -void ptx_cta_info::register_deleted_thread( ptx_thread_info *thd ) -{ - m_dangling_pointers.insert(thd); +void ptx_cta_info::register_deleted_thread(ptx_thread_info *thd) { + m_dangling_pointers.insert(thd); } -unsigned ptx_cta_info::get_sm_idx() const -{ - return m_sm_idx; -} +unsigned ptx_cta_info::get_sm_idx() const { return m_sm_idx; } -unsigned ptx_cta_info::get_bar_threads() const -{ - return m_bar_threads; -} +unsigned ptx_cta_info::get_bar_threads() const { return m_bar_threads; } -void ptx_cta_info::inc_bar_threads() -{ - m_bar_threads++; -} +void ptx_cta_info::inc_bar_threads() { m_bar_threads++; } -void ptx_cta_info::reset_bar_threads() -{ - m_bar_threads = 0; -} - -ptx_warp_info::ptx_warp_info() -{ - reset_done_threads(); -} +void ptx_cta_info::reset_bar_threads() { m_bar_threads = 0; } -unsigned ptx_warp_info::get_done_threads() const -{ - return m_done_threads; -} +ptx_warp_info::ptx_warp_info() { reset_done_threads(); } -void ptx_warp_info::inc_done_threads() -{ - m_done_threads++; -} +unsigned ptx_warp_info::get_done_threads() const { return m_done_threads; } -void ptx_warp_info::reset_done_threads() -{ - m_done_threads = 0; -} +void ptx_warp_info::inc_done_threads() { m_done_threads++; } +void ptx_warp_info::reset_done_threads() { m_done_threads = 0; } -ptx_thread_info::~ptx_thread_info() -{ - m_gpu->gpgpu_ctx->func_sim->g_ptx_thread_info_delete_count++; +ptx_thread_info::~ptx_thread_info() { + m_gpu->gpgpu_ctx->func_sim->g_ptx_thread_info_delete_count++; } -ptx_thread_info::ptx_thread_info( kernel_info_t &kernel ) - : m_kernel(kernel) -{ - m_uid = kernel.entry()->gpgpu_ctx->func_sim->g_ptx_thread_info_uid_next++; - m_core = NULL; - m_barrier_num = -1; - m_at_barrier = false; - m_valid = false; - m_gridid = 0; - m_thread_done = false; - m_cycle_done = 0; - m_PC=0; - m_icount = 0; - m_last_effective_address = 0; - m_last_memory_space = undefined_space; - m_branch_taken = 0; - m_shared_mem = NULL; - m_sstarr_mem = NULL; - m_warp_info = NULL; - m_cta_info = NULL; - m_local_mem = NULL; - m_symbol_table = NULL; - m_func_info = NULL; - m_hw_tid = -1; - m_hw_wid = -1; - m_hw_sid = -1; - m_last_dram_callback.function = NULL; - m_last_dram_callback.instruction = NULL; - m_regs.push_back( reg_map_t() ); - m_debug_trace_regs_modified.push_back( reg_map_t() ); - m_debug_trace_regs_read.push_back( reg_map_t() ); - m_callstack.push_back( stack_entry() ); - m_RPC = -1; - m_RPC_updated = false; - m_last_was_call = false; - m_enable_debug_trace = false; - m_local_mem_stack_pointer = 0; - m_gpu = NULL; - m_last_set_operand_value=ptx_reg_t(); +ptx_thread_info::ptx_thread_info(kernel_info_t &kernel) : m_kernel(kernel) { + m_uid = kernel.entry()->gpgpu_ctx->func_sim->g_ptx_thread_info_uid_next++; + m_core = NULL; + m_barrier_num = -1; + m_at_barrier = false; + m_valid = false; + m_gridid = 0; + m_thread_done = false; + m_cycle_done = 0; + m_PC = 0; + m_icount = 0; + m_last_effective_address = 0; + m_last_memory_space = undefined_space; + m_branch_taken = 0; + m_shared_mem = NULL; + m_sstarr_mem = NULL; + m_warp_info = NULL; + m_cta_info = NULL; + m_local_mem = NULL; + m_symbol_table = NULL; + m_func_info = NULL; + m_hw_tid = -1; + m_hw_wid = -1; + m_hw_sid = -1; + m_last_dram_callback.function = NULL; + m_last_dram_callback.instruction = NULL; + m_regs.push_back(reg_map_t()); + m_debug_trace_regs_modified.push_back(reg_map_t()); + m_debug_trace_regs_read.push_back(reg_map_t()); + m_callstack.push_back(stack_entry()); + m_RPC = -1; + m_RPC_updated = false; + m_last_was_call = false; + m_enable_debug_trace = false; + m_local_mem_stack_pointer = 0; + m_gpu = NULL; + m_last_set_operand_value = ptx_reg_t(); } -const ptx_version &ptx_thread_info::get_ptx_version() const -{ - return m_func_info->get_ptx_version(); +const ptx_version &ptx_thread_info::get_ptx_version() const { + return m_func_info->get_ptx_version(); } -void ptx_thread_info::set_done() -{ - assert( !m_at_barrier ); - m_thread_done = true; - m_cycle_done = m_gpu->gpu_sim_cycle; +void ptx_thread_info::set_done() { + assert(!m_at_barrier); + m_thread_done = true; + m_cycle_done = m_gpu->gpu_sim_cycle; } -unsigned ptx_thread_info::get_builtin( int builtin_id, unsigned dim_mod ) -{ - assert( m_valid ); - switch ((builtin_id&0xFFFF)) { - case CLOCK_REG: +unsigned ptx_thread_info::get_builtin(int builtin_id, unsigned dim_mod) { + assert(m_valid); + switch ((builtin_id & 0xFFFF)) { + case CLOCK_REG: return (unsigned)(m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); - case CLOCK64_REG: - abort(); // change return value to unsigned long long? - // GPGPUSim clock is 4 times slower - multiply by 4 - return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle)*4; - case HALFCLOCK_ID: + case CLOCK64_REG: + abort(); // change return value to unsigned long long? + // GPGPUSim clock is 4 times slower - multiply by 4 + return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) * 4; + case HALFCLOCK_ID: // GPGPUSim clock is 4 times slower - multiply by 4 - // Hardware clock counter is incremented at half the shader clock frequency - divide by 2 (Henry '10) - return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle)*2; - case CTAID_REG: - assert( dim_mod < 3 ); - if( dim_mod == 0 ) return m_ctaid.x; - if( dim_mod == 1 ) return m_ctaid.y; - if( dim_mod == 2 ) return m_ctaid.z; + // Hardware clock counter is incremented at half the shader clock + // frequency - divide by 2 (Henry '10) + return (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) * 2; + case CTAID_REG: + assert(dim_mod < 3); + if (dim_mod == 0) return m_ctaid.x; + if (dim_mod == 1) return m_ctaid.y; + if (dim_mod == 2) return m_ctaid.z; abort(); break; - case ENVREG_REG:{ - int index = builtin_id >> 16; - dim3 gdim = this->get_core()->get_kernel_info()->get_grid_dim(); - switch(index){ - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - return 0; - break; - case 6: - return gdim.x; - case 7: - return gdim.y; - case 8: - return gdim.z; - case 9: - if(gdim.z == 1 && gdim.y == 1) - return 1; - else if(gdim.z == 1) - return 2; - else - return 3; - break; - default: - break; - } - } - case GRIDID_REG: + case ENVREG_REG: { + int index = builtin_id >> 16; + dim3 gdim = this->get_core()->get_kernel_info()->get_grid_dim(); + switch (index) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + return 0; + break; + case 6: + return gdim.x; + case 7: + return gdim.y; + case 8: + return gdim.z; + case 9: + if (gdim.z == 1 && gdim.y == 1) + return 1; + else if (gdim.z == 1) + return 2; + else + return 3; + break; + default: + break; + } + } + case GRIDID_REG: return m_gridid; - case LANEID_REG: return get_hw_tid() % m_core->get_warp_size(); - case LANEMASK_EQ_REG: feature_not_implemented( "%lanemask_eq" ); return 0; - case LANEMASK_LE_REG: feature_not_implemented( "%lanemask_le" ); return 0; - case LANEMASK_LT_REG: feature_not_implemented( "%lanemask_lt" ); return 0; - case LANEMASK_GE_REG: feature_not_implemented( "%lanemask_ge" ); return 0; - case LANEMASK_GT_REG: feature_not_implemented( "%lanemask_gt" ); return 0; - case NCTAID_REG: - assert( dim_mod < 3 ); - if( dim_mod == 0 ) return m_nctaid.x; - if( dim_mod == 1 ) return m_nctaid.y; - if( dim_mod == 2 ) return m_nctaid.z; + case LANEID_REG: + return get_hw_tid() % m_core->get_warp_size(); + case LANEMASK_EQ_REG: + feature_not_implemented("%lanemask_eq"); + return 0; + case LANEMASK_LE_REG: + feature_not_implemented("%lanemask_le"); + return 0; + case LANEMASK_LT_REG: + feature_not_implemented("%lanemask_lt"); + return 0; + case LANEMASK_GE_REG: + feature_not_implemented("%lanemask_ge"); + return 0; + case LANEMASK_GT_REG: + feature_not_implemented("%lanemask_gt"); + return 0; + case NCTAID_REG: + assert(dim_mod < 3); + if (dim_mod == 0) return m_nctaid.x; + if (dim_mod == 1) return m_nctaid.y; + if (dim_mod == 2) return m_nctaid.z; abort(); break; - case NTID_REG: - assert( dim_mod < 3 ); - if( dim_mod == 0 ) return m_ntid.x; - if( dim_mod == 1 ) return m_ntid.y; - if( dim_mod == 2 ) return m_ntid.z; + case NTID_REG: + assert(dim_mod < 3); + if (dim_mod == 0) return m_ntid.x; + if (dim_mod == 1) return m_ntid.y; + if (dim_mod == 2) return m_ntid.z; abort(); break; - case NWARPID_REG: feature_not_implemented( "%nwarpid" ); return 0; - case PM_REG: feature_not_implemented( "%pm" ); return 0; - case SMID_REG: feature_not_implemented( "%smid" ); return 0; - case TID_REG: - assert( dim_mod < 3 ); - if( dim_mod == 0 ) return m_tid.x; - if( dim_mod == 1 ) return m_tid.y; - if( dim_mod == 2 ) return m_tid.z; + case NWARPID_REG: + feature_not_implemented("%nwarpid"); + return 0; + case PM_REG: + feature_not_implemented("%pm"); + return 0; + case SMID_REG: + feature_not_implemented("%smid"); + return 0; + case TID_REG: + assert(dim_mod < 3); + if (dim_mod == 0) return m_tid.x; + if (dim_mod == 1) return m_tid.y; + if (dim_mod == 2) return m_tid.z; abort(); break; - case WARPSZ_REG: return m_core->get_warp_size() ; - default: + case WARPSZ_REG: + return m_core->get_warp_size(); + default: assert(0); - } - return 0; + } + return 0; } -void ptx_thread_info::set_info( function_info *func ) -{ +void ptx_thread_info::set_info(function_info *func) { m_symbol_table = func->get_symtab(); m_func_info = func; m_PC = func->get_start_PC(); } -void ptx_thread_info::cpy_tid_to_reg( dim3 tid ) -{ - //copies %tid.x, %tid.y and %tid.z into $r0 - ptx_reg_t data; - data.s64=0; +void ptx_thread_info::cpy_tid_to_reg(dim3 tid) { + // copies %tid.x, %tid.y and %tid.z into $r0 + ptx_reg_t data; + data.s64 = 0; - data.u32=(tid.x + (tid.y<<16) + (tid.z<<26)); + data.u32 = (tid.x + (tid.y << 16) + (tid.z << 26)); - const symbol *r0 = m_symbol_table->lookup("$r0"); - if (r0){ - //No need to set pid if kernel doesn't use it - set_reg(r0,data); - } + const symbol *r0 = m_symbol_table->lookup("$r0"); + if (r0) { + // No need to set pid if kernel doesn't use it + set_reg(r0, data); + } } -void ptx_thread_info::print_insn( unsigned pc, FILE * fp ) const -{ - m_func_info->print_insn(pc,fp); +void ptx_thread_info::print_insn(unsigned pc, FILE *fp) const { + m_func_info->print_insn(pc, fp); } -static void print_reg( FILE *fp, std::string name, ptx_reg_t value, symbol_table *symtab ) -{ - const symbol *sym = symtab->lookup(name.c_str()); - fprintf(fp," %8s ", name.c_str() ); - if( sym == NULL ) { - fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 ); - return; - } - const type_info *t = sym->type(); - if( t == NULL ) { - fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 ); - return; - } - type_info_key ti = t->get_key(); +static void print_reg(FILE *fp, std::string name, ptx_reg_t value, + symbol_table *symtab) { + const symbol *sym = symtab->lookup(name.c_str()); + fprintf(fp, " %8s ", name.c_str()); + if (sym == NULL) { + fprintf(fp, "<unknown type> 0x%llx\n", (unsigned long long)value.u64); + return; + } + const type_info *t = sym->type(); + if (t == NULL) { + fprintf(fp, "<unknown type> 0x%llx\n", (unsigned long long)value.u64); + return; + } + type_info_key ti = t->get_key(); - switch ( ti.scalar_type() ) { - case S8_TYPE: fprintf(fp,".s8 %d\n", value.s8 ); break; - case S16_TYPE: fprintf(fp,".s16 %d\n", value.s16 ); break; - case S32_TYPE: fprintf(fp,".s32 %d\n", value.s32 ); break; - case S64_TYPE: fprintf(fp,".s64 %Ld\n", value.s64 ); break; - case U8_TYPE: fprintf(fp,".u8 %u [0x%02x]\n", value.u8, (unsigned) value.u8 ); break; - case U16_TYPE: fprintf(fp,".u16 %u [0x%04x]\n", value.u16, (unsigned) value.u16 ); break; - case U32_TYPE: fprintf(fp,".u32 %u [0x%08x]\n", value.u32, (unsigned) value.u32 ); break; - case U64_TYPE: fprintf(fp,".u64 %llu [0x%llx]\n", value.u64, value.u64 ); break; - case F16_TYPE: fprintf(fp,".f16 %f [0x%04x]\n", value.f16, (unsigned) value.u16 ); break; - case F32_TYPE: fprintf(fp,".f32 %.15lf [0x%08x]\n", value.f32, value.u32 ); break; - case F64_TYPE: fprintf(fp,".f64 %.15le [0x%016llx]\n", value.f64, value.u64 ); break; - case B8_TYPE: fprintf(fp,".b8 0x%02x\n", (unsigned) value.u8 ); break; - case B16_TYPE: fprintf(fp,".b16 0x%04x\n", (unsigned) value.u16 ); break; - case B32_TYPE: fprintf(fp,".b32 0x%08x\n", (unsigned) value.u32 ); break; - case B64_TYPE: fprintf(fp,".b64 0x%llx\n", (unsigned long long ) value.u64 ); break; - case PRED_TYPE: fprintf(fp,".pred %u\n", (unsigned) value.pred ); break; - default: - fprintf( fp, "non-scalar type\n" ); + switch (ti.scalar_type()) { + case S8_TYPE: + fprintf(fp, ".s8 %d\n", value.s8); + break; + case S16_TYPE: + fprintf(fp, ".s16 %d\n", value.s16); + break; + case S32_TYPE: + fprintf(fp, ".s32 %d\n", value.s32); + break; + case S64_TYPE: + fprintf(fp, ".s64 %Ld\n", value.s64); + break; + case U8_TYPE: + fprintf(fp, ".u8 %u [0x%02x]\n", value.u8, (unsigned)value.u8); + break; + case U16_TYPE: + fprintf(fp, ".u16 %u [0x%04x]\n", value.u16, (unsigned)value.u16); break; - } - fflush(fp); + case U32_TYPE: + fprintf(fp, ".u32 %u [0x%08x]\n", value.u32, (unsigned)value.u32); + break; + case U64_TYPE: + fprintf(fp, ".u64 %llu [0x%llx]\n", value.u64, value.u64); + break; + case F16_TYPE: + fprintf(fp, ".f16 %f [0x%04x]\n", value.f16, (unsigned)value.u16); + break; + case F32_TYPE: + fprintf(fp, ".f32 %.15lf [0x%08x]\n", value.f32, value.u32); + break; + case F64_TYPE: + fprintf(fp, ".f64 %.15le [0x%016llx]\n", value.f64, value.u64); + break; + case B8_TYPE: + fprintf(fp, ".b8 0x%02x\n", (unsigned)value.u8); + break; + case B16_TYPE: + fprintf(fp, ".b16 0x%04x\n", (unsigned)value.u16); + break; + case B32_TYPE: + fprintf(fp, ".b32 0x%08x\n", (unsigned)value.u32); + break; + case B64_TYPE: + fprintf(fp, ".b64 0x%llx\n", (unsigned long long)value.u64); + break; + case PRED_TYPE: + fprintf(fp, ".pred %u\n", (unsigned)value.pred); + break; + default: + fprintf(fp, "non-scalar type\n"); + break; + } + fflush(fp); } -static void print_reg( std::string name, ptx_reg_t value, symbol_table *symtab ) -{ - print_reg(stdout,name,value,symtab); +static void print_reg(std::string name, ptx_reg_t value, symbol_table *symtab) { + print_reg(stdout, name, value, symtab); } -void ptx_thread_info::callstack_push( unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) -{ - m_RPC = -1; - m_RPC_updated = true; - m_last_was_call = true; - assert( m_func_info != NULL ); - m_callstack.push_back( stack_entry(m_symbol_table,m_func_info,pc,rpc,return_var_src,return_var_dst,call_uid) ); - m_regs.push_back( reg_map_t() ); - m_debug_trace_regs_modified.push_back( reg_map_t() ); - m_debug_trace_regs_read.push_back( reg_map_t() ); - m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); +void ptx_thread_info::callstack_push(unsigned pc, unsigned rpc, + const symbol *return_var_src, + const symbol *return_var_dst, + unsigned call_uid) { + m_RPC = -1; + m_RPC_updated = true; + m_last_was_call = true; + assert(m_func_info != NULL); + m_callstack.push_back(stack_entry(m_symbol_table, m_func_info, pc, rpc, + return_var_src, return_var_dst, call_uid)); + m_regs.push_back(reg_map_t()); + m_debug_trace_regs_modified.push_back(reg_map_t()); + m_debug_trace_regs_read.push_back(reg_map_t()); + m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); } -//ptxplus version of callstack_push. -void ptx_thread_info::callstack_push_plus( unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) -{ - m_RPC = -1; - m_RPC_updated = true; - m_last_was_call = true; - assert( m_func_info != NULL ); - m_callstack.push_back( stack_entry(m_symbol_table,m_func_info,pc,rpc,return_var_src,return_var_dst,call_uid) ); - //m_regs.push_back( reg_map_t() ); - //m_debug_trace_regs_modified.push_back( reg_map_t() ); - //m_debug_trace_regs_read.push_back( reg_map_t() ); - m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); +// ptxplus version of callstack_push. +void ptx_thread_info::callstack_push_plus(unsigned pc, unsigned rpc, + const symbol *return_var_src, + const symbol *return_var_dst, + unsigned call_uid) { + m_RPC = -1; + m_RPC_updated = true; + m_last_was_call = true; + assert(m_func_info != NULL); + m_callstack.push_back(stack_entry(m_symbol_table, m_func_info, pc, rpc, + return_var_src, return_var_dst, call_uid)); + // m_regs.push_back( reg_map_t() ); + // m_debug_trace_regs_modified.push_back( reg_map_t() ); + // m_debug_trace_regs_read.push_back( reg_map_t() ); + m_local_mem_stack_pointer += m_func_info->local_mem_framesize(); } +bool ptx_thread_info::callstack_pop() { + const symbol *rv_src = m_callstack.back().m_return_var_src; + const symbol *rv_dst = m_callstack.back().m_return_var_dst; + assert(!((rv_src != NULL) ^ + (rv_dst != NULL))); // ensure caller and callee agree on whether + // there is a return value -bool ptx_thread_info::callstack_pop() -{ - const symbol *rv_src = m_callstack.back().m_return_var_src; - const symbol *rv_dst = m_callstack.back().m_return_var_dst; - assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value - - // read return value from callee frame - arg_buffer_t buffer(m_gpu->gpgpu_ctx); - if( rv_src != NULL ) - buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); + // read return value from callee frame + arg_buffer_t buffer(m_gpu->gpgpu_ctx); + if (rv_src != NULL) + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), + rv_dst); - m_symbol_table = m_callstack.back().m_symbol_table; - m_NPC = m_callstack.back().m_PC; - m_RPC_updated = true; - m_last_was_call = false; - m_RPC = m_callstack.back().m_RPC; - m_func_info = m_callstack.back().m_func_info; - if( m_func_info ) { - assert( m_local_mem_stack_pointer >= m_func_info->local_mem_framesize() ); - m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); - } - m_callstack.pop_back(); - m_regs.pop_back(); - m_debug_trace_regs_modified.pop_back(); - m_debug_trace_regs_read.pop_back(); + m_symbol_table = m_callstack.back().m_symbol_table; + m_NPC = m_callstack.back().m_PC; + m_RPC_updated = true; + m_last_was_call = false; + m_RPC = m_callstack.back().m_RPC; + m_func_info = m_callstack.back().m_func_info; + if (m_func_info) { + assert(m_local_mem_stack_pointer >= m_func_info->local_mem_framesize()); + m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); + } + m_callstack.pop_back(); + m_regs.pop_back(); + m_debug_trace_regs_modified.pop_back(); + m_debug_trace_regs_read.pop_back(); - // write return value into caller frame - if( rv_dst != NULL ) - copy_buffer_to_frame(this, buffer); + // write return value into caller frame + if (rv_dst != NULL) copy_buffer_to_frame(this, buffer); - return m_callstack.empty(); + return m_callstack.empty(); } -//ptxplus version of callstack_pop -bool ptx_thread_info::callstack_pop_plus() -{ - const symbol *rv_src = m_callstack.back().m_return_var_src; - const symbol *rv_dst = m_callstack.back().m_return_var_dst; - assert( !((rv_src != NULL) ^ (rv_dst != NULL)) ); // ensure caller and callee agree on whether there is a return value +// ptxplus version of callstack_pop +bool ptx_thread_info::callstack_pop_plus() { + const symbol *rv_src = m_callstack.back().m_return_var_src; + const symbol *rv_dst = m_callstack.back().m_return_var_dst; + assert(!((rv_src != NULL) ^ + (rv_dst != NULL))); // ensure caller and callee agree on whether + // there is a return value - // read return value from callee frame - arg_buffer_t buffer(m_gpu->gpgpu_ctx); - if( rv_src != NULL ) - buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), rv_dst ); + // read return value from callee frame + arg_buffer_t buffer(m_gpu->gpgpu_ctx); + if (rv_src != NULL) + buffer = copy_arg_to_buffer(this, operand_info(rv_src, m_gpu->gpgpu_ctx), + rv_dst); - m_symbol_table = m_callstack.back().m_symbol_table; - m_NPC = m_callstack.back().m_PC; - m_RPC_updated = true; - m_last_was_call = false; - m_RPC = m_callstack.back().m_RPC; - m_func_info = m_callstack.back().m_func_info; - if( m_func_info ) { - assert( m_local_mem_stack_pointer >= m_func_info->local_mem_framesize() ); - m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); - } - m_callstack.pop_back(); - //m_regs.pop_back(); - //m_debug_trace_regs_modified.pop_back(); - //m_debug_trace_regs_read.pop_back(); + m_symbol_table = m_callstack.back().m_symbol_table; + m_NPC = m_callstack.back().m_PC; + m_RPC_updated = true; + m_last_was_call = false; + m_RPC = m_callstack.back().m_RPC; + m_func_info = m_callstack.back().m_func_info; + if (m_func_info) { + assert(m_local_mem_stack_pointer >= m_func_info->local_mem_framesize()); + m_local_mem_stack_pointer -= m_func_info->local_mem_framesize(); + } + m_callstack.pop_back(); + // m_regs.pop_back(); + // m_debug_trace_regs_modified.pop_back(); + // m_debug_trace_regs_read.pop_back(); - // write return value into caller frame - if( rv_dst != NULL ) - copy_buffer_to_frame(this, buffer); + // write return value into caller frame + if (rv_dst != NULL) copy_buffer_to_frame(this, buffer); - return m_callstack.empty(); + return m_callstack.empty(); } -void ptx_thread_info::dump_callstack() const -{ - std::list<stack_entry>::const_iterator c=m_callstack.begin(); - std::list<reg_map_t>::const_iterator r=m_regs.begin(); +void ptx_thread_info::dump_callstack() const { + std::list<stack_entry>::const_iterator c = m_callstack.begin(); + std::list<reg_map_t>::const_iterator r = m_regs.begin(); - printf("\n\n"); - printf("Call stack for thread uid = %u (sc=%u, hwtid=%u)\n", m_uid, m_hw_sid, m_hw_tid ); - while( c != m_callstack.end() && r != m_regs.end() ) { - const stack_entry &c_e = *c; - const reg_map_t ®s = *r; - if( !c_e.m_valid ) { - printf(" <entry> #regs = %zu\n", regs.size() ); - } else { - printf(" %20s PC=%3u RV= (callee=\'%s\',caller=\'%s\') #regs = %zu\n", - c_e.m_func_info->get_name().c_str(), c_e.m_PC, - c_e.m_return_var_src->name().c_str(), - c_e.m_return_var_dst->name().c_str(), - regs.size() ); - } - c++; - r++; - } - if( c != m_callstack.end() || r != m_regs.end() ) { - printf(" *** mismatch in m_regs and m_callstack sizes ***\n" ); - } - printf("\n\n"); + printf("\n\n"); + printf("Call stack for thread uid = %u (sc=%u, hwtid=%u)\n", m_uid, m_hw_sid, + m_hw_tid); + while (c != m_callstack.end() && r != m_regs.end()) { + const stack_entry &c_e = *c; + const reg_map_t ®s = *r; + if (!c_e.m_valid) { + printf(" <entry> #regs = %zu\n", + regs.size()); + } else { + printf(" %20s PC=%3u RV= (callee=\'%s\',caller=\'%s\') #regs = %zu\n", + c_e.m_func_info->get_name().c_str(), c_e.m_PC, + c_e.m_return_var_src->name().c_str(), + c_e.m_return_var_dst->name().c_str(), regs.size()); + } + c++; + r++; + } + if (c != m_callstack.end() || r != m_regs.end()) { + printf(" *** mismatch in m_regs and m_callstack sizes ***\n"); + } + printf("\n\n"); +} + +std::string ptx_thread_info::get_location() const { + const ptx_instruction *pI = m_func_info->get_instruction(m_PC); + char buf[1024]; + snprintf(buf, 1024, "%s:%u", pI->source_file(), pI->source_line()); + return std::string(buf); } -std::string ptx_thread_info::get_location() const -{ - const ptx_instruction *pI = m_func_info->get_instruction(m_PC); - char buf[1024]; - snprintf(buf,1024,"%s:%u", pI->source_file(), pI->source_line() ); - return std::string(buf); +const ptx_instruction *ptx_thread_info::get_inst() const { + return m_func_info->get_instruction(m_PC); } -const ptx_instruction *ptx_thread_info::get_inst() const -{ - return m_func_info->get_instruction(m_PC); +const ptx_instruction *ptx_thread_info::get_inst(addr_t pc) const { + return m_func_info->get_instruction(pc); } -const ptx_instruction *ptx_thread_info::get_inst( addr_t pc ) const -{ - return m_func_info->get_instruction(pc); +void ptx_thread_info::dump_regs(FILE *fp) { + if (m_regs.empty()) return; + if (m_regs.back().empty()) return; + fprintf(fp, "Register File Contents:\n"); + fflush(fp); + reg_map_t::const_iterator r; + for (r = m_regs.back().begin(); r != m_regs.back().end(); ++r) { + const symbol *sym = r->first; + ptx_reg_t value = r->second; + std::string name = sym->name(); + print_reg(fp, name, value, m_symbol_table); + } } -void ptx_thread_info::dump_regs( FILE *fp ) -{ - if(m_regs.empty()) return; - if(m_regs.back().empty()) return; - fprintf(fp,"Register File Contents:\n"); - fflush(fp); - reg_map_t::const_iterator r; - for ( r=m_regs.back().begin(); r != m_regs.back().end(); ++r ) { +void ptx_thread_info::dump_modifiedregs(FILE *fp) { + if (!(m_debug_trace_regs_modified.empty() || + m_debug_trace_regs_modified.back().empty())) { + fprintf(fp, "Output Registers:\n"); + fflush(fp); + reg_map_t::iterator r; + for (r = m_debug_trace_regs_modified.back().begin(); + r != m_debug_trace_regs_modified.back().end(); ++r) { const symbol *sym = r->first; + std::string name = sym->name(); ptx_reg_t value = r->second; + print_reg(fp, name, value, m_symbol_table); + } + } + if (!(m_debug_trace_regs_read.empty() || + m_debug_trace_regs_read.back().empty())) { + fprintf(fp, "Input Registers:\n"); + fflush(fp); + reg_map_t::iterator r; + for (r = m_debug_trace_regs_read.back().begin(); + r != m_debug_trace_regs_read.back().end(); ++r) { + const symbol *sym = r->first; std::string name = sym->name(); - print_reg(fp,name,value,m_symbol_table); - } -} - -void ptx_thread_info::dump_modifiedregs(FILE *fp) -{ - if( !(m_debug_trace_regs_modified.empty() || - m_debug_trace_regs_modified.back().empty()) ) { - fprintf(fp,"Output Registers:\n"); - fflush(fp); - reg_map_t::iterator r; - for ( r=m_debug_trace_regs_modified.back().begin(); r != m_debug_trace_regs_modified.back().end(); ++r ) { - const symbol *sym = r->first; - std::string name = sym->name(); - ptx_reg_t value = r->second; - print_reg(fp,name,value,m_symbol_table); - } - } - if( !(m_debug_trace_regs_read.empty() || - m_debug_trace_regs_read.back().empty()) ) { - fprintf(fp,"Input Registers:\n"); - fflush(fp); - reg_map_t::iterator r; - for ( r=m_debug_trace_regs_read.back().begin(); r != m_debug_trace_regs_read.back().end(); ++r ) { - const symbol *sym = r->first; - std::string name = sym->name(); - ptx_reg_t value = r->second; - print_reg(fp,name,value,m_symbol_table); - } - } + ptx_reg_t value = r->second; + print_reg(fp, name, value, m_symbol_table); + } + } } -void ptx_thread_info::push_breakaddr(const operand_info &breakaddr) -{ - m_breakaddrs.push(breakaddr); +void ptx_thread_info::push_breakaddr(const operand_info &breakaddr) { + m_breakaddrs.push(breakaddr); } -const operand_info& ptx_thread_info::pop_breakaddr() -{ - if(m_breakaddrs.empty()) { - printf("empty breakaddrs stack"); - assert(0); - } - operand_info& breakaddr = m_breakaddrs.top(); - m_breakaddrs.pop(); - return breakaddr; +const operand_info &ptx_thread_info::pop_breakaddr() { + if (m_breakaddrs.empty()) { + printf("empty breakaddrs stack"); + assert(0); + } + operand_info &breakaddr = m_breakaddrs.top(); + m_breakaddrs.pop(); + return breakaddr; } -void ptx_thread_info::set_npc( const function_info *f ) -{ - m_NPC = f->get_start_PC(); - m_func_info = const_cast<function_info*>( f ); - m_symbol_table = m_func_info->get_symtab(); +void ptx_thread_info::set_npc(const function_info *f) { + m_NPC = f->get_start_PC(); + m_func_info = const_cast<function_info *>(f); + m_symbol_table = m_func_info->get_symtab(); } - -void feature_not_implemented( const char *f ) -{ - printf("GPGPU-Sim: feature '%s' not supported\n", f ); - abort(); +void feature_not_implemented(const char *f) { + printf("GPGPU-Sim: feature '%s' not supported\n", f); + abort(); } diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index c2b3cc8..f0c26ef 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -6,53 +6,51 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ptx_sim_h_INCLUDED #define ptx_sim_h_INCLUDED #include <stdlib.h> -#include "half.h" #include "../abstract_hardware_model.h" -#include "../tr1_hash_map.h" +#include "../tr1_hash_map.h" +#include "half.h" #include <assert.h> #include "opcodes.h" -#include <string> +#include <list> #include <map> #include <set> -#include <list> +#include <string> #include "memory.h" -#define GCC_VERSION (__GNUC__ * 10000 \ - + __GNUC_MINOR__ * 100 \ - + __GNUC_PATCHLEVEL__) - - +#define GCC_VERSION \ + (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) struct param_t { - const void *pdata; - int type; - size_t size; - size_t offset; + const void *pdata; + int type; + size_t size; + size_t offset; }; #include <stack> @@ -62,98 +60,93 @@ struct param_t { using half_float::half; union ptx_reg_t { - ptx_reg_t() { - bits.ms = 0; - bits.ls = 0; - u128.low=0; - u128.lowest=0; - u128.highest=0; - u128.high=0; - s8=0; - s16=0; - s32=0; - s64=0; - u8=0; - u16=0; - u64=0; - f16=0; - f32=0; - f64=0; - pred=0; - } - ptx_reg_t(unsigned x) - { - bits.ms = 0; - bits.ls = 0; - u128.low=0; - u128.lowest=0; - u128.highest=0; - u128.high=0; - s8=0; - s16=0; - s32=0; - s64=0; - u8=0; - u16=0; - u64=0; - f16=0; - f32=0; - f64=0; - pred=0; - u32 = x; - } - operator unsigned int() { return u32;} - operator unsigned short() { return u16;} - operator unsigned char() { return u8;} - operator unsigned long long() { return u64;} - - void mask_and( unsigned ms, unsigned ls ) - { - bits.ms &= ms; - bits.ls &= ls; - } + ptx_reg_t() { + bits.ms = 0; + bits.ls = 0; + u128.low = 0; + u128.lowest = 0; + u128.highest = 0; + u128.high = 0; + s8 = 0; + s16 = 0; + s32 = 0; + s64 = 0; + u8 = 0; + u16 = 0; + u64 = 0; + f16 = 0; + f32 = 0; + f64 = 0; + pred = 0; + } + ptx_reg_t(unsigned x) { + bits.ms = 0; + bits.ls = 0; + u128.low = 0; + u128.lowest = 0; + u128.highest = 0; + u128.high = 0; + s8 = 0; + s16 = 0; + s32 = 0; + s64 = 0; + u8 = 0; + u16 = 0; + u64 = 0; + f16 = 0; + f32 = 0; + f64 = 0; + pred = 0; + u32 = x; + } + operator unsigned int() { return u32; } + operator unsigned short() { return u16; } + operator unsigned char() { return u8; } + operator unsigned long long() { return u64; } - void mask_or( unsigned ms, unsigned ls ) - { - bits.ms |= ms; - bits.ls |= ls; - } - int get_bit( unsigned bit ) - { - if ( bit < 32 ) - return(bits.ls >> bit) & 1; - else - return(bits.ms >> (bit-32)) & 1; - } + void mask_and(unsigned ms, unsigned ls) { + bits.ms &= ms; + bits.ls &= ls; + } - signed char s8; - signed short s16; - signed int s32; - signed long long s64; - unsigned char u8; - unsigned short u16; - unsigned int u32; - unsigned long long u64; - //gcc 4.7.0 - #if GCC_VERSION >= 40700 - half f16; - #else - float f16; - #endif - float f32; - double f64; - struct { - unsigned ls; - unsigned ms; - } bits; - struct { - unsigned int lowest; - unsigned int low; - unsigned int high; - unsigned int highest; - } u128; - unsigned pred : 4; + void mask_or(unsigned ms, unsigned ls) { + bits.ms |= ms; + bits.ls |= ls; + } + int get_bit(unsigned bit) { + if (bit < 32) + return (bits.ls >> bit) & 1; + else + return (bits.ms >> (bit - 32)) & 1; + } + signed char s8; + signed short s16; + signed int s32; + signed long long s64; + unsigned char u8; + unsigned short u16; + unsigned int u32; + unsigned long long u64; +// gcc 4.7.0 +#if GCC_VERSION >= 40700 + half f16; +#else + float f16; +#endif + float f32; + double f64; + struct { + unsigned ls; + unsigned ms; + } bits; + struct { + unsigned int lowest; + unsigned int low; + unsigned int high; + unsigned int highest; + } u128; + unsigned pred : 4; }; class ptx_instruction; @@ -163,373 +156,375 @@ class function_info; class ptx_thread_info; class ptx_cta_info { -public: - ptx_cta_info( unsigned sm_idx, gpgpu_context* ctx ); - void add_thread( ptx_thread_info *thd ); - unsigned num_threads() const; - void check_cta_thread_status_and_reset(); - void register_thread_exit( ptx_thread_info *thd ); - void register_deleted_thread( ptx_thread_info *thd ); - unsigned get_sm_idx() const; - unsigned get_bar_threads() const; - void inc_bar_threads(); - void reset_bar_threads(); + public: + ptx_cta_info(unsigned sm_idx, gpgpu_context *ctx); + void add_thread(ptx_thread_info *thd); + unsigned num_threads() const; + void check_cta_thread_status_and_reset(); + void register_thread_exit(ptx_thread_info *thd); + void register_deleted_thread(ptx_thread_info *thd); + unsigned get_sm_idx() const; + unsigned get_bar_threads() const; + void inc_bar_threads(); + void reset_bar_threads(); -private: - // backward pointer - class gpgpu_context* gpgpu_ctx; - unsigned m_bar_threads; - unsigned long long m_uid; - unsigned m_sm_idx; - std::set<ptx_thread_info*> m_threads_in_cta; - std::set<ptx_thread_info*> m_threads_that_have_exited; - std::set<ptx_thread_info*> m_dangling_pointers; + private: + // backward pointer + class gpgpu_context *gpgpu_ctx; + unsigned m_bar_threads; + unsigned long long m_uid; + unsigned m_sm_idx; + std::set<ptx_thread_info *> m_threads_in_cta; + std::set<ptx_thread_info *> m_threads_that_have_exited; + std::set<ptx_thread_info *> m_dangling_pointers; }; class ptx_warp_info { -public: - ptx_warp_info(); // add get_core or something, or threads? - unsigned get_done_threads() const; - void inc_done_threads(); - void reset_done_threads(); + public: + ptx_warp_info(); // add get_core or something, or threads? + unsigned get_done_threads() const; + void inc_done_threads(); + void reset_done_threads(); -private: - unsigned m_done_threads; + private: + unsigned m_done_threads; }; class symbol; struct stack_entry { - stack_entry() { - m_symbol_table=NULL; - m_func_info=NULL; - m_PC=0; - m_RPC=-1; - m_return_var_src = NULL; - m_return_var_dst = NULL; - m_call_uid = 0; - m_valid = false; - } - stack_entry( symbol_table *s, function_info *f, unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ) - { - m_symbol_table=s; - m_func_info=f; - m_PC=pc; - m_RPC=rpc; - m_return_var_src = return_var_src; - m_return_var_dst = return_var_dst; - m_call_uid = call_uid; - m_valid = true; - } + stack_entry() { + m_symbol_table = NULL; + m_func_info = NULL; + m_PC = 0; + m_RPC = -1; + m_return_var_src = NULL; + m_return_var_dst = NULL; + m_call_uid = 0; + m_valid = false; + } + stack_entry(symbol_table *s, function_info *f, unsigned pc, unsigned rpc, + const symbol *return_var_src, const symbol *return_var_dst, + unsigned call_uid) { + m_symbol_table = s; + m_func_info = f; + m_PC = pc; + m_RPC = rpc; + m_return_var_src = return_var_src; + m_return_var_dst = return_var_dst; + m_call_uid = call_uid; + m_valid = true; + } - bool m_valid; - symbol_table *m_symbol_table; - function_info *m_func_info; - unsigned m_PC; - unsigned m_RPC; - const symbol *m_return_var_src; - const symbol *m_return_var_dst; - unsigned m_call_uid; + bool m_valid; + symbol_table *m_symbol_table; + function_info *m_func_info; + unsigned m_PC; + unsigned m_RPC; + const symbol *m_return_var_src; + const symbol *m_return_var_dst; + unsigned m_call_uid; }; class ptx_version { -public: - ptx_version() - { - m_valid = false; - m_ptx_version = 0; - m_ptx_extensions = 0; - m_sm_version_valid=false; - m_texmode_unified=true; - m_map_f64_to_f32 = true; - } - ptx_version(float ver, unsigned extensions) - { - m_valid = true; - m_ptx_version = ver; - m_ptx_extensions = extensions; - m_sm_version_valid=false; - m_texmode_unified=true; - } - void set_target( const char *sm_ver, const char *ext, const char *ext2 ) - { - assert( m_valid ); - m_sm_version_str = sm_ver; - check_target_extension(ext); - check_target_extension(ext2); - sscanf(sm_ver,"%u",&m_sm_version); - m_sm_version_valid=true; - } - float ver() const { assert(m_valid); return m_ptx_version; } - unsigned target() const { assert(m_valid&&m_sm_version_valid); return m_sm_version; } - unsigned extensions() const { assert(m_valid); return m_ptx_extensions; } -private: - void check_target_extension( const char *ext ) - { - if( ext ) { - if( !strcmp(ext,"texmode_independent") ) - m_texmode_unified=false; - else if( !strcmp(ext,"texmode_unified") ) - m_texmode_unified=true; - else if( !strcmp(ext,"map_f64_to_f32") ) - m_map_f64_to_f32 = true; - else abort(); - } - } + public: + ptx_version() { + m_valid = false; + m_ptx_version = 0; + m_ptx_extensions = 0; + m_sm_version_valid = false; + m_texmode_unified = true; + m_map_f64_to_f32 = true; + } + ptx_version(float ver, unsigned extensions) { + m_valid = true; + m_ptx_version = ver; + m_ptx_extensions = extensions; + m_sm_version_valid = false; + m_texmode_unified = true; + } + void set_target(const char *sm_ver, const char *ext, const char *ext2) { + assert(m_valid); + m_sm_version_str = sm_ver; + check_target_extension(ext); + check_target_extension(ext2); + sscanf(sm_ver, "%u", &m_sm_version); + m_sm_version_valid = true; + } + float ver() const { + assert(m_valid); + return m_ptx_version; + } + unsigned target() const { + assert(m_valid && m_sm_version_valid); + return m_sm_version; + } + unsigned extensions() const { + assert(m_valid); + return m_ptx_extensions; + } + + private: + void check_target_extension(const char *ext) { + if (ext) { + if (!strcmp(ext, "texmode_independent")) + m_texmode_unified = false; + else if (!strcmp(ext, "texmode_unified")) + m_texmode_unified = true; + else if (!strcmp(ext, "map_f64_to_f32")) + m_map_f64_to_f32 = true; + else + abort(); + } + } - bool m_valid; - float m_ptx_version; - unsigned m_sm_version_valid; - std::string m_sm_version_str; - bool m_texmode_unified; - bool m_map_f64_to_f32; - unsigned m_sm_version; - unsigned m_ptx_extensions; + bool m_valid; + float m_ptx_version; + unsigned m_sm_version_valid; + std::string m_sm_version_str; + bool m_texmode_unified; + bool m_map_f64_to_f32; + unsigned m_sm_version; + unsigned m_ptx_extensions; }; class ptx_thread_info { -public: - ~ptx_thread_info(); - ptx_thread_info( kernel_info_t &kernel ); + public: + ~ptx_thread_info(); + ptx_thread_info(kernel_info_t &kernel); - void init(gpgpu_t *gpu, core_t *core, unsigned sid, unsigned cta_id, unsigned wid, unsigned tid, bool fsim) - { - m_gpu = gpu; - m_core = core; - m_hw_sid=sid; - m_hw_ctaid=cta_id; - m_hw_wid=wid; - m_hw_tid=tid; - m_functionalSimulationMode = fsim; - } + void init(gpgpu_t *gpu, core_t *core, unsigned sid, unsigned cta_id, + unsigned wid, unsigned tid, bool fsim) { + m_gpu = gpu; + m_core = core; + m_hw_sid = sid; + m_hw_ctaid = cta_id; + m_hw_wid = wid; + m_hw_tid = tid; + m_functionalSimulationMode = fsim; + } - void ptx_fetch_inst( inst_t &inst ) const; - void ptx_exec_inst( warp_inst_t &inst, unsigned lane_id ); + void ptx_fetch_inst(inst_t &inst) const; + void ptx_exec_inst(warp_inst_t &inst, unsigned lane_id); - const ptx_version &get_ptx_version() const; - void set_reg( const symbol *reg, const ptx_reg_t &value ); - void print_reg_thread (char * fname); - void resume_reg_thread(char * fname, symbol_table * symtab); - ptx_reg_t get_reg( const symbol *reg ); - ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ); - void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ); - void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ); - void get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ); - void set_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4 ); - void set_wmma_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4, - const ptx_reg_t &data5, - const ptx_reg_t &data6, - const ptx_reg_t &data7, - const ptx_reg_t &data8 ); + const ptx_version &get_ptx_version() const; + void set_reg(const symbol *reg, const ptx_reg_t &value); + void print_reg_thread(char *fname); + void resume_reg_thread(char *fname, symbol_table *symtab); + ptx_reg_t get_reg(const symbol *reg); + ptx_reg_t get_operand_value(const operand_info &op, operand_info dstInfo, + unsigned opType, ptx_thread_info *thread, + int derefFlag); + void set_operand_value(const operand_info &dst, const ptx_reg_t &data, + unsigned type, ptx_thread_info *thread, + const ptx_instruction *pI); + void set_operand_value(const operand_info &dst, const ptx_reg_t &data, + unsigned type, ptx_thread_info *thread, + const ptx_instruction *pI, int overflow, int carry); + void get_vector_operand_values(const operand_info &op, ptx_reg_t *ptx_regs, + unsigned num_elements); + void set_vector_operand_values(const operand_info &dst, + const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4); + void set_wmma_vector_operand_values( + const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, + const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8); - function_info *func_info() - { - return m_func_info; - } - void print_insn( unsigned pc, FILE * fp ) const; - void set_info( function_info *func ); - unsigned get_uid() const - { - return m_uid; - } + function_info *func_info() { return m_func_info; } + void print_insn(unsigned pc, FILE *fp) const; + void set_info(function_info *func); + unsigned get_uid() const { return m_uid; } - dim3 get_ctaid() const { return m_ctaid; } - dim3 get_tid() const { return m_tid; } - dim3 get_ntid() const { return m_ntid; } - class gpgpu_sim *get_gpu() { return (gpgpu_sim*)m_gpu;} - unsigned get_hw_tid() const { return m_hw_tid;} - unsigned get_hw_ctaid() const { return m_hw_ctaid;} - unsigned get_hw_wid() const { return m_hw_wid;} - unsigned get_hw_sid() const { return m_hw_sid;} - core_t *get_core() { return m_core; } + dim3 get_ctaid() const { return m_ctaid; } + dim3 get_tid() const { return m_tid; } + dim3 get_ntid() const { return m_ntid; } + class gpgpu_sim *get_gpu() { + return (gpgpu_sim *)m_gpu; + } + unsigned get_hw_tid() const { return m_hw_tid; } + unsigned get_hw_ctaid() const { return m_hw_ctaid; } + unsigned get_hw_wid() const { return m_hw_wid; } + unsigned get_hw_sid() const { return m_hw_sid; } + core_t *get_core() { return m_core; } - unsigned get_icount() const { return m_icount;} - void set_valid() { m_valid = true;} - addr_t last_eaddr() const { return m_last_effective_address;} - memory_space_t last_space() const { return m_last_memory_space;} - dram_callback_t last_callback() const { return m_last_dram_callback;} - unsigned long long get_cta_uid() { return m_cta_info->get_sm_idx();} + unsigned get_icount() const { return m_icount; } + void set_valid() { m_valid = true; } + addr_t last_eaddr() const { return m_last_effective_address; } + memory_space_t last_space() const { return m_last_memory_space; } + dram_callback_t last_callback() const { return m_last_dram_callback; } + unsigned long long get_cta_uid() { return m_cta_info->get_sm_idx(); } - void set_single_thread_single_block() - { - m_ntid.x = 1; - m_ntid.y = 1; - m_ntid.z = 1; - m_ctaid.x = 0; - m_ctaid.y = 0; - m_ctaid.z = 0; - m_tid.x = 0; - m_tid.y = 0; - m_tid.z = 0; - m_nctaid.x = 1; - m_nctaid.y = 1; - m_nctaid.z = 1; - m_gridid = 0; - m_valid = true; - } - void set_tid( dim3 tid ) { m_tid = tid; } - void cpy_tid_to_reg( dim3 tid ); - void set_ctaid( dim3 ctaid ) { m_ctaid = ctaid; } - void set_ntid( dim3 tid ) { m_ntid = tid; } - void set_nctaid( dim3 cta_size ) { m_nctaid = cta_size; } + void set_single_thread_single_block() { + m_ntid.x = 1; + m_ntid.y = 1; + m_ntid.z = 1; + m_ctaid.x = 0; + m_ctaid.y = 0; + m_ctaid.z = 0; + m_tid.x = 0; + m_tid.y = 0; + m_tid.z = 0; + m_nctaid.x = 1; + m_nctaid.y = 1; + m_nctaid.z = 1; + m_gridid = 0; + m_valid = true; + } + void set_tid(dim3 tid) { m_tid = tid; } + void cpy_tid_to_reg(dim3 tid); + void set_ctaid(dim3 ctaid) { m_ctaid = ctaid; } + void set_ntid(dim3 tid) { m_ntid = tid; } + void set_nctaid(dim3 cta_size) { m_nctaid = cta_size; } - unsigned get_builtin( int builtin_id, unsigned dim_mod ); + unsigned get_builtin(int builtin_id, unsigned dim_mod); - void set_done(); - bool is_done() { return m_thread_done;} - unsigned donecycle() const { return m_cycle_done; } + void set_done(); + bool is_done() { return m_thread_done; } + unsigned donecycle() const { return m_cycle_done; } - unsigned next_instr() - { - m_icount++; - m_branch_taken = false; - return m_PC; - } - bool branch_taken() const - { - return m_branch_taken; - } - unsigned get_pc() const - { - return m_PC; - } - void set_npc( unsigned npc ) - { - m_NPC = npc; - } - void set_npc( const function_info *f ); - void callstack_push( unsigned npc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ); - bool callstack_pop(); - void callstack_push_plus( unsigned npc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid ); - bool callstack_pop_plus(); - void dump_callstack() const; - std::string get_location() const; - const ptx_instruction *get_inst() const; - const ptx_instruction *get_inst( addr_t pc ) const; - bool rpc_updated() const { return m_RPC_updated; } - bool last_was_call() const { return m_last_was_call; } - unsigned get_rpc() const { return m_RPC; } - void clearRPC() - { - m_RPC = -1; - m_RPC_updated = false; - m_last_was_call = false; - } - unsigned get_return_PC() - { - return m_callstack.back().m_PC; - } - void update_pc( ) - { - m_PC = m_NPC; - } - void dump_regs(FILE * fp); - void dump_modifiedregs(FILE *fp); - void clear_modifiedregs() { m_debug_trace_regs_modified.back().clear(); m_debug_trace_regs_read.back().clear(); } - function_info *get_finfo() { return m_func_info; } - const function_info *get_finfo() const { return m_func_info; } - void push_breakaddr(const operand_info &breakaddr); - const operand_info& pop_breakaddr(); - void enable_debug_trace() { m_enable_debug_trace = true; } - unsigned get_local_mem_stack_pointer() const { return m_local_mem_stack_pointer; } + unsigned next_instr() { + m_icount++; + m_branch_taken = false; + return m_PC; + } + bool branch_taken() const { return m_branch_taken; } + unsigned get_pc() const { return m_PC; } + void set_npc(unsigned npc) { m_NPC = npc; } + void set_npc(const function_info *f); + void callstack_push(unsigned npc, unsigned rpc, const symbol *return_var_src, + const symbol *return_var_dst, unsigned call_uid); + bool callstack_pop(); + void callstack_push_plus(unsigned npc, unsigned rpc, + const symbol *return_var_src, + const symbol *return_var_dst, unsigned call_uid); + bool callstack_pop_plus(); + void dump_callstack() const; + std::string get_location() const; + const ptx_instruction *get_inst() const; + const ptx_instruction *get_inst(addr_t pc) const; + bool rpc_updated() const { return m_RPC_updated; } + bool last_was_call() const { return m_last_was_call; } + unsigned get_rpc() const { return m_RPC; } + void clearRPC() { + m_RPC = -1; + m_RPC_updated = false; + m_last_was_call = false; + } + unsigned get_return_PC() { return m_callstack.back().m_PC; } + void update_pc() { m_PC = m_NPC; } + void dump_regs(FILE *fp); + void dump_modifiedregs(FILE *fp); + void clear_modifiedregs() { + m_debug_trace_regs_modified.back().clear(); + m_debug_trace_regs_read.back().clear(); + } + function_info *get_finfo() { return m_func_info; } + const function_info *get_finfo() const { return m_func_info; } + void push_breakaddr(const operand_info &breakaddr); + const operand_info &pop_breakaddr(); + void enable_debug_trace() { m_enable_debug_trace = true; } + unsigned get_local_mem_stack_pointer() const { + return m_local_mem_stack_pointer; + } - memory_space *get_global_memory() { return m_gpu->get_global_memory(); } - memory_space *get_tex_memory() { return m_gpu->get_tex_memory(); } - memory_space *get_surf_memory() { return m_gpu->get_surf_memory(); } - memory_space *get_param_memory() { return m_kernel.get_param_memory(); } - const gpgpu_functional_sim_config &get_config() const { return m_gpu->get_config(); } - bool isInFunctionalSimulationMode(){ return m_functionalSimulationMode;} - void exitCore() - { - //m_core is not used in case of functional simulation mode - if(!m_functionalSimulationMode) - m_core->warp_exit(m_hw_wid); - } - - void registerExit(){m_cta_info->register_thread_exit(this);} - unsigned get_reduction_value(unsigned ctaid, unsigned barid) {return m_core->get_reduction_value(ctaid,barid);} - void and_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->and_reduction(ctaid,barid,value);} - void or_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->or_reduction(ctaid,barid,value);} - void popc_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->popc_reduction(ctaid,barid,value);} + memory_space *get_global_memory() { return m_gpu->get_global_memory(); } + memory_space *get_tex_memory() { return m_gpu->get_tex_memory(); } + memory_space *get_surf_memory() { return m_gpu->get_surf_memory(); } + memory_space *get_param_memory() { return m_kernel.get_param_memory(); } + const gpgpu_functional_sim_config &get_config() const { + return m_gpu->get_config(); + } + bool isInFunctionalSimulationMode() { return m_functionalSimulationMode; } + void exitCore() { + // m_core is not used in case of functional simulation mode + if (!m_functionalSimulationMode) m_core->warp_exit(m_hw_wid); + } - //Jin: get corresponding kernel grid for CDP purpose - kernel_info_t & get_kernel() { return m_kernel; } + void registerExit() { m_cta_info->register_thread_exit(this); } + unsigned get_reduction_value(unsigned ctaid, unsigned barid) { + return m_core->get_reduction_value(ctaid, barid); + } + void and_reduction(unsigned ctaid, unsigned barid, bool value) { + m_core->and_reduction(ctaid, barid, value); + } + void or_reduction(unsigned ctaid, unsigned barid, bool value) { + m_core->or_reduction(ctaid, barid, value); + } + void popc_reduction(unsigned ctaid, unsigned barid, bool value) { + m_core->popc_reduction(ctaid, barid, value); + } -public: - addr_t m_last_effective_address; - bool m_branch_taken; - memory_space_t m_last_memory_space; - dram_callback_t m_last_dram_callback; - memory_space *m_shared_mem; - memory_space *m_sstarr_mem; - memory_space *m_local_mem; - ptx_warp_info *m_warp_info; - ptx_cta_info *m_cta_info; - ptx_reg_t m_last_set_operand_value; + // Jin: get corresponding kernel grid for CDP purpose + kernel_info_t &get_kernel() { return m_kernel; } -private: + public: + addr_t m_last_effective_address; + bool m_branch_taken; + memory_space_t m_last_memory_space; + dram_callback_t m_last_dram_callback; + memory_space *m_shared_mem; + memory_space *m_sstarr_mem; + memory_space *m_local_mem; + ptx_warp_info *m_warp_info; + ptx_cta_info *m_cta_info; + ptx_reg_t m_last_set_operand_value; - bool m_functionalSimulationMode; - unsigned m_uid; - kernel_info_t &m_kernel; - core_t *m_core; - gpgpu_t *m_gpu; - bool m_valid; - dim3 m_ntid; - dim3 m_tid; - dim3 m_nctaid; - dim3 m_ctaid; - unsigned m_gridid; - bool m_thread_done; - unsigned m_hw_sid; - unsigned m_hw_tid; - unsigned m_hw_wid; - unsigned m_hw_ctaid; + private: + bool m_functionalSimulationMode; + unsigned m_uid; + kernel_info_t &m_kernel; + core_t *m_core; + gpgpu_t *m_gpu; + bool m_valid; + dim3 m_ntid; + dim3 m_tid; + dim3 m_nctaid; + dim3 m_ctaid; + unsigned m_gridid; + bool m_thread_done; + unsigned m_hw_sid; + unsigned m_hw_tid; + unsigned m_hw_wid; + unsigned m_hw_ctaid; - unsigned m_icount; - unsigned m_PC; - unsigned m_NPC; - unsigned m_RPC; - bool m_RPC_updated; - bool m_last_was_call; - unsigned m_cycle_done; + unsigned m_icount; + unsigned m_PC; + unsigned m_NPC; + unsigned m_RPC; + bool m_RPC_updated; + bool m_last_was_call; + unsigned m_cycle_done; - int m_barrier_num; - bool m_at_barrier; + int m_barrier_num; + bool m_at_barrier; - symbol_table *m_symbol_table; - function_info *m_func_info; + symbol_table *m_symbol_table; + function_info *m_func_info; - std::list<stack_entry> m_callstack; - unsigned m_local_mem_stack_pointer; + std::list<stack_entry> m_callstack; + unsigned m_local_mem_stack_pointer; - typedef tr1_hash_map<const symbol*,ptx_reg_t> reg_map_t; - std::list<reg_map_t> m_regs; - std::list<reg_map_t> m_debug_trace_regs_modified; - std::list<reg_map_t> m_debug_trace_regs_read; - bool m_enable_debug_trace; + typedef tr1_hash_map<const symbol *, ptx_reg_t> reg_map_t; + std::list<reg_map_t> m_regs; + std::list<reg_map_t> m_debug_trace_regs_modified; + std::list<reg_map_t> m_debug_trace_regs_read; + bool m_enable_debug_trace; - std::stack<class operand_info, std::vector<operand_info> > m_breakaddrs; + std::stack<class operand_info, std::vector<operand_info> > m_breakaddrs; }; -addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ); -addr_t generic_to_shared( unsigned smid, addr_t addr ); -addr_t generic_to_global( addr_t addr ); -addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ); -addr_t shared_to_generic( unsigned smid, addr_t addr ); -addr_t global_to_generic( addr_t addr ); -bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ); -bool isspace_shared( unsigned smid, addr_t addr ); -bool isspace_global( addr_t addr ); -memory_space_t whichspace( addr_t addr ); +addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr); +addr_t generic_to_shared(unsigned smid, addr_t addr); +addr_t generic_to_global(addr_t addr); +addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr); +addr_t shared_to_generic(unsigned smid, addr_t addr); +addr_t global_to_generic(addr_t addr); +bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr); +bool isspace_shared(unsigned smid, addr_t addr); +bool isspace_global(addr_t addr); +memory_space_t whichspace(addr_t addr); extern unsigned g_ptx_thread_info_uid_next; diff --git a/src/debug.cc b/src/debug.cc index c00ff9e..29506bd 100644 --- a/src/debug.cc +++ b/src/debug.cc @@ -7,202 +7,212 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "debug.h" -#include "gpgpu-sim/shader.h" -#include "gpgpu-sim/gpu-sim.h" -#include "cuda-sim/ptx_sim.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" +#include "cuda-sim/ptx_sim.h" +#include "gpgpu-sim/gpu-sim.h" +#include "gpgpu-sim/shader.h" -#include <map> #include <stdio.h> #include <string.h> +#include <map> -void gpgpu_sim::hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI ) -{ - g_watchpoint_hits[watchpoint_num]=watchpoint_event(thd,pI); +void gpgpu_sim::hit_watchpoint(unsigned watchpoint_num, ptx_thread_info *thd, + const ptx_instruction *pI) { + g_watchpoint_hits[watchpoint_num] = watchpoint_event(thd, pI); } -/// interactive debugger +/// interactive debugger -void gpgpu_sim::gpgpu_debug() -{ - bool done=true; +void gpgpu_sim::gpgpu_debug() { + bool done = true; - static bool single_step=true; - static unsigned next_brkpt=1; - static std::map<unsigned,brk_pt> breakpoints; + static bool single_step = true; + static unsigned next_brkpt = 1; + static std::map<unsigned, brk_pt> breakpoints; - /// if single stepping, go to interactive debugger + /// if single stepping, go to interactive debugger - if( single_step ) - done=false; + if (single_step) done = false; - /// check if we've reached a breakpoint - const ptx_thread_info *brk_thd = NULL; - const ptx_instruction *brk_inst = NULL; + /// check if we've reached a breakpoint + const ptx_thread_info *brk_thd = NULL; + const ptx_instruction *brk_inst = NULL; - for( std::map<unsigned,brk_pt>::iterator i=breakpoints.begin(); i!=breakpoints.end(); i++) { - unsigned num=i->first; - brk_pt &b=i->second; - if( b.is_watchpoint() ) { - unsigned addr = b.get_addr(); - unsigned new_value; - m_global_mem->read(addr,4,&new_value); - if( new_value != b.get_value() || g_watchpoint_hits.find(num) != g_watchpoint_hits.end() ) { - printf( "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new value=%x)\n", - num,b.get_value(),new_value ); - std::map<unsigned,watchpoint_event>::iterator w=g_watchpoint_hits.find(num); - if( w==g_watchpoint_hits.end() ) - printf( "GPGPU-Sim PTX DBG: memory transfer modified value\n"); - else { - watchpoint_event wa = w->second; - brk_thd = wa.thread(); - brk_inst = wa.inst(); - printf( "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, hwtid=%u\n", - brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); - printf( "GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf( "\n" ); - g_watchpoint_hits.erase(w); - } - b.set_value(new_value); - done = false; - } - } else { - /* - for( unsigned sid=0; sid < m_n_shader; sid++ ) { - unsigned hw_thread_id = -1; - abort(); - ptx_thread_info *thread = m_sc[sid]->get_functional_thread(hw_thread_id); - if( thread_at_brkpt(thread, b) ) { - done = false; - printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s (sm=%u, hwtid=%u)\n", - num, b.location().c_str(), sid, hw_thread_id ); - brk_thd = thread; - brk_inst = brk_thd->get_inst(); - printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, hwtid=%u\n", - brk_thd->get_uid(),brk_thd->get_hw_sid(), brk_thd->get_hw_tid() ); - printf( "GPGPU-Sim PTX DBG: "); - brk_inst->print_insn(stdout); - printf( "\n" ); - } - } - */ + for (std::map<unsigned, brk_pt>::iterator i = breakpoints.begin(); + i != breakpoints.end(); i++) { + unsigned num = i->first; + brk_pt &b = i->second; + if (b.is_watchpoint()) { + unsigned addr = b.get_addr(); + unsigned new_value; + m_global_mem->read(addr, 4, &new_value); + if (new_value != b.get_value() || + g_watchpoint_hits.find(num) != g_watchpoint_hits.end()) { + printf( + "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new " + "value=%x)\n", + num, b.get_value(), new_value); + std::map<unsigned, watchpoint_event>::iterator w = + g_watchpoint_hits.find(num); + if (w == g_watchpoint_hits.end()) + printf("GPGPU-Sim PTX DBG: memory transfer modified value\n"); + else { + watchpoint_event wa = w->second; + brk_thd = wa.thread(); + brk_inst = wa.inst(); + printf( + "GPGPU-Sim PTX DBG: modified by thread uid=%u, sid=%u, " + "hwtid=%u\n", + brk_thd->get_uid(), brk_thd->get_hw_sid(), brk_thd->get_hw_tid()); + printf("GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf("\n"); + g_watchpoint_hits.erase(w); + } + b.set_value(new_value); + done = false; } - } + } else { + /* + for( unsigned sid=0; sid < m_n_shader; sid++ ) { + unsigned hw_thread_id = -1; + abort(); + ptx_thread_info *thread = + m_sc[sid]->get_functional_thread(hw_thread_id); if( thread_at_brkpt(thread, + b) ) { done = false; printf("GPGPU-Sim PTX DBG: reached breakpoint %u at %s + (sm=%u, hwtid=%u)\n", num, b.location().c_str(), sid, hw_thread_id ); + brk_thd = thread; + brk_inst = brk_thd->get_inst(); + printf( "GPGPU-Sim PTX DBG: reached by thread uid=%u, sid=%u, + hwtid=%u\n", brk_thd->get_uid(),brk_thd->get_hw_sid(), + brk_thd->get_hw_tid() ); printf( "GPGPU-Sim PTX DBG: "); + brk_inst->print_insn(stdout); + printf( "\n" ); + } + } + */ + } + } - if( done ) - assert( g_watchpoint_hits.empty() ); + if (done) assert(g_watchpoint_hits.empty()); - /// enter interactive debugger loop + /// enter interactive debugger loop - while (!done) { - printf("(ptx debugger) "); - fflush(stdout); - - char line[1024]; - fgets(line,1024,stdin); + while (!done) { + printf("(ptx debugger) "); + fflush(stdout); - char *tok = strtok(line," \t\n"); - if( !strcmp(tok,"dp") ) { - int shader_num = 0; - tok = strtok(NULL," \t\n"); - sscanf(tok,"%d",&shader_num); - dump_pipeline((0x40|0x4|0x1),shader_num,0); - printf("\n"); - fflush(stdout); - } else if( !strcmp(tok,"q") || !strcmp(tok,"quit") ) { - printf("\nreally quit GPGPU-Sim (y/n)?\n"); - fgets(line,1024,stdin); - tok = strtok(line," \t\n"); - if( !strcmp(tok,"y") ) { - exit(0); - } else { - printf("not quiting.\n"); - } - } else if( !strcmp(tok,"b") ) { - tok = strtok(NULL," \t\n"); - char brkpt[1024]; - sscanf(tok,"%s",brkpt); - tok = strtok(NULL," \t\n"); - unsigned uid; - sscanf(tok,"%u",&uid); - breakpoints[next_brkpt++] = brk_pt(brkpt,uid); - } else if( !strcmp(tok,"d") ) { - tok = strtok(NULL," \t\n"); - unsigned uid; - sscanf(tok,"%u",&uid); - breakpoints.erase(uid); - } else if( !strcmp(tok,"s") ) { - done = true; - } else if( !strcmp(tok,"c") ) { - single_step=false; - done = true; - } else if( !strcmp(tok,"w") ) { - tok = strtok(NULL," \t\n"); - unsigned addr; - sscanf(tok,"%x",&addr); - unsigned value; - m_global_mem->read(addr,4,&value); - m_global_mem->set_watch(addr,next_brkpt); - breakpoints[next_brkpt++] = brk_pt(addr,value); - } else if( !strcmp(tok,"l") ) { - if( brk_thd == NULL ) { - printf("no thread selected\n"); - } else { - addr_t pc = brk_thd->get_pc(); - addr_t start_pc = (pc<5)?0:(pc-5); - for( addr_t p=start_pc; p <= pc+5; p++ ) { - const ptx_instruction *i = brk_thd->get_inst(p); - if( i ) { - if( p != pc ) - printf( " " ); - else - printf( "==> " ); - i->print_insn(stdout); - printf( "\n" ); - } - } - } - } else if( !strcmp(tok,"h") ) { - printf("commands:\n"); - printf(" q - quit GPGPU-Sim\n"); - printf(" b <file>:<line> <thead uid> - set breakpoint\n"); - printf(" w <global address> - set watchpoint\n"); - printf(" del <n> - delete breakpoint\n"); - printf(" s - single step one shader cycle (all cores)\n"); - printf(" c - continue simulation without single stepping\n"); - printf(" l - list PTX around current breakpoint\n"); - printf(" dp <n> - display pipeline contents on SM <n>\n"); - printf(" h - print this message\n"); + char line[1024]; + fgets(line, 1024, stdin); + + char *tok = strtok(line, " \t\n"); + if (!strcmp(tok, "dp")) { + int shader_num = 0; + tok = strtok(NULL, " \t\n"); + sscanf(tok, "%d", &shader_num); + dump_pipeline((0x40 | 0x4 | 0x1), shader_num, 0); + printf("\n"); + fflush(stdout); + } else if (!strcmp(tok, "q") || !strcmp(tok, "quit")) { + printf("\nreally quit GPGPU-Sim (y/n)?\n"); + fgets(line, 1024, stdin); + tok = strtok(line, " \t\n"); + if (!strcmp(tok, "y")) { + exit(0); } else { - printf("\ncommand not understood.\n"); + printf("not quiting.\n"); } - fflush(stdout); - } + } else if (!strcmp(tok, "b")) { + tok = strtok(NULL, " \t\n"); + char brkpt[1024]; + sscanf(tok, "%s", brkpt); + tok = strtok(NULL, " \t\n"); + unsigned uid; + sscanf(tok, "%u", &uid); + breakpoints[next_brkpt++] = brk_pt(brkpt, uid); + } else if (!strcmp(tok, "d")) { + tok = strtok(NULL, " \t\n"); + unsigned uid; + sscanf(tok, "%u", &uid); + breakpoints.erase(uid); + } else if (!strcmp(tok, "s")) { + done = true; + } else if (!strcmp(tok, "c")) { + single_step = false; + done = true; + } else if (!strcmp(tok, "w")) { + tok = strtok(NULL, " \t\n"); + unsigned addr; + sscanf(tok, "%x", &addr); + unsigned value; + m_global_mem->read(addr, 4, &value); + m_global_mem->set_watch(addr, next_brkpt); + breakpoints[next_brkpt++] = brk_pt(addr, value); + } else if (!strcmp(tok, "l")) { + if (brk_thd == NULL) { + printf("no thread selected\n"); + } else { + addr_t pc = brk_thd->get_pc(); + addr_t start_pc = (pc < 5) ? 0 : (pc - 5); + for (addr_t p = start_pc; p <= pc + 5; p++) { + const ptx_instruction *i = brk_thd->get_inst(p); + if (i) { + if (p != pc) + printf(" "); + else + printf("==> "); + i->print_insn(stdout); + printf("\n"); + } + } + } + } else if (!strcmp(tok, "h")) { + printf("commands:\n"); + printf(" q - quit GPGPU-Sim\n"); + printf(" b <file>:<line> <thead uid> - set breakpoint\n"); + printf(" w <global address> - set watchpoint\n"); + printf(" del <n> - delete breakpoint\n"); + printf( + " s - single step one shader cycle (all " + "cores)\n"); + printf( + " c - continue simulation without single " + "stepping\n"); + printf( + " l - list PTX around current " + "breakpoint\n"); + printf( + " dp <n> - display pipeline contents on SM " + "<n>\n"); + printf(" h - print this message\n"); + } else { + printf("\ncommand not understood.\n"); + } + fflush(stdout); + } } -bool thread_at_brkpt( ptx_thread_info *thread, const class brk_pt &b ) -{ - return b.is_equal(thread->get_location(),thread->get_uid()); +bool thread_at_brkpt(ptx_thread_info *thread, const class brk_pt &b) { + return b.is_equal(thread->get_location(), thread->get_uid()); } - diff --git a/src/debug.h b/src/debug.h index 4e79a9f..0c4dea7 100644 --- a/src/debug.h +++ b/src/debug.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef PTX_DEBUG_INCLUDED #define PTX_DEBUG_INCLUDED @@ -33,58 +34,53 @@ #include <string> class brk_pt { -public: - brk_pt() { m_valid=false; } - brk_pt( const char *fileline, unsigned uid ) - { - m_valid = true; - m_watch = false; - m_fileline = std::string(fileline); - m_thread_uid=uid; - } - brk_pt( unsigned addr, unsigned value ) - { - m_valid = true; - m_watch = true; - m_addr = addr; - m_value = value; - } + public: + brk_pt() { m_valid = false; } + brk_pt(const char *fileline, unsigned uid) { + m_valid = true; + m_watch = false; + m_fileline = std::string(fileline); + m_thread_uid = uid; + } + brk_pt(unsigned addr, unsigned value) { + m_valid = true; + m_watch = true; + m_addr = addr; + m_value = value; + } - unsigned get_value() const { return m_value; } - addr_t get_addr() const { return m_addr; } - bool is_valid() const { return m_valid; } - bool is_watchpoint() const { return m_watch; } - bool is_equal( const std::string &fileline, unsigned uid ) const - { - if( m_watch ) - return false; - if( (m_thread_uid != (unsigned)-1) && (uid != m_thread_uid) ) - return false; - return m_fileline == fileline; - } - std::string location() const - { - char buffer[1024]; - sprintf(buffer,"%s thread uid = %u", m_fileline.c_str(), m_thread_uid); - return buffer; - } + unsigned get_value() const { return m_value; } + addr_t get_addr() const { return m_addr; } + bool is_valid() const { return m_valid; } + bool is_watchpoint() const { return m_watch; } + bool is_equal(const std::string &fileline, unsigned uid) const { + if (m_watch) return false; + if ((m_thread_uid != (unsigned)-1) && (uid != m_thread_uid)) return false; + return m_fileline == fileline; + } + std::string location() const { + char buffer[1024]; + sprintf(buffer, "%s thread uid = %u", m_fileline.c_str(), m_thread_uid); + return buffer; + } - unsigned set_value( unsigned val ) { return m_value=val; } -private: - bool m_valid; - bool m_watch; + unsigned set_value(unsigned val) { return m_value = val; } - // break point - std::string m_fileline; - unsigned m_thread_uid; + private: + bool m_valid; + bool m_watch; - // watch point - unsigned m_addr; - unsigned m_value; + // break point + std::string m_fileline; + unsigned m_thread_uid; + + // watch point + unsigned m_addr; + unsigned m_value; }; class ptx_thread_info; class ptx_instruction; -bool thread_at_brkpt( ptx_thread_info *thd_info, const class brk_pt &b ); +bool thread_at_brkpt(ptx_thread_info *thd_info, const class brk_pt &b); #endif diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc index e3713f3..655d790 100644 --- a/src/gpgpu-sim/addrdec.cc +++ b/src/gpgpu-sim/addrdec.cc @@ -7,550 +7,624 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. - -#include <string.h> #include "addrdec.h" -#include "gpu-sim.h" +#include <string.h> #include "../option_parser.h" +#include "gpu-sim.h" +static long int powli(long int x, long int y); +static unsigned int LOGB2_32(unsigned int v); +static new_addr_type addrdec_packbits(new_addr_type mask, new_addr_type val, + unsigned char high, unsigned char low); +static void addrdec_getmasklimit(new_addr_type mask, unsigned char *high, + unsigned char *low); - -static long int powli( long int x, long int y ); -static unsigned int LOGB2_32( unsigned int v ); -static new_addr_type addrdec_packbits( new_addr_type mask, new_addr_type val, unsigned char high, unsigned char low); -static void addrdec_getmasklimit(new_addr_type mask, unsigned char *high, unsigned char *low); - -linear_to_raw_address_translation::linear_to_raw_address_translation() -{ - addrdec_option = NULL; - ADDR_CHIP_S = 10; - memset(addrdec_mklow,0,N_ADDRDEC); - memset(addrdec_mkhigh,64,N_ADDRDEC); - addrdec_mask[0] = 0x0000000000001C00; - addrdec_mask[1] = 0x0000000000000300; - addrdec_mask[2] = 0x000000000FFF0000; - addrdec_mask[3] = 0x000000000000E0FF; - addrdec_mask[4] = 0x000000000000000F; +linear_to_raw_address_translation::linear_to_raw_address_translation() { + addrdec_option = NULL; + ADDR_CHIP_S = 10; + memset(addrdec_mklow, 0, N_ADDRDEC); + memset(addrdec_mkhigh, 64, N_ADDRDEC); + addrdec_mask[0] = 0x0000000000001C00; + addrdec_mask[1] = 0x0000000000000300; + addrdec_mask[2] = 0x000000000FFF0000; + addrdec_mask[3] = 0x000000000000E0FF; + addrdec_mask[4] = 0x000000000000000F; } -void linear_to_raw_address_translation::addrdec_setoption(option_parser_t opp) -{ - option_parser_register(opp, "-gpgpu_mem_addr_mapping", OPT_CSTR, &addrdec_option, - "mapping memory address to dram model {dramid@<start bit>;<memory address map>}", - NULL); - option_parser_register(opp, "-gpgpu_mem_addr_test", OPT_BOOL, &run_test, - "run sweep test to check address mapping for aliased address", +void linear_to_raw_address_translation::addrdec_setoption(option_parser_t opp) { + option_parser_register(opp, "-gpgpu_mem_addr_mapping", OPT_CSTR, + &addrdec_option, + "mapping memory address to dram model {dramid@<start " + "bit>;<memory address map>}", + NULL); + option_parser_register( + opp, "-gpgpu_mem_addr_test", OPT_BOOL, &run_test, + "run sweep test to check address mapping for aliased address", "0"); + option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, + &gpgpu_mem_address_mask, + "0 = old addressing mask, 1 = new addressing mask, 2 " + "= new add. mask + flipped bank sel and chip sel bits", + "0"); + option_parser_register( + opp, "-memory_partition_indexing", OPT_UINT32, &memory_partition_indexing, + "0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing", "0"); - option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, - "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits", - "0"); - option_parser_register(opp, "-memory_partition_indexing", OPT_UINT32, &memory_partition_indexing, - "0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing", - "0"); } -new_addr_type linear_to_raw_address_translation::partition_address( new_addr_type addr ) const -{ - if (!gap) { - return addrdec_packbits( ~(addrdec_mask[CHIP] | sub_partition_id_mask), addr, 64, 0 ); - } else { - // see addrdec_tlx for explanation - unsigned long long int partition_addr; - partition_addr = ( (addr>>ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; - partition_addr |= addr & ((1 << ADDR_CHIP_S) - 1); - // remove the part of address that constributes to the sub partition ID - partition_addr = addrdec_packbits( ~sub_partition_id_mask, partition_addr, 64, 0); - return partition_addr; - } +new_addr_type linear_to_raw_address_translation::partition_address( + new_addr_type addr) const { + if (!gap) { + return addrdec_packbits(~(addrdec_mask[CHIP] | sub_partition_id_mask), addr, + 64, 0); + } else { + // see addrdec_tlx for explanation + unsigned long long int partition_addr; + partition_addr = ((addr >> ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; + partition_addr |= addr & ((1 << ADDR_CHIP_S) - 1); + // remove the part of address that constributes to the sub partition ID + partition_addr = + addrdec_packbits(~sub_partition_id_mask, partition_addr, 64, 0); + return partition_addr; + } } -void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const -{ - unsigned long long int addr_for_chip,rest_of_addr; - if (!gap) { - tlx->chip = addrdec_packbits(addrdec_mask[CHIP], addr, addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]); - tlx->bk = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], addrdec_mklow[BK]); - tlx->row = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]); - tlx->col = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], addrdec_mklow[COL]); - tlx->burst= addrdec_packbits(addrdec_mask[BURST], addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); - } else { - // Split the given address at ADDR_CHIP_S into (MSBs,LSBs) - // - extract chip address using modulus of MSBs - // - recreate the rest of the address by stitching the quotient of MSBs and the LSBs - addr_for_chip = (addr>>ADDR_CHIP_S) % m_n_channel; - rest_of_addr = ( (addr>>ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; - rest_of_addr |= addr & ((1 << ADDR_CHIP_S) - 1); +void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, + addrdec_t *tlx) const { + unsigned long long int addr_for_chip, rest_of_addr; + if (!gap) { + tlx->chip = addrdec_packbits(addrdec_mask[CHIP], addr, addrdec_mkhigh[CHIP], + addrdec_mklow[CHIP]); + tlx->bk = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], + addrdec_mklow[BK]); + tlx->row = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], + addrdec_mklow[ROW]); + tlx->col = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], + addrdec_mklow[COL]); + tlx->burst = addrdec_packbits(addrdec_mask[BURST], addr, + addrdec_mkhigh[BURST], addrdec_mklow[BURST]); + } else { + // Split the given address at ADDR_CHIP_S into (MSBs,LSBs) + // - extract chip address using modulus of MSBs + // - recreate the rest of the address by stitching the quotient of MSBs and + // the LSBs + addr_for_chip = (addr >> ADDR_CHIP_S) % m_n_channel; + rest_of_addr = ((addr >> ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; + rest_of_addr |= addr & ((1 << ADDR_CHIP_S) - 1); - tlx->chip = addr_for_chip; - tlx->bk = addrdec_packbits(addrdec_mask[BK], rest_of_addr, addrdec_mkhigh[BK], addrdec_mklow[BK]); - tlx->row = addrdec_packbits(addrdec_mask[ROW], rest_of_addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]); - tlx->col = addrdec_packbits(addrdec_mask[COL], rest_of_addr, addrdec_mkhigh[COL], addrdec_mklow[COL]); - tlx->burst= addrdec_packbits(addrdec_mask[BURST], rest_of_addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); - } + tlx->chip = addr_for_chip; + tlx->bk = addrdec_packbits(addrdec_mask[BK], rest_of_addr, + addrdec_mkhigh[BK], addrdec_mklow[BK]); + tlx->row = addrdec_packbits(addrdec_mask[ROW], rest_of_addr, + addrdec_mkhigh[ROW], addrdec_mklow[ROW]); + tlx->col = addrdec_packbits(addrdec_mask[COL], rest_of_addr, + addrdec_mkhigh[COL], addrdec_mklow[COL]); + tlx->burst = addrdec_packbits(addrdec_mask[BURST], rest_of_addr, + addrdec_mkhigh[BURST], addrdec_mklow[BURST]); + } - switch(memory_partition_indexing){ - case CONSECUTIVE: - //Do nothing - break; - case BITWISE_PERMUTATION: - { - assert(!gap); - tlx->chip = (tlx->chip) ^ (tlx->row & (m_n_channel-1)); - assert(tlx->chip < m_n_channel); - break; - } - case IPOLY: - { - /* - * Set Indexing function from "Pseudo-randomly interleaved memory." - * Rau, B. R et al. - * ISCA 1991 - * - * equations are adopted from: - * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." - * Khairy et al. - * IEEE TPDS 2017. - */ - if(m_n_channel == 32) { - std::bitset<64> a(tlx->row); - std::bitset<5> chip(tlx->chip); - chip[0] = a[13]^a[12]^a[11]^a[10]^a[9]^a[6]^a[5]^a[3]^a[0]^chip[0]; - chip[1] = a[14]^a[13]^a[12]^a[11]^a[10]^a[7]^a[6]^a[4]^a[1]^chip[1]; - chip[2] = a[14]^a[10]^a[9]^a[8]^a[7]^a[6]^a[3]^a[2]^a[0]^chip[2]; - chip[3] = a[11]^a[10]^a[9]^a[8]^a[7]^a[4]^a[3]^a[1]^chip[3]; - chip[4] = a[12]^a[11]^a[10]^a[9]^a[8]^a[5]^a[4]^a[2]^chip[4]; - tlx->chip = chip.to_ulong(); - - } - else{ /* Else incorrect number of channels for the hashing function */ - assert("\nGPGPU-Sim memory_partition_indexing error: The number of channels should be " - "32 for the hashing IPOLY index function.\n" && 0); - } - assert(tlx->chip < m_n_channel); - break; - } - case PAE: - { - //Page Address Entropy - //random selected bits from the page and bank bits - //similar to - //Liu, Yuxi, et al. "Get Out of the Valley: Power-Efficient Address Mapping for GPUs." ISCA 2018 - std::bitset<64> a(tlx->row); - std::bitset<5> chip(tlx->chip); - std::bitset<4> b(tlx->bk); - chip[0] = a[13]^a[10]^a[9]^a[5]^a[0]^b[3]^b[0]^chip[0]; - chip[1] = a[12]^a[11]^a[6]^a[1]^b[3]^b[2]^b[1]^chip[1]; - chip[2] = a[14]^a[9]^a[8]^a[7]^a[2]^b[1]^chip[2]; - chip[3] = a[11]^a[10]^a[8]^a[3]^b[2]^b[3]^chip[3]; - chip[4] = a[12]^a[9]^a[8]^a[5]^a[4]^b[1]^b[0]^chip[4]; - tlx->chip = chip.to_ulong(); - assert(tlx->chip < m_n_channel); - break; - } - case RANDOM: - { - //This is an unrealistic hashing using software hashtable - //we generate a random set for each memory address and save the value in a big hashtable for future reuse - new_addr_type chip_address = (addr>>ADDR_CHIP_S); - tr1_hash_map<new_addr_type,unsigned>::const_iterator got = address_random_interleaving.find (chip_address); - if ( got == address_random_interleaving.end() ) { - unsigned new_chip_id = rand() % (m_n_channel*m_n_sub_partition_in_channel); - address_random_interleaving[chip_address] = new_chip_id; - tlx->chip = new_chip_id/m_n_sub_partition_in_channel; - tlx->sub_partition = new_chip_id; - } - else { - unsigned new_chip_id = got->second; - tlx->chip = new_chip_id/m_n_sub_partition_in_channel; - tlx->sub_partition = new_chip_id; - } + switch (memory_partition_indexing) { + case CONSECUTIVE: + // Do nothing + break; + case BITWISE_PERMUTATION: { + assert(!gap); + tlx->chip = (tlx->chip) ^ (tlx->row & (m_n_channel - 1)); + assert(tlx->chip < m_n_channel); + break; + } + case IPOLY: { + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * equations are adopted from: + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu + * cache management scheme." Khairy et al. IEEE TPDS 2017. + */ + if (m_n_channel == 32) { + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + chip[0] = a[13] ^ a[12] ^ a[11] ^ a[10] ^ a[9] ^ a[6] ^ a[5] ^ a[3] ^ + a[0] ^ chip[0]; + chip[1] = a[14] ^ a[13] ^ a[12] ^ a[11] ^ a[10] ^ a[7] ^ a[6] ^ a[4] ^ + a[1] ^ chip[1]; + chip[2] = a[14] ^ a[10] ^ a[9] ^ a[8] ^ a[7] ^ a[6] ^ a[3] ^ a[2] ^ + a[0] ^ chip[2]; + chip[3] = + a[11] ^ a[10] ^ a[9] ^ a[8] ^ a[7] ^ a[4] ^ a[3] ^ a[1] ^ chip[3]; + chip[4] = + a[12] ^ a[11] ^ a[10] ^ a[9] ^ a[8] ^ a[5] ^ a[4] ^ a[2] ^ chip[4]; + tlx->chip = chip.to_ulong(); - assert(tlx->chip < m_n_channel); - assert(tlx->sub_partition < m_n_channel * m_n_sub_partition_in_channel); - return; - break; - } - case CUSTOM: - /* No custom set function implemented */ - //Do you custom index here - break; - default: - assert("\nUndefined set index function.\n" && 0); - break; - } + } else { /* Else incorrect number of channels for the hashing function */ + assert( + "\nGPGPU-Sim memory_partition_indexing error: The number of " + "channels should be " + "32 for the hashing IPOLY index function.\n" && + 0); + } + assert(tlx->chip < m_n_channel); + break; + } + case PAE: { + // Page Address Entropy + // random selected bits from the page and bank bits + // similar to + // Liu, Yuxi, et al. "Get Out of the Valley: Power-Efficient Address + // Mapping for GPUs." ISCA 2018 + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + std::bitset<4> b(tlx->bk); + chip[0] = a[13] ^ a[10] ^ a[9] ^ a[5] ^ a[0] ^ b[3] ^ b[0] ^ chip[0]; + chip[1] = a[12] ^ a[11] ^ a[6] ^ a[1] ^ b[3] ^ b[2] ^ b[1] ^ chip[1]; + chip[2] = a[14] ^ a[9] ^ a[8] ^ a[7] ^ a[2] ^ b[1] ^ chip[2]; + chip[3] = a[11] ^ a[10] ^ a[8] ^ a[3] ^ b[2] ^ b[3] ^ chip[3]; + chip[4] = a[12] ^ a[9] ^ a[8] ^ a[5] ^ a[4] ^ b[1] ^ b[0] ^ chip[4]; + tlx->chip = chip.to_ulong(); + assert(tlx->chip < m_n_channel); + break; + } + case RANDOM: { + // This is an unrealistic hashing using software hashtable + // we generate a random set for each memory address and save the value in + // a big hashtable for future reuse + new_addr_type chip_address = (addr >> ADDR_CHIP_S); + tr1_hash_map<new_addr_type, unsigned>::const_iterator got = + address_random_interleaving.find(chip_address); + if (got == address_random_interleaving.end()) { + unsigned new_chip_id = + rand() % (m_n_channel * m_n_sub_partition_in_channel); + address_random_interleaving[chip_address] = new_chip_id; + tlx->chip = new_chip_id / m_n_sub_partition_in_channel; + tlx->sub_partition = new_chip_id; + } else { + unsigned new_chip_id = got->second; + tlx->chip = new_chip_id / m_n_sub_partition_in_channel; + tlx->sub_partition = new_chip_id; + } - // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID - unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; - tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel - + (tlx->bk & sub_partition_addr_mask); + assert(tlx->chip < m_n_channel); + assert(tlx->sub_partition < m_n_channel * m_n_sub_partition_in_channel); + return; + break; + } + case CUSTOM: + /* No custom set function implemented */ + // Do you custom index here + break; + default: + assert("\nUndefined set index function.\n" && 0); + break; + } + + // combine the chip address and the lower bits of DRAM bank address to form + // the subpartition ID + unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; + tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel + + (tlx->bk & sub_partition_addr_mask); } -void linear_to_raw_address_translation::addrdec_parseoption(const char *option) -{ - unsigned int dramid_start = 0; - int dramid_parsed = sscanf(option, "dramid@%d", &dramid_start); - if (dramid_parsed == 1) { - ADDR_CHIP_S = dramid_start; - } else { - ADDR_CHIP_S = -1; - } - - const char *cmapping = strchr(option, ';'); - if (cmapping == NULL) { - cmapping = option; - } else { - cmapping += 1; - } +void linear_to_raw_address_translation::addrdec_parseoption( + const char *option) { + unsigned int dramid_start = 0; + int dramid_parsed = sscanf(option, "dramid@%d", &dramid_start); + if (dramid_parsed == 1) { + ADDR_CHIP_S = dramid_start; + } else { + ADDR_CHIP_S = -1; + } - addrdec_mask[CHIP] = 0x0; - addrdec_mask[BK] = 0x0; - addrdec_mask[ROW] = 0x0; - addrdec_mask[COL] = 0x0; - addrdec_mask[BURST]= 0x0; - - int ofs = 63; - while ((*cmapping) != '\0') { - switch (*cmapping) { - case 'D': case 'd': - assert(dramid_parsed != 1); addrdec_mask[CHIP] |= (1ULL << ofs); ofs--; break; - case 'B': case 'b': addrdec_mask[BK] |= (1ULL << ofs); ofs--; break; - case 'R': case 'r': addrdec_mask[ROW] |= (1ULL << ofs); ofs--; break; - case 'C': case 'c': addrdec_mask[COL] |= (1ULL << ofs); ofs--; break; - case 'S': case 's': addrdec_mask[BURST] |= (1ULL << ofs); addrdec_mask[COL] |= (1ULL << ofs); ofs--; break; - // ignore bit - case '0': ofs--; break; - // ignore character - case '|': - case ' ': - case '.': break; - default: - fprintf(stderr, "ERROR: Invalid address mapping character '%c' in option '%s'\n", *cmapping, option); - } - cmapping += 1; - } + const char *cmapping = strchr(option, ';'); + if (cmapping == NULL) { + cmapping = option; + } else { + cmapping += 1; + } + + addrdec_mask[CHIP] = 0x0; + addrdec_mask[BK] = 0x0; + addrdec_mask[ROW] = 0x0; + addrdec_mask[COL] = 0x0; + addrdec_mask[BURST] = 0x0; + + int ofs = 63; + while ((*cmapping) != '\0') { + switch (*cmapping) { + case 'D': + case 'd': + assert(dramid_parsed != 1); + addrdec_mask[CHIP] |= (1ULL << ofs); + ofs--; + break; + case 'B': + case 'b': + addrdec_mask[BK] |= (1ULL << ofs); + ofs--; + break; + case 'R': + case 'r': + addrdec_mask[ROW] |= (1ULL << ofs); + ofs--; + break; + case 'C': + case 'c': + addrdec_mask[COL] |= (1ULL << ofs); + ofs--; + break; + case 'S': + case 's': + addrdec_mask[BURST] |= (1ULL << ofs); + addrdec_mask[COL] |= (1ULL << ofs); + ofs--; + break; + // ignore bit + case '0': + ofs--; + break; + // ignore character + case '|': + case ' ': + case '.': + break; + default: + fprintf( + stderr, + "ERROR: Invalid address mapping character '%c' in option '%s'\n", + *cmapping, option); + } + cmapping += 1; + } - if (ofs != -1) { - fprintf(stderr, "ERROR: Invalid address mapping length (%d) in option '%s'\n", 63 - ofs, option); - assert(ofs == -1); - } + if (ofs != -1) { + fprintf(stderr, + "ERROR: Invalid address mapping length (%d) in option '%s'\n", + 63 - ofs, option); + assert(ofs == -1); + } } -void linear_to_raw_address_translation::init(unsigned int n_channel, unsigned int n_sub_partition_in_channel) -{ - unsigned i; - unsigned long long int mask; - unsigned int nchipbits = ::LOGB2_32(n_channel); - m_n_channel = n_channel; - m_n_sub_partition_in_channel = n_sub_partition_in_channel; +void linear_to_raw_address_translation::init( + unsigned int n_channel, unsigned int n_sub_partition_in_channel) { + unsigned i; + unsigned long long int mask; + unsigned int nchipbits = ::LOGB2_32(n_channel); + m_n_channel = n_channel; + m_n_sub_partition_in_channel = n_sub_partition_in_channel; - gap = (n_channel - ::powli(2,nchipbits)); - if (gap) { - nchipbits++; - } - switch (gpgpu_mem_address_mask) { - case 0: - //old, added 2row bits, use #define ADDR_CHIP_S 10 + gap = (n_channel - ::powli(2, nchipbits)); + if (gap) { + nchipbits++; + } + switch (gpgpu_mem_address_mask) { + case 0: + // old, added 2row bits, use #define ADDR_CHIP_S 10 ADDR_CHIP_S = 10; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000000300; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x0000000000001CFF; + addrdec_mask[BK] = 0x0000000000000300; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001CFF; break; - case 1: + case 1: ADDR_CHIP_S = 13; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 2: + case 2: ADDR_CHIP_S = 11; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 3: + case 3: ADDR_CHIP_S = 11; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x000000000FFFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x000000000FFFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 14: + case 14: ADDR_CHIP_S = 14; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 15: + case 15: ADDR_CHIP_S = 15; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 16: + case 16: ADDR_CHIP_S = 16; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 6: + case 6: ADDR_CHIP_S = 6; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 5: + case 5: ADDR_CHIP_S = 5; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; - break; - case 100: + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 100: ADDR_CHIP_S = 1; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000000003; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x0000000000001FFC; + addrdec_mask[BK] = 0x0000000000000003; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001FFC; break; - case 103: + case 103: ADDR_CHIP_S = 3; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000000003; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x0000000000001FFC; + addrdec_mask[BK] = 0x0000000000000003; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001FFC; break; - case 106: + case 106: ADDR_CHIP_S = 6; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000001800; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x00000000000007FF; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; break; - case 160: - //old, added 2row bits, use #define ADDR_CHIP_S 10 + case 160: + // old, added 2row bits, use #define ADDR_CHIP_S 10 ADDR_CHIP_S = 6; addrdec_mask[CHIP] = 0x0000000000000000; - addrdec_mask[BK] = 0x0000000000000300; - addrdec_mask[ROW] = 0x0000000007FFE000; - addrdec_mask[COL] = 0x0000000000001CFF; + addrdec_mask[BK] = 0x0000000000000300; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001CFF; - default: + default: break; - } + } - if (addrdec_option != NULL) - addrdec_parseoption(addrdec_option); + if (addrdec_option != NULL) addrdec_parseoption(addrdec_option); - if (ADDR_CHIP_S != -1) { - if (!gap) { - // number of chip is power of two: - // - insert CHIP mask starting at the bit position ADDR_CHIP_S - mask = ((unsigned long long int)1 << ADDR_CHIP_S) - 1; - addrdec_mask[BK] = ((addrdec_mask[BK] & ~mask) << nchipbits) | (addrdec_mask[BK] & mask); - addrdec_mask[ROW] = ((addrdec_mask[ROW] & ~mask) << nchipbits) | (addrdec_mask[ROW] & mask); - addrdec_mask[COL] = ((addrdec_mask[COL] & ~mask) << nchipbits) | (addrdec_mask[COL] & mask); + if (ADDR_CHIP_S != -1) { + if (!gap) { + // number of chip is power of two: + // - insert CHIP mask starting at the bit position ADDR_CHIP_S + mask = ((unsigned long long int)1 << ADDR_CHIP_S) - 1; + addrdec_mask[BK] = + ((addrdec_mask[BK] & ~mask) << nchipbits) | (addrdec_mask[BK] & mask); + addrdec_mask[ROW] = ((addrdec_mask[ROW] & ~mask) << nchipbits) | + (addrdec_mask[ROW] & mask); + addrdec_mask[COL] = ((addrdec_mask[COL] & ~mask) << nchipbits) | + (addrdec_mask[COL] & mask); - for (i=ADDR_CHIP_S;i<(ADDR_CHIP_S+nchipbits);i++) { - mask = (unsigned long long int)1 << i; - addrdec_mask[CHIP] |= mask; - } - } // otherwise, no need to change the masks - } else { - // make sure n_channel is power of two when explicit dram id mask is used - assert((n_channel & (n_channel - 1)) == 0); - } - // make sure m_n_sub_partition_in_channel is power of two - assert((m_n_sub_partition_in_channel & (m_n_sub_partition_in_channel - 1)) == 0); + for (i = ADDR_CHIP_S; i < (ADDR_CHIP_S + nchipbits); i++) { + mask = (unsigned long long int)1 << i; + addrdec_mask[CHIP] |= mask; + } + } // otherwise, no need to change the masks + } else { + // make sure n_channel is power of two when explicit dram id mask is used + assert((n_channel & (n_channel - 1)) == 0); + } + // make sure m_n_sub_partition_in_channel is power of two + assert((m_n_sub_partition_in_channel & (m_n_sub_partition_in_channel - 1)) == + 0); - addrdec_getmasklimit(addrdec_mask[CHIP], &addrdec_mkhigh[CHIP], &addrdec_mklow[CHIP] ); - addrdec_getmasklimit(addrdec_mask[BK], &addrdec_mkhigh[BK], &addrdec_mklow[BK] ); - addrdec_getmasklimit(addrdec_mask[ROW], &addrdec_mkhigh[ROW], &addrdec_mklow[ROW] ); - addrdec_getmasklimit(addrdec_mask[COL], &addrdec_mkhigh[COL], &addrdec_mklow[COL] ); - addrdec_getmasklimit(addrdec_mask[BURST], &addrdec_mkhigh[BURST], &addrdec_mklow[BURST]); + addrdec_getmasklimit(addrdec_mask[CHIP], &addrdec_mkhigh[CHIP], + &addrdec_mklow[CHIP]); + addrdec_getmasklimit(addrdec_mask[BK], &addrdec_mkhigh[BK], + &addrdec_mklow[BK]); + addrdec_getmasklimit(addrdec_mask[ROW], &addrdec_mkhigh[ROW], + &addrdec_mklow[ROW]); + addrdec_getmasklimit(addrdec_mask[COL], &addrdec_mkhigh[COL], + &addrdec_mklow[COL]); + addrdec_getmasklimit(addrdec_mask[BURST], &addrdec_mkhigh[BURST], + &addrdec_mklow[BURST]); - printf("addr_dec_mask[CHIP] = %016llx \thigh:%d low:%d\n", addrdec_mask[CHIP], addrdec_mkhigh[CHIP], addrdec_mklow[CHIP] ); - printf("addr_dec_mask[BK] = %016llx \thigh:%d low:%d\n", addrdec_mask[BK], addrdec_mkhigh[BK], addrdec_mklow[BK] ); - printf("addr_dec_mask[ROW] = %016llx \thigh:%d low:%d\n", addrdec_mask[ROW], addrdec_mkhigh[ROW], addrdec_mklow[ROW] ); - printf("addr_dec_mask[COL] = %016llx \thigh:%d low:%d\n", addrdec_mask[COL], addrdec_mkhigh[COL], addrdec_mklow[COL] ); - printf("addr_dec_mask[BURST] = %016llx \thigh:%d low:%d\n", addrdec_mask[BURST], addrdec_mkhigh[BURST], addrdec_mklow[BURST]); + printf("addr_dec_mask[CHIP] = %016llx \thigh:%d low:%d\n", + addrdec_mask[CHIP], addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]); + printf("addr_dec_mask[BK] = %016llx \thigh:%d low:%d\n", addrdec_mask[BK], + addrdec_mkhigh[BK], addrdec_mklow[BK]); + printf("addr_dec_mask[ROW] = %016llx \thigh:%d low:%d\n", addrdec_mask[ROW], + addrdec_mkhigh[ROW], addrdec_mklow[ROW]); + printf("addr_dec_mask[COL] = %016llx \thigh:%d low:%d\n", addrdec_mask[COL], + addrdec_mkhigh[COL], addrdec_mklow[COL]); + printf("addr_dec_mask[BURST] = %016llx \thigh:%d low:%d\n", + addrdec_mask[BURST], addrdec_mkhigh[BURST], addrdec_mklow[BURST]); - // create the sub partition ID mask (for removing the sub partition ID from the partition address) - sub_partition_id_mask = 0; - if (m_n_sub_partition_in_channel > 1) { - unsigned n_sub_partition_log2 = LOGB2_32(m_n_sub_partition_in_channel); - unsigned pos=0; - for (unsigned i=addrdec_mklow[BK];i<addrdec_mkhigh[BK];i++) { - if ((addrdec_mask[BK] & ((unsigned long long int)1<<i)) != 0) { - sub_partition_id_mask |= ((unsigned long long int)1<<i); - pos++; - if (pos >= n_sub_partition_log2) - break; - } + // create the sub partition ID mask (for removing the sub partition ID from + // the partition address) + sub_partition_id_mask = 0; + if (m_n_sub_partition_in_channel > 1) { + unsigned n_sub_partition_log2 = LOGB2_32(m_n_sub_partition_in_channel); + unsigned pos = 0; + for (unsigned i = addrdec_mklow[BK]; i < addrdec_mkhigh[BK]; i++) { + if ((addrdec_mask[BK] & ((unsigned long long int)1 << i)) != 0) { + sub_partition_id_mask |= ((unsigned long long int)1 << i); + pos++; + if (pos >= n_sub_partition_log2) break; } - } - printf("sub_partition_id_mask = %016llx\n", sub_partition_id_mask); - - if (run_test) { - sweep_test(); - } + } + } + printf("sub_partition_id_mask = %016llx\n", sub_partition_id_mask); - if(memory_partition_indexing == RANDOM) - srand (1); + if (run_test) { + sweep_test(); + } + if (memory_partition_indexing == RANDOM) srand(1); } -#include "../tr1_hash_map.h" +#include "../tr1_hash_map.h" -bool operator==(const addrdec_t &x, const addrdec_t &y) -{ - return ( memcmp(&x, &y, sizeof(addrdec_t)) == 0 ); +bool operator==(const addrdec_t &x, const addrdec_t &y) { + return (memcmp(&x, &y, sizeof(addrdec_t)) == 0); } -bool operator<(const addrdec_t &x, const addrdec_t &y) -{ - if (x.chip >= y.chip) return false; - else if (x.bk >= y.bk) return false; - else if (x.row >= y.row) return false; - else if (x.col >= y.col) return false; - else if (x.burst >= y.burst) return false; - else return true; +bool operator<(const addrdec_t &x, const addrdec_t &y) { + if (x.chip >= y.chip) + return false; + else if (x.bk >= y.bk) + return false; + else if (x.row >= y.row) + return false; + else if (x.col >= y.col) + return false; + else if (x.burst >= y.burst) + return false; + else + return true; } -class hash_addrdec_t -{ -public: - size_t operator()(const addrdec_t &x) const { - return (x.chip ^ x.bk ^ x.row ^ x.col ^ x.burst); - } +class hash_addrdec_t { + public: + size_t operator()(const addrdec_t &x) const { + return (x.chip ^ x.bk ^ x.row ^ x.col ^ x.burst); + } }; -// a simple sweep test to ensure that two linear addresses are not mapped to the same raw address -void linear_to_raw_address_translation::sweep_test() const -{ - new_addr_type sweep_range = 16 * 1024 * 1024; +// a simple sweep test to ensure that two linear addresses are not mapped to the +// same raw address +void linear_to_raw_address_translation::sweep_test() const { + new_addr_type sweep_range = 16 * 1024 * 1024; #if tr1_hash_map_ismap == 1 - typedef tr1_hash_map<addrdec_t, new_addr_type> history_map_t; + typedef tr1_hash_map<addrdec_t, new_addr_type> history_map_t; #else - typedef tr1_hash_map<addrdec_t, new_addr_type, hash_addrdec_t> history_map_t; + typedef tr1_hash_map<addrdec_t, new_addr_type, hash_addrdec_t> history_map_t; #endif - history_map_t history_map; + history_map_t history_map; - for (new_addr_type raw_addr = 4; raw_addr < sweep_range; raw_addr += 4) { - addrdec_t tlx; - addrdec_tlx(raw_addr, &tlx); + for (new_addr_type raw_addr = 4; raw_addr < sweep_range; raw_addr += 4) { + addrdec_t tlx; + addrdec_tlx(raw_addr, &tlx); - history_map_t::iterator h = history_map.find(tlx); + history_map_t::iterator h = history_map.find(tlx); - if (h != history_map.end()) { - printf("[AddrDec] ** Error: address decoding mapping aliases two addresses to same partition with same intra-partition address: %llx %llx\n", h->second, raw_addr); - abort(); - } else { - assert((int)tlx.chip < m_n_channel); - // ensure that partition_address() returns the concatenated address - if ((ADDR_CHIP_S != -1 and raw_addr >= (1ULL << ADDR_CHIP_S)) or - (ADDR_CHIP_S == -1 and raw_addr >= (1ULL << addrdec_mklow[CHIP]))) { - assert(raw_addr != partition_address(raw_addr)); - } - history_map[tlx] = raw_addr; + if (h != history_map.end()) { + printf( + "[AddrDec] ** Error: address decoding mapping aliases two addresses " + "to same partition with same intra-partition address: %llx %llx\n", + h->second, raw_addr); + abort(); + } else { + assert((int)tlx.chip < m_n_channel); + // ensure that partition_address() returns the concatenated address + if ((ADDR_CHIP_S != -1 and raw_addr >= (1ULL << ADDR_CHIP_S)) or + (ADDR_CHIP_S == -1 and raw_addr >= (1ULL << addrdec_mklow[CHIP]))) { + assert(raw_addr != partition_address(raw_addr)); } + history_map[tlx] = raw_addr; + } - if ((raw_addr & 0xffff) == 0) printf("%llu scaned\n", raw_addr); - } + if ((raw_addr & 0xffff) == 0) printf("%llu scaned\n", raw_addr); + } } -void addrdec_t::print( FILE *fp ) const -{ - fprintf(fp,"\tchip:%x ", chip); - fprintf(fp,"\trow:%x ", row); - fprintf(fp,"\tcol:%x ", col); - fprintf(fp,"\tbk:%x ", bk); - fprintf(fp,"\tburst:%x ", burst); - fprintf(fp,"\tsub_partition:%x ", sub_partition); -} - +void addrdec_t::print(FILE *fp) const { + fprintf(fp, "\tchip:%x ", chip); + fprintf(fp, "\trow:%x ", row); + fprintf(fp, "\tcol:%x ", col); + fprintf(fp, "\tbk:%x ", bk); + fprintf(fp, "\tburst:%x ", burst); + fprintf(fp, "\tsub_partition:%x ", sub_partition); +} -static long int powli( long int x, long int y ) // compute x to the y +static long int powli(long int x, long int y) // compute x to the y { - long int r = 1; - int i; - for (i = 0; i < y; ++i ) { - r *= x; - } - return r; + long int r = 1; + int i; + for (i = 0; i < y; ++i) { + r *= x; + } + return r; } -static unsigned int LOGB2_32( unsigned int v ) -{ - unsigned int shift; - unsigned int r; +static unsigned int LOGB2_32(unsigned int v) { + unsigned int shift; + unsigned int r; - r = 0; + r = 0; - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; - return r; + return r; } -static new_addr_type addrdec_packbits( new_addr_type mask, new_addr_type val, unsigned char high, unsigned char low) -{ - unsigned pos=0; - new_addr_type result = 0; - for (unsigned i=low;i<high;i++) { - if ((mask & ((unsigned long long int)1<<i)) != 0) { - result |= ((val & ((unsigned long long int)1<<i)) >> i) << pos; - pos++; - } - } - return result; +static new_addr_type addrdec_packbits(new_addr_type mask, new_addr_type val, + unsigned char high, unsigned char low) { + unsigned pos = 0; + new_addr_type result = 0; + for (unsigned i = low; i < high; i++) { + if ((mask & ((unsigned long long int)1 << i)) != 0) { + result |= ((val & ((unsigned long long int)1 << i)) >> i) << pos; + pos++; + } + } + return result; } -static void addrdec_getmasklimit(new_addr_type mask, unsigned char *high, unsigned char *low) -{ - *high = 64; - *low = 0; - int i; - int low_found = 0; +static void addrdec_getmasklimit(new_addr_type mask, unsigned char *high, + unsigned char *low) { + *high = 64; + *low = 0; + int i; + int low_found = 0; - for (i=0;i<64;i++) { - if ((mask & ((unsigned long long int)1<<i)) != 0) { - if (low_found) { - *high = i + 1; - } else { - *high = i + 1; - *low = i; - low_found = 1; - } + for (i = 0; i < 64; i++) { + if ((mask & ((unsigned long long int)1 << i)) != 0) { + if (low_found) { + *high = i + 1; + } else { + *high = i + 1; + *low = i; + low_found = 1; } - } + } + } } diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h index c9a1420..03b84a4 100644 --- a/src/gpgpu-sim/addrdec.h +++ b/src/gpgpu-sim/addrdec.h @@ -7,27 +7,28 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include <assert.h> #include <stdio.h> #include <stdlib.h> -#include <assert.h> #include "../option_parser.h" #ifndef ADDRDEC_H @@ -35,66 +36,57 @@ #include "../abstract_hardware_model.h" -enum partition_index_function{ - CONSECUTIVE = 0, - BITWISE_PERMUTATION, - IPOLY, - PAE, - RANDOM, - CUSTOM +enum partition_index_function { + CONSECUTIVE = 0, + BITWISE_PERMUTATION, + IPOLY, + PAE, + RANDOM, + CUSTOM }; struct addrdec_t { - void print( FILE *fp ) const; - - unsigned chip; - unsigned bk; - unsigned row; - unsigned col; - unsigned burst; + void print(FILE *fp) const; - unsigned sub_partition; -}; + unsigned chip; + unsigned bk; + unsigned row; + unsigned col; + unsigned burst; + unsigned sub_partition; +}; class linear_to_raw_address_translation { -public: - linear_to_raw_address_translation(); - void addrdec_setoption(option_parser_t opp); - void init(unsigned int n_channel, unsigned int n_sub_partition_in_channel); - - // accessors - void addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const; - new_addr_type partition_address( new_addr_type addr ) const; + public: + linear_to_raw_address_translation(); + void addrdec_setoption(option_parser_t opp); + void init(unsigned int n_channel, unsigned int n_sub_partition_in_channel); -private: - void addrdec_parseoption(const char *option); - void sweep_test() const; // sanity check to ensure no overlapping + // accessors + void addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const; + new_addr_type partition_address(new_addr_type addr) const; - enum { - CHIP = 0, - BK = 1, - ROW = 2, - COL = 3, - BURST = 4, - N_ADDRDEC - }; + private: + void addrdec_parseoption(const char *option); + void sweep_test() const; // sanity check to ensure no overlapping - const char *addrdec_option; - int gpgpu_mem_address_mask; - partition_index_function memory_partition_indexing; - bool run_test; + enum { CHIP = 0, BK = 1, ROW = 2, COL = 3, BURST = 4, N_ADDRDEC }; - int ADDR_CHIP_S; - unsigned char addrdec_mklow[N_ADDRDEC]; - unsigned char addrdec_mkhigh[N_ADDRDEC]; - new_addr_type addrdec_mask[N_ADDRDEC]; - new_addr_type sub_partition_id_mask; + const char *addrdec_option; + int gpgpu_mem_address_mask; + partition_index_function memory_partition_indexing; + bool run_test; - unsigned int gap; - unsigned m_n_channel; - int m_n_sub_partition_in_channel; + int ADDR_CHIP_S; + unsigned char addrdec_mklow[N_ADDRDEC]; + unsigned char addrdec_mkhigh[N_ADDRDEC]; + new_addr_type addrdec_mask[N_ADDRDEC]; + new_addr_type sub_partition_id_mask; + unsigned int gap; + unsigned m_n_channel; + int m_n_sub_partition_in_channel; }; #endif diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h index 0caa5d4..1cf4185 100644 --- a/src/gpgpu-sim/delayqueue.h +++ b/src/gpgpu-sim/delayqueue.h @@ -7,26 +7,27 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include <stdio.h> #include <assert.h> +#include <stdio.h> #include <stdlib.h> #ifndef DELAYQUEUE_H @@ -37,157 +38,150 @@ template <class T> struct fifo_data { - T *m_data; - fifo_data *m_next; + T* m_data; + fifo_data* m_next; }; -template <class T> +template <class T> class fifo_pipeline { -public: - fifo_pipeline(const char* nm, unsigned int minlen, unsigned int maxlen ) - { - assert(maxlen); - m_name = nm; - m_min_len = minlen; - m_max_len = maxlen; - m_length = 0; - m_n_element = 0; - m_head = NULL; - m_tail = NULL; - for (unsigned i=0;i<m_min_len;i++) - push(NULL); - } + public: + fifo_pipeline(const char* nm, unsigned int minlen, unsigned int maxlen) { + assert(maxlen); + m_name = nm; + m_min_len = minlen; + m_max_len = maxlen; + m_length = 0; + m_n_element = 0; + m_head = NULL; + m_tail = NULL; + for (unsigned i = 0; i < m_min_len; i++) push(NULL); + } - ~fifo_pipeline() - { - while (m_head) { - m_tail = m_head; - m_head = m_head->m_next; - delete m_tail; - } - } + ~fifo_pipeline() { + while (m_head) { + m_tail = m_head; + m_head = m_head->m_next; + delete m_tail; + } + } - void push(T* data ) - { - assert(m_length < m_max_len); - if (m_head) { - if (m_tail->m_data || m_length < m_min_len) { - m_tail->m_next = new fifo_data<T>(); - m_tail = m_tail->m_next; - m_length++; - m_n_element++; - } - } else { - m_head = m_tail = new fifo_data<T>(); - m_length++; - m_n_element++; + void push(T* data) { + assert(m_length < m_max_len); + if (m_head) { + if (m_tail->m_data || m_length < m_min_len) { + m_tail->m_next = new fifo_data<T>(); + m_tail = m_tail->m_next; + m_length++; + m_n_element++; } - m_tail->m_next = NULL; - m_tail->m_data = data; - } + } else { + m_head = m_tail = new fifo_data<T>(); + m_length++; + m_n_element++; + } + m_tail->m_next = NULL; + m_tail->m_data = data; + } - T* pop() - { - fifo_data<T>* next; - T* data; - if (m_head) { - next = m_head->m_next; - data = m_head->m_data; - if ( m_head == m_tail ) { - assert( next == NULL ); - m_tail = NULL; - } - delete m_head; - m_head = next; - m_length--; - if (m_length == 0) { - assert( m_head == NULL ); - m_tail = m_head; - } - m_n_element--; - if (m_min_len && m_length < m_min_len) { - push(NULL); - m_n_element--; // uncount NULL elements inserted to create delays - } - } else { - data = NULL; + T* pop() { + fifo_data<T>* next; + T* data; + if (m_head) { + next = m_head->m_next; + data = m_head->m_data; + if (m_head == m_tail) { + assert(next == NULL); + m_tail = NULL; } - return data; - } - - T* top() const - { - if (m_head) { - return m_head->m_data; - } else { - return NULL; + delete m_head; + m_head = next; + m_length--; + if (m_length == 0) { + assert(m_head == NULL); + m_tail = m_head; } - } - - void set_min_length(unsigned int new_min_len) - { - if (new_min_len == m_min_len) return; - - if (new_min_len > m_min_len) { - m_min_len = new_min_len; - while (m_length < m_min_len) { - push(NULL); - m_n_element--; // uncount NULL elements inserted to create delays - } - } else { - // in this branch imply that the original min_len is larger then 0 - // ie. head != 0 - assert(m_head); - m_min_len = new_min_len; - while ((m_length > m_min_len) && (m_tail->m_data == 0)) { - fifo_data<T> *iter; - iter = m_head; - while (iter && (iter->m_next != m_tail)) - iter = iter->m_next; - if (!iter) { - // there is only one node, and that node is empty - assert(m_head->m_data == 0); - pop(); - } else { - // there are more than one node, and tail node is empty - assert(iter->m_next == m_tail); - delete m_tail; - m_tail = iter; - m_tail->m_next = 0; - m_length--; - } - } + m_n_element--; + if (m_min_len && m_length < m_min_len) { + push(NULL); + m_n_element--; // uncount NULL elements inserted to create delays } - } + } else { + data = NULL; + } + return data; + } + + T* top() const { + if (m_head) { + return m_head->m_data; + } else { + return NULL; + } + } - bool full() const { return (m_max_len && m_length >= m_max_len); } - bool is_avilable_size(unsigned size) const { return (m_max_len && m_length+size-1 >= m_max_len); } - bool empty() const { return m_head == NULL; } - unsigned get_n_element() const { return m_n_element; } - unsigned get_length() const { return m_length; } - unsigned get_max_len() const { return m_max_len; } + void set_min_length(unsigned int new_min_len) { + if (new_min_len == m_min_len) return; - void print() const - { - fifo_data<T>* ddp = m_head; - printf("%s(%d): ", m_name, m_length); - while (ddp) { - printf("%p ", ddp->m_data); - ddp = ddp->m_next; + if (new_min_len > m_min_len) { + m_min_len = new_min_len; + while (m_length < m_min_len) { + push(NULL); + m_n_element--; // uncount NULL elements inserted to create delays + } + } else { + // in this branch imply that the original min_len is larger then 0 + // ie. head != 0 + assert(m_head); + m_min_len = new_min_len; + while ((m_length > m_min_len) && (m_tail->m_data == 0)) { + fifo_data<T>* iter; + iter = m_head; + while (iter && (iter->m_next != m_tail)) iter = iter->m_next; + if (!iter) { + // there is only one node, and that node is empty + assert(m_head->m_data == 0); + pop(); + } else { + // there are more than one node, and tail node is empty + assert(iter->m_next == m_tail); + delete m_tail; + m_tail = iter; + m_tail->m_next = 0; + m_length--; + } } - printf("\n"); - } + } + } + + bool full() const { return (m_max_len && m_length >= m_max_len); } + bool is_avilable_size(unsigned size) const { + return (m_max_len && m_length + size - 1 >= m_max_len); + } + bool empty() const { return m_head == NULL; } + unsigned get_n_element() const { return m_n_element; } + unsigned get_length() const { return m_length; } + unsigned get_max_len() const { return m_max_len; } + + void print() const { + fifo_data<T>* ddp = m_head; + printf("%s(%d): ", m_name, m_length); + while (ddp) { + printf("%p ", ddp->m_data); + ddp = ddp->m_next; + } + printf("\n"); + } -private: - const char* m_name; + private: + const char* m_name; - unsigned int m_min_len; - unsigned int m_max_len; - unsigned int m_length; - unsigned int m_n_element; + unsigned int m_min_len; + unsigned int m_max_len; + unsigned int m_length; + unsigned int m_n_element; - fifo_data<T> *m_head; - fifo_data<T> *m_tail; + fifo_data<T>* m_head; + fifo_data<T>* m_tail; }; #endif diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index 9c33822..02356b2 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -8,31 +8,32 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "gpu-sim.h" -#include "gpu-misc.h" #include "dram.h" -#include "mem_latency_stat.h" #include "dram_sched.h" -#include "mem_fetch.h" +#include "gpu-misc.h" +#include "gpu-sim.h" #include "l2cache.h" +#include "mem_fetch.h" +#include "mem_latency_stat.h" #ifdef DRAM_VERIFY int PRINT_CYCLE = 0; @@ -41,869 +42,828 @@ int PRINT_CYCLE = 0; template class fifo_pipeline<mem_fetch>; template class fifo_pipeline<dram_req_t>; -dram_t::dram_t( unsigned int partition_id, const memory_config *config, memory_stats_t *stats, - memory_partition_unit *mp, gpgpu_sim* gpu ) -{ - id = partition_id; - m_memory_partition_unit = mp; - m_stats = stats; - m_config = config; - m_gpu = gpu; +dram_t::dram_t(unsigned int partition_id, const memory_config *config, + memory_stats_t *stats, memory_partition_unit *mp, + gpgpu_sim *gpu) { + id = partition_id; + m_memory_partition_unit = mp; + m_stats = stats; + m_config = config; + m_gpu = gpu; - //rowblp - access_num=0; - hits_num=0; - read_num=0; - write_num=0; - hits_read_num=0; - hits_write_num=0; - banks_1time=0; - banks_acess_total=0; - banks_acess_total_after=0; - banks_time_ready=0; - banks_access_ready_total=0; - issued_two=0; - issued_total=0; - issued_total_row=0; - issued_total_col=0; + // rowblp + access_num = 0; + hits_num = 0; + read_num = 0; + write_num = 0; + hits_read_num = 0; + hits_write_num = 0; + banks_1time = 0; + banks_acess_total = 0; + banks_acess_total_after = 0; + banks_time_ready = 0; + banks_access_ready_total = 0; + issued_two = 0; + issued_total = 0; + issued_total_row = 0; + issued_total_col = 0; - CCDc = 0; - RRDc = 0; - RTWc = 0; - WTRc = 0; + CCDc = 0; + RRDc = 0; + RTWc = 0; + WTRc = 0; - wasted_bw_row=0; - wasted_bw_col=0; - util_bw=0; - idle_bw=0; - RCDc_limit=0; - CCDLc_limit=0; - CCDLc_limit_alone=0; - CCDc_limit=0; - WTRc_limit=0; - WTRc_limit_alone=0; - RCDWRc_limit=0; - RTWc_limit=0; - RTWc_limit_alone=0; - rwq_limit=0; - write_to_read_ratio_blp_rw_average=0; - bkgrp_parallsim_rw=0; + wasted_bw_row = 0; + wasted_bw_col = 0; + util_bw = 0; + idle_bw = 0; + RCDc_limit = 0; + CCDLc_limit = 0; + CCDLc_limit_alone = 0; + CCDc_limit = 0; + WTRc_limit = 0; + WTRc_limit_alone = 0; + RCDWRc_limit = 0; + RTWc_limit = 0; + RTWc_limit_alone = 0; + rwq_limit = 0; + write_to_read_ratio_blp_rw_average = 0; + bkgrp_parallsim_rw = 0; - rw = READ; //read mode is default + rw = READ; // read mode is default - bkgrp = (bankgrp_t**) calloc(sizeof(bankgrp_t*), m_config->nbkgrp); - bkgrp[0] = (bankgrp_t*) calloc(sizeof(bank_t), m_config->nbkgrp); - for (unsigned i=1; i<m_config->nbkgrp; i++) { - bkgrp[i] = bkgrp[0] + i; - } - for (unsigned i=0; i<m_config->nbkgrp; i++) { - bkgrp[i]->CCDLc = 0; - bkgrp[i]->RTPLc = 0; - } + bkgrp = (bankgrp_t **)calloc(sizeof(bankgrp_t *), m_config->nbkgrp); + bkgrp[0] = (bankgrp_t *)calloc(sizeof(bank_t), m_config->nbkgrp); + for (unsigned i = 1; i < m_config->nbkgrp; i++) { + bkgrp[i] = bkgrp[0] + i; + } + for (unsigned i = 0; i < m_config->nbkgrp; i++) { + bkgrp[i]->CCDLc = 0; + bkgrp[i]->RTPLc = 0; + } - bk = (bank_t**) calloc(sizeof(bank_t*),m_config->nbk); - bk[0] = (bank_t*) calloc(sizeof(bank_t),m_config->nbk); - for (unsigned i=1;i<m_config->nbk;i++) - bk[i] = bk[0] + i; - for (unsigned i=0;i<m_config->nbk;i++) { - bk[i]->state = BANK_IDLE; - bk[i]->bkgrpindex = i/(m_config->nbk/m_config->nbkgrp); - } - prio = 0; + bk = (bank_t **)calloc(sizeof(bank_t *), m_config->nbk); + bk[0] = (bank_t *)calloc(sizeof(bank_t), m_config->nbk); + for (unsigned i = 1; i < m_config->nbk; i++) bk[i] = bk[0] + i; + for (unsigned i = 0; i < m_config->nbk; i++) { + bk[i]->state = BANK_IDLE; + bk[i]->bkgrpindex = i / (m_config->nbk / m_config->nbkgrp); + } + prio = 0; - rwq = new fifo_pipeline<dram_req_t>("rwq",m_config->CL,m_config->CL+1); - mrqq = new fifo_pipeline<dram_req_t>("mrqq",0,2); - returnq = new fifo_pipeline<mem_fetch>("dramreturnq",0,m_config->gpgpu_dram_return_queue_size==0?1024:m_config->gpgpu_dram_return_queue_size); - m_frfcfs_scheduler = NULL; - if ( m_config->scheduler_type == DRAM_FRFCFS) - m_frfcfs_scheduler = new frfcfs_scheduler(m_config,this,stats); - n_cmd = 0; - n_activity = 0; - n_nop = 0; - n_act = 0; - n_pre = 0; - n_rd = 0; - n_wr = 0; - n_wr_WB=0; - n_rd_L2_A=0; - n_req = 0; - max_mrqs_temp = 0; - bwutil = 0; - max_mrqs = 0; - ave_mrqs = 0; + rwq = new fifo_pipeline<dram_req_t>("rwq", m_config->CL, m_config->CL + 1); + mrqq = new fifo_pipeline<dram_req_t>("mrqq", 0, 2); + returnq = new fifo_pipeline<mem_fetch>( + "dramreturnq", 0, + m_config->gpgpu_dram_return_queue_size == 0 + ? 1024 + : m_config->gpgpu_dram_return_queue_size); + m_frfcfs_scheduler = NULL; + if (m_config->scheduler_type == DRAM_FRFCFS) + m_frfcfs_scheduler = new frfcfs_scheduler(m_config, this, stats); + n_cmd = 0; + n_activity = 0; + n_nop = 0; + n_act = 0; + n_pre = 0; + n_rd = 0; + n_wr = 0; + n_wr_WB = 0; + n_rd_L2_A = 0; + n_req = 0; + max_mrqs_temp = 0; + bwutil = 0; + max_mrqs = 0; + ave_mrqs = 0; - for (unsigned i=0;i<10;i++) { - dram_util_bins[i]=0; - dram_eff_bins[i]=0; - } - last_n_cmd = last_n_activity = last_bwutil = 0; + for (unsigned i = 0; i < 10; i++) { + dram_util_bins[i] = 0; + dram_eff_bins[i] = 0; + } + last_n_cmd = last_n_activity = last_bwutil = 0; - n_cmd_partial = 0; - n_activity_partial = 0; - n_nop_partial = 0; - n_act_partial = 0; - n_pre_partial = 0; - n_req_partial = 0; - ave_mrqs_partial = 0; - bwutil_partial = 0; - - if ( queue_limit() ) - mrqq_Dist = StatCreate("mrqq_length",1, queue_limit()); - else //queue length is unlimited; - mrqq_Dist = StatCreate("mrqq_length",1,64); //track up to 64 entries + n_cmd_partial = 0; + n_activity_partial = 0; + n_nop_partial = 0; + n_act_partial = 0; + n_pre_partial = 0; + n_req_partial = 0; + ave_mrqs_partial = 0; + bwutil_partial = 0; + if (queue_limit()) + mrqq_Dist = StatCreate("mrqq_length", 1, queue_limit()); + else // queue length is unlimited; + mrqq_Dist = StatCreate("mrqq_length", 1, 64); // track up to 64 entries } -bool dram_t::full(bool is_write) const -{ - if(m_config->scheduler_type == DRAM_FRFCFS){ - if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false; - if(m_config->seperate_write_queue_enabled){ - if(is_write) - return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size; - else - return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; - } - else - return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; - } - else return mrqq->full(); +bool dram_t::full(bool is_write) const { + if (m_config->scheduler_type == DRAM_FRFCFS) { + if (m_config->gpgpu_frfcfs_dram_sched_queue_size == 0) return false; + if (m_config->seperate_write_queue_enabled) { + if (is_write) + return m_frfcfs_scheduler->num_write_pending() >= + m_config->gpgpu_frfcfs_dram_write_queue_size; + else + return m_frfcfs_scheduler->num_pending() >= + m_config->gpgpu_frfcfs_dram_sched_queue_size; + } else + return m_frfcfs_scheduler->num_pending() >= + m_config->gpgpu_frfcfs_dram_sched_queue_size; + } else + return mrqq->full(); } -unsigned dram_t::que_length() const -{ - unsigned nreqs = 0; - if (m_config->scheduler_type == DRAM_FRFCFS) { - nreqs = m_frfcfs_scheduler->num_pending(); - } else { - nreqs = mrqq->get_length(); - } - return nreqs; +unsigned dram_t::que_length() const { + unsigned nreqs = 0; + if (m_config->scheduler_type == DRAM_FRFCFS) { + nreqs = m_frfcfs_scheduler->num_pending(); + } else { + nreqs = mrqq->get_length(); + } + return nreqs; } -bool dram_t::returnq_full() const -{ - return returnq->full(); -} +bool dram_t::returnq_full() const { return returnq->full(); } -unsigned int dram_t::queue_limit() const -{ - return m_config->gpgpu_frfcfs_dram_sched_queue_size; +unsigned int dram_t::queue_limit() const { + return m_config->gpgpu_frfcfs_dram_sched_queue_size; } +dram_req_t::dram_req_t(class mem_fetch *mf, unsigned banks, + unsigned dram_bnk_indexing_policy, + class gpgpu_sim *gpu) { + txbytes = 0; + dqbytes = 0; + data = mf; + m_gpu = gpu; -dram_req_t::dram_req_t( class mem_fetch *mf, unsigned banks, unsigned dram_bnk_indexing_policy, class gpgpu_sim* gpu) -{ - txbytes = 0; - dqbytes = 0; - data = mf; - m_gpu = gpu; - - const addrdec_t &tlx = mf->get_tlx_addr(); - - switch(dram_bnk_indexing_policy){ - case LINEAR_BK_INDEX: - { - bk = tlx.bk; - break; - } - case BITWISE_XORING_BK_INDEX: - { - //xoring bank bits with lower bits of the page - int lbank = log2(banks); - bk = tlx.bk ^ (tlx.row & ((1<<lbank)-1)); - break; - } - case CUSTOM_BK_INDEX: - /* No custom set function implemented */ - //Do you custom index here - break; - default: - assert("\nUndefined bank index function.\n" && 0); - break; - } + const addrdec_t &tlx = mf->get_tlx_addr(); + switch (dram_bnk_indexing_policy) { + case LINEAR_BK_INDEX: { + bk = tlx.bk; + break; + } + case BITWISE_XORING_BK_INDEX: { + // xoring bank bits with lower bits of the page + int lbank = log2(banks); + bk = tlx.bk ^ (tlx.row & ((1 << lbank) - 1)); + break; + } + case CUSTOM_BK_INDEX: + /* No custom set function implemented */ + // Do you custom index here + break; + default: + assert("\nUndefined bank index function.\n" && 0); + break; + } - row = tlx.row; - col = tlx.col; - nbytes = mf->get_data_size(); + row = tlx.row; + col = tlx.col; + nbytes = mf->get_data_size(); - timestamp = m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle; - addr = mf->get_addr(); - insertion_time = (unsigned) m_gpu->gpu_sim_cycle; - rw = data->get_is_write()?WRITE:READ; + timestamp = m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle; + addr = mf->get_addr(); + insertion_time = (unsigned)m_gpu->gpu_sim_cycle; + rw = data->get_is_write() ? WRITE : READ; } -void dram_t::push( class mem_fetch *data ) -{ - assert(id == data->get_tlx_addr().chip); // Ensure request is in correct memory partition +void dram_t::push(class mem_fetch *data) { + assert(id == data->get_tlx_addr() + .chip); // Ensure request is in correct memory partition - dram_req_t *mrq = new dram_req_t(data,m_config->nbk,m_config->dram_bnk_indexing_policy,m_memory_partition_unit->get_mgpu()); + dram_req_t *mrq = + new dram_req_t(data, m_config->nbk, m_config->dram_bnk_indexing_policy, + m_memory_partition_unit->get_mgpu()); - data->set_status(IN_PARTITION_MC_INTERFACE_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - mrqq->push(mrq); + data->set_status(IN_PARTITION_MC_INTERFACE_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + mrqq->push(mrq); - // stats... - n_req += 1; - n_req_partial += 1; - if ( m_config->scheduler_type == DRAM_FRFCFS) { - unsigned nreqs = m_frfcfs_scheduler->num_pending(); - if ( nreqs > max_mrqs_temp) - max_mrqs_temp = nreqs; - } else { - max_mrqs_temp = (max_mrqs_temp > mrqq->get_length())? max_mrqs_temp : mrqq->get_length(); - } - m_stats->memlatstat_dram_access(data); + // stats... + n_req += 1; + n_req_partial += 1; + if (m_config->scheduler_type == DRAM_FRFCFS) { + unsigned nreqs = m_frfcfs_scheduler->num_pending(); + if (nreqs > max_mrqs_temp) max_mrqs_temp = nreqs; + } else { + max_mrqs_temp = (max_mrqs_temp > mrqq->get_length()) ? max_mrqs_temp + : mrqq->get_length(); + } + m_stats->memlatstat_dram_access(data); } -void dram_t::scheduler_fifo() -{ - if (!mrqq->empty()) { - unsigned int bkn; - dram_req_t *head_mrqq = mrqq->top(); - head_mrqq->data->set_status(IN_PARTITION_MC_BANK_ARB_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - bkn = head_mrqq->bk; - if (!bk[bkn]->mrq) - bk[bkn]->mrq = mrqq->pop(); - } +void dram_t::scheduler_fifo() { + if (!mrqq->empty()) { + unsigned int bkn; + dram_req_t *head_mrqq = mrqq->top(); + head_mrqq->data->set_status( + IN_PARTITION_MC_BANK_ARB_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + bkn = head_mrqq->bk; + if (!bk[bkn]->mrq) bk[bkn]->mrq = mrqq->pop(); + } } +#define DEC2ZERO(x) x = (x) ? (x - 1) : 0; +#define SWAP(a, b) \ + a ^= b; \ + b ^= a; \ + a ^= b; -#define DEC2ZERO(x) x = (x)? (x-1) : 0; -#define SWAP(a,b) a ^= b; b ^= a; a ^= b; - -void dram_t::cycle() -{ - - if( !returnq->full() ) { - dram_req_t *cmd = rwq->pop(); - if( cmd ) { -#ifdef DRAM_VIEWCMD - printf("\tDQ: BK%d Row:%03x Col:%03x", cmd->bk, cmd->row, cmd->col + cmd->dqbytes); +void dram_t::cycle() { + if (!returnq->full()) { + dram_req_t *cmd = rwq->pop(); + if (cmd) { +#ifdef DRAM_VIEWCMD + printf("\tDQ: BK%d Row:%03x Col:%03x", cmd->bk, cmd->row, + cmd->col + cmd->dqbytes); #endif - cmd->dqbytes += m_config->dram_atom_size; + cmd->dqbytes += m_config->dram_atom_size; - if (cmd->dqbytes >= cmd->nbytes) { - mem_fetch *data = cmd->data; - data->set_status(IN_PARTITION_MC_RETURNQ,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - if( data->get_access_type() != L1_WRBK_ACC && data->get_access_type() != L2_WRBK_ACC ) { - data->set_reply(); - returnq->push(data); - } else { - m_memory_partition_unit->set_done(data); - delete data; - } - delete cmd; - } -#ifdef DRAM_VIEWCMD - printf("\n"); + if (cmd->dqbytes >= cmd->nbytes) { + mem_fetch *data = cmd->data; + data->set_status(IN_PARTITION_MC_RETURNQ, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + if (data->get_access_type() != L1_WRBK_ACC && + data->get_access_type() != L2_WRBK_ACC) { + data->set_reply(); + returnq->push(data); + } else { + m_memory_partition_unit->set_done(data); + delete data; + } + delete cmd; + } +#ifdef DRAM_VIEWCMD + printf("\n"); #endif - } - } + } + } - /* check if the upcoming request is on an idle bank */ - /* Should we modify this so that multiple requests are checked? */ + /* check if the upcoming request is on an idle bank */ + /* Should we modify this so that multiple requests are checked? */ - switch (m_config->scheduler_type) { - case DRAM_FIFO: scheduler_fifo(); break; - case DRAM_FRFCFS: scheduler_frfcfs(); break; - default: - printf("Error: Unknown DRAM scheduler type\n"); - assert(0); - } - if ( m_config->scheduler_type == DRAM_FRFCFS) { - unsigned nreqs = m_frfcfs_scheduler->num_pending(); - if ( nreqs > max_mrqs) { - max_mrqs = nreqs; - } - ave_mrqs += nreqs; - ave_mrqs_partial += nreqs; - } else { - if (mrqq->get_length() > max_mrqs) { - max_mrqs = mrqq->get_length(); - } - ave_mrqs += mrqq->get_length(); - ave_mrqs_partial += mrqq->get_length(); - } - - unsigned k=m_config->nbk; - bool issued = false; + switch (m_config->scheduler_type) { + case DRAM_FIFO: + scheduler_fifo(); + break; + case DRAM_FRFCFS: + scheduler_frfcfs(); + break; + default: + printf("Error: Unknown DRAM scheduler type\n"); + assert(0); + } + if (m_config->scheduler_type == DRAM_FRFCFS) { + unsigned nreqs = m_frfcfs_scheduler->num_pending(); + if (nreqs > max_mrqs) { + max_mrqs = nreqs; + } + ave_mrqs += nreqs; + ave_mrqs_partial += nreqs; + } else { + if (mrqq->get_length() > max_mrqs) { + max_mrqs = mrqq->get_length(); + } + ave_mrqs += mrqq->get_length(); + ave_mrqs_partial += mrqq->get_length(); + } - //collect row buffer locality, BLP and other statistics - ///////////////////////////////////////////////////////////////////////// - unsigned int memory_pending=0; - for (unsigned i=0;i<m_config->nbk;i++) { - if (bk[i]->mrq) - memory_pending++; - } - banks_1time += memory_pending; - if(memory_pending >0) - banks_acess_total++; + unsigned k = m_config->nbk; + bool issued = false; - unsigned int memory_pending_rw=0; - unsigned read_blp_rw=0; - unsigned write_blp_rw=0; - std::bitset<8> bnkgrp_rw_found; //assume max we have 8 bank groups + // collect row buffer locality, BLP and other statistics + ///////////////////////////////////////////////////////////////////////// + unsigned int memory_pending = 0; + for (unsigned i = 0; i < m_config->nbk; i++) { + if (bk[i]->mrq) memory_pending++; + } + banks_1time += memory_pending; + if (memory_pending > 0) banks_acess_total++; - for (unsigned j=0;j<m_config->nbk;j++) { - unsigned grp = get_bankgrp_number(j); - if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && - (bk[j]->state == BANK_ACTIVE)))) - { - memory_pending_rw++; - read_blp_rw++; - bnkgrp_rw_found.set(grp); - } - else if - (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && - (bk[j]->state == BANK_ACTIVE)))) - { - memory_pending_rw++; - write_blp_rw++; - bnkgrp_rw_found.set(grp); - } - } - banks_time_rw += memory_pending_rw; - bkgrp_parallsim_rw += bnkgrp_rw_found.count(); - if(memory_pending_rw >0) - { - write_to_read_ratio_blp_rw_average += (double)write_blp_rw/(write_blp_rw+read_blp_rw); - banks_access_rw_total++; - } + unsigned int memory_pending_rw = 0; + unsigned read_blp_rw = 0; + unsigned write_blp_rw = 0; + std::bitset<8> bnkgrp_rw_found; // assume max we have 8 bank groups - unsigned int memory_Pending_ready=0; - for (unsigned j=0;j<m_config->nbk;j++) { - unsigned grp = get_bankgrp_number(j); - if (bk[j]->mrq && ((!CCDc && !bk[j]->RCDc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full()) - || - (!CCDc && !bk[j]->RCDWRc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full()))) - { - memory_Pending_ready++; - } - } - banks_time_ready += memory_Pending_ready; - if(memory_Pending_ready >0) - banks_access_ready_total++; - /////////////////////////////////////////////////////////////////////////////////// + for (unsigned j = 0; j < m_config->nbk; j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && + (((bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) { + memory_pending_rw++; + read_blp_rw++; + bnkgrp_rw_found.set(grp); + } else if (bk[j]->mrq && + (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (bk[j]->state == BANK_ACTIVE)))) { + memory_pending_rw++; + write_blp_rw++; + bnkgrp_rw_found.set(grp); + } + } + banks_time_rw += memory_pending_rw; + bkgrp_parallsim_rw += bnkgrp_rw_found.count(); + if (memory_pending_rw > 0) { + write_to_read_ratio_blp_rw_average += + (double)write_blp_rw / (write_blp_rw + read_blp_rw); + banks_access_rw_total++; + } - bool issued_col_cmd = false; - bool issued_row_cmd = false; + unsigned int memory_Pending_ready = 0; + for (unsigned j = 0; j < m_config->nbk; j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && + ((!CCDc && !bk[j]->RCDc && !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && + (WTRc == 0) && (bk[j]->state == BANK_ACTIVE) && !rwq->full()) || + (!CCDc && !bk[j]->RCDWRc && !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == WRITE) && + (RTWc == 0) && (bk[j]->state == BANK_ACTIVE) && !rwq->full()))) { + memory_Pending_ready++; + } + } + banks_time_ready += memory_Pending_ready; + if (memory_Pending_ready > 0) banks_access_ready_total++; + /////////////////////////////////////////////////////////////////////////////////// - if(m_config->dual_bus_interface) - { - //dual bus interface - //issue one row command and one column command - for (unsigned i=0;i<m_config->nbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - issued_col_cmd = issue_col_command(j); - if(issued_col_cmd) break; - } - for (unsigned i=0;i<m_config->nbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - issued_row_cmd = issue_row_command(j); - if(issued_row_cmd) break; - } - for (unsigned i=0;i<m_config->nbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - if(!bk[j]->mrq) { - if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc - && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; - bk[j]->n_idle++; - } - } - } - else - { - //single bus interface - //issue only one row/column command - for (unsigned i=0;i<m_config->nbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - if(!issued_col_cmd) - issued_col_cmd = issue_col_command(j); + bool issued_col_cmd = false; + bool issued_row_cmd = false; - if(!issued_col_cmd && !issued_row_cmd) - issued_row_cmd = issue_row_command(j); + if (m_config->dual_bus_interface) { + // dual bus interface + // issue one row command and one column command + for (unsigned i = 0; i < m_config->nbk; i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_col_cmd = issue_col_command(j); + if (issued_col_cmd) break; + } + for (unsigned i = 0; i < m_config->nbk; i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_row_cmd = issue_row_command(j); + if (issued_row_cmd) break; + } + for (unsigned i = 0; i < m_config->nbk; i++) { + unsigned j = (i + prio) % m_config->nbk; + if (!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc && + !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) + k--; + bk[j]->n_idle++; + } + } + } else { + // single bus interface + // issue only one row/column command + for (unsigned i = 0; i < m_config->nbk; i++) { + unsigned j = (i + prio) % m_config->nbk; + if (!issued_col_cmd) issued_col_cmd = issue_col_command(j); - if(!bk[j]->mrq) { - if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc - && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; - bk[j]->n_idle++; - } + if (!issued_col_cmd && !issued_row_cmd) + issued_row_cmd = issue_row_command(j); - } - } + if (!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc && + !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) + k--; + bk[j]->n_idle++; + } + } + } - issued = issued_row_cmd || issued_col_cmd; - if (!issued) { - n_nop++; - n_nop_partial++; + issued = issued_row_cmd || issued_col_cmd; + if (!issued) { + n_nop++; + n_nop_partial++; #ifdef DRAM_VIEWCMD - printf("\tNOP "); + printf("\tNOP "); #endif - } - if (k) { - n_activity++; - n_activity_partial++; - } - n_cmd++; - n_cmd_partial++; - if(issued) - { - issued_total++; - if(issued_col_cmd && issued_row_cmd) - issued_two++; - } - if(issued_col_cmd) issued_total_col++; - if(issued_row_cmd) issued_total_row++; + } + if (k) { + n_activity++; + n_activity_partial++; + } + n_cmd++; + n_cmd_partial++; + if (issued) { + issued_total++; + if (issued_col_cmd && issued_row_cmd) issued_two++; + } + if (issued_col_cmd) issued_total_col++; + if (issued_row_cmd) issued_total_row++; + // Collect some statistics + // check the limitation, see where BW is wasted? + ///////////////////////////////////////////////////////// + unsigned int memory_pending_found = 0; + for (unsigned i = 0; i < m_config->nbk; i++) { + if (bk[i]->mrq) memory_pending_found++; + } + if (memory_pending_found > 0) banks_acess_total_after++; - //Collect some statistics - //check the limitation, see where BW is wasted? - ///////////////////////////////////////////////////////// - unsigned int memory_pending_found=0; - for (unsigned i=0;i<m_config->nbk;i++) { - if (bk[i]->mrq) - memory_pending_found++; - } - if(memory_pending_found>0) - banks_acess_total_after++; - - bool memory_pending_rw_found=false; - for (unsigned j=0;j<m_config->nbk;j++) { - if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && - (bk[j]->state == BANK_ACTIVE)) - || - ( - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && - (bk[j]->state == BANK_ACTIVE)))) - memory_pending_rw_found=true; - } - + bool memory_pending_rw_found = false; + for (unsigned j = 0; j < m_config->nbk; j++) { + if (bk[j]->mrq && + (((bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)) || + ((bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE)))) + memory_pending_rw_found = true; + } - if(issued_col_cmd || CCDc) - util_bw++; - else if (memory_pending_rw_found) - { - wasted_bw_col++; - for (unsigned j=0;j<m_config->nbk;j++) { - unsigned grp = get_bankgrp_number(j); - //read - if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && - (bk[j]->state == BANK_ACTIVE)))) - { - if(bk[j]->RCDc) RCDc_limit++; - if(bkgrp[grp]->CCDLc) CCDLc_limit++; - if(WTRc) WTRc_limit++; - if(CCDc) CCDc_limit++; - if(rwq->full()) rwq_limit++; - if(bkgrp[grp]->CCDLc && !WTRc) CCDLc_limit_alone++; - if(!bkgrp[grp]->CCDLc && WTRc) WTRc_limit_alone++; - } - //write - else if (bk[j]->mrq && ((bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && - (bk[j]->state == BANK_ACTIVE))) - { - if(bk[j]->RCDWRc) RCDWRc_limit++; - if(bkgrp[grp]->CCDLc) CCDLc_limit++; - if(RTWc) RTWc_limit++; - if(CCDc) CCDc_limit++; - if(rwq->full()) rwq_limit++; - if(bkgrp[grp]->CCDLc && !RTWc) CCDLc_limit_alone++; - if(!bkgrp[grp]->CCDLc && RTWc) RTWc_limit_alone++; - } - } - } - else if (memory_pending_found) - wasted_bw_row++; - else if (!memory_pending_found) - idle_bw++; - else - assert(1); + if (issued_col_cmd || CCDc) + util_bw++; + else if (memory_pending_rw_found) { + wasted_bw_col++; + for (unsigned j = 0; j < m_config->nbk; j++) { + unsigned grp = get_bankgrp_number(j); + // read + if (bk[j]->mrq && + (((bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) { + if (bk[j]->RCDc) RCDc_limit++; + if (bkgrp[grp]->CCDLc) CCDLc_limit++; + if (WTRc) WTRc_limit++; + if (CCDc) CCDc_limit++; + if (rwq->full()) rwq_limit++; + if (bkgrp[grp]->CCDLc && !WTRc) CCDLc_limit_alone++; + if (!bkgrp[grp]->CCDLc && WTRc) WTRc_limit_alone++; + } + // write + else if (bk[j]->mrq && + ((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (bk[j]->state == BANK_ACTIVE))) { + if (bk[j]->RCDWRc) RCDWRc_limit++; + if (bkgrp[grp]->CCDLc) CCDLc_limit++; + if (RTWc) RTWc_limit++; + if (CCDc) CCDc_limit++; + if (rwq->full()) rwq_limit++; + if (bkgrp[grp]->CCDLc && !RTWc) CCDLc_limit_alone++; + if (!bkgrp[grp]->CCDLc && RTWc) RTWc_limit_alone++; + } + } + } else if (memory_pending_found) + wasted_bw_row++; + else if (!memory_pending_found) + idle_bw++; + else + assert(1); - ///////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////// - // decrements counters once for each time dram_issueCMD is called - DEC2ZERO(RRDc); - DEC2ZERO(CCDc); - DEC2ZERO(RTWc); - DEC2ZERO(WTRc); - for (unsigned j=0;j<m_config->nbk;j++) { - DEC2ZERO(bk[j]->RCDc); - DEC2ZERO(bk[j]->RASc); - DEC2ZERO(bk[j]->RCc); - DEC2ZERO(bk[j]->RPc); - DEC2ZERO(bk[j]->RCDWRc); - DEC2ZERO(bk[j]->WTPc); - DEC2ZERO(bk[j]->RTPc); - } - for (unsigned j=0; j<m_config->nbkgrp; j++) { - DEC2ZERO(bkgrp[j]->CCDLc); - DEC2ZERO(bkgrp[j]->RTPLc); - } + // decrements counters once for each time dram_issueCMD is called + DEC2ZERO(RRDc); + DEC2ZERO(CCDc); + DEC2ZERO(RTWc); + DEC2ZERO(WTRc); + for (unsigned j = 0; j < m_config->nbk; j++) { + DEC2ZERO(bk[j]->RCDc); + DEC2ZERO(bk[j]->RASc); + DEC2ZERO(bk[j]->RCc); + DEC2ZERO(bk[j]->RPc); + DEC2ZERO(bk[j]->RCDWRc); + DEC2ZERO(bk[j]->WTPc); + DEC2ZERO(bk[j]->RTPc); + } + for (unsigned j = 0; j < m_config->nbkgrp; j++) { + DEC2ZERO(bkgrp[j]->CCDLc); + DEC2ZERO(bkgrp[j]->RTPLc); + } #ifdef DRAM_VISUALIZE - visualize(); + visualize(); #endif } -bool dram_t::issue_col_command(int j) -{ - bool issued = false; - unsigned grp = get_bankgrp_number(j); - if (bk[j]->mrq) { //if currently servicing a memory request - bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - // correct row activated for a READ - if ( !issued && !CCDc && !bk[j]->RCDc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==WRITE) { - rw=READ; - rwq->set_min_length(m_config->CL); - } - rwq->push(bk[j]->mrq); - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - RTWc = m_config->tRTW; - bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio; - bkgrp[grp]->RTPLc = m_config->tRTPL; - issued = true; - if(bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R) - n_rd_L2_A++; - else - n_rd++; +bool dram_t::issue_col_command(int j) { + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { // if currently servicing a memory request + bk[j]->mrq->data->set_status( + IN_PARTITION_DRAM, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + // correct row activated for a READ + if (!issued && !CCDc && !bk[j]->RCDc && !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && + (WTRc == 0) && (bk[j]->state == BANK_ACTIVE) && !rwq->full()) { + if (rw == WRITE) { + rw = READ; + rwq->set_min_length(m_config->CL); + } + rwq->push(bk[j]->mrq); + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + RTWc = m_config->tRTW; + bk[j]->RTPc = m_config->BL / m_config->data_command_freq_ratio; + bkgrp[grp]->RTPLc = m_config->tRTPL; + issued = true; + if (bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R) + n_rd_L2_A++; + else + n_rd++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; - bk[j]->n_access++; + bwutil += m_config->BL / m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL / m_config->data_command_freq_ratio; + bk[j]->n_access++; #ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tRD Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); + PRINT_CYCLE = 1; + printf("\tRD Bk:%d Row:%03x Col:%03x \n", j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); #endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } else - // correct row activated for a WRITE - if ( !issued && !CCDc && !bk[j]->RCDWRc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==READ) { - rw=WRITE; - rwq->set_min_length(m_config->WL); - } - rwq->push(bk[j]->mrq); + // transfer done + if (!(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes)) { + bk[j]->mrq = NULL; + } + } else + // correct row activated for a WRITE + if (!issued && !CCDc && !bk[j]->RCDWRc && !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == WRITE) && + (RTWc == 0) && (bk[j]->state == BANK_ACTIVE) && !rwq->full()) { + if (rw == READ) { + rw = WRITE; + rwq->set_min_length(m_config->WL); + } + rwq->push(bk[j]->mrq); - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - WTRc = m_config->tWTR; - bk[j]->WTPc = m_config->tWTP; - issued = true; + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + WTRc = m_config->tWTR; + bk[j]->WTPc = m_config->tWTP; + issued = true; - if(bk[j]->mrq->data->get_access_type() == L2_WRBK_ACC) - n_wr_WB++; - else - n_wr++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; + if (bk[j]->mrq->data->get_access_type() == L2_WRBK_ACC) + n_wr_WB++; + else + n_wr++; + bwutil += m_config->BL / m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL / m_config->data_command_freq_ratio; #ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tWR Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); + PRINT_CYCLE = 1; + printf("\tWR Bk:%d Row:%03x Col:%03x \n", j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); #endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } - + // transfer done + if (!(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes)) { + bk[j]->mrq = NULL; + } } + } - return issued; + return issued; } -bool dram_t::issue_row_command(int j) -{ - bool issued = false; - unsigned grp = get_bankgrp_number(j); - if (bk[j]->mrq) { //if currently servicing a memory request - bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - // bank is idle - //else - if ( !issued && !RRDc && - (bk[j]->state == BANK_IDLE) && - !bk[j]->RPc && !bk[j]->RCc) { // +bool dram_t::issue_row_command(int j) { + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { // if currently servicing a memory request + bk[j]->mrq->data->set_status( + IN_PARTITION_DRAM, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + // bank is idle + // else + if (!issued && !RRDc && (bk[j]->state == BANK_IDLE) && !bk[j]->RPc && + !bk[j]->RCc) { // #ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tACT BK:%d NewRow:%03x From:%03x \n", - j,bk[j]->mrq->row,bk[j]->curr_row); + PRINT_CYCLE = 1; + printf("\tACT BK:%d NewRow:%03x From:%03x \n", j, bk[j]->mrq->row, + bk[j]->curr_row); #endif - // activate the row with current memory request - bk[j]->curr_row = bk[j]->mrq->row; - bk[j]->state = BANK_ACTIVE; - RRDc = m_config->tRRD; - bk[j]->RCDc = m_config->tRCD; - bk[j]->RCDWRc = m_config->tRCDWR; - bk[j]->RASc = m_config->tRAS; - bk[j]->RCc = m_config->tRC; - prio = (j + 1) % m_config->nbk; - issued = true; - n_act_partial++; - n_act++; - } + // activate the row with current memory request + bk[j]->curr_row = bk[j]->mrq->row; + bk[j]->state = BANK_ACTIVE; + RRDc = m_config->tRRD; + bk[j]->RCDc = m_config->tRCD; + bk[j]->RCDWRc = m_config->tRCDWR; + bk[j]->RASc = m_config->tRAS; + bk[j]->RCc = m_config->tRC; + prio = (j + 1) % m_config->nbk; + issued = true; + n_act_partial++; + n_act++; + } - else - // different row activated - if ( (!issued) && - (bk[j]->curr_row != bk[j]->mrq->row) && - (bk[j]->state == BANK_ACTIVE) && - (!bk[j]->RASc && !bk[j]->WTPc && - !bk[j]->RTPc && - !bkgrp[grp]->RTPLc) ) { - // make the bank idle again - bk[j]->state = BANK_IDLE; - bk[j]->RPc = m_config->tRP; - prio = (j + 1) % m_config->nbk; - issued = true; - n_pre++; - n_pre_partial++; + else + // different row activated + if ((!issued) && (bk[j]->curr_row != bk[j]->mrq->row) && + (bk[j]->state == BANK_ACTIVE) && + (!bk[j]->RASc && !bk[j]->WTPc && !bk[j]->RTPc && + !bkgrp[grp]->RTPLc)) { + // make the bank idle again + bk[j]->state = BANK_IDLE; + bk[j]->RPc = m_config->tRP; + prio = (j + 1) % m_config->nbk; + issued = true; + n_pre++; + n_pre_partial++; #ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row); + PRINT_CYCLE = 1; + printf("\tPRE BK:%d Row:%03x \n", j, bk[j]->curr_row); #endif - } } - return issued; + } + return issued; } - -//if mrq is being serviced by dram, gets popped after CL latency fulfilled -class mem_fetch* dram_t::return_queue_pop() -{ - return returnq->pop(); +// if mrq is being serviced by dram, gets popped after CL latency fulfilled +class mem_fetch *dram_t::return_queue_pop() { + return returnq->pop(); } -class mem_fetch* dram_t::return_queue_top() -{ - return returnq->top(); +class mem_fetch *dram_t::return_queue_top() { + return returnq->top(); } +void dram_t::print(FILE *simFile) const { + unsigned i; + fprintf(simFile, "DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", id, m_config->nbk, + m_config->busW, m_config->BL, m_config->CL); + fprintf(simFile, "tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n", + m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, + m_config->tRP, m_config->tRC); + fprintf( + simFile, + "n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref_event=%llu n_req=%llu " + "n_rd=%llu n_rd_L2_A=%llu n_write=%llu n_wr_bk=%llu bw_util=%.4g\n", + n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB, + (float)bwutil / n_cmd); + fprintf(simFile, "n_activity=%llu dram_eff=%.4g\n", n_activity, + (float)bwutil / n_activity); + for (i = 0; i < m_config->nbk; i++) { + fprintf(simFile, "bk%d: %da %di ", i, bk[i]->n_access, bk[i]->n_idle); + } + fprintf(simFile, "\n"); + fprintf(simFile, + "\n------------------------------------------------------------------" + "------\n"); -void dram_t::print( FILE* simFile) const -{ - unsigned i; - fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", - id, m_config->nbk, m_config->busW, m_config->BL, m_config->CL ); - fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n", - m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC ); - fprintf(simFile,"n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref_event=%llu n_req=%llu n_rd=%llu n_rd_L2_A=%llu n_write=%llu n_wr_bk=%llu bw_util=%.4g\n", - n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB, - (float)bwutil/n_cmd); - fprintf(simFile,"n_activity=%llu dram_eff=%.4g\n", - n_activity, (float)bwutil/n_activity); - for (i=0;i<m_config->nbk;i++) { - fprintf(simFile, "bk%d: %da %di ",i,bk[i]->n_access,bk[i]->n_idle); - } - fprintf(simFile, "\n"); - fprintf(simFile, "\n------------------------------------------------------------------------\n"); + printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num); + printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num); + printf("\nRow_Buffer_Locality_write = %.6f", + (float)hits_write_num / write_num); + printf("\nBank_Level_Parallism = %.6f", + (float)banks_1time / banks_acess_total); + printf("\nBank_Level_Parallism_Col = %.6f", + (float)banks_time_rw / banks_access_rw_total); + printf("\nBank_Level_Parallism_Ready = %.6f", + (float)banks_time_ready / banks_access_ready_total); + printf("\nwrite_to_read_ratio_blp_rw_average = %.6f", + write_to_read_ratio_blp_rw_average / banks_access_rw_total); + printf("\nGrpLevelPara = %.6f \n", + (float)bkgrp_parallsim_rw / banks_access_rw_total); - printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num); - printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num); - printf("\nRow_Buffer_Locality_write = %.6f", (float)hits_write_num / write_num); - printf("\nBank_Level_Parallism = %.6f", (float)banks_1time / banks_acess_total); - printf("\nBank_Level_Parallism_Col = %.6f", (float)banks_time_rw / banks_access_rw_total); - printf("\nBank_Level_Parallism_Ready = %.6f", (float)banks_time_ready /banks_access_ready_total); - printf("\nwrite_to_read_ratio_blp_rw_average = %.6f", write_to_read_ratio_blp_rw_average /banks_access_rw_total); - printf("\nGrpLevelPara = %.6f \n", (float)bkgrp_parallsim_rw /banks_access_rw_total); + printf("\nBW Util details:\n"); + printf("bwutil = %.6f \n", (float)bwutil / n_cmd); + printf("total_CMD = %llu \n", n_cmd); + printf("util_bw = %llu \n", util_bw); + printf("Wasted_Col = %llu \n", wasted_bw_col); + printf("Wasted_Row = %llu \n", wasted_bw_row); + printf("Idle = %llu \n", idle_bw); - printf("\nBW Util details:\n"); - printf("bwutil = %.6f \n", (float)bwutil/n_cmd); - printf("total_CMD = %llu \n", n_cmd); - printf("util_bw = %llu \n", util_bw); - printf("Wasted_Col = %llu \n", wasted_bw_col); - printf("Wasted_Row = %llu \n", wasted_bw_row); - printf("Idle = %llu \n", idle_bw); + printf("\nBW Util Bottlenecks: \n"); + printf("RCDc_limit = %llu \n", RCDc_limit); + printf("RCDWRc_limit = %llu \n", RCDWRc_limit); + printf("WTRc_limit = %llu \n", WTRc_limit); + printf("RTWc_limit = %llu \n", RTWc_limit); + printf("CCDLc_limit = %llu \n", CCDLc_limit); + printf("rwq = %llu \n", rwq_limit); + printf("CCDLc_limit_alone = %llu \n", CCDLc_limit_alone); + printf("WTRc_limit_alone = %llu \n", WTRc_limit_alone); + printf("RTWc_limit_alone = %llu \n", RTWc_limit_alone); - printf("\nBW Util Bottlenecks: \n"); - printf("RCDc_limit = %llu \n", RCDc_limit); - printf("RCDWRc_limit = %llu \n", RCDWRc_limit); - printf("WTRc_limit = %llu \n", WTRc_limit); - printf("RTWc_limit = %llu \n", RTWc_limit); - printf("CCDLc_limit = %llu \n", CCDLc_limit); - printf("rwq = %llu \n", rwq_limit); - printf("CCDLc_limit_alone = %llu \n", CCDLc_limit_alone); - printf("WTRc_limit_alone = %llu \n", WTRc_limit_alone); - printf("RTWc_limit_alone = %llu \n", RTWc_limit_alone); + printf("\nCommands details: \n"); + printf("total_CMD = %llu \n", n_cmd); + printf("n_nop = %llu \n", n_nop); + printf("Read = %llu \n", n_rd); + printf("Write = %llu \n", n_wr); + printf("L2_Alloc = %llu \n", n_rd_L2_A); + printf("L2_WB = %llu \n", n_wr_WB); + printf("n_act = %llu \n", n_act); + printf("n_pre = %llu \n", n_pre); + printf("n_ref = %llu \n", n_ref); + printf("n_req = %llu \n", n_req); + printf("total_req = %llu \n", n_rd + n_wr + n_rd_L2_A + n_wr_WB); - printf("\nCommands details: \n"); - printf("total_CMD = %llu \n", n_cmd); - printf("n_nop = %llu \n", n_nop); - printf("Read = %llu \n", n_rd); - printf("Write = %llu \n",n_wr); - printf("L2_Alloc = %llu \n", n_rd_L2_A); - printf("L2_WB = %llu \n", n_wr_WB); - printf("n_act = %llu \n", n_act); - printf("n_pre = %llu \n", n_pre); - printf("n_ref = %llu \n", n_ref); - printf("n_req = %llu \n", n_req ); - printf("total_req = %llu \n", n_rd+n_wr+n_rd_L2_A+n_wr_WB); + printf("\nDual Bus Interface Util: \n"); + printf("issued_total_row = %llu \n", issued_total_row); + printf("issued_total_col = %llu \n", issued_total_col); + printf("Row_Bus_Util = %.6f \n", (float)issued_total_row / n_cmd); + printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd); + printf("Either_Row_CoL_Bus_Util = %.6f \n", (float)issued_total / n_cmd); + printf("Issued_on_Two_Bus_Simul_Util = %.6f \n", (float)issued_two / n_cmd); + printf("issued_two_Eff = %.6f \n", (float)issued_two / issued_total); + printf("queue_avg = %.6f \n\n", (float)ave_mrqs / n_cmd); - printf("\nDual Bus Interface Util: \n"); - printf("issued_total_row = %llu \n", issued_total_row); - printf("issued_total_col = %llu \n", issued_total_col); - printf("Row_Bus_Util = %.6f \n", (float)issued_total_row / n_cmd); - printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd); - printf("Either_Row_CoL_Bus_Util = %.6f \n", (float)issued_total / n_cmd); - printf("Issued_on_Two_Bus_Simul_Util = %.6f \n", (float)issued_two /n_cmd); - printf("issued_two_Eff = %.6f \n", (float)issued_two /issued_total); - printf("queue_avg = %.6f \n\n", (float)ave_mrqs/n_cmd ); - - fprintf(simFile, "\n"); - fprintf(simFile, "dram_util_bins:"); - for (i=0;i<10;i++) fprintf(simFile, " %d", dram_util_bins[i]); - fprintf(simFile, "\ndram_eff_bins:"); - for (i=0;i<10;i++) fprintf(simFile, " %d", dram_eff_bins[i]); - fprintf(simFile, "\n"); - if(m_config->scheduler_type== DRAM_FRFCFS) - fprintf(simFile, "mrqq: max=%d avg=%g\n", max_mrqs, (float)ave_mrqs/n_cmd); + fprintf(simFile, "\n"); + fprintf(simFile, "dram_util_bins:"); + for (i = 0; i < 10; i++) fprintf(simFile, " %d", dram_util_bins[i]); + fprintf(simFile, "\ndram_eff_bins:"); + for (i = 0; i < 10; i++) fprintf(simFile, " %d", dram_eff_bins[i]); + fprintf(simFile, "\n"); + if (m_config->scheduler_type == DRAM_FRFCFS) + fprintf(simFile, "mrqq: max=%d avg=%g\n", max_mrqs, + (float)ave_mrqs / n_cmd); } -void dram_t::visualize() const -{ - printf("RRDc=%d CCDc=%d mrqq.Length=%d rwq.Length=%d\n", - RRDc, CCDc, mrqq->get_length(),rwq->get_length()); - for (unsigned i=0;i<m_config->nbk;i++) { - printf("BK%d: state=%c curr_row=%03x, %2d %2d %2d %2d %p ", - i, bk[i]->state, bk[i]->curr_row, - bk[i]->RCDc, bk[i]->RASc, - bk[i]->RPc, bk[i]->RCc, - bk[i]->mrq ); - if (bk[i]->mrq) - printf("txf: %d %d", bk[i]->mrq->nbytes, bk[i]->mrq->txbytes); - printf("\n"); - } - if ( m_frfcfs_scheduler ) - m_frfcfs_scheduler->print(stdout); +void dram_t::visualize() const { + printf("RRDc=%d CCDc=%d mrqq.Length=%d rwq.Length=%d\n", RRDc, CCDc, + mrqq->get_length(), rwq->get_length()); + for (unsigned i = 0; i < m_config->nbk; i++) { + printf("BK%d: state=%c curr_row=%03x, %2d %2d %2d %2d %p ", i, bk[i]->state, + bk[i]->curr_row, bk[i]->RCDc, bk[i]->RASc, bk[i]->RPc, bk[i]->RCc, + bk[i]->mrq); + if (bk[i]->mrq) + printf("txf: %d %d", bk[i]->mrq->nbytes, bk[i]->mrq->txbytes); + printf("\n"); + } + if (m_frfcfs_scheduler) m_frfcfs_scheduler->print(stdout); } -void dram_t::print_stat( FILE* simFile ) -{ - fprintf(simFile,"DRAM (%u): n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref=%llu n_req=%llu n_rd=%llu n_write=%llu bw_util=%.4g ", - id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr, - (float)bwutil/n_cmd); - fprintf(simFile, "mrqq: %d %.4g mrqsmax=%llu ", max_mrqs, (float)ave_mrqs/n_cmd, max_mrqs_temp); - fprintf(simFile, "\n"); - fprintf(simFile, "dram_util_bins:"); - for (unsigned i=0;i<10;i++) fprintf(simFile, " %d", dram_util_bins[i]); - fprintf(simFile, "\ndram_eff_bins:"); - for (unsigned i=0;i<10;i++) fprintf(simFile, " %d", dram_eff_bins[i]); - fprintf(simFile, "\n"); - max_mrqs_temp = 0; +void dram_t::print_stat(FILE *simFile) { + fprintf(simFile, + "DRAM (%u): n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref=%llu " + "n_req=%llu n_rd=%llu n_write=%llu bw_util=%.4g ", + id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr, + (float)bwutil / n_cmd); + fprintf(simFile, "mrqq: %d %.4g mrqsmax=%llu ", max_mrqs, + (float)ave_mrqs / n_cmd, max_mrqs_temp); + fprintf(simFile, "\n"); + fprintf(simFile, "dram_util_bins:"); + for (unsigned i = 0; i < 10; i++) fprintf(simFile, " %d", dram_util_bins[i]); + fprintf(simFile, "\ndram_eff_bins:"); + for (unsigned i = 0; i < 10; i++) fprintf(simFile, " %d", dram_eff_bins[i]); + fprintf(simFile, "\n"); + max_mrqs_temp = 0; } -void dram_t::visualizer_print( gzFile visualizer_file ) -{ - // dram specific statistics - gzprintf(visualizer_file,"dramncmd: %u %u\n",id, n_cmd_partial); - gzprintf(visualizer_file,"dramnop: %u %u\n",id,n_nop_partial); - gzprintf(visualizer_file,"dramnact: %u %u\n",id,n_act_partial); - gzprintf(visualizer_file,"dramnpre: %u %u\n",id,n_pre_partial); - gzprintf(visualizer_file,"dramnreq: %u %u\n",id,n_req_partial); - gzprintf(visualizer_file,"dramavemrqs: %u %u\n",id, - n_cmd_partial?(ave_mrqs_partial/n_cmd_partial ):0); +void dram_t::visualizer_print(gzFile visualizer_file) { + // dram specific statistics + gzprintf(visualizer_file, "dramncmd: %u %u\n", id, n_cmd_partial); + gzprintf(visualizer_file, "dramnop: %u %u\n", id, n_nop_partial); + gzprintf(visualizer_file, "dramnact: %u %u\n", id, n_act_partial); + gzprintf(visualizer_file, "dramnpre: %u %u\n", id, n_pre_partial); + gzprintf(visualizer_file, "dramnreq: %u %u\n", id, n_req_partial); + gzprintf(visualizer_file, "dramavemrqs: %u %u\n", id, + n_cmd_partial ? (ave_mrqs_partial / n_cmd_partial) : 0); - // utilization and efficiency - gzprintf(visualizer_file,"dramutil: %u %u\n", - id,n_cmd_partial?100*bwutil_partial/n_cmd_partial:0); - gzprintf(visualizer_file,"drameff: %u %u\n", - id,n_activity_partial?100*bwutil_partial/n_activity_partial:0); + // utilization and efficiency + gzprintf(visualizer_file, "dramutil: %u %u\n", id, + n_cmd_partial ? 100 * bwutil_partial / n_cmd_partial : 0); + gzprintf(visualizer_file, "drameff: %u %u\n", id, + n_activity_partial ? 100 * bwutil_partial / n_activity_partial : 0); - // reset for next interval - bwutil_partial = 0; - n_activity_partial = 0; - ave_mrqs_partial = 0; - n_cmd_partial = 0; - n_nop_partial = 0; - n_act_partial = 0; - n_pre_partial = 0; - n_req_partial = 0; + // reset for next interval + bwutil_partial = 0; + n_activity_partial = 0; + ave_mrqs_partial = 0; + n_cmd_partial = 0; + n_nop_partial = 0; + n_act_partial = 0; + n_pre_partial = 0; + n_req_partial = 0; - - // dram access type classification - for (unsigned j = 0; j < m_config->nbk; j++) { - gzprintf(visualizer_file,"dramglobal_acc_r: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[GLOBAL_ACC_R][id][j]); - gzprintf(visualizer_file,"dramglobal_acc_w: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[GLOBAL_ACC_W][id][j]); - gzprintf(visualizer_file,"dramlocal_acc_r: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[LOCAL_ACC_R][id][j]); - gzprintf(visualizer_file,"dramlocal_acc_w: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[LOCAL_ACC_W][id][j]); - gzprintf(visualizer_file,"dramconst_acc_r: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[CONST_ACC_R][id][j]); - gzprintf(visualizer_file,"dramtexture_acc_r: %u %u %u\n", id, j, - m_stats->mem_access_type_stats[TEXTURE_ACC_R][id][j]); - } + // dram access type classification + for (unsigned j = 0; j < m_config->nbk; j++) { + gzprintf(visualizer_file, "dramglobal_acc_r: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[GLOBAL_ACC_R][id][j]); + gzprintf(visualizer_file, "dramglobal_acc_w: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[GLOBAL_ACC_W][id][j]); + gzprintf(visualizer_file, "dramlocal_acc_r: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[LOCAL_ACC_R][id][j]); + gzprintf(visualizer_file, "dramlocal_acc_w: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[LOCAL_ACC_W][id][j]); + gzprintf(visualizer_file, "dramconst_acc_r: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[CONST_ACC_R][id][j]); + gzprintf(visualizer_file, "dramtexture_acc_r: %u %u %u\n", id, j, + m_stats->mem_access_type_stats[TEXTURE_ACC_R][id][j]); + } } - -void dram_t::set_dram_power_stats( unsigned &cmd, - unsigned &activity, - unsigned &nop, - unsigned &act, - unsigned &pre, - unsigned &rd, - unsigned &wr, - unsigned &req) const{ - - // Point power performance counters to low-level DRAM counters - cmd = n_cmd; - activity = n_activity; - nop = n_nop; - act = n_act; - pre = n_pre; - rd = n_rd; - wr = n_wr; - req = n_req; +void dram_t::set_dram_power_stats(unsigned &cmd, unsigned &activity, + unsigned &nop, unsigned &act, unsigned &pre, + unsigned &rd, unsigned &wr, + unsigned &req) const { + // Point power performance counters to low-level DRAM counters + cmd = n_cmd; + activity = n_activity; + nop = n_nop; + act = n_act; + pre = n_pre; + rd = n_rd; + wr = n_wr; + req = n_req; } -unsigned dram_t::get_bankgrp_number(unsigned i) -{ - if(m_config->dram_bnkgrp_indexing_policy == HIGHER_BITS) { //higher bits - return i >> m_config->bk_tag_length; - } - else if (m_config->dram_bnkgrp_indexing_policy == LOWER_BITS) { //lower bits - return i & ((m_config->nbkgrp - 1)); - } - else { - assert(1); - } +unsigned dram_t::get_bankgrp_number(unsigned i) { + if (m_config->dram_bnkgrp_indexing_policy == HIGHER_BITS) { // higher bits + return i >> m_config->bk_tag_length; + } else if (m_config->dram_bnkgrp_indexing_policy == + LOWER_BITS) { // lower bits + return i & ((m_config->nbkgrp - 1)); + } else { + assert(1); + } } diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 0bd9725..2e39a43 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Ivan Sham, Ali Bakhoda, +// Copyright (c) 2009-2011, Tor M. Aamodt, Ivan Sham, Ali Bakhoda, // George L. Yuan, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -8,248 +8,239 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef DRAM_H #define DRAM_H -#include "delayqueue.h" -#include <set> -#include <vector> +#include <stdio.h> +#include <stdlib.h> +#include <zlib.h> #include <bitset> +#include <fstream> +#include <iomanip> +#include <set> #include <sstream> #include <string> -#include <fstream> -#include <zlib.h> -#include <stdio.h> -#include <stdlib.h> -#include<iomanip> +#include <vector> +#include "delayqueue.h" -#define READ 'R' //define read and write states +#define READ 'R' // define read and write states #define WRITE 'W' #define BANK_IDLE 'I' #define BANK_ACTIVE 'A' class dram_req_t { -public: - dram_req_t( class mem_fetch *data , unsigned banks, unsigned dram_bnk_indexing_policy, class gpgpu_sim* gpu); + public: + dram_req_t(class mem_fetch *data, unsigned banks, + unsigned dram_bnk_indexing_policy, class gpgpu_sim *gpu); - unsigned int row; - unsigned int col; - unsigned int bk; - unsigned int nbytes; - unsigned int txbytes; - unsigned int dqbytes; - unsigned int age; - unsigned int timestamp; - unsigned char rw; //is the request a read or a write? - unsigned long long int addr; - unsigned int insertion_time; - class mem_fetch * data; - class gpgpu_sim * m_gpu; + unsigned int row; + unsigned int col; + unsigned int bk; + unsigned int nbytes; + unsigned int txbytes; + unsigned int dqbytes; + unsigned int age; + unsigned int timestamp; + unsigned char rw; // is the request a read or a write? + unsigned long long int addr; + unsigned int insertion_time; + class mem_fetch *data; + class gpgpu_sim *m_gpu; }; -struct bankgrp_t -{ - unsigned int CCDLc; - unsigned int RTPLc; +struct bankgrp_t { + unsigned int CCDLc; + unsigned int RTPLc; }; -struct bank_t -{ - unsigned int RCDc; - unsigned int RCDWRc; - unsigned int RASc; - unsigned int RPc; - unsigned int RCc; - unsigned int WTPc; // write to precharge - unsigned int RTPc; // read to precharge +struct bank_t { + unsigned int RCDc; + unsigned int RCDWRc; + unsigned int RASc; + unsigned int RPc; + unsigned int RCc; + unsigned int WTPc; // write to precharge + unsigned int RTPc; // read to precharge - unsigned char rw; //is the bank reading or writing? - unsigned char state; //is the bank active or idle? - unsigned int curr_row; + unsigned char rw; // is the bank reading or writing? + unsigned char state; // is the bank active or idle? + unsigned int curr_row; - dram_req_t *mrq; + dram_req_t *mrq; - unsigned int n_access; - unsigned int n_writes; - unsigned int n_idle; + unsigned int n_access; + unsigned int n_writes; + unsigned int n_idle; - unsigned int bkgrpindex; + unsigned int bkgrpindex; }; -enum bank_index_function{ - LINEAR_BK_INDEX = 0, - BITWISE_XORING_BK_INDEX, - CUSTOM_BK_INDEX +enum bank_index_function { + LINEAR_BK_INDEX = 0, + BITWISE_XORING_BK_INDEX, + CUSTOM_BK_INDEX }; -enum bank_grp_bits_position{ - HIGHER_BITS = 0, - LOWER_BITS -}; +enum bank_grp_bits_position { HIGHER_BITS = 0, LOWER_BITS }; class mem_fetch; class memory_config; -class dram_t -{ -public: - dram_t( unsigned int parition_id, const memory_config *config, class memory_stats_t *stats, - class memory_partition_unit *mp, class gpgpu_sim* gpu ); - - bool full(bool is_write) const; - void print( FILE* simFile ) const; - void visualize() const; - void print_stat( FILE* simFile ); - unsigned que_length() const; - bool returnq_full() const; - unsigned int queue_limit() const; - void visualizer_print( gzFile visualizer_file ); - - class mem_fetch* return_queue_pop(); - class mem_fetch* return_queue_top(); +class dram_t { + public: + dram_t(unsigned int parition_id, const memory_config *config, + class memory_stats_t *stats, class memory_partition_unit *mp, + class gpgpu_sim *gpu); - void push( class mem_fetch *data ); - void cycle(); - void dram_log (int task); + bool full(bool is_write) const; + void print(FILE *simFile) const; + void visualize() const; + void print_stat(FILE *simFile); + unsigned que_length() const; + bool returnq_full() const; + unsigned int queue_limit() const; + void visualizer_print(gzFile visualizer_file); - class memory_partition_unit *m_memory_partition_unit; - class gpgpu_sim* m_gpu; - unsigned int id; + class mem_fetch *return_queue_pop(); + class mem_fetch *return_queue_top(); - // Power Model - void set_dram_power_stats(unsigned &cmd, - unsigned &activity, - unsigned &nop, - unsigned &act, - unsigned &pre, - unsigned &rd, - unsigned &wr, - unsigned &req) const; + void push(class mem_fetch *data); + void cycle(); + void dram_log(int task); + class memory_partition_unit *m_memory_partition_unit; + class gpgpu_sim *m_gpu; + unsigned int id; + // Power Model + void set_dram_power_stats(unsigned &cmd, unsigned &activity, unsigned &nop, + unsigned &act, unsigned &pre, unsigned &rd, + unsigned &wr, unsigned &req) const; - const memory_config *m_config; + const memory_config *m_config; -private: - bankgrp_t **bkgrp; + private: + bankgrp_t **bkgrp; - bank_t **bk; - unsigned int prio; + bank_t **bk; + unsigned int prio; - unsigned get_bankgrp_number(unsigned i); + unsigned get_bankgrp_number(unsigned i); - void scheduler_fifo(); - void scheduler_frfcfs(); + void scheduler_fifo(); + void scheduler_frfcfs(); - bool issue_col_command(int j); - bool issue_row_command(int j); + bool issue_col_command(int j); + bool issue_row_command(int j); - unsigned int RRDc; - unsigned int CCDc; - unsigned int RTWc; //read to write penalty applies across banks - unsigned int WTRc; //write to read penalty applies across banks + unsigned int RRDc; + unsigned int CCDc; + unsigned int RTWc; // read to write penalty applies across banks + unsigned int WTRc; // write to read penalty applies across banks - unsigned char rw; //was last request a read or write? (important for RTW, WTR) + unsigned char + rw; // was last request a read or write? (important for RTW, WTR) - unsigned int pending_writes; + unsigned int pending_writes; - fifo_pipeline<dram_req_t> *rwq; - fifo_pipeline<dram_req_t> *mrqq; - //buffer to hold packets when DRAM processing is over - //should be filled with dram clock and popped with l2or icnt clock - fifo_pipeline<mem_fetch> *returnq; + fifo_pipeline<dram_req_t> *rwq; + fifo_pipeline<dram_req_t> *mrqq; + // buffer to hold packets when DRAM processing is over + // should be filled with dram clock and popped with l2or icnt clock + fifo_pipeline<mem_fetch> *returnq; - unsigned int dram_util_bins[10]; - unsigned int dram_eff_bins[10]; - unsigned int last_n_cmd, last_n_activity, last_bwutil; + unsigned int dram_util_bins[10]; + unsigned int dram_eff_bins[10]; + unsigned int last_n_cmd, last_n_activity, last_bwutil; - unsigned long long n_cmd; - unsigned long long n_activity; - unsigned long long n_nop; - unsigned long long n_act; - unsigned long long n_pre; - unsigned long long n_ref; - unsigned long long n_rd; - unsigned long long n_rd_L2_A; - unsigned long long n_wr; - unsigned long long n_wr_WB; - unsigned long long n_req; - unsigned long long max_mrqs_temp; + unsigned long long n_cmd; + unsigned long long n_activity; + unsigned long long n_nop; + unsigned long long n_act; + unsigned long long n_pre; + unsigned long long n_ref; + unsigned long long n_rd; + unsigned long long n_rd_L2_A; + unsigned long long n_wr; + unsigned long long n_wr_WB; + unsigned long long n_req; + unsigned long long max_mrqs_temp; - //some statistics to see where BW is wasted? - unsigned long long wasted_bw_row; - unsigned long long wasted_bw_col; - unsigned long long util_bw; - unsigned long long idle_bw; - unsigned long long RCDc_limit; - unsigned long long CCDLc_limit; - unsigned long long CCDLc_limit_alone; - unsigned long long CCDc_limit; - unsigned long long WTRc_limit; - unsigned long long WTRc_limit_alone; - unsigned long long RCDWRc_limit; - unsigned long long RTWc_limit; - unsigned long long RTWc_limit_alone; - unsigned long long rwq_limit; + // some statistics to see where BW is wasted? + unsigned long long wasted_bw_row; + unsigned long long wasted_bw_col; + unsigned long long util_bw; + unsigned long long idle_bw; + unsigned long long RCDc_limit; + unsigned long long CCDLc_limit; + unsigned long long CCDLc_limit_alone; + unsigned long long CCDc_limit; + unsigned long long WTRc_limit; + unsigned long long WTRc_limit_alone; + unsigned long long RCDWRc_limit; + unsigned long long RTWc_limit; + unsigned long long RTWc_limit_alone; + unsigned long long rwq_limit; - //row locality, BLP and other statistics - unsigned long long access_num; - unsigned long long read_num; - unsigned long long write_num; - unsigned long long hits_num; - unsigned long long hits_read_num; - unsigned long long hits_write_num; - unsigned long long banks_1time; - unsigned long long banks_acess_total; - unsigned long long banks_acess_total_after; - unsigned long long banks_time_rw; - unsigned long long banks_access_rw_total; - unsigned long long banks_time_ready; - unsigned long long banks_access_ready_total; - unsigned long long issued_two; - unsigned long long issued_total; - unsigned long long issued_total_row; - unsigned long long issued_total_col; - double write_to_read_ratio_blp_rw_average; - unsigned long long bkgrp_parallsim_rw; + // row locality, BLP and other statistics + unsigned long long access_num; + unsigned long long read_num; + unsigned long long write_num; + unsigned long long hits_num; + unsigned long long hits_read_num; + unsigned long long hits_write_num; + unsigned long long banks_1time; + unsigned long long banks_acess_total; + unsigned long long banks_acess_total_after; + unsigned long long banks_time_rw; + unsigned long long banks_access_rw_total; + unsigned long long banks_time_ready; + unsigned long long banks_access_ready_total; + unsigned long long issued_two; + unsigned long long issued_total; + unsigned long long issued_total_row; + unsigned long long issued_total_col; + double write_to_read_ratio_blp_rw_average; + unsigned long long bkgrp_parallsim_rw; - unsigned int bwutil; - unsigned int max_mrqs; - unsigned int ave_mrqs; + unsigned int bwutil; + unsigned int max_mrqs; + unsigned int ave_mrqs; - class frfcfs_scheduler* m_frfcfs_scheduler; + class frfcfs_scheduler *m_frfcfs_scheduler; - unsigned int n_cmd_partial; - unsigned int n_activity_partial; - unsigned int n_nop_partial; - unsigned int n_act_partial; - unsigned int n_pre_partial; - unsigned int n_req_partial; - unsigned int ave_mrqs_partial; - unsigned int bwutil_partial; + unsigned int n_cmd_partial; + unsigned int n_activity_partial; + unsigned int n_nop_partial; + unsigned int n_act_partial; + unsigned int n_pre_partial; + unsigned int n_req_partial; + unsigned int ave_mrqs_partial; + unsigned int bwutil_partial; - class memory_stats_t *m_stats; - class Stats* mrqq_Dist; //memory request queue inside DRAM + class memory_stats_t *m_stats; + class Stats *mrqq_Dist; // memory request queue inside DRAM - friend class frfcfs_scheduler; + friend class frfcfs_scheduler; }; #endif /*DRAM_H*/ diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index 6ee6271..1df8a5a 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -7,243 +7,252 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "dram_sched.h" +#include "../abstract_hardware_model.h" #include "gpu-misc.h" #include "gpu-sim.h" -#include "../abstract_hardware_model.h" #include "mem_latency_stat.h" -frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats ) -{ - m_config = config; - m_stats = stats; - m_num_pending = 0; - m_num_write_pending = 0; - m_dram = dm; - m_queue = new std::list<dram_req_t*>[m_config->nbk]; - m_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ]; - m_last_row = new std::list<std::list<dram_req_t*>::iterator>*[ m_config->nbk ]; - curr_row_service_time = new unsigned[m_config->nbk]; - row_service_timestamp = new unsigned[m_config->nbk]; - for ( unsigned i=0; i < m_config->nbk; i++ ) { - m_queue[i].clear(); - m_bins[i].clear(); - m_last_row[i] = NULL; - curr_row_service_time[i] = 0; - row_service_timestamp[i] = 0; - } - if(m_config->seperate_write_queue_enabled) { - m_write_queue = new std::list<dram_req_t*>[m_config->nbk]; - m_write_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ]; - m_last_write_row = new std::list<std::list<dram_req_t*>::iterator>*[ m_config->nbk ]; - - for ( unsigned i=0; i < m_config->nbk; i++ ) { - m_write_queue[i].clear(); - m_write_bins[i].clear(); - m_last_write_row[i] = NULL; - } - } - m_mode = READ_MODE; +frfcfs_scheduler::frfcfs_scheduler(const memory_config *config, dram_t *dm, + memory_stats_t *stats) { + m_config = config; + m_stats = stats; + m_num_pending = 0; + m_num_write_pending = 0; + m_dram = dm; + m_queue = new std::list<dram_req_t *>[m_config->nbk]; + m_bins = new std::map< + unsigned, std::list<std::list<dram_req_t *>::iterator> >[m_config->nbk]; + m_last_row = + new std::list<std::list<dram_req_t *>::iterator> *[m_config->nbk]; + curr_row_service_time = new unsigned[m_config->nbk]; + row_service_timestamp = new unsigned[m_config->nbk]; + for (unsigned i = 0; i < m_config->nbk; i++) { + m_queue[i].clear(); + m_bins[i].clear(); + m_last_row[i] = NULL; + curr_row_service_time[i] = 0; + row_service_timestamp[i] = 0; + } + if (m_config->seperate_write_queue_enabled) { + m_write_queue = new std::list<dram_req_t *>[m_config->nbk]; + m_write_bins = new std::map< + unsigned, std::list<std::list<dram_req_t *>::iterator> >[m_config->nbk]; + m_last_write_row = + new std::list<std::list<dram_req_t *>::iterator> *[m_config->nbk]; + for (unsigned i = 0; i < m_config->nbk; i++) { + m_write_queue[i].clear(); + m_write_bins[i].clear(); + m_last_write_row[i] = NULL; + } + } + m_mode = READ_MODE; } -void frfcfs_scheduler::add_req( dram_req_t *req ) -{ - if(m_config->seperate_write_queue_enabled && req->data->is_write()) { - assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size); - m_num_write_pending++; - m_write_queue[req->bk].push_front(req); - std::list<dram_req_t*>::iterator ptr = m_write_queue[req->bk].begin(); - m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front +void frfcfs_scheduler::add_req(dram_req_t *req) { + if (m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size); + m_num_write_pending++; + m_write_queue[req->bk].push_front(req); + std::list<dram_req_t *>::iterator ptr = m_write_queue[req->bk].begin(); + m_write_bins[req->bk][req->row].push_front(ptr); // newest reqs to the + // front } else { - assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size); - m_num_pending++; - m_queue[req->bk].push_front(req); - std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin(); - m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size); + m_num_pending++; + m_queue[req->bk].push_front(req); + std::list<dram_req_t *>::iterator ptr = m_queue[req->bk].begin(); + m_bins[req->bk][req->row].push_front(ptr); // newest reqs to the front } } -void frfcfs_scheduler::data_collection(unsigned int bank) -{ - if (m_dram->m_gpu->gpu_sim_cycle > row_service_timestamp[bank]) { - curr_row_service_time[bank] = m_dram->m_gpu->gpu_sim_cycle - row_service_timestamp[bank]; - if (curr_row_service_time[bank] > m_stats->max_servicetime2samerow[m_dram->id][bank]) - m_stats->max_servicetime2samerow[m_dram->id][bank] = curr_row_service_time[bank]; - } - curr_row_service_time[bank] = 0; - row_service_timestamp[bank] = m_dram->m_gpu->gpu_sim_cycle; - if (m_stats->concurrent_row_access[m_dram->id][bank] > m_stats->max_conc_access2samerow[m_dram->id][bank]) { - m_stats->max_conc_access2samerow[m_dram->id][bank] = m_stats->concurrent_row_access[m_dram->id][bank]; - } - m_stats->concurrent_row_access[m_dram->id][bank] = 0; - m_stats->num_activates[m_dram->id][bank]++; +void frfcfs_scheduler::data_collection(unsigned int bank) { + if (m_dram->m_gpu->gpu_sim_cycle > row_service_timestamp[bank]) { + curr_row_service_time[bank] = + m_dram->m_gpu->gpu_sim_cycle - row_service_timestamp[bank]; + if (curr_row_service_time[bank] > + m_stats->max_servicetime2samerow[m_dram->id][bank]) + m_stats->max_servicetime2samerow[m_dram->id][bank] = + curr_row_service_time[bank]; + } + curr_row_service_time[bank] = 0; + row_service_timestamp[bank] = m_dram->m_gpu->gpu_sim_cycle; + if (m_stats->concurrent_row_access[m_dram->id][bank] > + m_stats->max_conc_access2samerow[m_dram->id][bank]) { + m_stats->max_conc_access2samerow[m_dram->id][bank] = + m_stats->concurrent_row_access[m_dram->id][bank]; + } + m_stats->concurrent_row_access[m_dram->id][bank] = 0; + m_stats->num_activates[m_dram->id][bank]++; } -dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) -{ - //row - bool rowhit = true; - std::list<dram_req_t*> *m_current_queue = m_queue; - std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> > *m_current_bins = m_bins ; - std::list<std::list<dram_req_t*>::iterator> **m_current_last_row = m_last_row; +dram_req_t *frfcfs_scheduler::schedule(unsigned bank, unsigned curr_row) { + // row + bool rowhit = true; + std::list<dram_req_t *> *m_current_queue = m_queue; + std::map<unsigned, std::list<std::list<dram_req_t *>::iterator> > + *m_current_bins = m_bins; + std::list<std::list<dram_req_t *>::iterator> **m_current_last_row = + m_last_row; - if(m_config->seperate_write_queue_enabled) { - if(m_mode == READ_MODE && - ((m_num_write_pending >= m_config->write_high_watermark ) - // || (m_queue[bank].empty() && !m_write_queue[bank].empty()) - )) { - m_mode = WRITE_MODE; - } - else if(m_mode == WRITE_MODE && - (( m_num_write_pending < m_config->write_low_watermark ) - // || (!m_queue[bank].empty() && m_write_queue[bank].empty()) - )){ - m_mode = READ_MODE; - } - } + if (m_config->seperate_write_queue_enabled) { + if (m_mode == READ_MODE && + ((m_num_write_pending >= m_config->write_high_watermark) + // || (m_queue[bank].empty() && !m_write_queue[bank].empty()) + )) { + m_mode = WRITE_MODE; + } else if (m_mode == WRITE_MODE && + ((m_num_write_pending < m_config->write_low_watermark) + // || (!m_queue[bank].empty() && m_write_queue[bank].empty()) + )) { + m_mode = READ_MODE; + } + } - if(m_mode == WRITE_MODE) { - m_current_queue = m_write_queue; - m_current_bins = m_write_bins ; - m_current_last_row = m_last_write_row; - } + if (m_mode == WRITE_MODE) { + m_current_queue = m_write_queue; + m_current_bins = m_write_bins; + m_current_last_row = m_last_write_row; + } - if ( m_current_last_row[bank] == NULL ) { - if ( m_current_queue[bank].empty() ) - return NULL; + if (m_current_last_row[bank] == NULL) { + if (m_current_queue[bank].empty()) return NULL; - std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row ); - if ( bin_ptr == m_current_bins[bank].end()) { - dram_req_t *req = m_current_queue[bank].back(); - bin_ptr = m_current_bins[bank].find( req->row ); - assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go??? - m_current_last_row[bank] = &(bin_ptr->second); - data_collection(bank); - rowhit = false; - } else { - m_current_last_row[bank] = &(bin_ptr->second); - rowhit = true; - } - } - std::list<dram_req_t*>::iterator next = m_current_last_row[bank]->back(); - dram_req_t *req = (*next); + std::map<unsigned, std::list<std::list<dram_req_t *>::iterator> >::iterator + bin_ptr = m_current_bins[bank].find(curr_row); + if (bin_ptr == m_current_bins[bank].end()) { + dram_req_t *req = m_current_queue[bank].back(); + bin_ptr = m_current_bins[bank].find(req->row); + assert(bin_ptr != + m_current_bins[bank].end()); // where did the request go??? + m_current_last_row[bank] = &(bin_ptr->second); + data_collection(bank); + rowhit = false; + } else { + m_current_last_row[bank] = &(bin_ptr->second); + rowhit = true; + } + } + std::list<dram_req_t *>::iterator next = m_current_last_row[bank]->back(); + dram_req_t *req = (*next); - //rowblp stats - m_dram->access_num++; - bool is_write = req->data->is_write(); - if(is_write) - m_dram->write_num++; - else - m_dram->read_num++; + // rowblp stats + m_dram->access_num++; + bool is_write = req->data->is_write(); + if (is_write) + m_dram->write_num++; + else + m_dram->read_num++; - if(rowhit) { - m_dram->hits_num++; - if(is_write) - m_dram->hits_write_num++; - else - m_dram->hits_read_num++; - } + if (rowhit) { + m_dram->hits_num++; + if (is_write) + m_dram->hits_write_num++; + else + m_dram->hits_read_num++; + } - m_stats->concurrent_row_access[m_dram->id][bank]++; - m_stats->row_access[m_dram->id][bank]++; - m_current_last_row[bank]->pop_back(); + m_stats->concurrent_row_access[m_dram->id][bank]++; + m_stats->row_access[m_dram->id][bank]++; + m_current_last_row[bank]->pop_back(); - m_current_queue[bank].erase(next); - if ( m_current_last_row[bank]->empty() ) { - m_current_bins[bank].erase( req->row ); - m_current_last_row[bank] = NULL; - } + m_current_queue[bank].erase(next); + if (m_current_last_row[bank]->empty()) { + m_current_bins[bank].erase(req->row); + m_current_last_row[bank] = NULL; + } #ifdef DEBUG_FAST_IDEAL_SCHED - if ( req ) - printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", - (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row ); + if (req) + printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", + (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row); #endif - if(m_config->seperate_write_queue_enabled && req->data->is_write()) { - assert( req != NULL && m_num_write_pending != 0 ); - m_num_write_pending--; - } - else { - assert( req != NULL && m_num_pending != 0 ); - m_num_pending--; - } + if (m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert(req != NULL && m_num_write_pending != 0); + m_num_write_pending--; + } else { + assert(req != NULL && m_num_pending != 0); + m_num_pending--; + } - return req; + return req; } - -void frfcfs_scheduler::print( FILE *fp ) -{ - for ( unsigned b=0; b < m_config->nbk; b++ ) { - printf(" %u: queue length = %u\n", b, (unsigned)m_queue[b].size() ); - } +void frfcfs_scheduler::print(FILE *fp) { + for (unsigned b = 0; b < m_config->nbk; b++) { + printf(" %u: queue length = %u\n", b, (unsigned)m_queue[b].size()); + } } -void dram_t::scheduler_frfcfs() -{ - unsigned mrq_latency; - frfcfs_scheduler *sched = m_frfcfs_scheduler; - while ( !mrqq->empty() ) { - dram_req_t *req = mrqq->pop(); - - // Power stats - //if(req->data->get_type() != READ_REPLY && req->data->get_type() != WRITE_ACK) - m_stats->total_n_access++; +void dram_t::scheduler_frfcfs() { + unsigned mrq_latency; + frfcfs_scheduler *sched = m_frfcfs_scheduler; + while (!mrqq->empty()) { + dram_req_t *req = mrqq->pop(); - if(req->data->get_type() == WRITE_REQUEST){ - m_stats->total_n_writes++; - }else if(req->data->get_type() == READ_REQUEST){ - m_stats->total_n_reads++; - } + // Power stats + // if(req->data->get_type() != READ_REPLY && req->data->get_type() != + // WRITE_ACK) + m_stats->total_n_access++; - req->data->set_status(IN_PARTITION_MC_INPUT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - sched->add_req(req); - } + if (req->data->get_type() == WRITE_REQUEST) { + m_stats->total_n_writes++; + } else if (req->data->get_type() == READ_REQUEST) { + m_stats->total_n_reads++; + } - dram_req_t *req; - unsigned i; - for ( i=0; i < m_config->nbk; i++ ) { - unsigned b = (i+prio)%m_config->nbk; - if ( !bk[b]->mrq ) { + req->data->set_status(IN_PARTITION_MC_INPUT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + sched->add_req(req); + } - req = sched->schedule(b, bk[b]->curr_row); + dram_req_t *req; + unsigned i; + for (i = 0; i < m_config->nbk; i++) { + unsigned b = (i + prio) % m_config->nbk; + if (!bk[b]->mrq) { + req = sched->schedule(b, bk[b]->curr_row); - if ( req ) { - req->data->set_status(IN_PARTITION_MC_BANK_ARB_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - prio = (prio+1)%m_config->nbk; - bk[b]->mrq = req; - if (m_config->gpgpu_memlatency_stat) { - mrq_latency = m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle - bk[b]->mrq->timestamp; - m_stats->tot_mrq_latency += mrq_latency; - m_stats->tot_mrq_num++; - bk[b]->mrq->timestamp =m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle; - m_stats->mrq_lat_table[LOGB2(mrq_latency)]++; - if (mrq_latency > m_stats->max_mrq_latency) { - m_stats->max_mrq_latency = mrq_latency; - } - } + if (req) { + req->data->set_status(IN_PARTITION_MC_BANK_ARB_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + prio = (prio + 1) % m_config->nbk; + bk[b]->mrq = req; + if (m_config->gpgpu_memlatency_stat) { + mrq_latency = m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle - + bk[b]->mrq->timestamp; + m_stats->tot_mrq_latency += mrq_latency; + m_stats->tot_mrq_num++; + bk[b]->mrq->timestamp = + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle; + m_stats->mrq_lat_table[LOGB2(mrq_latency)]++; + if (mrq_latency > m_stats->max_mrq_latency) { + m_stats->max_mrq_latency = mrq_latency; + } + } - break; - } + break; } - } + } + } } diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h index 63f5831..4f1ef3d 100644 --- a/src/gpgpu-sim/dram_sched.h +++ b/src/gpgpu-sim/dram_sched.h @@ -7,66 +7,67 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef dram_sched_h_INCLUDED #define dram_sched_h_INCLUDED -#include "dram.h" -#include "shader.h" -#include "gpu-sim.h" -#include "gpu-misc.h" #include <list> #include <map> +#include "dram.h" +#include "gpu-misc.h" +#include "gpu-sim.h" +#include "shader.h" -enum memory_mode { - READ_MODE = 0, - WRITE_MODE -}; +enum memory_mode { READ_MODE = 0, WRITE_MODE }; class frfcfs_scheduler { -public: - frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats ); - void add_req( dram_req_t *req ); - void data_collection(unsigned bank); - dram_req_t *schedule( unsigned bank, unsigned curr_row ); - void print( FILE *fp ); - unsigned num_pending() const { return m_num_pending;} - unsigned num_write_pending() const { return m_num_write_pending;} + public: + frfcfs_scheduler(const memory_config *config, dram_t *dm, + memory_stats_t *stats); + void add_req(dram_req_t *req); + void data_collection(unsigned bank); + dram_req_t *schedule(unsigned bank, unsigned curr_row); + void print(FILE *fp); + unsigned num_pending() const { return m_num_pending; } + unsigned num_write_pending() const { return m_num_write_pending; } -private: - const memory_config *m_config; - dram_t *m_dram; - unsigned m_num_pending; - unsigned m_num_write_pending; - std::list<dram_req_t*> *m_queue; - std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> > *m_bins; - std::list<std::list<dram_req_t*>::iterator> **m_last_row; - unsigned *curr_row_service_time; //one set of variables for each bank. - unsigned *row_service_timestamp; //tracks when scheduler began servicing current row + private: + const memory_config *m_config; + dram_t *m_dram; + unsigned m_num_pending; + unsigned m_num_write_pending; + std::list<dram_req_t *> *m_queue; + std::map<unsigned, std::list<std::list<dram_req_t *>::iterator> > *m_bins; + std::list<std::list<dram_req_t *>::iterator> **m_last_row; + unsigned *curr_row_service_time; // one set of variables for each bank. + unsigned *row_service_timestamp; // tracks when scheduler began servicing + // current row - std::list<dram_req_t*> *m_write_queue; - std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> > *m_write_bins; - std::list<std::list<dram_req_t*>::iterator> **m_last_write_row; + std::list<dram_req_t *> *m_write_queue; + std::map<unsigned, std::list<std::list<dram_req_t *>::iterator> > + *m_write_bins; + std::list<std::list<dram_req_t *>::iterator> **m_last_write_row; - enum memory_mode m_mode; - memory_stats_t *m_stats; + enum memory_mode m_mode; + memory_stats_t *m_stats; }; #endif diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index 1e99fec..156c174 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -7,1637 +7,1631 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "gpu-cache.h" +#include <assert.h> #include "gpu-sim.h" #include "stat-tool.h" -#include <assert.h> -// used to allocate memory that is large enough to adapt the changes in cache size across kernels +// used to allocate memory that is large enough to adapt the changes in cache +// size across kernels -const char * cache_request_status_str(enum cache_request_status status) -{ - static const char * static_cache_request_status_str[] = { - "HIT", - "HIT_RESERVED", - "MISS", - "RESERVATION_FAIL", - "SECTOR_MISS" - }; +const char *cache_request_status_str(enum cache_request_status status) { + static const char *static_cache_request_status_str[] = { + "HIT", "HIT_RESERVED", "MISS", "RESERVATION_FAIL", "SECTOR_MISS"}; - assert(sizeof(static_cache_request_status_str) / sizeof(const char*) == NUM_CACHE_REQUEST_STATUS); - assert(status < NUM_CACHE_REQUEST_STATUS); + assert(sizeof(static_cache_request_status_str) / sizeof(const char *) == + NUM_CACHE_REQUEST_STATUS); + assert(status < NUM_CACHE_REQUEST_STATUS); - return static_cache_request_status_str[status]; + return static_cache_request_status_str[status]; } -const char * cache_fail_status_str(enum cache_reservation_fail_reason status) -{ - static const char * static_cache_reservation_fail_reason_str[] = { - "LINE_ALLOC_FAIL", - "MISS_QUEUE_FULL", - "MSHR_ENRTY_FAIL", - "MSHR_MERGE_ENRTY_FAIL", - "MSHR_RW_PENDING" - }; +const char *cache_fail_status_str(enum cache_reservation_fail_reason status) { + static const char *static_cache_reservation_fail_reason_str[] = { + "LINE_ALLOC_FAIL", "MISS_QUEUE_FULL", "MSHR_ENRTY_FAIL", + "MSHR_MERGE_ENRTY_FAIL", "MSHR_RW_PENDING"}; - assert(sizeof(static_cache_reservation_fail_reason_str) / sizeof(const char*) == NUM_CACHE_RESERVATION_FAIL_STATUS); - assert(status < NUM_CACHE_RESERVATION_FAIL_STATUS); + assert(sizeof(static_cache_reservation_fail_reason_str) / + sizeof(const char *) == + NUM_CACHE_RESERVATION_FAIL_STATUS); + assert(status < NUM_CACHE_RESERVATION_FAIL_STATUS); - return static_cache_reservation_fail_reason_str[status]; + return static_cache_reservation_fail_reason_str[status]; } -unsigned l1d_cache_config::set_bank(new_addr_type addr) const{ - - if(m_cache_type == SECTOR) - return (addr >> m_sector_sz_log2) & (l1_banks-1); - else - return (addr >> m_line_sz_log2) & (l1_banks-1); +unsigned l1d_cache_config::set_bank(new_addr_type addr) const { + if (m_cache_type == SECTOR) + return (addr >> m_sector_sz_log2) & (l1_banks - 1); + else + return (addr >> m_line_sz_log2) & (l1_banks - 1); } -unsigned l1d_cache_config::set_index(new_addr_type addr) const{ - unsigned set_index = m_nset; // Default to linear set index function - unsigned lower_xor = 0; - unsigned upper_xor = 0; +unsigned l1d_cache_config::set_index(new_addr_type addr) const { + unsigned set_index = m_nset; // Default to linear set index function + unsigned lower_xor = 0; + unsigned upper_xor = 0; - switch(m_set_index_function){ + switch (m_set_index_function) { case FERMI_HASH_SET_FUNCTION: case BITWISE_XORING_FUNCTION: - /* - * Set Indexing function from "A Detailed GPU Cache Model Based on Reuse Distance Theory" - * Cedric Nugteren et al. - * HPCA 2014 - */ - if(m_nset == 32 || m_nset == 64){ - // Lower xor value is bits 7-11 - lower_xor = (addr >> m_line_sz_log2) & 0x1F; + /* + * Set Indexing function from "A Detailed GPU Cache Model Based on Reuse + * Distance Theory" Cedric Nugteren et al. HPCA 2014 + */ + if (m_nset == 32 || m_nset == 64) { + // Lower xor value is bits 7-11 + lower_xor = (addr >> m_line_sz_log2) & 0x1F; - // Upper xor value is bits 13, 14, 15, 17, and 19 - upper_xor = (addr & 0xE000) >> 13; // Bits 13, 14, 15 - upper_xor |= (addr & 0x20000) >> 14; // Bit 17 - upper_xor |= (addr & 0x80000) >> 15; // Bit 19 + // Upper xor value is bits 13, 14, 15, 17, and 19 + upper_xor = (addr & 0xE000) >> 13; // Bits 13, 14, 15 + upper_xor |= (addr & 0x20000) >> 14; // Bit 17 + upper_xor |= (addr & 0x80000) >> 15; // Bit 19 - set_index = (lower_xor ^ upper_xor); + set_index = (lower_xor ^ upper_xor); - // 48KB cache prepends the set_index with bit 12 - if(m_nset == 64) - set_index |= (addr & 0x1000) >> 7; + // 48KB cache prepends the set_index with bit 12 + if (m_nset == 64) set_index |= (addr & 0x1000) >> 7; - }else{ /* Else incorrect number of sets for the hashing function */ - assert("\nGPGPU-Sim cache configuration error: The number of sets should be " - "32 or 64 for the hashing set index function.\n" && 0); - } - break; + } else { /* Else incorrect number of sets for the hashing function */ + assert( + "\nGPGPU-Sim cache configuration error: The number of sets should " + "be " + "32 or 64 for the hashing set index function.\n" && + 0); + } + break; case HASH_IPOLY_FUNCTION: - /* - * Set Indexing function from "Pseudo-randomly interleaved memory." - * Rau, B. R et al. - * ISCA 1991 - * - * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." - * Khairy et al. - * IEEE TPDS 2017. - */ - if(m_nset == 32 || m_nset == 64){ - std::bitset<64> a(addr); - std::bitset<6> index; - index[0] = a[25]^a[24]^a[23]^a[22]^a[21]^a[18]^a[17]^a[15]^a[12]^a[7]; //10 - index[1] = a[26]^a[25]^a[24]^a[23]^a[22]^a[19]^a[18]^a[16]^a[13]^a[8]; //10 - index[2] = a[26]^a[22]^a[21]^a[20]^a[19]^a[18]^a[15]^a[14]^a[12]^a[9]; //10 - index[3] = a[23]^a[22]^a[21]^a[20]^a[19]^a[16]^a[15]^a[13]^a[10]; //9 - index[4] = a[24]^a[23]^a[22]^a[21]^a[20]^a[17]^a[16]^a[14]^a[11]; //9 + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu + * cache management scheme." Khairy et al. IEEE TPDS 2017. + */ + if (m_nset == 32 || m_nset == 64) { + std::bitset<64> a(addr); + std::bitset<6> index; + index[0] = a[25] ^ a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[18] ^ a[17] ^ + a[15] ^ a[12] ^ a[7]; // 10 + index[1] = a[26] ^ a[25] ^ a[24] ^ a[23] ^ a[22] ^ a[19] ^ a[18] ^ + a[16] ^ a[13] ^ a[8]; // 10 + index[2] = a[26] ^ a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[18] ^ a[15] ^ + a[14] ^ a[12] ^ a[9]; // 10 + index[3] = a[23] ^ a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[15] ^ + a[13] ^ a[10]; // 9 + index[4] = a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[20] ^ a[17] ^ a[16] ^ + a[14] ^ a[11]; // 9 - if(m_nset == 64) - index[5] = a[12]; + if (m_nset == 64) index[5] = a[12]; - set_index = index.to_ulong(); + set_index = index.to_ulong(); - }else{ /* Else incorrect number of sets for the hashing function */ - assert("\nGPGPU-Sim cache configuration error: The number of sets should be " - "32 or 64 for the hashing set index function.\n" && 0); - } - break; + } else { /* Else incorrect number of sets for the hashing function */ + assert( + "\nGPGPU-Sim cache configuration error: The number of sets should " + "be " + "32 or 64 for the hashing set index function.\n" && + 0); + } + break; case CUSTOM_SET_FUNCTION: - /* No custom set function implemented */ - break; + /* No custom set function implemented */ + break; case LINEAR_SET_FUNCTION: - set_index = (addr >> m_line_sz_log2) & (m_nset-1); - break; + set_index = (addr >> m_line_sz_log2) & (m_nset - 1); + break; default: - assert("\nUndefined set index function.\n" && 0); - break; - } + assert("\nUndefined set index function.\n" && 0); + break; + } - // Linear function selected or custom set index function not implemented - assert((set_index < m_nset) && "\nError: Set index out of bounds. This is caused by " - "an incorrect or unimplemented custom set index function.\n"); + // Linear function selected or custom set index function not implemented + assert((set_index < m_nset) && + "\nError: Set index out of bounds. This is caused by " + "an incorrect or unimplemented custom set index function.\n"); - return set_index; + return set_index; } -void l2_cache_config::init(linear_to_raw_address_translation *address_mapping){ - cache_config::init(m_config_string,FuncCachePreferNone); - m_address_mapping = address_mapping; +void l2_cache_config::init(linear_to_raw_address_translation *address_mapping) { + cache_config::init(m_config_string, FuncCachePreferNone); + m_address_mapping = address_mapping; } -unsigned l2_cache_config::set_index(new_addr_type addr) const{ - if(!m_address_mapping){ - return(addr >> m_line_sz_log2) & (m_nset-1); - }else{ - // Calculate set index without memory partition bits to reduce set camping - new_addr_type part_addr = m_address_mapping->partition_address(addr); - return(part_addr >> m_line_sz_log2) & (m_nset -1); - } +unsigned l2_cache_config::set_index(new_addr_type addr) const { + if (!m_address_mapping) { + return (addr >> m_line_sz_log2) & (m_nset - 1); + } else { + // Calculate set index without memory partition bits to reduce set camping + new_addr_type part_addr = m_address_mapping->partition_address(addr); + return (part_addr >> m_line_sz_log2) & (m_nset - 1); + } } -tag_array::~tag_array() -{ - unsigned cache_lines_num = m_config.get_max_num_lines(); - for(unsigned i=0; i<cache_lines_num; ++i) - delete m_lines[i]; - delete[] m_lines; +tag_array::~tag_array() { + unsigned cache_lines_num = m_config.get_max_num_lines(); + for (unsigned i = 0; i < cache_lines_num; ++i) delete m_lines[i]; + delete[] m_lines; } -tag_array::tag_array( cache_config &config, - int core_id, - int type_id, - cache_block_t** new_lines) - : m_config( config ), - m_lines( new_lines ) -{ - init( core_id, type_id ); +tag_array::tag_array(cache_config &config, int core_id, int type_id, + cache_block_t **new_lines) + : m_config(config), m_lines(new_lines) { + init(core_id, type_id); } -void tag_array::update_cache_parameters(cache_config &config) -{ - m_config=config; +void tag_array::update_cache_parameters(cache_config &config) { + m_config = config; } -tag_array::tag_array( cache_config &config, - int core_id, - int type_id ) - : m_config( config ) -{ - //assert( m_config.m_write_policy == READ_ONLY ); Old assert - unsigned cache_lines_num = config.get_max_num_lines(); - m_lines = new cache_block_t*[cache_lines_num]; - if(config.m_cache_type == NORMAL) - { - for(unsigned i=0; i<cache_lines_num; ++i) - m_lines[i] = new line_cache_block(); - } - else if(config.m_cache_type == SECTOR) - { - for(unsigned i=0; i<cache_lines_num; ++i) - m_lines[i] = new sector_cache_block(); - } - else - assert(0); +tag_array::tag_array(cache_config &config, int core_id, int type_id) + : m_config(config) { + // assert( m_config.m_write_policy == READ_ONLY ); Old assert + unsigned cache_lines_num = config.get_max_num_lines(); + m_lines = new cache_block_t *[cache_lines_num]; + if (config.m_cache_type == NORMAL) { + for (unsigned i = 0; i < cache_lines_num; ++i) + m_lines[i] = new line_cache_block(); + } else if (config.m_cache_type == SECTOR) { + for (unsigned i = 0; i < cache_lines_num; ++i) + m_lines[i] = new sector_cache_block(); + } else + assert(0); - init( core_id, type_id ); + init(core_id, type_id); } -void tag_array::init( int core_id, int type_id ) -{ - m_access = 0; - m_miss = 0; - m_pending_hit = 0; - m_res_fail = 0; - m_sector_miss = 0; - // initialize snapshot counters for visualizer - m_prev_snapshot_access = 0; - m_prev_snapshot_miss = 0; - m_prev_snapshot_pending_hit = 0; - m_core_id = core_id; - m_type_id = type_id; - is_used = false; +void tag_array::init(int core_id, int type_id) { + m_access = 0; + m_miss = 0; + m_pending_hit = 0; + m_res_fail = 0; + m_sector_miss = 0; + // initialize snapshot counters for visualizer + m_prev_snapshot_access = 0; + m_prev_snapshot_miss = 0; + m_prev_snapshot_pending_hit = 0; + m_core_id = core_id; + m_type_id = type_id; + is_used = false; } -void tag_array::add_pending_line(mem_fetch *mf){ - assert(mf); - new_addr_type addr = m_config.block_addr(mf->get_addr()); - line_table::const_iterator i = pending_lines.find(addr); - if ( i == pending_lines.end() ) { - pending_lines[addr] = mf->get_inst().get_uid(); - } +void tag_array::add_pending_line(mem_fetch *mf) { + assert(mf); + new_addr_type addr = m_config.block_addr(mf->get_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if (i == pending_lines.end()) { + pending_lines[addr] = mf->get_inst().get_uid(); + } } -void tag_array::remove_pending_line(mem_fetch *mf){ - assert(mf); - new_addr_type addr = m_config.block_addr(mf->get_addr()); - line_table::const_iterator i = pending_lines.find(addr); - if ( i != pending_lines.end() ) { - pending_lines.erase(addr); - } +void tag_array::remove_pending_line(mem_fetch *mf) { + assert(mf); + new_addr_type addr = m_config.block_addr(mf->get_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if (i != pending_lines.end()) { + pending_lines.erase(addr); + } } -enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_fetch* mf, bool probe_mode) const { - mem_access_sector_mask_t mask = mf->get_access_sector_mask(); - return probe(addr, idx, mask, probe_mode, mf); +enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx, + mem_fetch *mf, + bool probe_mode) const { + mem_access_sector_mask_t mask = mf->get_access_sector_mask(); + return probe(addr, idx, mask, probe_mode, mf); } +enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx, + mem_access_sector_mask_t mask, + bool probe_mode, + mem_fetch *mf) const { + // assert( m_config.m_write_policy == READ_ONLY ); + unsigned set_index = m_config.set_index(addr); + new_addr_type tag = m_config.tag(addr); -enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask, bool probe_mode, mem_fetch* mf) const { - //assert( m_config.m_write_policy == READ_ONLY ); - unsigned set_index = m_config.set_index(addr); - new_addr_type tag = m_config.tag(addr); - - unsigned invalid_line = (unsigned)-1; - unsigned valid_line = (unsigned)-1; - unsigned long long valid_timestamp = (unsigned)-1; - - bool all_reserved = true; + unsigned invalid_line = (unsigned)-1; + unsigned valid_line = (unsigned)-1; + unsigned long long valid_timestamp = (unsigned)-1; - // check for hit or pending hit - for (unsigned way=0; way<m_config.m_assoc; way++) { - unsigned index = set_index*m_config.m_assoc+way; - cache_block_t *line = m_lines[index]; - if (line->m_tag == tag) { - if ( line->get_status(mask) == RESERVED ) { - idx = index; - return HIT_RESERVED; - } else if ( line->get_status(mask) == VALID ) { - idx = index; - return HIT; - } else if ( line->get_status(mask) == MODIFIED) { - if(line->is_readable(mask)) { - idx = index; - return HIT; - } - else { - idx = index; - return SECTOR_MISS; - } + bool all_reserved = true; - } else if ( line->is_valid_line() && line->get_status(mask) == INVALID ) { - idx = index; - return SECTOR_MISS; - }else { - assert( line->get_status(mask) == INVALID ); - } - } - if (!line->is_reserved_line()) { - all_reserved = false; - if (line->is_invalid_line()) { - invalid_line = index; - } else { - // valid line : keep track of most appropriate replacement candidate - if ( m_config.m_replacement_policy == LRU ) { - if ( line->get_last_access_time() < valid_timestamp ) { - valid_timestamp = line->get_last_access_time(); - valid_line = index; - } - } else if ( m_config.m_replacement_policy == FIFO ) { - if ( line->get_alloc_time() < valid_timestamp ) { - valid_timestamp = line->get_alloc_time(); - valid_line = index; - } - } - } + // check for hit or pending hit + for (unsigned way = 0; way < m_config.m_assoc; way++) { + unsigned index = set_index * m_config.m_assoc + way; + cache_block_t *line = m_lines[index]; + if (line->m_tag == tag) { + if (line->get_status(mask) == RESERVED) { + idx = index; + return HIT_RESERVED; + } else if (line->get_status(mask) == VALID) { + idx = index; + return HIT; + } else if (line->get_status(mask) == MODIFIED) { + if (line->is_readable(mask)) { + idx = index; + return HIT; + } else { + idx = index; + return SECTOR_MISS; } + + } else if (line->is_valid_line() && line->get_status(mask) == INVALID) { + idx = index; + return SECTOR_MISS; + } else { + assert(line->get_status(mask) == INVALID); + } } - if ( all_reserved ) { - assert( m_config.m_alloc_policy == ON_MISS ); - return RESERVATION_FAIL; // miss and not enough space in cache to allocate on miss + if (!line->is_reserved_line()) { + all_reserved = false; + if (line->is_invalid_line()) { + invalid_line = index; + } else { + // valid line : keep track of most appropriate replacement candidate + if (m_config.m_replacement_policy == LRU) { + if (line->get_last_access_time() < valid_timestamp) { + valid_timestamp = line->get_last_access_time(); + valid_line = index; + } + } else if (m_config.m_replacement_policy == FIFO) { + if (line->get_alloc_time() < valid_timestamp) { + valid_timestamp = line->get_alloc_time(); + valid_line = index; + } + } + } } + } + if (all_reserved) { + assert(m_config.m_alloc_policy == ON_MISS); + return RESERVATION_FAIL; // miss and not enough space in cache to allocate + // on miss + } - if ( invalid_line != (unsigned)-1 ) { - idx = invalid_line; - } else if ( valid_line != (unsigned)-1) { - idx = valid_line; - } else abort(); // if an unreserved block exists, it is either invalid or replaceable - + if (invalid_line != (unsigned)-1) { + idx = invalid_line; + } else if (valid_line != (unsigned)-1) { + idx = valid_line; + } else + abort(); // if an unreserved block exists, it is either invalid or + // replaceable - if(probe_mode && m_config.is_streaming()){ - line_table::const_iterator i = pending_lines.find(m_config.block_addr(addr)); - assert(mf); - if ( !mf->is_write() && i != pending_lines.end() ) { - if(i->second != mf->get_inst().get_uid()) - return SECTOR_MISS; - } + if (probe_mode && m_config.is_streaming()) { + line_table::const_iterator i = + pending_lines.find(m_config.block_addr(addr)); + assert(mf); + if (!mf->is_write() && i != pending_lines.end()) { + if (i->second != mf->get_inst().get_uid()) return SECTOR_MISS; } + } - return MISS; + return MISS; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf) -{ - bool wb=false; - evicted_block_info evicted; - enum cache_request_status result = access(addr,time,idx,wb,evicted,mf); - assert(!wb); - return result; +enum cache_request_status tag_array::access(new_addr_type addr, unsigned time, + unsigned &idx, mem_fetch *mf) { + bool wb = false; + evicted_block_info evicted; + enum cache_request_status result = access(addr, time, idx, wb, evicted, mf); + assert(!wb); + return result; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf ) -{ - m_access++; - is_used = true; - shader_cache_access_log(m_core_id, m_type_id, 0); // log accesses to cache - enum cache_request_status status = probe(addr,idx,mf); - switch (status) { - case HIT_RESERVED: - m_pending_hit++; - case HIT: - m_lines[idx]->set_last_access_time(time, mf->get_access_sector_mask()); - break; +enum cache_request_status tag_array::access(new_addr_type addr, unsigned time, + unsigned &idx, bool &wb, + evicted_block_info &evicted, + mem_fetch *mf) { + m_access++; + is_used = true; + shader_cache_access_log(m_core_id, m_type_id, 0); // log accesses to cache + enum cache_request_status status = probe(addr, idx, mf); + switch (status) { + case HIT_RESERVED: + m_pending_hit++; + case HIT: + m_lines[idx]->set_last_access_time(time, mf->get_access_sector_mask()); + break; case MISS: - m_miss++; - shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses - if ( m_config.m_alloc_policy == ON_MISS ) { - if( m_lines[idx]->is_modified_line()) { - wb = true; - evicted.set_info(m_lines[idx]->m_block_addr, m_lines[idx]->get_modified_size()); - } - m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mf->get_access_sector_mask()); + m_miss++; + shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses + if (m_config.m_alloc_policy == ON_MISS) { + if (m_lines[idx]->is_modified_line()) { + wb = true; + evicted.set_info(m_lines[idx]->m_block_addr, + m_lines[idx]->get_modified_size()); } - break; + m_lines[idx]->allocate(m_config.tag(addr), m_config.block_addr(addr), + time, mf->get_access_sector_mask()); + } + break; case SECTOR_MISS: - assert(m_config.m_cache_type == SECTOR); - m_sector_miss++; - shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses - if ( m_config.m_alloc_policy == ON_MISS ) { - ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mf->get_access_sector_mask() ); - } - break; + assert(m_config.m_cache_type == SECTOR); + m_sector_miss++; + shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses + if (m_config.m_alloc_policy == ON_MISS) { + ((sector_cache_block *)m_lines[idx]) + ->allocate_sector(time, mf->get_access_sector_mask()); + } + break; case RESERVATION_FAIL: - m_res_fail++; - shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses - break; + m_res_fail++; + shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses + break; default: - fprintf( stderr, "tag_array::access - Error: Unknown" - "cache_request_status %d\n", status ); - abort(); - } - return status; + fprintf(stderr, + "tag_array::access - Error: Unknown" + "cache_request_status %d\n", + status); + abort(); + } + return status; } -void tag_array::fill( new_addr_type addr, unsigned time, mem_fetch* mf) -{ - fill(addr, time, mf->get_access_sector_mask()); +void tag_array::fill(new_addr_type addr, unsigned time, mem_fetch *mf) { + fill(addr, time, mf->get_access_sector_mask()); } -void tag_array::fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) -{ - //assert( m_config.m_alloc_policy == ON_FILL ); - unsigned idx; - enum cache_request_status status = probe(addr,idx,mask); - //assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request - if(status==MISS) - m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mask ); - else if (status==SECTOR_MISS) { - assert(m_config.m_cache_type == SECTOR); - ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mask ); - } +void tag_array::fill(new_addr_type addr, unsigned time, + mem_access_sector_mask_t mask) { + // assert( m_config.m_alloc_policy == ON_FILL ); + unsigned idx; + enum cache_request_status status = probe(addr, idx, mask); + // assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented + // redundant memory request + if (status == MISS) + m_lines[idx]->allocate(m_config.tag(addr), m_config.block_addr(addr), time, + mask); + else if (status == SECTOR_MISS) { + assert(m_config.m_cache_type == SECTOR); + ((sector_cache_block *)m_lines[idx])->allocate_sector(time, mask); + } - m_lines[idx]->fill(time, mask); + m_lines[idx]->fill(time, mask); } -void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf) -{ - assert( m_config.m_alloc_policy == ON_MISS ); - m_lines[index]->fill(time, mf->get_access_sector_mask()); +void tag_array::fill(unsigned index, unsigned time, mem_fetch *mf) { + assert(m_config.m_alloc_policy == ON_MISS); + m_lines[index]->fill(time, mf->get_access_sector_mask()); } +// TODO: we need write back the flushed data to the upper level +void tag_array::flush() { + if (!is_used) return; -//TODO: we need write back the flushed data to the upper level -void tag_array::flush() -{ - if(!is_used) - return; - - for (unsigned i=0; i < m_config.get_num_lines(); i++) - if(m_lines[i]->is_modified_line()) { - for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) - m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; - } + for (unsigned i = 0; i < m_config.get_num_lines(); i++) + if (m_lines[i]->is_modified_line()) { + for (unsigned j = 0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)); + } - is_used = false; + is_used = false; } -void tag_array::invalidate() -{ - if(!is_used) - return; +void tag_array::invalidate() { + if (!is_used) return; - for (unsigned i=0; i < m_config.get_num_lines(); i++) - for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) - m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + for (unsigned i = 0; i < m_config.get_num_lines(); i++) + for (unsigned j = 0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)); - is_used = false; + is_used = false; } -float tag_array::windowed_miss_rate( ) const -{ - unsigned n_access = m_access - m_prev_snapshot_access; - unsigned n_miss = (m_miss+m_sector_miss) - m_prev_snapshot_miss; - // unsigned n_pending_hit = m_pending_hit - m_prev_snapshot_pending_hit; +float tag_array::windowed_miss_rate() const { + unsigned n_access = m_access - m_prev_snapshot_access; + unsigned n_miss = (m_miss + m_sector_miss) - m_prev_snapshot_miss; + // unsigned n_pending_hit = m_pending_hit - m_prev_snapshot_pending_hit; - float missrate = 0.0f; - if (n_access != 0) - missrate = (float) (n_miss+m_sector_miss) / n_access; - return missrate; + float missrate = 0.0f; + if (n_access != 0) missrate = (float)(n_miss + m_sector_miss) / n_access; + return missrate; } -void tag_array::new_window() -{ - m_prev_snapshot_access = m_access; - m_prev_snapshot_miss = m_miss; - m_prev_snapshot_miss = m_miss + m_sector_miss; - m_prev_snapshot_pending_hit = m_pending_hit; +void tag_array::new_window() { + m_prev_snapshot_access = m_access; + m_prev_snapshot_miss = m_miss; + m_prev_snapshot_miss = m_miss + m_sector_miss; + m_prev_snapshot_pending_hit = m_pending_hit; } -void tag_array::print( FILE *stream, unsigned &total_access, unsigned &total_misses ) const -{ - m_config.print(stream); - fprintf( stream, "\t\tAccess = %d, Miss = %d, Sector_Miss = %d, Total_Miss = %d (%.3g), PendingHit = %d (%.3g)\n", - m_access, m_miss, m_sector_miss, (m_miss+m_sector_miss), (float) (m_miss+m_sector_miss) / m_access, - m_pending_hit, (float) m_pending_hit / m_access); - total_misses+=(m_miss+m_sector_miss); - total_access+=m_access; +void tag_array::print(FILE *stream, unsigned &total_access, + unsigned &total_misses) const { + m_config.print(stream); + fprintf(stream, + "\t\tAccess = %d, Miss = %d, Sector_Miss = %d, Total_Miss = %d " + "(%.3g), PendingHit = %d (%.3g)\n", + m_access, m_miss, m_sector_miss, (m_miss + m_sector_miss), + (float)(m_miss + m_sector_miss) / m_access, m_pending_hit, + (float)m_pending_hit / m_access); + total_misses += (m_miss + m_sector_miss); + total_access += m_access; } -void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsigned &total_hit_res, unsigned &total_res_fail) const{ - // Update statistics from the tag array - total_access = m_access; - total_misses = (m_miss+m_sector_miss); - total_hit_res = m_pending_hit; - total_res_fail = m_res_fail; +void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, + unsigned &total_hit_res, + unsigned &total_res_fail) const { + // Update statistics from the tag array + total_access = m_access; + total_misses = (m_miss + m_sector_miss); + total_hit_res = m_pending_hit; + total_res_fail = m_res_fail; } - -bool was_write_sent( const std::list<cache_event> &events ) -{ - for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( (*e).m_cache_event_type == WRITE_REQUEST_SENT ) - return true; - } - return false; +bool was_write_sent(const std::list<cache_event> &events) { + for (std::list<cache_event>::const_iterator e = events.begin(); + e != events.end(); e++) { + if ((*e).m_cache_event_type == WRITE_REQUEST_SENT) return true; + } + return false; } -bool was_writeback_sent( const std::list<cache_event> &events, cache_event& wb_event) -{ - for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( (*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT ) - wb_event = *e; - return true; - } - return false; +bool was_writeback_sent(const std::list<cache_event> &events, + cache_event &wb_event) { + for (std::list<cache_event>::const_iterator e = events.begin(); + e != events.end(); e++) { + if ((*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT) wb_event = *e; + return true; + } + return false; } -bool was_read_sent( const std::list<cache_event> &events ) -{ - for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( (*e).m_cache_event_type == READ_REQUEST_SENT ) - return true; - } - return false; +bool was_read_sent(const std::list<cache_event> &events) { + for (std::list<cache_event>::const_iterator e = events.begin(); + e != events.end(); e++) { + if ((*e).m_cache_event_type == READ_REQUEST_SENT) return true; + } + return false; } -bool was_writeallocate_sent( const std::list<cache_event> &events ) -{ - for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( (*e).m_cache_event_type == WRITE_ALLOCATE_SENT ) - return true; - } - return false; +bool was_writeallocate_sent(const std::list<cache_event> &events) { + for (std::list<cache_event>::const_iterator e = events.begin(); + e != events.end(); e++) { + if ((*e).m_cache_event_type == WRITE_ALLOCATE_SENT) return true; + } + return false; } -/****************************************************************** MSHR ******************************************************************/ +/****************************************************************** MSHR + * ******************************************************************/ /// Checks if there is a pending request to the lower memory level already -bool mshr_table::probe( new_addr_type block_addr ) const{ - table::const_iterator a = m_data.find(block_addr); - return a != m_data.end(); +bool mshr_table::probe(new_addr_type block_addr) const { + table::const_iterator a = m_data.find(block_addr); + return a != m_data.end(); } /// Checks if there is space for tracking a new memory access -bool mshr_table::full( new_addr_type block_addr ) const{ - table::const_iterator i=m_data.find(block_addr); - if ( i != m_data.end() ) - return i->second.m_list.size() >= m_max_merged; - else - return m_data.size() >= m_num_entries; +bool mshr_table::full(new_addr_type block_addr) const { + table::const_iterator i = m_data.find(block_addr); + if (i != m_data.end()) + return i->second.m_list.size() >= m_max_merged; + else + return m_data.size() >= m_num_entries; } /// Add or merge this access -void mshr_table::add( new_addr_type block_addr, mem_fetch *mf ){ - m_data[block_addr].m_list.push_back(mf); - assert( m_data.size() <= m_num_entries ); - assert( m_data[block_addr].m_list.size() <= m_max_merged ); - // indicate that this MSHR entry contains an atomic operation - if ( mf->isatomic() ) { - m_data[block_addr].m_has_atomic = true; - } +void mshr_table::add(new_addr_type block_addr, mem_fetch *mf) { + m_data[block_addr].m_list.push_back(mf); + assert(m_data.size() <= m_num_entries); + assert(m_data[block_addr].m_list.size() <= m_max_merged); + // indicate that this MSHR entry contains an atomic operation + if (mf->isatomic()) { + m_data[block_addr].m_has_atomic = true; + } } /// check is_read_after_write_pending -bool mshr_table::is_read_after_write_pending( new_addr_type block_addr){ - std::list<mem_fetch*> my_list = m_data[block_addr].m_list; - bool write_found = false; - for (std::list<mem_fetch*>::iterator it=my_list.begin(); it != my_list.end(); ++it) - { - if((*it)->is_write()) //Pending Write Request - write_found = true; - else if(write_found) //Pending Read Request and we found previous Write - return true; - } - - return false; +bool mshr_table::is_read_after_write_pending(new_addr_type block_addr) { + std::list<mem_fetch *> my_list = m_data[block_addr].m_list; + bool write_found = false; + for (std::list<mem_fetch *>::iterator it = my_list.begin(); + it != my_list.end(); ++it) { + if ((*it)->is_write()) // Pending Write Request + write_found = true; + else if (write_found) // Pending Read Request and we found previous Write + return true; + } + return false; } /// Accept a new cache fill response: mark entry ready for processing -void mshr_table::mark_ready( new_addr_type block_addr, bool &has_atomic ){ - assert( !busy() ); - table::iterator a = m_data.find(block_addr); - assert( a != m_data.end() ); - m_current_response.push_back( block_addr ); - has_atomic = a->second.m_has_atomic; - assert( m_current_response.size() <= m_data.size() ); +void mshr_table::mark_ready(new_addr_type block_addr, bool &has_atomic) { + assert(!busy()); + table::iterator a = m_data.find(block_addr); + assert(a != m_data.end()); + m_current_response.push_back(block_addr); + has_atomic = a->second.m_has_atomic; + assert(m_current_response.size() <= m_data.size()); } /// Returns next ready access -mem_fetch *mshr_table::next_access(){ - assert( access_ready() ); - new_addr_type block_addr = m_current_response.front(); - assert( !m_data[block_addr].m_list.empty() ); - mem_fetch *result = m_data[block_addr].m_list.front(); - m_data[block_addr].m_list.pop_front(); - if ( m_data[block_addr].m_list.empty() ) { - // release entry - m_data.erase(block_addr); - m_current_response.pop_front(); - } - return result; +mem_fetch *mshr_table::next_access() { + assert(access_ready()); + new_addr_type block_addr = m_current_response.front(); + assert(!m_data[block_addr].m_list.empty()); + mem_fetch *result = m_data[block_addr].m_list.front(); + m_data[block_addr].m_list.pop_front(); + if (m_data[block_addr].m_list.empty()) { + // release entry + m_data.erase(block_addr); + m_current_response.pop_front(); + } + return result; } -void mshr_table::display( FILE *fp ) const{ - fprintf(fp,"MSHR contents\n"); - for ( table::const_iterator e=m_data.begin(); e!=m_data.end(); ++e ) { - unsigned block_addr = e->first; - fprintf(fp,"MSHR: tag=0x%06x, atomic=%d %zu entries : ", block_addr, e->second.m_has_atomic, e->second.m_list.size()); - if ( !e->second.m_list.empty() ) { - mem_fetch *mf = e->second.m_list.front(); - fprintf(fp,"%p :",mf); - mf->print(fp); - } else { - fprintf(fp," no memory requests???\n"); - } +void mshr_table::display(FILE *fp) const { + fprintf(fp, "MSHR contents\n"); + for (table::const_iterator e = m_data.begin(); e != m_data.end(); ++e) { + unsigned block_addr = e->first; + fprintf(fp, "MSHR: tag=0x%06x, atomic=%d %zu entries : ", block_addr, + e->second.m_has_atomic, e->second.m_list.size()); + if (!e->second.m_list.empty()) { + mem_fetch *mf = e->second.m_list.front(); + fprintf(fp, "%p :", mf); + mf->print(fp); + } else { + fprintf(fp, " no memory requests???\n"); } + } } -/***************************************************************** Caches *****************************************************************/ -cache_stats::cache_stats(){ - m_stats.resize(NUM_MEM_ACCESS_TYPE); - m_stats_pw.resize(NUM_MEM_ACCESS_TYPE); - m_fail_stats.resize(NUM_MEM_ACCESS_TYPE); - for(unsigned i=0; i<NUM_MEM_ACCESS_TYPE; ++i){ - m_stats[i].resize(NUM_CACHE_REQUEST_STATUS, 0); - m_stats_pw[i].resize(NUM_CACHE_REQUEST_STATUS, 0); - m_fail_stats[i].resize(NUM_CACHE_RESERVATION_FAIL_STATUS, 0); - } - m_cache_port_available_cycles = 0; - m_cache_data_port_busy_cycles = 0; - m_cache_fill_port_busy_cycles = 0; +/***************************************************************** Caches + * *****************************************************************/ +cache_stats::cache_stats() { + m_stats.resize(NUM_MEM_ACCESS_TYPE); + m_stats_pw.resize(NUM_MEM_ACCESS_TYPE); + m_fail_stats.resize(NUM_MEM_ACCESS_TYPE); + for (unsigned i = 0; i < NUM_MEM_ACCESS_TYPE; ++i) { + m_stats[i].resize(NUM_CACHE_REQUEST_STATUS, 0); + m_stats_pw[i].resize(NUM_CACHE_REQUEST_STATUS, 0); + m_fail_stats[i].resize(NUM_CACHE_RESERVATION_FAIL_STATUS, 0); + } + m_cache_port_available_cycles = 0; + m_cache_data_port_busy_cycles = 0; + m_cache_fill_port_busy_cycles = 0; } -void cache_stats::clear(){ - /// - /// Zero out all current cache statistics - /// - for(unsigned i=0; i<NUM_MEM_ACCESS_TYPE; ++i){ - std::fill(m_stats[i].begin(), m_stats[i].end(), 0); - std::fill(m_fail_stats[i].begin(), m_fail_stats[i].end(), 0); - } - m_cache_port_available_cycles = 0; - m_cache_data_port_busy_cycles = 0; - m_cache_fill_port_busy_cycles = 0; +void cache_stats::clear() { + /// + /// Zero out all current cache statistics + /// + for (unsigned i = 0; i < NUM_MEM_ACCESS_TYPE; ++i) { + std::fill(m_stats[i].begin(), m_stats[i].end(), 0); + std::fill(m_fail_stats[i].begin(), m_fail_stats[i].end(), 0); + } + m_cache_port_available_cycles = 0; + m_cache_data_port_busy_cycles = 0; + m_cache_fill_port_busy_cycles = 0; } -void cache_stats::clear_pw(){ - /// - /// Zero out per-window cache statistics - /// - for(unsigned i=0; i<NUM_MEM_ACCESS_TYPE; ++i){ - std::fill(m_stats_pw[i].begin(), m_stats_pw[i].end(), 0); - } +void cache_stats::clear_pw() { + /// + /// Zero out per-window cache statistics + /// + for (unsigned i = 0; i < NUM_MEM_ACCESS_TYPE; ++i) { + std::fill(m_stats_pw[i].begin(), m_stats_pw[i].end(), 0); + } } -void cache_stats::inc_stats(int access_type, int access_outcome){ - /// - /// Increment the stat corresponding to (access_type, access_outcome) by 1. - /// - if(!check_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or access outcome"); +void cache_stats::inc_stats(int access_type, int access_outcome) { + /// + /// Increment the stat corresponding to (access_type, access_outcome) by 1. + /// + if (!check_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or access outcome"); - m_stats[access_type][access_outcome]++; + m_stats[access_type][access_outcome]++; } -void cache_stats::inc_stats_pw(int access_type, int access_outcome){ - /// - /// Increment the corresponding per-window cache stat - /// - if(!check_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or access outcome"); - m_stats_pw[access_type][access_outcome]++; +void cache_stats::inc_stats_pw(int access_type, int access_outcome) { + /// + /// Increment the corresponding per-window cache stat + /// + if (!check_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or access outcome"); + m_stats_pw[access_type][access_outcome]++; } -void cache_stats::inc_fail_stats(int access_type, int fail_outcome){ - - if(!check_fail_valid(access_type, fail_outcome)) - assert(0 && "Unknown cache access type or access fail"); +void cache_stats::inc_fail_stats(int access_type, int fail_outcome) { + if (!check_fail_valid(access_type, fail_outcome)) + assert(0 && "Unknown cache access type or access fail"); - m_fail_stats[access_type][fail_outcome]++; + m_fail_stats[access_type][fail_outcome]++; } - -enum cache_request_status cache_stats::select_stats_status(enum cache_request_status probe, enum cache_request_status access) const { - /// - /// This function selects how the cache access outcome should be counted. HIT_RESERVED is considered as a MISS - /// in the cores, however, it should be counted as a HIT_RESERVED in the caches. - /// - if(probe == HIT_RESERVED && access != RESERVATION_FAIL) - return probe; - else if(probe == SECTOR_MISS && access == MISS) - return probe; - else - return access; +enum cache_request_status cache_stats::select_stats_status( + enum cache_request_status probe, enum cache_request_status access) const { + /// + /// This function selects how the cache access outcome should be counted. + /// HIT_RESERVED is considered as a MISS in the cores, however, it should be + /// counted as a HIT_RESERVED in the caches. + /// + if (probe == HIT_RESERVED && access != RESERVATION_FAIL) + return probe; + else if (probe == SECTOR_MISS && access == MISS) + return probe; + else + return access; } -unsigned long long &cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome){ - /// - /// Simple method to read/modify the stat corresponding to (access_type, access_outcome) - /// Used overloaded () to avoid the need for separate read/write member functions - /// - if(fail_outcome) { - if(!check_fail_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or fail outcome"); +unsigned long long &cache_stats::operator()(int access_type, int access_outcome, + bool fail_outcome) { + /// + /// Simple method to read/modify the stat corresponding to (access_type, + /// access_outcome) Used overloaded () to avoid the need for separate + /// read/write member functions + /// + if (fail_outcome) { + if (!check_fail_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or fail outcome"); - return m_fail_stats[access_type][access_outcome]; - } - else { - if(!check_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or access outcome"); + return m_fail_stats[access_type][access_outcome]; + } else { + if (!check_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or access outcome"); - return m_stats[access_type][access_outcome]; - } + return m_stats[access_type][access_outcome]; + } } -unsigned long long cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome) const{ - /// - /// Const accessor into m_stats. - /// - if(fail_outcome) { - if(!check_fail_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or fail outcome"); +unsigned long long cache_stats::operator()(int access_type, int access_outcome, + bool fail_outcome) const { + /// + /// Const accessor into m_stats. + /// + if (fail_outcome) { + if (!check_fail_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or fail outcome"); - return m_fail_stats[access_type][access_outcome]; - } - else { - if(!check_valid(access_type, access_outcome)) - assert(0 && "Unknown cache access type or access outcome"); + return m_fail_stats[access_type][access_outcome]; + } else { + if (!check_valid(access_type, access_outcome)) + assert(0 && "Unknown cache access type or access outcome"); - return m_stats[access_type][access_outcome]; - } + return m_stats[access_type][access_outcome]; + } } -cache_stats cache_stats::operator+(const cache_stats &cs){ - /// - /// Overloaded + operator to allow for simple stat accumulation - /// - cache_stats ret; - for(unsigned type=0; type<NUM_MEM_ACCESS_TYPE; ++type){ - for(unsigned status=0; status<NUM_CACHE_REQUEST_STATUS; ++status){ - ret(type, status, false) = m_stats[type][status] + cs(type, status, false); - } - for(unsigned status=0; status<NUM_CACHE_RESERVATION_FAIL_STATUS; ++status){ - ret(type, status, true) = m_fail_stats[type][status] + cs(type, status, true); - } - } - ret.m_cache_port_available_cycles = m_cache_port_available_cycles + cs.m_cache_port_available_cycles; - ret.m_cache_data_port_busy_cycles = m_cache_data_port_busy_cycles + cs.m_cache_data_port_busy_cycles; - ret.m_cache_fill_port_busy_cycles = m_cache_fill_port_busy_cycles + cs.m_cache_fill_port_busy_cycles; - return ret; +cache_stats cache_stats::operator+(const cache_stats &cs) { + /// + /// Overloaded + operator to allow for simple stat accumulation + /// + cache_stats ret; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + ret(type, status, false) = + m_stats[type][status] + cs(type, status, false); + } + for (unsigned status = 0; status < NUM_CACHE_RESERVATION_FAIL_STATUS; + ++status) { + ret(type, status, true) = + m_fail_stats[type][status] + cs(type, status, true); + } + } + ret.m_cache_port_available_cycles = + m_cache_port_available_cycles + cs.m_cache_port_available_cycles; + ret.m_cache_data_port_busy_cycles = + m_cache_data_port_busy_cycles + cs.m_cache_data_port_busy_cycles; + ret.m_cache_fill_port_busy_cycles = + m_cache_fill_port_busy_cycles + cs.m_cache_fill_port_busy_cycles; + return ret; } -cache_stats &cache_stats::operator+=(const cache_stats &cs){ - /// - /// Overloaded += operator to allow for simple stat accumulation - /// - for(unsigned type=0; type<NUM_MEM_ACCESS_TYPE; ++type){ - for(unsigned status=0; status<NUM_CACHE_REQUEST_STATUS; ++status){ - m_stats[type][status] += cs(type, status, false); - } - for(unsigned status=0; status<NUM_CACHE_REQUEST_STATUS; ++status){ - m_stats_pw[type][status] += cs(type, status, false); - } - for(unsigned status=0; status<NUM_CACHE_RESERVATION_FAIL_STATUS; ++status){ - m_fail_stats[type][status] += cs(type, status, true); - } +cache_stats &cache_stats::operator+=(const cache_stats &cs) { + /// + /// Overloaded += operator to allow for simple stat accumulation + /// + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + m_stats[type][status] += cs(type, status, false); + } + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + m_stats_pw[type][status] += cs(type, status, false); + } + for (unsigned status = 0; status < NUM_CACHE_RESERVATION_FAIL_STATUS; + ++status) { + m_fail_stats[type][status] += cs(type, status, true); } - m_cache_port_available_cycles += cs.m_cache_port_available_cycles; - m_cache_data_port_busy_cycles += cs.m_cache_data_port_busy_cycles; - m_cache_fill_port_busy_cycles += cs.m_cache_fill_port_busy_cycles; - return *this; + } + m_cache_port_available_cycles += cs.m_cache_port_available_cycles; + m_cache_data_port_busy_cycles += cs.m_cache_data_port_busy_cycles; + m_cache_fill_port_busy_cycles += cs.m_cache_fill_port_busy_cycles; + return *this; } -void cache_stats::print_stats(FILE *fout, const char *cache_name) const{ - /// - /// Print out each non-zero cache statistic for every memory access type and status - /// "cache_name" defaults to "Cache_stats" when no argument is provided, otherwise - /// the provided name is used. - /// The printed format is "<cache_name>[<request_type>][<request_status>] = <stat_value>" - /// - std::vector< unsigned > total_access; - total_access.resize(NUM_MEM_ACCESS_TYPE, 0); - std::string m_cache_name = cache_name; - for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { - for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { - fprintf(fout, "\t%s[%s][%s] = %llu\n", - m_cache_name.c_str(), - mem_access_type_str((enum mem_access_type)type), - cache_request_status_str((enum cache_request_status)status), - m_stats[type][status]); +void cache_stats::print_stats(FILE *fout, const char *cache_name) const { + /// + /// Print out each non-zero cache statistic for every memory access type and + /// status "cache_name" defaults to "Cache_stats" when no argument is + /// provided, otherwise the provided name is used. The printed format is + /// "<cache_name>[<request_type>][<request_status>] = <stat_value>" + /// + std::vector<unsigned> total_access; + total_access.resize(NUM_MEM_ACCESS_TYPE, 0); + std::string m_cache_name = cache_name; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + fprintf(fout, "\t%s[%s][%s] = %llu\n", m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + cache_request_status_str((enum cache_request_status)status), + m_stats[type][status]); - if(status != RESERVATION_FAIL) - total_access[type]+= m_stats[type][status]; - } - } - for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { - if(total_access[type] > 0) - fprintf(fout, "\t%s[%s][%s] = %u\n", - m_cache_name.c_str(), - mem_access_type_str((enum mem_access_type)type), - "TOTAL_ACCESS", - total_access[type]); + if (status != RESERVATION_FAIL) + total_access[type] += m_stats[type][status]; } + } + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + if (total_access[type] > 0) + fprintf(fout, "\t%s[%s][%s] = %u\n", m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), "TOTAL_ACCESS", + total_access[type]); + } } -void cache_stats::print_fail_stats(FILE *fout, const char *cache_name) const{ - std::string m_cache_name = cache_name; - for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { - for (unsigned fail = 0; fail < NUM_CACHE_RESERVATION_FAIL_STATUS; ++fail) { - if(m_fail_stats[type][fail] > 0){ - fprintf(fout, "\t%s[%s][%s] = %llu\n", - m_cache_name.c_str(), - mem_access_type_str((enum mem_access_type)type), - cache_fail_status_str((enum cache_reservation_fail_reason)fail), - m_fail_stats[type][fail]); - } - } - } +void cache_stats::print_fail_stats(FILE *fout, const char *cache_name) const { + std::string m_cache_name = cache_name; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned fail = 0; fail < NUM_CACHE_RESERVATION_FAIL_STATUS; ++fail) { + if (m_fail_stats[type][fail] > 0) { + fprintf(fout, "\t%s[%s][%s] = %llu\n", m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + cache_fail_status_str((enum cache_reservation_fail_reason)fail), + m_fail_stats[type][fail]); + } + } + } } -void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const -{ - float data_port_util = 0.0f; - if (port_available_cycles > 0) { - data_port_util = (float) data_port_busy_cycles / port_available_cycles; - } - fprintf(fout, "%s_data_port_util = %.3f\n", cache_name, data_port_util); - float fill_port_util = 0.0f; - if (port_available_cycles > 0) { - fill_port_util = (float) fill_port_busy_cycles / port_available_cycles; - } - fprintf(fout, "%s_fill_port_util = %.3f\n", cache_name, fill_port_util); +void cache_sub_stats::print_port_stats(FILE *fout, + const char *cache_name) const { + float data_port_util = 0.0f; + if (port_available_cycles > 0) { + data_port_util = (float)data_port_busy_cycles / port_available_cycles; + } + fprintf(fout, "%s_data_port_util = %.3f\n", cache_name, data_port_util); + float fill_port_util = 0.0f; + if (port_available_cycles > 0) { + fill_port_util = (float)fill_port_busy_cycles / port_available_cycles; + } + fprintf(fout, "%s_fill_port_util = %.3f\n", cache_name, fill_port_util); } -unsigned long long cache_stats::get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const{ - /// - /// Returns a sum of the stats corresponding to each "access_type" and "access_status" pair. - /// "access_type" is an array of "num_access_type" mem_access_types. - /// "access_status" is an array of "num_access_status" cache_request_statuses. - /// - unsigned long long total=0; - for(unsigned type =0; type < num_access_type; ++type){ - for(unsigned status=0; status < num_access_status; ++status){ - if(!check_valid((int)access_type[type], (int)access_status[status])) - assert(0 && "Unknown cache access type or access outcome"); - total += m_stats[access_type[type]][access_status[status]]; - } +unsigned long long cache_stats::get_stats( + enum mem_access_type *access_type, unsigned num_access_type, + enum cache_request_status *access_status, + unsigned num_access_status) const { + /// + /// Returns a sum of the stats corresponding to each "access_type" and + /// "access_status" pair. "access_type" is an array of "num_access_type" + /// mem_access_types. "access_status" is an array of "num_access_status" + /// cache_request_statuses. + /// + unsigned long long total = 0; + for (unsigned type = 0; type < num_access_type; ++type) { + for (unsigned status = 0; status < num_access_status; ++status) { + if (!check_valid((int)access_type[type], (int)access_status[status])) + assert(0 && "Unknown cache access type or access outcome"); + total += m_stats[access_type[type]][access_status[status]]; } - return total; + } + return total; } -void cache_stats::get_sub_stats(struct cache_sub_stats &css) const{ - /// - /// Overwrites "css" with the appropriate statistics from this cache. - /// - struct cache_sub_stats t_css; - t_css.clear(); +void cache_stats::get_sub_stats(struct cache_sub_stats &css) const { + /// + /// Overwrites "css" with the appropriate statistics from this cache. + /// + struct cache_sub_stats t_css; + t_css.clear(); - for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { - for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { - if(status == HIT || status == MISS || status == SECTOR_MISS || status == HIT_RESERVED) - t_css.accesses += m_stats[type][status]; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + if (status == HIT || status == MISS || status == SECTOR_MISS || + status == HIT_RESERVED) + t_css.accesses += m_stats[type][status]; - if(status == MISS || status == SECTOR_MISS) - t_css.misses += m_stats[type][status]; + if (status == MISS || status == SECTOR_MISS) + t_css.misses += m_stats[type][status]; - if(status == HIT_RESERVED) - t_css.pending_hits += m_stats[type][status]; + if (status == HIT_RESERVED) t_css.pending_hits += m_stats[type][status]; - if(status == RESERVATION_FAIL) - t_css.res_fails += m_stats[type][status]; - } + if (status == RESERVATION_FAIL) t_css.res_fails += m_stats[type][status]; } + } - t_css.port_available_cycles = m_cache_port_available_cycles; - t_css.data_port_busy_cycles = m_cache_data_port_busy_cycles; - t_css.fill_port_busy_cycles = m_cache_fill_port_busy_cycles; + t_css.port_available_cycles = m_cache_port_available_cycles; + t_css.data_port_busy_cycles = m_cache_data_port_busy_cycles; + t_css.fill_port_busy_cycles = m_cache_fill_port_busy_cycles; - css = t_css; + css = t_css; } -void cache_stats::get_sub_stats_pw(struct cache_sub_stats_pw &css) const{ - /// - /// Overwrites "css" with the appropriate statistics from this cache. - /// - struct cache_sub_stats_pw t_css; - t_css.clear(); +void cache_stats::get_sub_stats_pw(struct cache_sub_stats_pw &css) const { + /// + /// Overwrites "css" with the appropriate statistics from this cache. + /// + struct cache_sub_stats_pw t_css; + t_css.clear(); - for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { - for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { - if(status == HIT || status == MISS || status == SECTOR_MISS || status == HIT_RESERVED) - t_css.accesses += m_stats_pw[type][status]; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { + if (status == HIT || status == MISS || status == SECTOR_MISS || + status == HIT_RESERVED) + t_css.accesses += m_stats_pw[type][status]; - if(status == HIT){ - if(type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R){ - t_css.read_hits += m_stats_pw[type][status]; - } else if(type == GLOBAL_ACC_W){ - t_css.write_hits += m_stats_pw[type][status]; - } - } + if (status == HIT) { + if (type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R) { + t_css.read_hits += m_stats_pw[type][status]; + } else if (type == GLOBAL_ACC_W) { + t_css.write_hits += m_stats_pw[type][status]; + } + } - if(status == MISS || status == SECTOR_MISS){ - if(type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R){ - t_css.read_misses += m_stats_pw[type][status]; - } else if(type == GLOBAL_ACC_W){ - t_css.write_misses += m_stats_pw[type][status]; - } - } + if (status == MISS || status == SECTOR_MISS) { + if (type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R) { + t_css.read_misses += m_stats_pw[type][status]; + } else if (type == GLOBAL_ACC_W) { + t_css.write_misses += m_stats_pw[type][status]; + } + } - if(status == HIT_RESERVED){ - if(type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R){ - t_css.read_pending_hits += m_stats_pw[type][status]; - } else if(type == GLOBAL_ACC_W){ - t_css.write_pending_hits += m_stats_pw[type][status]; - } - } + if (status == HIT_RESERVED) { + if (type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R) { + t_css.read_pending_hits += m_stats_pw[type][status]; + } else if (type == GLOBAL_ACC_W) { + t_css.write_pending_hits += m_stats_pw[type][status]; + } + } - if(status == RESERVATION_FAIL){ - if(type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R){ - t_css.read_res_fails += m_stats_pw[type][status]; - } else if(type == GLOBAL_ACC_W){ - t_css.write_res_fails += m_stats_pw[type][status]; - } - } + if (status == RESERVATION_FAIL) { + if (type == GLOBAL_ACC_R || type == CONST_ACC_R || type == INST_ACC_R) { + t_css.read_res_fails += m_stats_pw[type][status]; + } else if (type == GLOBAL_ACC_W) { + t_css.write_res_fails += m_stats_pw[type][status]; } + } } + } - css = t_css; + css = t_css; } -bool cache_stats::check_valid(int type, int status) const{ - /// - /// Verify a valid access_type/access_status - /// - if((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (status >= 0) && (status < NUM_CACHE_REQUEST_STATUS)) - return true; - else - return false; +bool cache_stats::check_valid(int type, int status) const { + /// + /// Verify a valid access_type/access_status + /// + if ((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (status >= 0) && + (status < NUM_CACHE_REQUEST_STATUS)) + return true; + else + return false; } -bool cache_stats::check_fail_valid(int type, int fail) const{ - /// - /// Verify a valid access_type/access_status - /// - if((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (fail >= 0) && (fail < NUM_CACHE_RESERVATION_FAIL_STATUS)) - return true; - else - return false; +bool cache_stats::check_fail_valid(int type, int fail) const { + /// + /// Verify a valid access_type/access_status + /// + if ((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (fail >= 0) && + (fail < NUM_CACHE_RESERVATION_FAIL_STATUS)) + return true; + else + return false; } -void cache_stats::sample_cache_port_utility(bool data_port_busy, bool fill_port_busy) -{ - m_cache_port_available_cycles += 1; - if (data_port_busy) { - m_cache_data_port_busy_cycles += 1; - } - if (fill_port_busy) { - m_cache_fill_port_busy_cycles += 1; - } +void cache_stats::sample_cache_port_utility(bool data_port_busy, + bool fill_port_busy) { + m_cache_port_available_cycles += 1; + if (data_port_busy) { + m_cache_data_port_busy_cycles += 1; + } + if (fill_port_busy) { + m_cache_fill_port_busy_cycles += 1; + } } -baseline_cache::bandwidth_management::bandwidth_management(cache_config &config) -: m_config(config) -{ - m_data_port_occupied_cycles = 0; - m_fill_port_occupied_cycles = 0; +baseline_cache::bandwidth_management::bandwidth_management(cache_config &config) + : m_config(config) { + m_data_port_occupied_cycles = 0; + m_fill_port_occupied_cycles = 0; } -/// use the data port based on the outcome and events generated by the mem_fetch request -void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cache_request_status outcome, const std::list<cache_event> &events) -{ - unsigned data_size = mf->get_data_size(); - unsigned port_width = m_config.m_data_port_width; - switch (outcome) { +/// use the data port based on the outcome and events generated by the mem_fetch +/// request +void baseline_cache::bandwidth_management::use_data_port( + mem_fetch *mf, enum cache_request_status outcome, + const std::list<cache_event> &events) { + unsigned data_size = mf->get_data_size(); + unsigned port_width = m_config.m_data_port_width; + switch (outcome) { case HIT: { - unsigned data_cycles = data_size / port_width + ((data_size % port_width > 0)? 1 : 0); - m_data_port_occupied_cycles += data_cycles; - } break; + unsigned data_cycles = + data_size / port_width + ((data_size % port_width > 0) ? 1 : 0); + m_data_port_occupied_cycles += data_cycles; + } break; case HIT_RESERVED: case MISS: { - // the data array is accessed to read out the entire line for write-back - // in case of sector cache we need to write bank only the modified sectors - cache_event ev(WRITE_BACK_REQUEST_SENT); - if (was_writeback_sent(events, ev)) { - unsigned data_cycles = ev.m_evicted_block.m_modified_size / port_width; - m_data_port_occupied_cycles += data_cycles; - } - } break; + // the data array is accessed to read out the entire line for write-back + // in case of sector cache we need to write bank only the modified sectors + cache_event ev(WRITE_BACK_REQUEST_SENT); + if (was_writeback_sent(events, ev)) { + unsigned data_cycles = ev.m_evicted_block.m_modified_size / port_width; + m_data_port_occupied_cycles += data_cycles; + } + } break; case SECTOR_MISS: case RESERVATION_FAIL: - // Does not consume any port bandwidth - break; - default: - assert(0); - break; - } + // Does not consume any port bandwidth + break; + default: + assert(0); + break; + } } -/// use the fill port -void baseline_cache::bandwidth_management::use_fill_port(mem_fetch *mf) -{ - // assume filling the entire line with the returned request - unsigned fill_cycles = m_config.get_atom_sz() / m_config.m_data_port_width; - m_fill_port_occupied_cycles += fill_cycles; +/// use the fill port +void baseline_cache::bandwidth_management::use_fill_port(mem_fetch *mf) { + // assume filling the entire line with the returned request + unsigned fill_cycles = m_config.get_atom_sz() / m_config.m_data_port_width; + m_fill_port_occupied_cycles += fill_cycles; } -/// called every cache cycle to free up the ports -void baseline_cache::bandwidth_management::replenish_port_bandwidth() -{ - if (m_data_port_occupied_cycles > 0) { - m_data_port_occupied_cycles -= 1; - } - assert(m_data_port_occupied_cycles >= 0); +/// called every cache cycle to free up the ports +void baseline_cache::bandwidth_management::replenish_port_bandwidth() { + if (m_data_port_occupied_cycles > 0) { + m_data_port_occupied_cycles -= 1; + } + assert(m_data_port_occupied_cycles >= 0); - if (m_fill_port_occupied_cycles > 0) { - m_fill_port_occupied_cycles -= 1; - } - assert(m_fill_port_occupied_cycles >= 0); + if (m_fill_port_occupied_cycles > 0) { + m_fill_port_occupied_cycles -= 1; + } + assert(m_fill_port_occupied_cycles >= 0); } -/// query for data port availability -bool baseline_cache::bandwidth_management::data_port_free() const -{ - return (m_data_port_occupied_cycles == 0); +/// query for data port availability +bool baseline_cache::bandwidth_management::data_port_free() const { + return (m_data_port_occupied_cycles == 0); } -/// query for fill port availability -bool baseline_cache::bandwidth_management::fill_port_free() const -{ - return (m_fill_port_occupied_cycles == 0); +/// query for fill port availability +bool baseline_cache::bandwidth_management::fill_port_free() const { + return (m_fill_port_occupied_cycles == 0); } /// Sends next request to lower level of memory -void baseline_cache::cycle(){ - if ( !m_miss_queue.empty() ) { - mem_fetch *mf = m_miss_queue.front(); - if ( !m_memport->full(mf->size(),mf->get_is_write()) ) { - m_miss_queue.pop_front(); - m_memport->push(mf); - } +void baseline_cache::cycle() { + if (!m_miss_queue.empty()) { + mem_fetch *mf = m_miss_queue.front(); + if (!m_memport->full(mf->size(), mf->get_is_write())) { + m_miss_queue.pop_front(); + m_memport->push(mf); } - bool data_port_busy = !m_bandwidth_management.data_port_free(); - bool fill_port_busy = !m_bandwidth_management.fill_port_free(); - m_stats.sample_cache_port_utility(data_port_busy, fill_port_busy); - m_bandwidth_management.replenish_port_bandwidth(); + } + bool data_port_busy = !m_bandwidth_management.data_port_free(); + bool fill_port_busy = !m_bandwidth_management.fill_port_free(); + m_stats.sample_cache_port_utility(data_port_busy, fill_port_busy); + m_bandwidth_management.replenish_port_bandwidth(); } -/// Interface for response from lower memory level (model bandwidth restictions in caller) -void baseline_cache::fill(mem_fetch *mf, unsigned time){ - - if(m_config.m_mshr_type == SECTOR_ASSOC) { - assert(mf->get_original_mf()); - extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); - assert( e != m_extra_mf_fields.end() ); +/// Interface for response from lower memory level (model bandwidth restictions +/// in caller) +void baseline_cache::fill(mem_fetch *mf, unsigned time) { + if (m_config.m_mshr_type == SECTOR_ASSOC) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = + m_extra_mf_fields.find(mf->get_original_mf()); + assert(e != m_extra_mf_fields.end()); e->second.pending_read--; - if(e->second.pending_read > 0) { - //wait for the other requests to come back - delete mf; - return; - } else { - mem_fetch *temp = mf; - mf = mf->get_original_mf(); - delete temp; - } - } - - extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); - assert( e != m_extra_mf_fields.end() ); - assert( e->second.m_valid ); - mf->set_data_size( e->second.m_data_size ); - mf->set_addr( e->second.m_addr ); - if ( m_config.m_alloc_policy == ON_MISS ) - m_tag_array->fill(e->second.m_cache_index,time,mf); - else if ( m_config.m_alloc_policy == ON_FILL ) { - m_tag_array->fill(e->second.m_block_addr,time,mf); - if(m_config.is_streaming()) - m_tag_array->remove_pending_line(mf); - } - else abort(); - bool has_atomic = false; - m_mshrs.mark_ready(e->second.m_block_addr, has_atomic); - if (has_atomic) { - assert(m_config.m_alloc_policy == ON_MISS); - cache_block_t* block = m_tag_array->get_block(e->second.m_cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()); // mark line as dirty for atomic operation + if (e->second.pending_read > 0) { + // wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; } - m_extra_mf_fields.erase(mf); - m_bandwidth_management.use_fill_port(mf); + } + + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); + assert(e != m_extra_mf_fields.end()); + assert(e->second.m_valid); + mf->set_data_size(e->second.m_data_size); + mf->set_addr(e->second.m_addr); + if (m_config.m_alloc_policy == ON_MISS) + m_tag_array->fill(e->second.m_cache_index, time, mf); + else if (m_config.m_alloc_policy == ON_FILL) { + m_tag_array->fill(e->second.m_block_addr, time, mf); + if (m_config.is_streaming()) m_tag_array->remove_pending_line(mf); + } else + abort(); + bool has_atomic = false; + m_mshrs.mark_ready(e->second.m_block_addr, has_atomic); + if (has_atomic) { + assert(m_config.m_alloc_policy == ON_MISS); + cache_block_t *block = m_tag_array->get_block(e->second.m_cache_index); + block->set_status(MODIFIED, + mf->get_access_sector_mask()); // mark line as dirty for + // atomic operation + } + m_extra_mf_fields.erase(mf); + m_bandwidth_management.use_fill_port(mf); } /// Checks if mf is waiting to be filled by lower memory level -bool baseline_cache::waiting_for_fill( mem_fetch *mf ){ - extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); - return e != m_extra_mf_fields.end(); +bool baseline_cache::waiting_for_fill(mem_fetch *mf) { + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); + return e != m_extra_mf_fields.end(); } -void baseline_cache::print(FILE *fp, unsigned &accesses, unsigned &misses) const{ - fprintf( fp, "Cache %s:\t", m_name.c_str() ); - m_tag_array->print(fp,accesses,misses); +void baseline_cache::print(FILE *fp, unsigned &accesses, + unsigned &misses) const { + fprintf(fp, "Cache %s:\t", m_name.c_str()); + m_tag_array->print(fp, accesses, misses); } -void baseline_cache::display_state( FILE *fp ) const{ - fprintf(fp,"Cache %s:\n", m_name.c_str() ); - m_mshrs.display(fp); - fprintf(fp,"\n"); +void baseline_cache::display_state(FILE *fp) const { + fprintf(fp, "Cache %s:\n", m_name.c_str()); + m_mshrs.display(fp); + fprintf(fp, "\n"); } /// Read miss handler without writeback -void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, std::list<cache_event> &events, bool read_only, bool wa){ - - bool wb=false; - evicted_block_info e; - send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, e, events, read_only, wa); +void baseline_cache::send_read_request(new_addr_type addr, + new_addr_type block_addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, bool &do_miss, + std::list<cache_event> &events, + bool read_only, bool wa) { + bool wb = false; + evicted_block_info e; + send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, e, + events, read_only, wa); } /// Read miss handler. Check MSHR hit or MSHR available -void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list<cache_event> &events, bool read_only, bool wa){ - - new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); - bool mshr_hit = m_mshrs.probe(mshr_addr); - bool mshr_avail = !m_mshrs.full(mshr_addr); - if ( mshr_hit && mshr_avail ) { - if(read_only) - m_tag_array->access(block_addr,time,cache_index,mf); - else - m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); - - m_mshrs.add(mshr_addr,mf); - do_miss = true; +void baseline_cache::send_read_request(new_addr_type addr, + new_addr_type block_addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, bool &do_miss, bool &wb, + evicted_block_info &evicted, + std::list<cache_event> &events, + bool read_only, bool wa) { + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if (mshr_hit && mshr_avail) { + if (read_only) + m_tag_array->access(block_addr, time, cache_index, mf); + else + m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf); - } else if ( !mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size) ) { - if(read_only) - m_tag_array->access(block_addr,time,cache_index,mf); - else - m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + m_mshrs.add(mshr_addr, mf); + do_miss = true; - m_mshrs.add(mshr_addr,mf); - if(m_config.is_streaming() && m_config.m_cache_type == SECTOR){ - m_tag_array->add_pending_line(mf); - } - m_extra_mf_fields[mf] = extra_mf_fields(mshr_addr,mf->get_addr(),cache_index, mf->get_data_size(), m_config); - mf->set_data_size( m_config.get_atom_sz() ); - mf->set_addr( mshr_addr ); - m_miss_queue.push_back(mf); - mf->set_status(m_miss_queue_status,time); - if(!wa) - events.push_back(cache_event(READ_REQUEST_SENT)); + } else if (!mshr_hit && mshr_avail && + (m_miss_queue.size() < m_config.m_miss_queue_size)) { + if (read_only) + m_tag_array->access(block_addr, time, cache_index, mf); + else + m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf); - do_miss = true; + m_mshrs.add(mshr_addr, mf); + if (m_config.is_streaming() && m_config.m_cache_type == SECTOR) { + m_tag_array->add_pending_line(mf); } - else if(mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); - else if (!mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); - else - assert(0); -} + m_extra_mf_fields[mf] = extra_mf_fields( + mshr_addr, mf->get_addr(), cache_index, mf->get_data_size(), m_config); + mf->set_data_size(m_config.get_atom_sz()); + mf->set_addr(mshr_addr); + m_miss_queue.push_back(mf); + mf->set_status(m_miss_queue_status, time); + if (!wa) events.push_back(cache_event(READ_REQUEST_SENT)); + do_miss = true; + } else if (mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); +} /// Sends write request to lower level memory (write or writeback) -void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned time, std::list<cache_event> &events){ - - events.push_back(request); - m_miss_queue.push_back(mf); - mf->set_status(m_miss_queue_status,time); +void data_cache::send_write_request(mem_fetch *mf, cache_event request, + unsigned time, + std::list<cache_event> &events) { + events.push_back(request); + m_miss_queue.push_back(mf); + mf->set_status(m_miss_queue_status, time); } - /****** Write-hit functions (Set by config file) ******/ /// Write-back hit: Mark block as modified -cache_request_status data_cache::wr_hit_wb(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){ - new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()); +cache_request_status data_cache::wr_hit_wb(new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events, + enum cache_request_status status) { + new_addr_type block_addr = m_config.block_addr(addr); + m_tag_array->access(block_addr, time, cache_index, mf); // update LRU state + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); - return HIT; + return HIT; } /// Write-through hit: Directly send request to lower level memory -cache_request_status data_cache::wr_hit_wt(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){ - if(miss_queue_full(0)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; // cannot handle request this cycle - } +cache_request_status data_cache::wr_hit_wt(new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events, + enum cache_request_status status) { + if (miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } - new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()); + new_addr_type block_addr = m_config.block_addr(addr); + m_tag_array->access(block_addr, time, cache_index, mf); // update LRU state + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); - // generate a write-through - send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); + // generate a write-through + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); - return HIT; + return HIT; } -/// Write-evict hit: Send request to lower level memory and invalidate corresponding block -cache_request_status data_cache::wr_hit_we(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){ - if(miss_queue_full(0)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; // cannot handle request this cycle - } +/// Write-evict hit: Send request to lower level memory and invalidate +/// corresponding block +cache_request_status data_cache::wr_hit_we(new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events, + enum cache_request_status status) { + if (miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } - // generate a write-through/evict - cache_block_t* block = m_tag_array->get_block(cache_index); - send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); + // generate a write-through/evict + cache_block_t *block = m_tag_array->get_block(cache_index); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); - // Invalidate block - block->set_status(INVALID, mf->get_access_sector_mask()); + // Invalidate block + block->set_status(INVALID, mf->get_access_sector_mask()); - return HIT; + return HIT; } /// Global write-evict, local write-back: Useful for private caches -enum cache_request_status data_cache::wr_hit_global_we_local_wb(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){ - bool evict = (mf->get_access_type() == GLOBAL_ACC_W); // evict a line that hits on global memory write - if(evict) - return wr_hit_we(addr, cache_index, mf, time, events, status); // Write-evict - else - return wr_hit_wb(addr, cache_index, mf, time, events, status); // Write-back +enum cache_request_status data_cache::wr_hit_global_we_local_wb( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + bool evict = (mf->get_access_type() == + GLOBAL_ACC_W); // evict a line that hits on global memory write + if (evict) + return wr_hit_we(addr, cache_index, mf, time, events, + status); // Write-evict + else + return wr_hit_wb(addr, cache_index, mf, time, events, + status); // Write-back } /****** Write-miss functions (Set by config file) ******/ /// Write-allocate miss: Send write request to lower level memory // and send a read request for the same block -enum cache_request_status -data_cache::wr_miss_wa_naive( new_addr_type addr, - unsigned cache_index, mem_fetch *mf, - unsigned time, std::list<cache_event> &events, - enum cache_request_status status ) -{ - new_addr_type block_addr = m_config.block_addr(addr); - new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); +enum cache_request_status data_cache::wr_miss_wa_naive( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); - // Write allocate, maximum 3 requests (write miss, read request, write back request) - // Conservatively ensure the worst-case request can be handled this cycle - bool mshr_hit = m_mshrs.probe(mshr_addr); - bool mshr_avail = !m_mshrs.full(mshr_addr); - if(miss_queue_full(2) - || (!(mshr_hit && mshr_avail) - && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { - //check what is the exactly the failure reason - if(miss_queue_full(2) ) - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - else if(mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); - else if (!mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); - else - assert(0); + // Write allocate, maximum 3 requests (write miss, read request, write back + // request) Conservatively ensure the worst-case request can be handled this + // cycle + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if (miss_queue_full(2) || + (!(mshr_hit && mshr_avail) && + !(!mshr_hit && mshr_avail && + (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + // check what is the exactly the failure reason + if (miss_queue_full(2)) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if (mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); - return RESERVATION_FAIL; - } + return RESERVATION_FAIL; + } - send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); - // Tries to send write allocate request, returns true on success and false on failure - //if(!send_write_allocate(mf, addr, block_addr, cache_index, time, events)) - // return RESERVATION_FAIL; + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); + // Tries to send write allocate request, returns true on success and false on + // failure + // if(!send_write_allocate(mf, addr, block_addr, cache_index, time, events)) + // return RESERVATION_FAIL; - const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, - mf->get_addr(), - m_config.get_atom_sz(), - false, // Now performing a read - mf->get_access_warp_mask(), - mf->get_access_byte_mask(), - mf->get_access_sector_mask(), - m_gpu->gpgpu_ctx); + const mem_access_t *ma = + new mem_access_t(m_wr_alloc_type, mf->get_addr(), m_config.get_atom_sz(), + false, // Now performing a read + mf->get_access_warp_mask(), mf->get_access_byte_mask(), + mf->get_access_sector_mask(), m_gpu->gpgpu_ctx); - mem_fetch *n_mf = new mem_fetch( *ma, - NULL, - mf->get_ctrl_size(), - mf->get_wid(), - mf->get_sid(), - mf->get_tpc(), - mf->get_mem_config(), - m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); + mem_fetch *n_mf = + new mem_fetch(*ma, NULL, mf->get_ctrl_size(), mf->get_wid(), + mf->get_sid(), mf->get_tpc(), mf->get_mem_config(), + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); - bool do_miss = false; - bool wb = false; - evicted_block_info evicted; + bool do_miss = false; + bool wb = false; + evicted_block_info evicted; - // Send read request resulting from write miss - send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, - evicted, events, false, true); + // Send read request resulting from write miss + send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, + evicted, events, false, true); - events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); - if( do_miss ){ - // If evicted block is modified and not a write-through - // (already modified lower level) - if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { - assert(status == MISS); //SECTOR_MISS and HIT_RESERVED should not send write back - mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,evicted.m_modified_size,true,m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); - send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); - } - return MISS; + if (do_miss) { + // If evicted block is modified and not a write-through + // (already modified lower level) + if (wb && (m_config.m_write_policy != WRITE_THROUGH)) { + assert(status == + MISS); // SECTOR_MISS and HIT_RESERVED should not send write back + mem_fetch *wb = m_memfetch_creator->alloc( + evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), + time, events); } + return MISS; + } - return RESERVATION_FAIL; + return RESERVATION_FAIL; } +enum cache_request_status data_cache::wr_miss_wa_fetch_on_write( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); -enum cache_request_status -data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, - unsigned cache_index, mem_fetch *mf, - unsigned time, std::list<cache_event> &events, - enum cache_request_status status ) -{ - new_addr_type block_addr = m_config.block_addr(addr); - new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); - - if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) - { - //if the request writes to the whole cache line/sector, then, write and set cache line Modified. - //and no need to send read request to memory or reserve mshr - - if(miss_queue_full(0)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; // cannot handle request this cycle - } - - bool wb = false; - evicted_block_info evicted; + if (mf->get_access_byte_mask().count() == m_config.get_atom_sz()) { + // if the request writes to the whole cache line/sector, then, write and set + // cache line Modified. and no need to send read request to memory or reserve + // mshr - cache_request_status status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); - assert(status != HIT); - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()); - if(status == HIT_RESERVED) - block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + if (miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } - if( status != RESERVATION_FAIL ){ - // If evicted block is modified and not a write-through - // (already modified lower level) - if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { - mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,evicted.m_modified_size,true,m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); - send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); - } - return MISS; - } - return RESERVATION_FAIL; - } - else - { - bool mshr_hit = m_mshrs.probe(mshr_addr); - bool mshr_avail = !m_mshrs.full(mshr_addr); - if(miss_queue_full(1) - || (!(mshr_hit && mshr_avail) - && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { - //check what is the exactly the failure reason - if(miss_queue_full(1) ) - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - else if(mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); - else if (!mshr_hit && !mshr_avail) - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); - else - assert(0); + bool wb = false; + evicted_block_info evicted; - return RESERVATION_FAIL; - } + cache_request_status status = + m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf); + assert(status != HIT); + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if (status == HIT_RESERVED) + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + if (status != RESERVATION_FAIL) { + // If evicted block is modified and not a write-through + // (already modified lower level) + if (wb && (m_config.m_write_policy != WRITE_THROUGH)) { + mem_fetch *wb = m_memfetch_creator->alloc( + evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), + time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } else { + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if (miss_queue_full(1) || + (!(mshr_hit && mshr_avail) && + !(!mshr_hit && mshr_avail && + (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + // check what is the exactly the failure reason + if (miss_queue_full(1)) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if (mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); - //prevent Write - Read - Write in pending mshr - //allowing another write will override the value of the first write, and the pending read request will read incorrect result from the second write - if(m_mshrs.probe(mshr_addr) && m_mshrs.is_read_after_write_pending(mshr_addr) && mf->is_write()) - { - //assert(0); - m_stats.inc_fail_stats(mf->get_access_type(), MSHR_RW_PENDING); - return RESERVATION_FAIL; - } + return RESERVATION_FAIL; + } - const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, - mf->get_addr(), - m_config.get_atom_sz(), - false, // Now performing a read - mf->get_access_warp_mask(), - mf->get_access_byte_mask(), - mf->get_access_sector_mask(), - m_gpu->gpgpu_ctx); + // prevent Write - Read - Write in pending mshr + // allowing another write will override the value of the first write, and + // the pending read request will read incorrect result from the second write + if (m_mshrs.probe(mshr_addr) && + m_mshrs.is_read_after_write_pending(mshr_addr) && mf->is_write()) { + // assert(0); + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_RW_PENDING); + return RESERVATION_FAIL; + } - mem_fetch *n_mf = new mem_fetch( *ma, - NULL, - mf->get_ctrl_size(), - mf->get_wid(), - mf->get_sid(), - mf->get_tpc(), - mf->get_mem_config(), - m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle, - NULL, - mf); + const mem_access_t *ma = new mem_access_t( + m_wr_alloc_type, mf->get_addr(), m_config.get_atom_sz(), + false, // Now performing a read + mf->get_access_warp_mask(), mf->get_access_byte_mask(), + mf->get_access_sector_mask(), m_gpu->gpgpu_ctx); + mem_fetch *n_mf = new mem_fetch( + *ma, NULL, mf->get_ctrl_size(), mf->get_wid(), mf->get_sid(), + mf->get_tpc(), mf->get_mem_config(), + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, NULL, mf); - new_addr_type block_addr = m_config.block_addr(addr); - bool do_miss = false; - bool wb = false; - evicted_block_info evicted; - send_read_request( addr, - block_addr, - cache_index, - n_mf, time, do_miss, wb, evicted, events, false, true); + new_addr_type block_addr = m_config.block_addr(addr); + bool do_miss = false; + bool wb = false; + evicted_block_info evicted; + send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, + evicted, events, false, true); - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_modified_on_fill(true, mf->get_access_sector_mask()); + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); - events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); - if( do_miss ){ - // If evicted block is modified and not a write-through - // (already modified lower level) - if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ - mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,evicted.m_modified_size,true,m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); - send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); - } - return MISS; - } - return RESERVATION_FAIL; - } + if (do_miss) { + // If evicted block is modified and not a write-through + // (already modified lower level) + if (wb && (m_config.m_write_policy != WRITE_THROUGH)) { + mem_fetch *wb = m_memfetch_creator->alloc( + evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), + time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } } -enum cache_request_status -data_cache::wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, - unsigned cache_index, mem_fetch *mf, - unsigned time, std::list<cache_event> &events, - enum cache_request_status status ) -{ - - new_addr_type block_addr = m_config.block_addr(addr); +enum cache_request_status data_cache::wr_miss_wa_lazy_fetch_on_read( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + new_addr_type block_addr = m_config.block_addr(addr); - //if the request writes to the whole cache line/sector, then, write and set cache line Modified. - //and no need to send read request to memory or reserve mshr + // if the request writes to the whole cache line/sector, then, write and set + // cache line Modified. and no need to send read request to memory or reserve + // mshr - if(miss_queue_full(0)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; // cannot handle request this cycle - } + if (miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } - bool wb = false; - evicted_block_info evicted; + bool wb = false; + evicted_block_info evicted; - cache_request_status m_status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); - assert(m_status != HIT); - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()); - if(m_status == HIT_RESERVED) { - block->set_ignore_on_fill(true, mf->get_access_sector_mask()); - block->set_modified_on_fill(true, mf->get_access_sector_mask()); - } + cache_request_status m_status = + m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf); + assert(m_status != HIT); + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if (m_status == HIT_RESERVED) { + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); + } - if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) - { - block->set_m_readable(true, mf->get_access_sector_mask()); - } else - { - block->set_m_readable(false, mf->get_access_sector_mask()); - } + if (mf->get_access_byte_mask().count() == m_config.get_atom_sz()) { + block->set_m_readable(true, mf->get_access_sector_mask()); + } else { + block->set_m_readable(false, mf->get_access_sector_mask()); + } - if( m_status != RESERVATION_FAIL ){ - // If evicted block is modified and not a write-through - // (already modified lower level) - if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { - mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,evicted.m_modified_size,true,m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); - send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); - } - return MISS; - } - return RESERVATION_FAIL; + if (m_status != RESERVATION_FAIL) { + // If evicted block is modified and not a write-through + // (already modified lower level) + if (wb && (m_config.m_write_policy != WRITE_THROUGH)) { + mem_fetch *wb = m_memfetch_creator->alloc( + evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), + time, events); + } + return MISS; + } + return RESERVATION_FAIL; } /// No write-allocate miss: Simply send write request to lower level memory -enum cache_request_status -data_cache::wr_miss_no_wa( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ) -{ - if(miss_queue_full(0)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; // cannot handle request this cycle - } +enum cache_request_status data_cache::wr_miss_no_wa( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + if (miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + // on miss, generate write through (no write buffering -- too many threads for + // that) + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); - // on miss, generate write through (no write buffering -- too many threads for that) - send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); - - return MISS; + return MISS; } /****** Read hit functions (Set by config file) ******/ /// Baseline read hit: Update LRU status of block. // Special case for atomic instructions -> Mark block as modified -enum cache_request_status -data_cache::rd_hit_base( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ) -{ - new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index,mf); - // Atomics treated as global read/write requests - Perform read, mark line as - // MODIFIED - if(mf->isatomic()){ - assert(mf->get_access_type() == GLOBAL_ACC_R); - cache_block_t* block = m_tag_array->get_block(cache_index); - block->set_status(MODIFIED, mf->get_access_sector_mask()) ; // mark line as dirty - } - return HIT; +enum cache_request_status data_cache::rd_hit_base( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + new_addr_type block_addr = m_config.block_addr(addr); + m_tag_array->access(block_addr, time, cache_index, mf); + // Atomics treated as global read/write requests - Perform read, mark line as + // MODIFIED + if (mf->isatomic()) { + assert(mf->get_access_type() == GLOBAL_ACC_R); + cache_block_t *block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, + mf->get_access_sector_mask()); // mark line as dirty + } + return HIT; } /****** Read miss functions (Set by config file) ******/ /// Baseline read miss: Send read request to lower level memory, // perform write-back as necessary -enum cache_request_status -data_cache::rd_miss_base( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ){ - if(miss_queue_full(1)) { - // cannot handle request this cycle - // (might need to generate two requests) - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; - } +enum cache_request_status data_cache::rd_miss_base( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status) { + if (miss_queue_full(1)) { + // cannot handle request this cycle + // (might need to generate two requests) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; + } - new_addr_type block_addr = m_config.block_addr(addr); - bool do_miss = false; - bool wb = false; - evicted_block_info evicted; - send_read_request( addr, - block_addr, - cache_index, - mf, time, do_miss, wb, evicted, events, false, false); + new_addr_type block_addr = m_config.block_addr(addr); + bool do_miss = false; + bool wb = false; + evicted_block_info evicted; + send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, + evicted, events, false, false); - if( do_miss ){ - // If evicted block is modified and not a write-through - // (already modified lower level) - if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ - mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,evicted.m_modified_size,true,m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle); - send_write_request(wb, WRITE_BACK_REQUEST_SENT, time, events); - } - return MISS; + if (do_miss) { + // If evicted block is modified and not a write-through + // (already modified lower level) + if (wb && (m_config.m_write_policy != WRITE_THROUGH)) { + mem_fetch *wb = m_memfetch_creator->alloc( + evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + send_write_request(wb, WRITE_BACK_REQUEST_SENT, time, events); } - return RESERVATION_FAIL; + return MISS; + } + return RESERVATION_FAIL; } /// Access cache for read_only_cache: returns RESERVATION_FAIL if // request could not be accepted (for any reason) -enum cache_request_status -read_only_cache::access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ) -{ - assert( mf->get_data_size() <= m_config.get_atom_sz()); - assert(m_config.m_write_policy == READ_ONLY); - assert(!mf->get_is_write()); - new_addr_type block_addr = m_config.block_addr(addr); - unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tag_array->probe(block_addr,cache_index,mf); - enum cache_request_status cache_status = RESERVATION_FAIL; +enum cache_request_status read_only_cache::access( + new_addr_type addr, mem_fetch *mf, unsigned time, + std::list<cache_event> &events) { + assert(mf->get_data_size() <= m_config.get_atom_sz()); + assert(m_config.m_write_policy == READ_ONLY); + assert(!mf->get_is_write()); + new_addr_type block_addr = m_config.block_addr(addr); + unsigned cache_index = (unsigned)-1; + enum cache_request_status status = + m_tag_array->probe(block_addr, cache_index, mf); + enum cache_request_status cache_status = RESERVATION_FAIL; - if ( status == HIT ) { - cache_status = m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state - }else if ( status != RESERVATION_FAIL ) { - if(!miss_queue_full(0)){ - bool do_miss=false; - send_read_request(addr, block_addr, cache_index, mf, time, do_miss, events, true, false); - if(do_miss) - cache_status = MISS; - else - cache_status = RESERVATION_FAIL; - }else{ - cache_status = RESERVATION_FAIL; - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - } - }else { - m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); + if (status == HIT) { + cache_status = m_tag_array->access(block_addr, time, cache_index, + mf); // update LRU state + } else if (status != RESERVATION_FAIL) { + if (!miss_queue_full(0)) { + bool do_miss = false; + send_read_request(addr, block_addr, cache_index, mf, time, do_miss, + events, true, false); + if (do_miss) + cache_status = MISS; + else + cache_status = RESERVATION_FAIL; + } else { + cache_status = RESERVATION_FAIL; + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); } + } else { + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); + } - m_stats.inc_stats(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); - m_stats.inc_stats_pw(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); - return cache_status; + m_stats.inc_stats(mf->get_access_type(), + m_stats.select_stats_status(status, cache_status)); + m_stats.inc_stats_pw(mf->get_access_type(), + m_stats.select_stats_status(status, cache_status)); + return cache_status; } //! A general function that takes the result of a tag_array probe // and performs the correspding functions based on the cache configuration // The access fucntion calls this function -enum cache_request_status -data_cache::process_tag_probe( bool wr, - enum cache_request_status probe_status, - new_addr_type addr, - unsigned cache_index, - mem_fetch* mf, - unsigned time, - std::list<cache_event>& events ) -{ - // Each function pointer ( m_[rd/wr]_[hit/miss] ) is set in the - // data_cache constructor to reflect the corresponding cache configuration - // options. Function pointers were used to avoid many long conditional - // branches resulting from many cache configuration options. - cache_request_status access_status = probe_status; - if(wr){ // Write - if(probe_status == HIT){ - access_status = (this->*m_wr_hit)( addr, - cache_index, - mf, time, events, probe_status ); - }else if ( (probe_status != RESERVATION_FAIL) || (probe_status == RESERVATION_FAIL && m_config.m_write_alloc_policy == NO_WRITE_ALLOCATE) ) { - access_status = (this->*m_wr_miss)( addr, - cache_index, - mf, time, events, probe_status ); - }else { - //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) - m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); - } - }else{ // Read - if(probe_status == HIT){ - access_status = (this->*m_rd_hit)( addr, - cache_index, - mf, time, events, probe_status ); - }else if ( probe_status != RESERVATION_FAIL ) { - access_status = (this->*m_rd_miss)( addr, - cache_index, - mf, time, events, probe_status ); - }else { - //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) - m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); - } +enum cache_request_status data_cache::process_tag_probe( + bool wr, enum cache_request_status probe_status, new_addr_type addr, + unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events) { + // Each function pointer ( m_[rd/wr]_[hit/miss] ) is set in the + // data_cache constructor to reflect the corresponding cache configuration + // options. Function pointers were used to avoid many long conditional + // branches resulting from many cache configuration options. + cache_request_status access_status = probe_status; + if (wr) { // Write + if (probe_status == HIT) { + access_status = + (this->*m_wr_hit)(addr, cache_index, mf, time, events, probe_status); + } else if ((probe_status != RESERVATION_FAIL) || + (probe_status == RESERVATION_FAIL && + m_config.m_write_alloc_policy == NO_WRITE_ALLOCATE)) { + access_status = + (this->*m_wr_miss)(addr, cache_index, mf, time, events, probe_status); + } else { + // the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all + // lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); + } + } else { // Read + if (probe_status == HIT) { + access_status = + (this->*m_rd_hit)(addr, cache_index, mf, time, events, probe_status); + } else if (probe_status != RESERVATION_FAIL) { + access_status = + (this->*m_rd_miss)(addr, cache_index, mf, time, events, probe_status); + } else { + // the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all + // lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } + } - m_bandwidth_management.use_data_port(mf, access_status, events); - return access_status; + m_bandwidth_management.use_data_port(mf, access_status, events); + return access_status; } // Both the L1 and L2 currently use the same access function. @@ -1645,51 +1639,41 @@ data_cache::process_tag_probe( bool wr, // of caching policies. // Both the L1 and L2 override this function to provide a means of // performing actions specific to each cache when such actions are implemnted. -enum cache_request_status -data_cache::access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ) -{ - - assert( mf->get_data_size() <= m_config.get_atom_sz()); - bool wr = mf->get_is_write(); - new_addr_type block_addr = m_config.block_addr(addr); - unsigned cache_index = (unsigned)-1; - enum cache_request_status probe_status - = m_tag_array->probe( block_addr, cache_index, mf, true); - enum cache_request_status access_status - = process_tag_probe( wr, probe_status, addr, cache_index, mf, time, events ); - m_stats.inc_stats(mf->get_access_type(), - m_stats.select_stats_status(probe_status, access_status)); - m_stats.inc_stats_pw(mf->get_access_type(), - m_stats.select_stats_status(probe_status, access_status)); - return access_status; +enum cache_request_status data_cache::access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) { + assert(mf->get_data_size() <= m_config.get_atom_sz()); + bool wr = mf->get_is_write(); + new_addr_type block_addr = m_config.block_addr(addr); + unsigned cache_index = (unsigned)-1; + enum cache_request_status probe_status = + m_tag_array->probe(block_addr, cache_index, mf, true); + enum cache_request_status access_status = + process_tag_probe(wr, probe_status, addr, cache_index, mf, time, events); + m_stats.inc_stats(mf->get_access_type(), + m_stats.select_stats_status(probe_status, access_status)); + m_stats.inc_stats_pw(mf->get_access_type(), m_stats.select_stats_status( + probe_status, access_status)); + return access_status; } /// This is meant to model the first level data cache in Fermi. /// It is write-evict (global) or write-back (local) at the /// granularity of individual blocks (Set by GPGPU-Sim configuration file) /// (the policy used in fermi according to the CUDA manual) -enum cache_request_status -l1_cache::access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ) -{ - return data_cache::access( addr, mf, time, events ); +enum cache_request_status l1_cache::access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) { + return data_cache::access(addr, mf, time, events); } // The l2 cache access function calls the base data_cache access // implementation. When the L2 needs to diverge from L1, L2 specific // changes should be made here. -enum cache_request_status -l2_cache::access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ) -{ - return data_cache::access( addr, mf, time, events ); +enum cache_request_status l2_cache::access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) { + return data_cache::access(addr, mf, time, events); } /// Access function for tex_cache @@ -1697,141 +1681,144 @@ l2_cache::access( new_addr_type addr, /// otherwise returns HIT_RESERVED or MISS; NOTE: *never* returns HIT /// since unlike a normal CPU cache, a "HIT" in texture cache does not /// mean the data is ready (still need to get through fragment fifo) -enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf, - unsigned time, std::list<cache_event> &events ) -{ - if ( m_fragment_fifo.full() || m_request_fifo.full() || m_rob.full() ) - return RESERVATION_FAIL; +enum cache_request_status tex_cache::access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) { + if (m_fragment_fifo.full() || m_request_fifo.full() || m_rob.full()) + return RESERVATION_FAIL; - assert( mf->get_data_size() <= m_config.get_line_sz()); + assert(mf->get_data_size() <= m_config.get_line_sz()); - // at this point, we will accept the request : access tags and immediately allocate line - new_addr_type block_addr = m_config.block_addr(addr); - unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tags.access(block_addr,time,cache_index,mf); - enum cache_request_status cache_status = RESERVATION_FAIL; - assert( status != RESERVATION_FAIL ); - assert( status != HIT_RESERVED ); // as far as tags are concerned: HIT or MISS - m_fragment_fifo.push( fragment_entry(mf,cache_index,status==MISS,mf->get_data_size()) ); - if ( status == MISS ) { - // we need to send a memory request... - unsigned rob_index = m_rob.push( rob_entry(cache_index, mf, block_addr) ); - m_extra_mf_fields[mf] = extra_mf_fields(rob_index, m_config); - mf->set_data_size(m_config.get_line_sz()); - m_tags.fill(cache_index,time,mf); // mark block as valid - m_request_fifo.push(mf); - mf->set_status(m_request_queue_status,time); - events.push_back(cache_event(READ_REQUEST_SENT)); - cache_status = MISS; - } else { - // the value *will* *be* in the cache already - cache_status = HIT_RESERVED; - } - m_stats.inc_stats(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); - m_stats.inc_stats_pw(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); - return cache_status; + // at this point, we will accept the request : access tags and immediately + // allocate line + new_addr_type block_addr = m_config.block_addr(addr); + unsigned cache_index = (unsigned)-1; + enum cache_request_status status = + m_tags.access(block_addr, time, cache_index, mf); + enum cache_request_status cache_status = RESERVATION_FAIL; + assert(status != RESERVATION_FAIL); + assert(status != HIT_RESERVED); // as far as tags are concerned: HIT or MISS + m_fragment_fifo.push( + fragment_entry(mf, cache_index, status == MISS, mf->get_data_size())); + if (status == MISS) { + // we need to send a memory request... + unsigned rob_index = m_rob.push(rob_entry(cache_index, mf, block_addr)); + m_extra_mf_fields[mf] = extra_mf_fields(rob_index, m_config); + mf->set_data_size(m_config.get_line_sz()); + m_tags.fill(cache_index, time, mf); // mark block as valid + m_request_fifo.push(mf); + mf->set_status(m_request_queue_status, time); + events.push_back(cache_event(READ_REQUEST_SENT)); + cache_status = MISS; + } else { + // the value *will* *be* in the cache already + cache_status = HIT_RESERVED; + } + m_stats.inc_stats(mf->get_access_type(), + m_stats.select_stats_status(status, cache_status)); + m_stats.inc_stats_pw(mf->get_access_type(), + m_stats.select_stats_status(status, cache_status)); + return cache_status; } -void tex_cache::cycle(){ - // send next request to lower level of memory - if ( !m_request_fifo.empty() ) { - mem_fetch *mf = m_request_fifo.peek(); - if ( !m_memport->full(mf->get_ctrl_size(),false) ) { - m_request_fifo.pop(); - m_memport->push(mf); - } +void tex_cache::cycle() { + // send next request to lower level of memory + if (!m_request_fifo.empty()) { + mem_fetch *mf = m_request_fifo.peek(); + if (!m_memport->full(mf->get_ctrl_size(), false)) { + m_request_fifo.pop(); + m_memport->push(mf); } - // read ready lines from cache - if ( !m_fragment_fifo.empty() && !m_result_fifo.full() ) { - const fragment_entry &e = m_fragment_fifo.peek(); - if ( e.m_miss ) { - // check head of reorder buffer to see if data is back from memory - unsigned rob_index = m_rob.next_pop_index(); - const rob_entry &r = m_rob.peek(rob_index); - assert( r.m_request == e.m_request ); - //assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) ); - if ( r.m_ready ) { - assert( r.m_index == e.m_cache_index ); - m_cache[r.m_index].m_valid = true; - m_cache[r.m_index].m_block_addr = r.m_block_addr; - m_result_fifo.push(e.m_request); - m_rob.pop(); - m_fragment_fifo.pop(); - } - } else { - // hit: - assert( m_cache[e.m_cache_index].m_valid ); - assert( m_cache[e.m_cache_index].m_block_addr - == m_config.block_addr(e.m_request->get_addr()) ); - m_result_fifo.push( e.m_request ); - m_fragment_fifo.pop(); - } + } + // read ready lines from cache + if (!m_fragment_fifo.empty() && !m_result_fifo.full()) { + const fragment_entry &e = m_fragment_fifo.peek(); + if (e.m_miss) { + // check head of reorder buffer to see if data is back from memory + unsigned rob_index = m_rob.next_pop_index(); + const rob_entry &r = m_rob.peek(rob_index); + assert(r.m_request == e.m_request); + // assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) + // ); + if (r.m_ready) { + assert(r.m_index == e.m_cache_index); + m_cache[r.m_index].m_valid = true; + m_cache[r.m_index].m_block_addr = r.m_block_addr; + m_result_fifo.push(e.m_request); + m_rob.pop(); + m_fragment_fifo.pop(); + } + } else { + // hit: + assert(m_cache[e.m_cache_index].m_valid); + assert(m_cache[e.m_cache_index].m_block_addr == + m_config.block_addr(e.m_request->get_addr())); + m_result_fifo.push(e.m_request); + m_fragment_fifo.pop(); } + } } /// Place returning cache block into reorder buffer -void tex_cache::fill( mem_fetch *mf, unsigned time ) -{ - if(m_config.m_mshr_type == SECTOR_TEX_FIFO) { - assert(mf->get_original_mf()); - extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); - assert( e != m_extra_mf_fields.end() ); +void tex_cache::fill(mem_fetch *mf, unsigned time) { + if (m_config.m_mshr_type == SECTOR_TEX_FIFO) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = + m_extra_mf_fields.find(mf->get_original_mf()); + assert(e != m_extra_mf_fields.end()); e->second.pending_read--; - if(e->second.pending_read > 0) { - //wait for the other requests to come back - delete mf; - return; - } else { - mem_fetch *temp = mf; - mf = mf->get_original_mf(); - delete temp; - } - } + if (e->second.pending_read > 0) { + // wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; + } + } - extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); - assert( e != m_extra_mf_fields.end() ); - assert( e->second.m_valid ); - assert( !m_rob.empty() ); - mf->set_status(m_rob_status,time); + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); + assert(e != m_extra_mf_fields.end()); + assert(e->second.m_valid); + assert(!m_rob.empty()); + mf->set_status(m_rob_status, time); - unsigned rob_index = e->second.m_rob_index; - rob_entry &r = m_rob.peek(rob_index); - assert( !r.m_ready ); - r.m_ready = true; - r.m_time = time; - assert( r.m_block_addr == m_config.block_addr(mf->get_addr()) ); + unsigned rob_index = e->second.m_rob_index; + rob_entry &r = m_rob.peek(rob_index); + assert(!r.m_ready); + r.m_ready = true; + r.m_time = time; + assert(r.m_block_addr == m_config.block_addr(mf->get_addr())); } -void tex_cache::display_state( FILE *fp ) const -{ - fprintf(fp,"%s (texture cache) state:\n", m_name.c_str() ); - fprintf(fp,"fragment fifo entries = %u / %u\n", - m_fragment_fifo.size(), m_fragment_fifo.capacity() ); - fprintf(fp,"reorder buffer entries = %u / %u\n", - m_rob.size(), m_rob.capacity() ); - fprintf(fp,"request fifo entries = %u / %u\n", - m_request_fifo.size(), m_request_fifo.capacity() ); - if ( !m_rob.empty() ) - fprintf(fp,"reorder buffer contents:\n"); - for ( int n=m_rob.size()-1; n>=0; n-- ) { - unsigned index = (m_rob.next_pop_index() + n)%m_rob.capacity(); - const rob_entry &r = m_rob.peek(index); - fprintf(fp, "tex rob[%3d] : %s ", - index, (r.m_ready?"ready ":"pending") ); - if ( r.m_ready ) - fprintf(fp,"@%6u", r.m_time ); - else - fprintf(fp," "); - fprintf(fp,"[idx=%4u]",r.m_index); - r.m_request->print(fp,false); - } - if ( !m_fragment_fifo.empty() ) { - fprintf(fp,"fragment fifo (oldest) :"); - fragment_entry &f = m_fragment_fifo.peek(); - fprintf(fp,"%s: ", f.m_miss?"miss":"hit "); - f.m_request->print(fp,false); - } +void tex_cache::display_state(FILE *fp) const { + fprintf(fp, "%s (texture cache) state:\n", m_name.c_str()); + fprintf(fp, "fragment fifo entries = %u / %u\n", m_fragment_fifo.size(), + m_fragment_fifo.capacity()); + fprintf(fp, "reorder buffer entries = %u / %u\n", m_rob.size(), + m_rob.capacity()); + fprintf(fp, "request fifo entries = %u / %u\n", m_request_fifo.size(), + m_request_fifo.capacity()); + if (!m_rob.empty()) fprintf(fp, "reorder buffer contents:\n"); + for (int n = m_rob.size() - 1; n >= 0; n--) { + unsigned index = (m_rob.next_pop_index() + n) % m_rob.capacity(); + const rob_entry &r = m_rob.peek(index); + fprintf(fp, "tex rob[%3d] : %s ", index, + (r.m_ready ? "ready " : "pending")); + if (r.m_ready) + fprintf(fp, "@%6u", r.m_time); + else + fprintf(fp, " "); + fprintf(fp, "[idx=%4u]", r.m_index); + r.m_request->print(fp, false); + } + if (!m_fragment_fifo.empty()) { + fprintf(fp, "fragment fifo (oldest) :"); + fragment_entry &f = m_fragment_fifo.peek(); + fprintf(fp, "%s: ", f.m_miss ? "miss" : "hit "); + f.m_request->print(fp, false); + } } /******************************************************************************************************************************************/ - diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h index 6f39221..647ed7a 100644 --- a/src/gpgpu-sim/gpu-cache.h +++ b/src/gpgpu-sim/gpu-cache.h @@ -7,487 +7,444 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef GPU_CACHE_H #define GPU_CACHE_H #include <stdio.h> #include <stdlib.h> -#include "gpu-misc.h" -#include "mem_fetch.h" #include "../abstract_hardware_model.h" #include "../tr1_hash_map.h" +#include "gpu-misc.h" +#include "mem_fetch.h" -#include "addrdec.h" #include <iostream> +#include "addrdec.h" #define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 -enum cache_block_state { - INVALID=0, - RESERVED, - VALID, - MODIFIED -}; +enum cache_block_state { INVALID = 0, RESERVED, VALID, MODIFIED }; enum cache_request_status { - HIT = 0, - HIT_RESERVED, - MISS, - RESERVATION_FAIL, - SECTOR_MISS, - NUM_CACHE_REQUEST_STATUS + HIT = 0, + HIT_RESERVED, + MISS, + RESERVATION_FAIL, + SECTOR_MISS, + NUM_CACHE_REQUEST_STATUS }; enum cache_reservation_fail_reason { - LINE_ALLOC_FAIL= 0,// all line are reserved - MISS_QUEUE_FULL, // MISS queue (i.e. interconnect or DRAM) is full - MSHR_ENRTY_FAIL, - MSHR_MERGE_ENRTY_FAIL, - MSHR_RW_PENDING, - NUM_CACHE_RESERVATION_FAIL_STATUS + LINE_ALLOC_FAIL = 0, // all line are reserved + MISS_QUEUE_FULL, // MISS queue (i.e. interconnect or DRAM) is full + MSHR_ENRTY_FAIL, + MSHR_MERGE_ENRTY_FAIL, + MSHR_RW_PENDING, + NUM_CACHE_RESERVATION_FAIL_STATUS }; enum cache_event_type { - WRITE_BACK_REQUEST_SENT, - READ_REQUEST_SENT, - WRITE_REQUEST_SENT, - WRITE_ALLOCATE_SENT + WRITE_BACK_REQUEST_SENT, + READ_REQUEST_SENT, + WRITE_REQUEST_SENT, + WRITE_ALLOCATE_SENT }; struct evicted_block_info { - new_addr_type m_block_addr; - unsigned m_modified_size; - evicted_block_info() { - m_block_addr = 0; - m_modified_size = 0; - } - void set_info(new_addr_type block_addr, unsigned modified_size){ - m_block_addr = block_addr; - m_modified_size = modified_size; - } + new_addr_type m_block_addr; + unsigned m_modified_size; + evicted_block_info() { + m_block_addr = 0; + m_modified_size = 0; + } + void set_info(new_addr_type block_addr, unsigned modified_size) { + m_block_addr = block_addr; + m_modified_size = modified_size; + } }; struct cache_event { - enum cache_event_type m_cache_event_type; - evicted_block_info m_evicted_block; //if it was write_back event, fill the the evicted block info + enum cache_event_type m_cache_event_type; + evicted_block_info m_evicted_block; // if it was write_back event, fill the + // the evicted block info - cache_event(enum cache_event_type m_cache_event){ - m_cache_event_type = m_cache_event; - } + cache_event(enum cache_event_type m_cache_event) { + m_cache_event_type = m_cache_event; + } - cache_event(enum cache_event_type cache_event, evicted_block_info evicted_block){ - m_cache_event_type = cache_event; - m_evicted_block = evicted_block; - } + cache_event(enum cache_event_type cache_event, + evicted_block_info evicted_block) { + m_cache_event_type = cache_event; + m_evicted_block = evicted_block; + } }; -const char * cache_request_status_str(enum cache_request_status status); +const char *cache_request_status_str(enum cache_request_status status); struct cache_block_t { - cache_block_t() - { - m_tag=0; - m_block_addr=0; - } - - virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) = 0; - virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) = 0; + cache_block_t() { + m_tag = 0; + m_block_addr = 0; + } - virtual bool is_invalid_line() = 0; - virtual bool is_valid_line() = 0; - virtual bool is_reserved_line() = 0; - virtual bool is_modified_line() = 0; - - virtual enum cache_block_state get_status( mem_access_sector_mask_t sector_mask) = 0; - virtual void set_status(enum cache_block_state m_status, mem_access_sector_mask_t sector_mask) = 0; + virtual void allocate(new_addr_type tag, new_addr_type block_addr, + unsigned time, + mem_access_sector_mask_t sector_mask) = 0; + virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask) = 0; - virtual unsigned long long get_last_access_time() = 0; - virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask) = 0; - virtual unsigned long long get_alloc_time() = 0; - virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) = 0; - virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) = 0; - virtual unsigned get_modified_size() = 0; - virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)=0; - virtual bool is_readable(mem_access_sector_mask_t sector_mask)=0; - virtual void print_status()=0; - virtual ~cache_block_t() {} + virtual bool is_invalid_line() = 0; + virtual bool is_valid_line() = 0; + virtual bool is_reserved_line() = 0; + virtual bool is_modified_line() = 0; + virtual enum cache_block_state get_status( + mem_access_sector_mask_t sector_mask) = 0; + virtual void set_status(enum cache_block_state m_status, + mem_access_sector_mask_t sector_mask) = 0; - new_addr_type m_tag; - new_addr_type m_block_addr; + virtual unsigned long long get_last_access_time() = 0; + virtual void set_last_access_time(unsigned long long time, + mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned long long get_alloc_time() = 0; + virtual void set_ignore_on_fill(bool m_ignore, + mem_access_sector_mask_t sector_mask) = 0; + virtual void set_modified_on_fill(bool m_modified, + mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned get_modified_size() = 0; + virtual void set_m_readable(bool readable, + mem_access_sector_mask_t sector_mask) = 0; + virtual bool is_readable(mem_access_sector_mask_t sector_mask) = 0; + virtual void print_status() = 0; + virtual ~cache_block_t() {} + new_addr_type m_tag; + new_addr_type m_block_addr; }; -struct line_cache_block: public cache_block_t { - line_cache_block() - { - m_alloc_time=0; - m_fill_time=0; - m_last_access_time=0; - m_status=INVALID; - m_ignore_on_fill_status = false; - m_set_modified_on_fill = false; - m_readable = true; - } - void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) - { - m_tag=tag; - m_block_addr=block_addr; - m_alloc_time=time; - m_last_access_time=time; - m_fill_time=0; - m_status=RESERVED; - m_ignore_on_fill_status = false; - m_set_modified_on_fill = false; - } - void fill( unsigned time, mem_access_sector_mask_t sector_mask ) - { - //if(!m_ignore_on_fill_status) - // assert( m_status == RESERVED ); - - m_status = m_set_modified_on_fill? MODIFIED : VALID; +struct line_cache_block : public cache_block_t { + line_cache_block() { + m_alloc_time = 0; + m_fill_time = 0; + m_last_access_time = 0; + m_status = INVALID; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + m_readable = true; + } + void allocate(new_addr_type tag, new_addr_type block_addr, unsigned time, + mem_access_sector_mask_t sector_mask) { + m_tag = tag; + m_block_addr = block_addr; + m_alloc_time = time; + m_last_access_time = time; + m_fill_time = 0; + m_status = RESERVED; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + } + void fill(unsigned time, mem_access_sector_mask_t sector_mask) { + // if(!m_ignore_on_fill_status) + // assert( m_status == RESERVED ); - m_fill_time=time; - } - virtual bool is_invalid_line() - { - return m_status == INVALID; - } - virtual bool is_valid_line() - { - return m_status == VALID; - } - virtual bool is_reserved_line() - { - return m_status == RESERVED; - } - virtual bool is_modified_line() - { - return m_status == MODIFIED; - } + m_status = m_set_modified_on_fill ? MODIFIED : VALID; - virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) - { - return m_status; - } - virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) - { - m_status = status; - } - virtual unsigned long long get_last_access_time() - { - return m_last_access_time; - } - virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask) - { - m_last_access_time = time; - } - virtual unsigned long long get_alloc_time() - { - return m_alloc_time; - } - virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) - { - m_ignore_on_fill_status = m_ignore; - } - virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) - { - m_set_modified_on_fill = m_modified; - } - virtual unsigned get_modified_size() - { - return SECTOR_CHUNCK_SIZE * SECTOR_SIZE; //i.e. cache line size - } - virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) - { - m_readable = readable; - } - virtual bool is_readable(mem_access_sector_mask_t sector_mask) { - return m_readable; - } - virtual void print_status() { - printf("m_block_addr is %llu, status = %u\n", m_block_addr, m_status); - } + m_fill_time = time; + } + virtual bool is_invalid_line() { return m_status == INVALID; } + virtual bool is_valid_line() { return m_status == VALID; } + virtual bool is_reserved_line() { return m_status == RESERVED; } + virtual bool is_modified_line() { return m_status == MODIFIED; } + virtual enum cache_block_state get_status( + mem_access_sector_mask_t sector_mask) { + return m_status; + } + virtual void set_status(enum cache_block_state status, + mem_access_sector_mask_t sector_mask) { + m_status = status; + } + virtual unsigned long long get_last_access_time() { + return m_last_access_time; + } + virtual void set_last_access_time(unsigned long long time, + mem_access_sector_mask_t sector_mask) { + m_last_access_time = time; + } + virtual unsigned long long get_alloc_time() { return m_alloc_time; } + virtual void set_ignore_on_fill(bool m_ignore, + mem_access_sector_mask_t sector_mask) { + m_ignore_on_fill_status = m_ignore; + } + virtual void set_modified_on_fill(bool m_modified, + mem_access_sector_mask_t sector_mask) { + m_set_modified_on_fill = m_modified; + } + virtual unsigned get_modified_size() { + return SECTOR_CHUNCK_SIZE * SECTOR_SIZE; // i.e. cache line size + } + virtual void set_m_readable(bool readable, + mem_access_sector_mask_t sector_mask) { + m_readable = readable; + } + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + return m_readable; + } + virtual void print_status() { + printf("m_block_addr is %llu, status = %u\n", m_block_addr, m_status); + } -private: - unsigned long long m_alloc_time; - unsigned long long m_last_access_time; - unsigned long long m_fill_time; - cache_block_state m_status; - bool m_ignore_on_fill_status; - bool m_set_modified_on_fill; - bool m_readable; + private: + unsigned long long m_alloc_time; + unsigned long long m_last_access_time; + unsigned long long m_fill_time; + cache_block_state m_status; + bool m_ignore_on_fill_status; + bool m_set_modified_on_fill; + bool m_readable; }; struct sector_cache_block : public cache_block_t { - sector_cache_block() - { - init(); - } - - void init() { - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - m_sector_alloc_time[i]= 0; - m_sector_fill_time[i]= 0; - m_last_sector_access_time[i]= 0; - m_status[i]= INVALID; - m_ignore_on_fill_status[i] = false; - m_set_modified_on_fill[i] = false; - m_readable[i] = true; - } - m_line_alloc_time=0; - m_line_last_access_time=0; - m_line_fill_time=0; - } + sector_cache_block() { init(); } - virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) - { - allocate_line( tag, block_addr, time, sector_mask ); + void init() { + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + m_sector_alloc_time[i] = 0; + m_sector_fill_time[i] = 0; + m_last_sector_access_time[i] = 0; + m_status[i] = INVALID; + m_ignore_on_fill_status[i] = false; + m_set_modified_on_fill[i] = false; + m_readable[i] = true; } + m_line_alloc_time = 0; + m_line_last_access_time = 0; + m_line_fill_time = 0; + } + + virtual void allocate(new_addr_type tag, new_addr_type block_addr, + unsigned time, mem_access_sector_mask_t sector_mask) { + allocate_line(tag, block_addr, time, sector_mask); + } - void allocate_line( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) - { - //allocate a new line - //assert(m_block_addr != 0 && m_block_addr != block_addr); - init(); - m_tag=tag; - m_block_addr=block_addr; + void allocate_line(new_addr_type tag, new_addr_type block_addr, unsigned time, + mem_access_sector_mask_t sector_mask) { + // allocate a new line + // assert(m_block_addr != 0 && m_block_addr != block_addr); + init(); + m_tag = tag; + m_block_addr = block_addr; - unsigned sidx = get_sector_index(sector_mask); + unsigned sidx = get_sector_index(sector_mask); - //set sector stats - m_sector_alloc_time[sidx]=time; - m_last_sector_access_time[sidx]=time; - m_sector_fill_time[sidx]=0; - m_status[sidx]=RESERVED; - m_ignore_on_fill_status[sidx] = false; - m_set_modified_on_fill[sidx] = false; + // set sector stats + m_sector_alloc_time[sidx] = time; + m_last_sector_access_time[sidx] = time; + m_sector_fill_time[sidx] = 0; + m_status[sidx] = RESERVED; + m_ignore_on_fill_status[sidx] = false; + m_set_modified_on_fill[sidx] = false; - //set line stats - m_line_alloc_time=time; //only set this for the first allocated sector - m_line_last_access_time=time; - m_line_fill_time=0; - } + // set line stats + m_line_alloc_time = time; // only set this for the first allocated sector + m_line_last_access_time = time; + m_line_fill_time = 0; + } - void allocate_sector(unsigned time, mem_access_sector_mask_t sector_mask ) - { - //allocate invalid sector of this allocated valid line - assert(is_valid_line()); - unsigned sidx = get_sector_index(sector_mask); + void allocate_sector(unsigned time, mem_access_sector_mask_t sector_mask) { + // allocate invalid sector of this allocated valid line + assert(is_valid_line()); + unsigned sidx = get_sector_index(sector_mask); - //set sector stats - m_sector_alloc_time[sidx]=time; - m_last_sector_access_time[sidx]=time; - m_sector_fill_time[sidx]=0; - if(m_status[sidx]==MODIFIED) //this should be the case only for fetch-on-write policy //TO DO - m_set_modified_on_fill[sidx] = true; - else - m_set_modified_on_fill[sidx] = false; + // set sector stats + m_sector_alloc_time[sidx] = time; + m_last_sector_access_time[sidx] = time; + m_sector_fill_time[sidx] = 0; + if (m_status[sidx] == MODIFIED) // this should be the case only for + // fetch-on-write policy //TO DO + m_set_modified_on_fill[sidx] = true; + else + m_set_modified_on_fill[sidx] = false; - m_status[sidx]=RESERVED; - m_ignore_on_fill_status[sidx] = false; - //m_set_modified_on_fill[sidx] = false; - m_readable[sidx] = true; + m_status[sidx] = RESERVED; + m_ignore_on_fill_status[sidx] = false; + // m_set_modified_on_fill[sidx] = false; + m_readable[sidx] = true; - //set line stats - m_line_last_access_time=time; - m_line_fill_time=0; - } + // set line stats + m_line_last_access_time = time; + m_line_fill_time = 0; + } - virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); + virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); // if(!m_ignore_on_fill_status[sidx]) // assert( m_status[sidx] == RESERVED ); - m_status[sidx] = m_set_modified_on_fill[sidx]? MODIFIED : VALID; + m_status[sidx] = m_set_modified_on_fill[sidx] ? MODIFIED : VALID; - m_sector_fill_time[sidx]=time; - m_line_fill_time=time; + m_sector_fill_time[sidx] = time; + m_line_fill_time = time; + } + virtual bool is_invalid_line() { + // all the sectors should be invalid + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] != INVALID) return false; } - virtual bool is_invalid_line() { - //all the sectors should be invalid - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - if (m_status[i] != INVALID) - return false; - } - return true; + return true; + } + virtual bool is_valid_line() { return !(is_invalid_line()); } + virtual bool is_reserved_line() { + // if any of the sector is reserved, then the line is reserved + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == RESERVED) return true; } - virtual bool is_valid_line() { return !(is_invalid_line()); } - virtual bool is_reserved_line() { - //if any of the sector is reserved, then the line is reserved - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - if (m_status[i] == RESERVED) - return true; - } - return false; + return false; + } + virtual bool is_modified_line() { + // if any of the sector is modified, then the line is modified + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) return true; } - virtual bool is_modified_line() { - //if any of the sector is modified, then the line is modified - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - if (m_status[i] == MODIFIED) - return true; - } - return false; - } - - virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); + return false; + } - return m_status[sidx]; - } + virtual enum cache_block_state get_status( + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); - virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); - m_status[sidx] = status; - } + return m_status[sidx]; + } - virtual unsigned long long get_last_access_time() - { - return m_line_last_access_time; - } + virtual void set_status(enum cache_block_state status, + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + m_status[sidx] = status; + } - virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); + virtual unsigned long long get_last_access_time() { + return m_line_last_access_time; + } - m_last_sector_access_time[sidx] = time; - m_line_last_access_time = time; - } + virtual void set_last_access_time(unsigned long long time, + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); - virtual unsigned long long get_alloc_time() - { - return m_line_alloc_time; - } + m_last_sector_access_time[sidx] = time; + m_line_last_access_time = time; + } - virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); - m_ignore_on_fill_status[sidx] = m_ignore; - } + virtual unsigned long long get_alloc_time() { return m_line_alloc_time; } - virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); - m_set_modified_on_fill[sidx] = m_modified; - } + virtual void set_ignore_on_fill(bool m_ignore, + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + m_ignore_on_fill_status[sidx] = m_ignore; + } - virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) - { - unsigned sidx = get_sector_index(sector_mask); - m_readable[sidx] = readable; - } + virtual void set_modified_on_fill(bool m_modified, + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + m_set_modified_on_fill[sidx] = m_modified; + } - virtual bool is_readable(mem_access_sector_mask_t sector_mask) { - unsigned sidx = get_sector_index(sector_mask); - return m_readable[sidx]; - } + virtual void set_m_readable(bool readable, + mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + m_readable[sidx] = readable; + } - virtual unsigned get_modified_size() - { - unsigned modified=0; - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - if (m_status[i] == MODIFIED) - modified++; - } - return modified * SECTOR_SIZE; - } + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + return m_readable[sidx]; + } - virtual void print_status() { - printf("m_block_addr is %llu, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]); + virtual unsigned get_modified_size() { + unsigned modified = 0; + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) modified++; } + return modified * SECTOR_SIZE; + } + virtual void print_status() { + printf("m_block_addr is %llu, status = %u %u %u %u\n", m_block_addr, + m_status[0], m_status[1], m_status[2], m_status[3]); + } -private: - unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE]; - unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE]; - unsigned m_sector_fill_time[SECTOR_CHUNCK_SIZE]; - unsigned m_line_alloc_time; - unsigned m_line_last_access_time; - unsigned m_line_fill_time; - cache_block_state m_status[SECTOR_CHUNCK_SIZE]; - bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE]; - bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE]; - bool m_readable[SECTOR_CHUNCK_SIZE]; + private: + unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE]; + unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE]; + unsigned m_sector_fill_time[SECTOR_CHUNCK_SIZE]; + unsigned m_line_alloc_time; + unsigned m_line_last_access_time; + unsigned m_line_fill_time; + cache_block_state m_status[SECTOR_CHUNCK_SIZE]; + bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE]; + bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE]; + bool m_readable[SECTOR_CHUNCK_SIZE]; - unsigned get_sector_index(mem_access_sector_mask_t sector_mask) - { - assert(sector_mask.count() == 1); - for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { - if(sector_mask.to_ulong() & (1<<i)) - return i; - } + unsigned get_sector_index(mem_access_sector_mask_t sector_mask) { + assert(sector_mask.count() == 1); + for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; ++i) { + if (sector_mask.to_ulong() & (1 << i)) return i; } + } }; -enum replacement_policy_t { - LRU, - FIFO -}; +enum replacement_policy_t { LRU, FIFO }; enum write_policy_t { - READ_ONLY, - WRITE_BACK, - WRITE_THROUGH, - WRITE_EVICT, - LOCAL_WB_GLOBAL_WT -}; - -enum allocation_policy_t { - ON_MISS, - ON_FILL, - STREAMING + READ_ONLY, + WRITE_BACK, + WRITE_THROUGH, + WRITE_EVICT, + LOCAL_WB_GLOBAL_WT }; +enum allocation_policy_t { ON_MISS, ON_FILL, STREAMING }; enum write_allocate_policy_t { - NO_WRITE_ALLOCATE, - WRITE_ALLOCATE, - FETCH_ON_WRITE, - LAZY_FETCH_ON_READ + NO_WRITE_ALLOCATE, + WRITE_ALLOCATE, + FETCH_ON_WRITE, + LAZY_FETCH_ON_READ }; enum mshr_config_t { - TEX_FIFO, // Tex cache - ASSOC, // normal cache - SECTOR_TEX_FIFO, //Tex cache sends requests to high-level sector cache - SECTOR_ASSOC // normal cache sends requests to high-level sector cache + TEX_FIFO, // Tex cache + ASSOC, // normal cache + SECTOR_TEX_FIFO, // Tex cache sends requests to high-level sector cache + SECTOR_ASSOC // normal cache sends requests to high-level sector cache }; -enum set_index_function{ - LINEAR_SET_FUNCTION = 0, - BITWISE_XORING_FUNCTION, - HASH_IPOLY_FUNCTION, - FERMI_HASH_SET_FUNCTION, - CUSTOM_SET_FUNCTION +enum set_index_function { + LINEAR_SET_FUNCTION = 0, + BITWISE_XORING_FUNCTION, + HASH_IPOLY_FUNCTION, + FERMI_HASH_SET_FUNCTION, + CUSTOM_SET_FUNCTION }; -enum cache_type{ - NORMAL = 0, - SECTOR -}; +enum cache_type { NORMAL = 0, SECTOR }; #define MAX_WARP_PER_SHADER 64 #define INCT_TOTAL_BUFFER 64 @@ -496,542 +453,605 @@ enum cache_type{ #define MAX_WARP_PER_SHADER 64 class cache_config { -public: - cache_config() - { - m_valid = false; - m_disabled = false; - m_config_string = NULL; // set by option parser - m_config_stringPrefL1 = NULL; - m_config_stringPrefShared = NULL; - m_data_port_width = 0; - m_set_index_function = LINEAR_SET_FUNCTION; - m_is_streaming = false; - } - void init(char * config, FuncCache status) - { - cache_status= status; - assert( config ); - char ct, rp, wp, ap, mshr_type, wap, sif; + public: + cache_config() { + m_valid = false; + m_disabled = false; + m_config_string = NULL; // set by option parser + m_config_stringPrefL1 = NULL; + m_config_stringPrefShared = NULL; + m_data_port_width = 0; + m_set_index_function = LINEAR_SET_FUNCTION; + m_is_streaming = false; + } + void init(char *config, FuncCache status) { + cache_status = status; + assert(config); + char ct, rp, wp, ap, mshr_type, wap, sif; + int ntok = + sscanf(config, "%c:%u:%u:%u,%c:%c:%c:%c:%c,%c:%u:%u,%u:%u,%u", &ct, + &m_nset, &m_line_sz, &m_assoc, &rp, &wp, &ap, &wap, &sif, + &mshr_type, &m_mshr_entries, &m_mshr_max_merge, + &m_miss_queue_size, &m_result_fifo_entries, &m_data_port_width); - int ntok = sscanf(config,"%c:%u:%u:%u,%c:%c:%c:%c:%c,%c:%u:%u,%u:%u,%u", - &ct, &m_nset, &m_line_sz, &m_assoc, &rp, &wp, &ap, &wap, - &sif,&mshr_type,&m_mshr_entries,&m_mshr_max_merge, - &m_miss_queue_size, &m_result_fifo_entries, - &m_data_port_width); - - if ( ntok < 12 ) { - if ( !strcmp(config,"none") ) { - m_disabled = true; - return; - } - exit_parse_error(); - } - - switch (ct) { - case 'N': m_cache_type = NORMAL; break; - case 'S': m_cache_type = SECTOR; break; - default: exit_parse_error(); - } - switch (rp) { - case 'L': m_replacement_policy = LRU; break; - case 'F': m_replacement_policy = FIFO; break; - default: exit_parse_error(); - } - switch (rp) { - case 'L': m_replacement_policy = LRU; break; - case 'F': m_replacement_policy = FIFO; break; - default: exit_parse_error(); - } - switch (wp) { - case 'R': m_write_policy = READ_ONLY; break; - case 'B': m_write_policy = WRITE_BACK; break; - case 'T': m_write_policy = WRITE_THROUGH; break; - case 'E': m_write_policy = WRITE_EVICT; break; - case 'L': m_write_policy = LOCAL_WB_GLOBAL_WT; break; - default: exit_parse_error(); - } - switch (ap) { - case 'm': m_alloc_policy = ON_MISS; break; - case 'f': m_alloc_policy = ON_FILL; break; - case 's': m_alloc_policy = STREAMING; break; - default: exit_parse_error(); - } - if(m_alloc_policy == STREAMING) { - //For streaming cache, we set the alloc policy to be on-fill to remove all line_alloc_fail stalls - //we set the MSHRs to be equal to max allocated cache lines. This is possible by moving TAG to be shared between cache line and MSHR enrty (i.e. for each cache line, there is an MSHR rntey associated with it) - //This is the easiest think we can think about to model (mimic) L1 streaming cache in Pascal and Volta - //Based on our microbenchmakrs, MSHRs entries have been increasing substantially in Pascal and Volta - //For more information about streaming cache, see: - // http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf - // https://ieeexplore.ieee.org/document/8344474/ - m_is_streaming = true; - m_alloc_policy = ON_FILL; - m_mshr_entries = m_nset*m_assoc*MAX_DEFAULT_CACHE_SIZE_MULTIBLIER; - if(m_cache_type == SECTOR) - m_mshr_entries *= SECTOR_CHUNCK_SIZE; - m_mshr_max_merge = MAX_WARP_PER_SM; - } - switch (mshr_type) { - case 'F': m_mshr_type = TEX_FIFO; assert(ntok==14); break; - case 'T': m_mshr_type = SECTOR_TEX_FIFO; assert(ntok==14); break; - case 'A': m_mshr_type = ASSOC; break; - case 'S' : m_mshr_type = SECTOR_ASSOC; break; - default: exit_parse_error(); - } - m_line_sz_log2 = LOGB2(m_line_sz); - m_nset_log2 = LOGB2(m_nset); - m_valid = true; - m_atom_sz = (m_cache_type == SECTOR)? SECTOR_SIZE : m_line_sz; - m_sector_sz_log2 = LOGB2(SECTOR_SIZE); - original_m_assoc = m_assoc; - - //For more details about difference between FETCH_ON_WRITE and WRITE VALIDAE policies - //Read: Jouppi, Norman P. "Cache write policies and performance". ISCA 93. - //WRITE_ALLOCATE is the old write policy in GPGPU-sim 3.x, that send WRITE and READ for every write request - switch(wap){ - case 'N': m_write_alloc_policy = NO_WRITE_ALLOCATE; break; - case 'W': m_write_alloc_policy = WRITE_ALLOCATE; break; - case 'F': m_write_alloc_policy = FETCH_ON_WRITE; break; - case 'L': m_write_alloc_policy = LAZY_FETCH_ON_READ; break; - default: exit_parse_error(); - } - - // detect invalid configuration - if (m_alloc_policy == ON_FILL and m_write_policy == WRITE_BACK) { - // A writeback cache with allocate-on-fill policy will inevitably lead to deadlock: - // The deadlock happens when an incoming cache-fill evicts a dirty - // line, generating a writeback request. If the memory subsystem - // is congested, the interconnection network may not have - // sufficient buffer for the writeback request. This stalls the - // incoming cache-fill. The stall may propagate through the memory - // subsystem back to the output port of the same core, creating a - // deadlock where the wrtieback request and the incoming cache-fill - // are stalling each other. - assert(0 && "Invalid cache configuration: Writeback cache cannot allocate new line on fill. "); - } - - if((m_write_alloc_policy == FETCH_ON_WRITE || m_write_alloc_policy == LAZY_FETCH_ON_READ )&& m_alloc_policy == ON_FILL) - { - assert(0 && "Invalid cache configuration: FETCH_ON_WRITE and LAZY_FETCH_ON_READ cannot work properly with ON_FILL policy. Cache must be ON_MISS. "); - } - if(m_cache_type == SECTOR) - { - assert(m_line_sz / SECTOR_SIZE == SECTOR_CHUNCK_SIZE && m_line_sz % SECTOR_SIZE == 0); - } - - // default: port to data array width and granularity = line size - if (m_data_port_width == 0) { - m_data_port_width = m_line_sz; - } - assert(m_line_sz % m_data_port_width == 0); + if (ntok < 12) { + if (!strcmp(config, "none")) { + m_disabled = true; + return; + } + exit_parse_error(); + } - switch(sif){ - case 'H': m_set_index_function = FERMI_HASH_SET_FUNCTION; break; - case 'P': m_set_index_function = HASH_IPOLY_FUNCTION; break; - case 'C': m_set_index_function = CUSTOM_SET_FUNCTION; break; - case 'L': m_set_index_function = LINEAR_SET_FUNCTION; break; - default: exit_parse_error(); - } + switch (ct) { + case 'N': + m_cache_type = NORMAL; + break; + case 'S': + m_cache_type = SECTOR; + break; + default: + exit_parse_error(); + } + switch (rp) { + case 'L': + m_replacement_policy = LRU; + break; + case 'F': + m_replacement_policy = FIFO; + break; + default: + exit_parse_error(); + } + switch (rp) { + case 'L': + m_replacement_policy = LRU; + break; + case 'F': + m_replacement_policy = FIFO; + break; + default: + exit_parse_error(); } - bool disabled() const { return m_disabled;} - unsigned get_line_sz() const - { - assert( m_valid ); - return m_line_sz; + switch (wp) { + case 'R': + m_write_policy = READ_ONLY; + break; + case 'B': + m_write_policy = WRITE_BACK; + break; + case 'T': + m_write_policy = WRITE_THROUGH; + break; + case 'E': + m_write_policy = WRITE_EVICT; + break; + case 'L': + m_write_policy = LOCAL_WB_GLOBAL_WT; + break; + default: + exit_parse_error(); } - unsigned get_atom_sz() const - { - assert( m_valid ); - return m_atom_sz; - } - unsigned get_num_lines() const - { - assert( m_valid ); - return m_nset * m_assoc; + switch (ap) { + case 'm': + m_alloc_policy = ON_MISS; + break; + case 'f': + m_alloc_policy = ON_FILL; + break; + case 's': + m_alloc_policy = STREAMING; + break; + default: + exit_parse_error(); } - unsigned get_max_num_lines() const - { - assert( m_valid ); - return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER * m_nset * original_m_assoc; + if (m_alloc_policy == STREAMING) { + // For streaming cache, we set the alloc policy to be on-fill to remove + // all line_alloc_fail stalls we set the MSHRs to be equal to max allocated + // cache lines. This is possible by moving TAG to be shared between cache + // line and MSHR enrty (i.e. for each cache line, there is an MSHR rntey + // associated with it) This is the easiest think we can think about to + // model (mimic) L1 streaming cache in Pascal and Volta Based on our + // microbenchmakrs, MSHRs entries have been increasing substantially in + // Pascal and Volta For more information about streaming cache, see: + // http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf + // https://ieeexplore.ieee.org/document/8344474/ + m_is_streaming = true; + m_alloc_policy = ON_FILL; + m_mshr_entries = m_nset * m_assoc * MAX_DEFAULT_CACHE_SIZE_MULTIBLIER; + if (m_cache_type == SECTOR) m_mshr_entries *= SECTOR_CHUNCK_SIZE; + m_mshr_max_merge = MAX_WARP_PER_SM; } - void print( FILE *fp ) const - { - fprintf( fp, "Size = %d B (%d Set x %d-way x %d byte line)\n", - m_line_sz * m_nset * m_assoc, - m_nset, m_assoc, m_line_sz ); + switch (mshr_type) { + case 'F': + m_mshr_type = TEX_FIFO; + assert(ntok == 14); + break; + case 'T': + m_mshr_type = SECTOR_TEX_FIFO; + assert(ntok == 14); + break; + case 'A': + m_mshr_type = ASSOC; + break; + case 'S': + m_mshr_type = SECTOR_ASSOC; + break; + default: + exit_parse_error(); } + m_line_sz_log2 = LOGB2(m_line_sz); + m_nset_log2 = LOGB2(m_nset); + m_valid = true; + m_atom_sz = (m_cache_type == SECTOR) ? SECTOR_SIZE : m_line_sz; + m_sector_sz_log2 = LOGB2(SECTOR_SIZE); + original_m_assoc = m_assoc; - virtual unsigned set_index( new_addr_type addr ) const - { - if(m_set_index_function != LINEAR_SET_FUNCTION){ - printf("\nGPGPU-Sim cache configuration error: Hashing or " - "custom set index function selected in configuration " - "file for a cache that has not overloaded the set_index " - "function\n"); - abort(); - } - return(addr >> m_line_sz_log2) & (m_nset-1); + // For more details about difference between FETCH_ON_WRITE and WRITE + // VALIDAE policies Read: Jouppi, Norman P. "Cache write policies and + // performance". ISCA 93. WRITE_ALLOCATE is the old write policy in + // GPGPU-sim 3.x, that send WRITE and READ for every write request + switch (wap) { + case 'N': + m_write_alloc_policy = NO_WRITE_ALLOCATE; + break; + case 'W': + m_write_alloc_policy = WRITE_ALLOCATE; + break; + case 'F': + m_write_alloc_policy = FETCH_ON_WRITE; + break; + case 'L': + m_write_alloc_policy = LAZY_FETCH_ON_READ; + break; + default: + exit_parse_error(); } - new_addr_type tag( new_addr_type addr ) const - { - // For generality, the tag includes both index and tag. This allows for more complex set index - // calculations that can result in different indexes mapping to the same set, thus the full - // tag + index is required to check for hit/miss. Tag is now identical to the block address. + // detect invalid configuration + if (m_alloc_policy == ON_FILL and m_write_policy == WRITE_BACK) { + // A writeback cache with allocate-on-fill policy will inevitably lead to + // deadlock: The deadlock happens when an incoming cache-fill evicts a + // dirty line, generating a writeback request. If the memory subsystem is + // congested, the interconnection network may not have sufficient buffer + // for the writeback request. This stalls the incoming cache-fill. The + // stall may propagate through the memory subsystem back to the output + // port of the same core, creating a deadlock where the wrtieback request + // and the incoming cache-fill are stalling each other. + assert(0 && + "Invalid cache configuration: Writeback cache cannot allocate new " + "line on fill. "); + } - //return addr >> (m_line_sz_log2+m_nset_log2); - return addr & ~(new_addr_type)(m_line_sz-1); + if ((m_write_alloc_policy == FETCH_ON_WRITE || + m_write_alloc_policy == LAZY_FETCH_ON_READ) && + m_alloc_policy == ON_FILL) { + assert( + 0 && + "Invalid cache configuration: FETCH_ON_WRITE and LAZY_FETCH_ON_READ " + "cannot work properly with ON_FILL policy. Cache must be ON_MISS. "); + } + if (m_cache_type == SECTOR) { + assert(m_line_sz / SECTOR_SIZE == SECTOR_CHUNCK_SIZE && + m_line_sz % SECTOR_SIZE == 0); } - new_addr_type block_addr( new_addr_type addr ) const - { - return addr & ~(new_addr_type)(m_line_sz-1); + + // default: port to data array width and granularity = line size + if (m_data_port_width == 0) { + m_data_port_width = m_line_sz; } - new_addr_type mshr_addr( new_addr_type addr ) const - { - return addr & ~(new_addr_type)(m_atom_sz-1); - } - enum mshr_config_t get_mshr_type() const - { - return m_mshr_type; - } - void set_assoc(unsigned n) - { - //set new assoc. L1 cache dynamically resized in Volta - m_assoc = n; - } - unsigned get_nset() const - { - assert( m_valid ); - return m_nset; - } - unsigned get_total_size_inKB() const - { - assert( m_valid ); - return (m_assoc*m_nset*m_line_sz)/1024; - } - bool is_streaming() { - return m_is_streaming; + assert(m_line_sz % m_data_port_width == 0); + + switch (sif) { + case 'H': + m_set_index_function = FERMI_HASH_SET_FUNCTION; + break; + case 'P': + m_set_index_function = HASH_IPOLY_FUNCTION; + break; + case 'C': + m_set_index_function = CUSTOM_SET_FUNCTION; + break; + case 'L': + m_set_index_function = LINEAR_SET_FUNCTION; + break; + default: + exit_parse_error(); } - FuncCache get_cache_status() {return cache_status;} - char *m_config_string; - char *m_config_stringPrefL1; - char *m_config_stringPrefShared; - FuncCache cache_status; + } + bool disabled() const { return m_disabled; } + unsigned get_line_sz() const { + assert(m_valid); + return m_line_sz; + } + unsigned get_atom_sz() const { + assert(m_valid); + return m_atom_sz; + } + unsigned get_num_lines() const { + assert(m_valid); + return m_nset * m_assoc; + } + unsigned get_max_num_lines() const { + assert(m_valid); + return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER * m_nset * original_m_assoc; + } + void print(FILE *fp) const { + fprintf(fp, "Size = %d B (%d Set x %d-way x %d byte line)\n", + m_line_sz * m_nset * m_assoc, m_nset, m_assoc, m_line_sz); + } -protected: - void exit_parse_error() - { - printf("GPGPU-Sim uArch: cache configuration parsing error (%s)\n", m_config_string ); - abort(); + virtual unsigned set_index(new_addr_type addr) const { + if (m_set_index_function != LINEAR_SET_FUNCTION) { + printf( + "\nGPGPU-Sim cache configuration error: Hashing or " + "custom set index function selected in configuration " + "file for a cache that has not overloaded the set_index " + "function\n"); + abort(); } + return (addr >> m_line_sz_log2) & (m_nset - 1); + } - bool m_valid; - bool m_disabled; - unsigned m_line_sz; - unsigned m_line_sz_log2; - unsigned m_nset; - unsigned m_nset_log2; - unsigned m_assoc; - unsigned m_atom_sz; - unsigned m_sector_sz_log2; - unsigned original_m_assoc; - bool m_is_streaming; + new_addr_type tag(new_addr_type addr) const { + // For generality, the tag includes both index and tag. This allows for more + // complex set index calculations that can result in different indexes + // mapping to the same set, thus the full tag + index is required to check + // for hit/miss. Tag is now identical to the block address. + + // return addr >> (m_line_sz_log2+m_nset_log2); + return addr & ~(new_addr_type)(m_line_sz - 1); + } + new_addr_type block_addr(new_addr_type addr) const { + return addr & ~(new_addr_type)(m_line_sz - 1); + } + new_addr_type mshr_addr(new_addr_type addr) const { + return addr & ~(new_addr_type)(m_atom_sz - 1); + } + enum mshr_config_t get_mshr_type() const { return m_mshr_type; } + void set_assoc(unsigned n) { + // set new assoc. L1 cache dynamically resized in Volta + m_assoc = n; + } + unsigned get_nset() const { + assert(m_valid); + return m_nset; + } + unsigned get_total_size_inKB() const { + assert(m_valid); + return (m_assoc * m_nset * m_line_sz) / 1024; + } + bool is_streaming() { return m_is_streaming; } + FuncCache get_cache_status() { return cache_status; } + char *m_config_string; + char *m_config_stringPrefL1; + char *m_config_stringPrefShared; + FuncCache cache_status; + + protected: + void exit_parse_error() { + printf("GPGPU-Sim uArch: cache configuration parsing error (%s)\n", + m_config_string); + abort(); + } + + bool m_valid; + bool m_disabled; + unsigned m_line_sz; + unsigned m_line_sz_log2; + unsigned m_nset; + unsigned m_nset_log2; + unsigned m_assoc; + unsigned m_atom_sz; + unsigned m_sector_sz_log2; + unsigned original_m_assoc; + bool m_is_streaming; - enum replacement_policy_t m_replacement_policy; // 'L' = LRU, 'F' = FIFO - enum write_policy_t m_write_policy; // 'T' = write through, 'B' = write back, 'R' = read only - enum allocation_policy_t m_alloc_policy; // 'm' = allocate on miss, 'f' = allocate on fill - enum mshr_config_t m_mshr_type; - enum cache_type m_cache_type; + enum replacement_policy_t m_replacement_policy; // 'L' = LRU, 'F' = FIFO + enum write_policy_t + m_write_policy; // 'T' = write through, 'B' = write back, 'R' = read only + enum allocation_policy_t + m_alloc_policy; // 'm' = allocate on miss, 'f' = allocate on fill + enum mshr_config_t m_mshr_type; + enum cache_type m_cache_type; - write_allocate_policy_t m_write_alloc_policy; // 'W' = Write allocate, 'N' = No write allocate + write_allocate_policy_t + m_write_alloc_policy; // 'W' = Write allocate, 'N' = No write allocate - union { - unsigned m_mshr_entries; - unsigned m_fragment_fifo_entries; - }; - union { - unsigned m_mshr_max_merge; - unsigned m_request_fifo_entries; - }; - union { - unsigned m_miss_queue_size; - unsigned m_rob_entries; - }; - unsigned m_result_fifo_entries; - unsigned m_data_port_width; //< number of byte the cache can access per cycle - enum set_index_function m_set_index_function; // Hash, linear, or custom set index function + union { + unsigned m_mshr_entries; + unsigned m_fragment_fifo_entries; + }; + union { + unsigned m_mshr_max_merge; + unsigned m_request_fifo_entries; + }; + union { + unsigned m_miss_queue_size; + unsigned m_rob_entries; + }; + unsigned m_result_fifo_entries; + unsigned m_data_port_width; //< number of byte the cache can access per cycle + enum set_index_function + m_set_index_function; // Hash, linear, or custom set index function - friend class tag_array; - friend class baseline_cache; - friend class read_only_cache; - friend class tex_cache; - friend class data_cache; - friend class l1_cache; - friend class l2_cache; - friend class memory_sub_partition; + friend class tag_array; + friend class baseline_cache; + friend class read_only_cache; + friend class tex_cache; + friend class data_cache; + friend class l1_cache; + friend class l2_cache; + friend class memory_sub_partition; }; -class l1d_cache_config : public cache_config{ -public: - l1d_cache_config() : cache_config(){} - virtual unsigned set_index(new_addr_type addr) const; - unsigned set_bank(new_addr_type addr) const; - unsigned l1_latency; - unsigned l1_banks; +class l1d_cache_config : public cache_config { + public: + l1d_cache_config() : cache_config() {} + virtual unsigned set_index(new_addr_type addr) const; + unsigned set_bank(new_addr_type addr) const; + unsigned l1_latency; + unsigned l1_banks; }; class l2_cache_config : public cache_config { -public: - l2_cache_config() : cache_config(){} - void init(linear_to_raw_address_translation *address_mapping); - virtual unsigned set_index(new_addr_type addr) const; + public: + l2_cache_config() : cache_config() {} + void init(linear_to_raw_address_translation *address_mapping); + virtual unsigned set_index(new_addr_type addr) const; -private: - linear_to_raw_address_translation *m_address_mapping; + private: + linear_to_raw_address_translation *m_address_mapping; }; class tag_array { -public: - // Use this constructor - tag_array(cache_config &config, int core_id, int type_id ); - ~tag_array(); + public: + // Use this constructor + tag_array(cache_config &config, int core_id, int type_id); + ~tag_array(); - enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_fetch* mf, bool probe_mode=false ) const; - enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask, bool probe_mode=false, mem_fetch* mf = NULL ) const; - enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf ); - enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf ); + enum cache_request_status probe(new_addr_type addr, unsigned &idx, + mem_fetch *mf, bool probe_mode = false) const; + enum cache_request_status probe(new_addr_type addr, unsigned &idx, + mem_access_sector_mask_t mask, + bool probe_mode = false, + mem_fetch *mf = NULL) const; + enum cache_request_status access(new_addr_type addr, unsigned time, + unsigned &idx, mem_fetch *mf); + enum cache_request_status access(new_addr_type addr, unsigned time, + unsigned &idx, bool &wb, + evicted_block_info &evicted, mem_fetch *mf); - void fill( new_addr_type addr, unsigned time, mem_fetch* mf ); - void fill( unsigned idx, unsigned time, mem_fetch* mf ); - void fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ); + void fill(new_addr_type addr, unsigned time, mem_fetch *mf); + void fill(unsigned idx, unsigned time, mem_fetch *mf); + void fill(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask); - unsigned size() const { return m_config.get_num_lines();} - cache_block_t* get_block(unsigned idx) { return m_lines[idx];} + unsigned size() const { return m_config.get_num_lines(); } + cache_block_t *get_block(unsigned idx) { return m_lines[idx]; } - void flush(); // flush all written entries - void invalidate(); // invalidate all entries - void new_window(); + void flush(); // flush all written entries + void invalidate(); // invalidate all entries + void new_window(); - void print( FILE *stream, unsigned &total_access, unsigned &total_misses ) const; - float windowed_miss_rate( ) const; - void get_stats(unsigned &total_access, unsigned &total_misses, unsigned &total_hit_res, unsigned &total_res_fail) const; + void print(FILE *stream, unsigned &total_access, + unsigned &total_misses) const; + float windowed_miss_rate() const; + void get_stats(unsigned &total_access, unsigned &total_misses, + unsigned &total_hit_res, unsigned &total_res_fail) const; - void update_cache_parameters(cache_config &config); - void add_pending_line(mem_fetch *mf); - void remove_pending_line(mem_fetch *mf); -protected: - // This constructor is intended for use only from derived classes that wish to - // avoid unnecessary memory allocation that takes place in the - // other tag_array constructor - tag_array( cache_config &config, - int core_id, - int type_id, - cache_block_t** new_lines ); - void init( int core_id, int type_id ); + void update_cache_parameters(cache_config &config); + void add_pending_line(mem_fetch *mf); + void remove_pending_line(mem_fetch *mf); -protected: + protected: + // This constructor is intended for use only from derived classes that wish to + // avoid unnecessary memory allocation that takes place in the + // other tag_array constructor + tag_array(cache_config &config, int core_id, int type_id, + cache_block_t **new_lines); + void init(int core_id, int type_id); - cache_config &m_config; + protected: + cache_config &m_config; - cache_block_t **m_lines; /* nbanks x nset x assoc lines in total */ + cache_block_t **m_lines; /* nbanks x nset x assoc lines in total */ - unsigned m_access; - unsigned m_miss; - unsigned m_pending_hit; // number of cache miss that hit a line that is allocated but not filled - unsigned m_res_fail; - unsigned m_sector_miss; + unsigned m_access; + unsigned m_miss; + unsigned m_pending_hit; // number of cache miss that hit a line that is + // allocated but not filled + unsigned m_res_fail; + unsigned m_sector_miss; - // performance counters for calculating the amount of misses within a time window - unsigned m_prev_snapshot_access; - unsigned m_prev_snapshot_miss; - unsigned m_prev_snapshot_pending_hit; + // performance counters for calculating the amount of misses within a time + // window + unsigned m_prev_snapshot_access; + unsigned m_prev_snapshot_miss; + unsigned m_prev_snapshot_pending_hit; - int m_core_id; // which shader core is using this - int m_type_id; // what kind of cache is this (normal, texture, constant) + int m_core_id; // which shader core is using this + int m_type_id; // what kind of cache is this (normal, texture, constant) - bool is_used; //a flag if the whole cache has ever been accessed before + bool is_used; // a flag if the whole cache has ever been accessed before - typedef tr1_hash_map<new_addr_type,unsigned> line_table; - line_table pending_lines; + typedef tr1_hash_map<new_addr_type, unsigned> line_table; + line_table pending_lines; }; class mshr_table { -public: - mshr_table( unsigned num_entries, unsigned max_merged) - : m_num_entries(num_entries), - m_max_merged(max_merged) + public: + mshr_table(unsigned num_entries, unsigned max_merged) + : m_num_entries(num_entries), + m_max_merged(max_merged) #if (tr1_hash_map_ismap == 0) - ,m_data(2*num_entries) + , + m_data(2 * num_entries) #endif - { - } - - /// Checks if there is a pending request to the lower memory level already - bool probe( new_addr_type block_addr ) const; - /// Checks if there is space for tracking a new memory access - bool full( new_addr_type block_addr ) const; - /// Add or merge this access - void add( new_addr_type block_addr, mem_fetch *mf ); - /// Returns true if cannot accept new fill responses - bool busy() const {return false;} - /// Accept a new cache fill response: mark entry ready for processing - void mark_ready( new_addr_type block_addr, bool &has_atomic ); - /// Returns true if ready accesses exist - bool access_ready() const {return !m_current_response.empty();} - /// Returns next ready access - mem_fetch *next_access(); - void display( FILE *fp ) const; - // Returns true if there is a pending read after write - bool is_read_after_write_pending(new_addr_type block_addr); + { + } - void check_mshr_parameters( unsigned num_entries, unsigned max_merged ) - { - assert(m_num_entries==num_entries && "Change of MSHR parameters between kernels is not allowed"); - assert(m_max_merged==max_merged && "Change of MSHR parameters between kernels is not allowed"); - } + /// Checks if there is a pending request to the lower memory level already + bool probe(new_addr_type block_addr) const; + /// Checks if there is space for tracking a new memory access + bool full(new_addr_type block_addr) const; + /// Add or merge this access + void add(new_addr_type block_addr, mem_fetch *mf); + /// Returns true if cannot accept new fill responses + bool busy() const { return false; } + /// Accept a new cache fill response: mark entry ready for processing + void mark_ready(new_addr_type block_addr, bool &has_atomic); + /// Returns true if ready accesses exist + bool access_ready() const { return !m_current_response.empty(); } + /// Returns next ready access + mem_fetch *next_access(); + void display(FILE *fp) const; + // Returns true if there is a pending read after write + bool is_read_after_write_pending(new_addr_type block_addr); -private: + void check_mshr_parameters(unsigned num_entries, unsigned max_merged) { + assert(m_num_entries == num_entries && + "Change of MSHR parameters between kernels is not allowed"); + assert(m_max_merged == max_merged && + "Change of MSHR parameters between kernels is not allowed"); + } - // finite sized, fully associative table, with a finite maximum number of merged requests - const unsigned m_num_entries; - const unsigned m_max_merged; + private: + // finite sized, fully associative table, with a finite maximum number of + // merged requests + const unsigned m_num_entries; + const unsigned m_max_merged; - struct mshr_entry { - std::list<mem_fetch*> m_list; - bool m_has_atomic; - mshr_entry() : m_has_atomic(false) { } - }; - typedef tr1_hash_map<new_addr_type,mshr_entry> table; - typedef tr1_hash_map<new_addr_type,mshr_entry> line_table; - table m_data; - line_table pending_lines; + struct mshr_entry { + std::list<mem_fetch *> m_list; + bool m_has_atomic; + mshr_entry() : m_has_atomic(false) {} + }; + typedef tr1_hash_map<new_addr_type, mshr_entry> table; + typedef tr1_hash_map<new_addr_type, mshr_entry> line_table; + table m_data; + line_table pending_lines; - // it may take several cycles to process the merged requests - bool m_current_response_ready; - std::list<new_addr_type> m_current_response; + // it may take several cycles to process the merged requests + bool m_current_response_ready; + std::list<new_addr_type> m_current_response; }; - -/***************************************************************** Caches *****************************************************************/ +/***************************************************************** Caches + * *****************************************************************/ /// -/// Simple struct to maintain cache accesses, misses, pending hits, and reservation fails. +/// Simple struct to maintain cache accesses, misses, pending hits, and +/// reservation fails. /// -struct cache_sub_stats{ - unsigned long long accesses; - unsigned long long misses; - unsigned long long pending_hits; - unsigned long long res_fails; +struct cache_sub_stats { + unsigned long long accesses; + unsigned long long misses; + unsigned long long pending_hits; + unsigned long long res_fails; - unsigned long long port_available_cycles; - unsigned long long data_port_busy_cycles; - unsigned long long fill_port_busy_cycles; + unsigned long long port_available_cycles; + unsigned long long data_port_busy_cycles; + unsigned long long fill_port_busy_cycles; - cache_sub_stats(){ - clear(); - } - void clear(){ - accesses = 0; - misses = 0; - pending_hits = 0; - res_fails = 0; - port_available_cycles = 0; - data_port_busy_cycles = 0; - fill_port_busy_cycles = 0; - } - cache_sub_stats &operator+=(const cache_sub_stats &css){ - /// - /// Overloading += operator to easily accumulate stats - /// - accesses += css.accesses; - misses += css.misses; - pending_hits += css.pending_hits; - res_fails += css.res_fails; - port_available_cycles += css.port_available_cycles; - data_port_busy_cycles += css.data_port_busy_cycles; - fill_port_busy_cycles += css.fill_port_busy_cycles; - return *this; - } + cache_sub_stats() { clear(); } + void clear() { + accesses = 0; + misses = 0; + pending_hits = 0; + res_fails = 0; + port_available_cycles = 0; + data_port_busy_cycles = 0; + fill_port_busy_cycles = 0; + } + cache_sub_stats &operator+=(const cache_sub_stats &css) { + /// + /// Overloading += operator to easily accumulate stats + /// + accesses += css.accesses; + misses += css.misses; + pending_hits += css.pending_hits; + res_fails += css.res_fails; + port_available_cycles += css.port_available_cycles; + data_port_busy_cycles += css.data_port_busy_cycles; + fill_port_busy_cycles += css.fill_port_busy_cycles; + return *this; + } - cache_sub_stats operator+(const cache_sub_stats &cs){ - /// - /// Overloading + operator to easily accumulate stats - /// - cache_sub_stats ret; - ret.accesses = accesses + cs.accesses; - ret.misses = misses + cs.misses; - ret.pending_hits = pending_hits + cs.pending_hits; - ret.res_fails = res_fails + cs.res_fails; - ret.port_available_cycles = port_available_cycles + cs.port_available_cycles; - ret.data_port_busy_cycles = data_port_busy_cycles + cs.data_port_busy_cycles; - ret.fill_port_busy_cycles = fill_port_busy_cycles + cs.fill_port_busy_cycles; - return ret; - } + cache_sub_stats operator+(const cache_sub_stats &cs) { + /// + /// Overloading + operator to easily accumulate stats + /// + cache_sub_stats ret; + ret.accesses = accesses + cs.accesses; + ret.misses = misses + cs.misses; + ret.pending_hits = pending_hits + cs.pending_hits; + ret.res_fails = res_fails + cs.res_fails; + ret.port_available_cycles = + port_available_cycles + cs.port_available_cycles; + ret.data_port_busy_cycles = + data_port_busy_cycles + cs.data_port_busy_cycles; + ret.fill_port_busy_cycles = + fill_port_busy_cycles + cs.fill_port_busy_cycles; + return ret; + } - void print_port_stats(FILE *fout, const char *cache_name) const; + void print_port_stats(FILE *fout, const char *cache_name) const; }; - // Used for collecting AerialVision per-window statistics -struct cache_sub_stats_pw{ - unsigned accesses; - unsigned write_misses; - unsigned write_hits; - unsigned write_pending_hits; - unsigned write_res_fails; +struct cache_sub_stats_pw { + unsigned accesses; + unsigned write_misses; + unsigned write_hits; + unsigned write_pending_hits; + unsigned write_res_fails; - unsigned read_misses; - unsigned read_hits; - unsigned read_pending_hits; - unsigned read_res_fails; - - cache_sub_stats_pw(){ - clear(); - } - void clear(){ - accesses = 0; - write_misses = 0; - write_hits = 0; - write_pending_hits = 0; - write_res_fails = 0; - read_misses = 0; - read_hits = 0; - read_pending_hits = 0; - read_res_fails = 0; - } - cache_sub_stats_pw &operator+=(const cache_sub_stats_pw &css){ - /// - /// Overloading += operator to easily accumulate stats - /// - accesses += css.accesses; - write_misses += css.write_misses; - read_misses += css.read_misses; - write_pending_hits += css.write_pending_hits; - read_pending_hits += css.read_pending_hits; - write_res_fails += css.write_res_fails; - read_res_fails += css.read_res_fails; - return *this; - } + unsigned read_misses; + unsigned read_hits; + unsigned read_pending_hits; + unsigned read_res_fails; - cache_sub_stats_pw operator+(const cache_sub_stats_pw &cs){ - /// - /// Overloading + operator to easily accumulate stats - /// - cache_sub_stats_pw ret; - ret.accesses = accesses + cs.accesses; - ret.write_misses = write_misses + cs.write_misses; - ret.read_misses = read_misses + cs.read_misses; - ret.write_pending_hits = write_pending_hits + cs.write_pending_hits; - ret.read_pending_hits = read_pending_hits + cs.read_pending_hits; - ret.write_res_fails = write_res_fails + cs.write_res_fails; - ret.read_res_fails = read_res_fails + cs.read_res_fails; - return ret; - } + cache_sub_stats_pw() { clear(); } + void clear() { + accesses = 0; + write_misses = 0; + write_hits = 0; + write_pending_hits = 0; + write_res_fails = 0; + read_misses = 0; + read_hits = 0; + read_pending_hits = 0; + read_res_fails = 0; + } + cache_sub_stats_pw &operator+=(const cache_sub_stats_pw &css) { + /// + /// Overloading += operator to easily accumulate stats + /// + accesses += css.accesses; + write_misses += css.write_misses; + read_misses += css.read_misses; + write_pending_hits += css.write_pending_hits; + read_pending_hits += css.read_pending_hits; + write_res_fails += css.write_res_fails; + read_res_fails += css.read_res_fails; + return *this; + } + cache_sub_stats_pw operator+(const cache_sub_stats_pw &cs) { + /// + /// Overloading + operator to easily accumulate stats + /// + cache_sub_stats_pw ret; + ret.accesses = accesses + cs.accesses; + ret.write_misses = write_misses + cs.write_misses; + ret.read_misses = read_misses + cs.read_misses; + ret.write_pending_hits = write_pending_hits + cs.write_pending_hits; + ret.read_pending_hits = read_pending_hits + cs.read_pending_hits; + ret.write_res_fails = write_res_fails + cs.write_res_fails; + ret.read_res_fails = read_res_fails + cs.read_res_fails; + return ret; + } }; - /// /// Cache_stats /// Used to record statistics for each cache. @@ -1039,484 +1059,468 @@ struct cache_sub_stats_pw{ /// 'cache_request_status' : [mem_access_type][cache_request_status] /// class cache_stats { -public: - cache_stats(); - void clear(); - // Clear AerialVision cache stats after each window - void clear_pw(); - void inc_stats(int access_type, int access_outcome); - // Increment AerialVision cache stats - void inc_stats_pw(int access_type, int access_outcome); - void inc_fail_stats(int access_type, int fail_outcome); - enum cache_request_status select_stats_status(enum cache_request_status probe, enum cache_request_status access) const; - unsigned long long &operator()(int access_type, int access_outcome, bool fail_outcome); - unsigned long long operator()(int access_type, int access_outcome, bool fail_outcome) const; - cache_stats operator+(const cache_stats &cs); - cache_stats &operator+=(const cache_stats &cs); - void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const; - void print_fail_stats(FILE *fout, const char *cache_name = "Cache_fail_stats") const; + public: + cache_stats(); + void clear(); + // Clear AerialVision cache stats after each window + void clear_pw(); + void inc_stats(int access_type, int access_outcome); + // Increment AerialVision cache stats + void inc_stats_pw(int access_type, int access_outcome); + void inc_fail_stats(int access_type, int fail_outcome); + enum cache_request_status select_stats_status( + enum cache_request_status probe, enum cache_request_status access) const; + unsigned long long &operator()(int access_type, int access_outcome, + bool fail_outcome); + unsigned long long operator()(int access_type, int access_outcome, + bool fail_outcome) const; + cache_stats operator+(const cache_stats &cs); + cache_stats &operator+=(const cache_stats &cs); + void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const; + void print_fail_stats(FILE *fout, + const char *cache_name = "Cache_fail_stats") const; - unsigned long long get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const; - void get_sub_stats(struct cache_sub_stats &css) const; + unsigned long long get_stats(enum mem_access_type *access_type, + unsigned num_access_type, + enum cache_request_status *access_status, + unsigned num_access_status) const; + void get_sub_stats(struct cache_sub_stats &css) const; - // Get per-window cache stats for AerialVision - void get_sub_stats_pw(struct cache_sub_stats_pw &css) const; + // Get per-window cache stats for AerialVision + void get_sub_stats_pw(struct cache_sub_stats_pw &css) const; - void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); -private: - bool check_valid(int type, int status) const; - bool check_fail_valid(int type, int fail) const; + void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); + private: + bool check_valid(int type, int status) const; + bool check_fail_valid(int type, int fail) const; - std::vector< std::vector<unsigned long long> > m_stats; - // AerialVision cache stats (per-window) - std::vector< std::vector<unsigned long long> > m_stats_pw; - std::vector< std::vector<unsigned long long> > m_fail_stats; + std::vector<std::vector<unsigned long long> > m_stats; + // AerialVision cache stats (per-window) + std::vector<std::vector<unsigned long long> > m_stats_pw; + std::vector<std::vector<unsigned long long> > m_fail_stats; - unsigned long long m_cache_port_available_cycles; - unsigned long long m_cache_data_port_busy_cycles; - unsigned long long m_cache_fill_port_busy_cycles; + unsigned long long m_cache_port_available_cycles; + unsigned long long m_cache_data_port_busy_cycles; + unsigned long long m_cache_fill_port_busy_cycles; }; class cache_t { -public: - virtual ~cache_t() {} - virtual enum cache_request_status access( new_addr_type addr, mem_fetch *mf, unsigned time, std::list<cache_event> &events ) = 0; + public: + virtual ~cache_t() {} + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) = 0; - // accessors for cache bandwidth availability - virtual bool data_port_free() const = 0; - virtual bool fill_port_free() const = 0; + // accessors for cache bandwidth availability + virtual bool data_port_free() const = 0; + virtual bool fill_port_free() const = 0; }; -bool was_write_sent( const std::list<cache_event> &events ); -bool was_read_sent( const std::list<cache_event> &events ); -bool was_writeallocate_sent( const std::list<cache_event> &events ); +bool was_write_sent(const std::list<cache_event> &events); +bool was_read_sent(const std::list<cache_event> &events); +bool was_writeallocate_sent(const std::list<cache_event> &events); /// Baseline cache /// Implements common functions for read_only_cache and data_cache /// Each subclass implements its own 'access' function class baseline_cache : public cache_t { -public: - baseline_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, - enum mem_fetch_status status ) - : m_config(config), m_tag_array(new tag_array(config,core_id,type_id)), - m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), - m_bandwidth_management(config) - { - init( name, config, memport, status ); - } + public: + baseline_cache(const char *name, cache_config &config, int core_id, + int type_id, mem_fetch_interface *memport, + enum mem_fetch_status status) + : m_config(config), + m_tag_array(new tag_array(config, core_id, type_id)), + m_mshrs(config.m_mshr_entries, config.m_mshr_max_merge), + m_bandwidth_management(config) { + init(name, config, memport, status); + } - void init( const char *name, - const cache_config &config, - mem_fetch_interface *memport, - enum mem_fetch_status status ) - { - m_name = name; - assert(config.m_mshr_type == ASSOC || config.m_mshr_type == SECTOR_ASSOC); - m_memport=memport; - m_miss_queue_status = status; - } + void init(const char *name, const cache_config &config, + mem_fetch_interface *memport, enum mem_fetch_status status) { + m_name = name; + assert(config.m_mshr_type == ASSOC || config.m_mshr_type == SECTOR_ASSOC); + m_memport = memport; + m_miss_queue_status = status; + } - virtual ~baseline_cache() - { - delete m_tag_array; - } + virtual ~baseline_cache() { delete m_tag_array; } - void update_cache_parameters(cache_config &config) - { - m_config=config; - m_tag_array->update_cache_parameters(config); - m_mshrs.check_mshr_parameters(config.m_mshr_entries,config.m_mshr_max_merge); - } + void update_cache_parameters(cache_config &config) { + m_config = config; + m_tag_array->update_cache_parameters(config); + m_mshrs.check_mshr_parameters(config.m_mshr_entries, + config.m_mshr_max_merge); + } - virtual enum cache_request_status access( new_addr_type addr, mem_fetch *mf, unsigned time, std::list<cache_event> &events ) = 0; - /// Sends next request to lower level of memory - void cycle(); - /// Interface for response from lower memory level (model bandwidth restictions in caller) - void fill( mem_fetch *mf, unsigned time ); - /// Checks if mf is waiting to be filled by lower memory level - bool waiting_for_fill( mem_fetch *mf ); - /// Are any (accepted) accesses that had to wait for memory now ready? (does not include accesses that "HIT") - bool access_ready() const {return m_mshrs.access_ready();} - /// Pop next ready access (does not include accesses that "HIT") - mem_fetch *next_access(){return m_mshrs.next_access();} - // flash invalidate all entries in cache - void flush(){m_tag_array->flush();} - void invalidate(){m_tag_array->invalidate();} - void print(FILE *fp, unsigned &accesses, unsigned &misses) const; - void display_state( FILE *fp ) const; + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events) = 0; + /// Sends next request to lower level of memory + void cycle(); + /// Interface for response from lower memory level (model bandwidth + /// restictions in caller) + void fill(mem_fetch *mf, unsigned time); + /// Checks if mf is waiting to be filled by lower memory level + bool waiting_for_fill(mem_fetch *mf); + /// Are any (accepted) accesses that had to wait for memory now ready? (does + /// not include accesses that "HIT") + bool access_ready() const { return m_mshrs.access_ready(); } + /// Pop next ready access (does not include accesses that "HIT") + mem_fetch *next_access() { return m_mshrs.next_access(); } + // flash invalidate all entries in cache + void flush() { m_tag_array->flush(); } + void invalidate() { m_tag_array->invalidate(); } + void print(FILE *fp, unsigned &accesses, unsigned &misses) const; + void display_state(FILE *fp) const; - // Stat collection - const cache_stats &get_stats() const { - return m_stats; - } - unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const{ - return m_stats.get_stats(access_type, num_access_type, access_status, num_access_status); - } - void get_sub_stats(struct cache_sub_stats &css) const { - m_stats.get_sub_stats(css); - } - // Clear per-window stats for AerialVision support - void clear_pw(){ - m_stats.clear_pw(); - } - // Per-window sub stats for AerialVision support - void get_sub_stats_pw(struct cache_sub_stats_pw &css) const { - m_stats.get_sub_stats_pw(css); - } + // Stat collection + const cache_stats &get_stats() const { return m_stats; } + unsigned get_stats(enum mem_access_type *access_type, + unsigned num_access_type, + enum cache_request_status *access_status, + unsigned num_access_status) const { + return m_stats.get_stats(access_type, num_access_type, access_status, + num_access_status); + } + void get_sub_stats(struct cache_sub_stats &css) const { + m_stats.get_sub_stats(css); + } + // Clear per-window stats for AerialVision support + void clear_pw() { m_stats.clear_pw(); } + // Per-window sub stats for AerialVision support + void get_sub_stats_pw(struct cache_sub_stats_pw &css) const { + m_stats.get_sub_stats_pw(css); + } - // accessors for cache bandwidth availability - bool data_port_free() const { return m_bandwidth_management.data_port_free(); } - bool fill_port_free() const { return m_bandwidth_management.fill_port_free(); } + // accessors for cache bandwidth availability + bool data_port_free() const { + return m_bandwidth_management.data_port_free(); + } + bool fill_port_free() const { + return m_bandwidth_management.fill_port_free(); + } - // This is a gapping hole we are poking in the system to quickly handle - // filling the cache on cudamemcopies. We don't care about anything other than - // L2 state after the memcopy - so just force the tag array to act as though - // something is read or written without doing anything else. - void force_tag_access( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) - { - m_tag_array->fill( addr, time, mask ); - } + // This is a gapping hole we are poking in the system to quickly handle + // filling the cache on cudamemcopies. We don't care about anything other than + // L2 state after the memcopy - so just force the tag array to act as though + // something is read or written without doing anything else. + void force_tag_access(new_addr_type addr, unsigned time, + mem_access_sector_mask_t mask) { + m_tag_array->fill(addr, time, mask); + } -protected: - // Constructor that can be used by derived classes with custom tag arrays - baseline_cache( const char *name, - cache_config &config, - int core_id, - int type_id, - mem_fetch_interface *memport, - enum mem_fetch_status status, - tag_array* new_tag_array ) - : m_config(config), - m_tag_array( new_tag_array ), - m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), - m_bandwidth_management(config) - { - init( name, config, memport, status ); - } + protected: + // Constructor that can be used by derived classes with custom tag arrays + baseline_cache(const char *name, cache_config &config, int core_id, + int type_id, mem_fetch_interface *memport, + enum mem_fetch_status status, tag_array *new_tag_array) + : m_config(config), + m_tag_array(new_tag_array), + m_mshrs(config.m_mshr_entries, config.m_mshr_max_merge), + m_bandwidth_management(config) { + init(name, config, memport, status); + } -protected: - std::string m_name; - cache_config &m_config; - tag_array* m_tag_array; - mshr_table m_mshrs; - std::list<mem_fetch*> m_miss_queue; - enum mem_fetch_status m_miss_queue_status; - mem_fetch_interface *m_memport; + protected: + std::string m_name; + cache_config &m_config; + tag_array *m_tag_array; + mshr_table m_mshrs; + std::list<mem_fetch *> m_miss_queue; + enum mem_fetch_status m_miss_queue_status; + mem_fetch_interface *m_memport; - struct extra_mf_fields { - extra_mf_fields() { m_valid = false;} - extra_mf_fields( new_addr_type a, new_addr_type ad, unsigned i, unsigned d, const cache_config& m_config) - { - m_valid = true; - m_block_addr = a; - m_addr = ad; - m_cache_index = i; - m_data_size = d; - pending_read = m_config.m_mshr_type == SECTOR_ASSOC? m_config.m_line_sz/SECTOR_SIZE : 0; + struct extra_mf_fields { + extra_mf_fields() { m_valid = false; } + extra_mf_fields(new_addr_type a, new_addr_type ad, unsigned i, unsigned d, + const cache_config &m_config) { + m_valid = true; + m_block_addr = a; + m_addr = ad; + m_cache_index = i; + m_data_size = d; + pending_read = m_config.m_mshr_type == SECTOR_ASSOC + ? m_config.m_line_sz / SECTOR_SIZE + : 0; + } + bool m_valid; + new_addr_type m_block_addr; + new_addr_type m_addr; + unsigned m_cache_index; + unsigned m_data_size; + // this variable is used when a load request generates multiple load + // transactions For example, a read request from non-sector L1 request sends + // a request to sector L2 + unsigned pending_read; + }; - } - bool m_valid; - new_addr_type m_block_addr; - new_addr_type m_addr; - unsigned m_cache_index; - unsigned m_data_size; - //this variable is used when a load request generates multiple load transactions - //For example, a read request from non-sector L1 request sends a request to sector L2 - unsigned pending_read; - }; + typedef std::map<mem_fetch *, extra_mf_fields> extra_mf_fields_lookup; - typedef std::map<mem_fetch*,extra_mf_fields> extra_mf_fields_lookup; + extra_mf_fields_lookup m_extra_mf_fields; - extra_mf_fields_lookup m_extra_mf_fields; + cache_stats m_stats; - cache_stats m_stats; + /// Checks whether this request can be handled on this cycle. num_miss equals + /// max # of misses to be handled on this cycle + bool miss_queue_full(unsigned num_miss) { + return ((m_miss_queue.size() + num_miss) >= m_config.m_miss_queue_size); + } + /// Read miss handler without writeback + void send_read_request(new_addr_type addr, new_addr_type block_addr, + unsigned cache_index, mem_fetch *mf, unsigned time, + bool &do_miss, std::list<cache_event> &events, + bool read_only, bool wa); + /// Read miss handler. Check MSHR hit or MSHR available + void send_read_request(new_addr_type addr, new_addr_type block_addr, + unsigned cache_index, mem_fetch *mf, unsigned time, + bool &do_miss, bool &wb, evicted_block_info &evicted, + std::list<cache_event> &events, bool read_only, + bool wa); - /// Checks whether this request can be handled on this cycle. num_miss equals max # of misses to be handled on this cycle - bool miss_queue_full(unsigned num_miss){ - return ( (m_miss_queue.size()+num_miss) >= m_config.m_miss_queue_size ); - } - /// Read miss handler without writeback - void send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, std::list<cache_event> &events, bool read_only, bool wa); - /// Read miss handler. Check MSHR hit or MSHR available - void send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list<cache_event> &events, bool read_only, bool wa); + /// Sub-class containing all metadata for port bandwidth management + class bandwidth_management { + public: + bandwidth_management(cache_config &config); - /// Sub-class containing all metadata for port bandwidth management - class bandwidth_management - { - public: - bandwidth_management(cache_config &config); + /// use the data port based on the outcome and events generated by the + /// mem_fetch request + void use_data_port(mem_fetch *mf, enum cache_request_status outcome, + const std::list<cache_event> &events); - /// use the data port based on the outcome and events generated by the mem_fetch request - void use_data_port(mem_fetch *mf, enum cache_request_status outcome, const std::list<cache_event> &events); + /// use the fill port + void use_fill_port(mem_fetch *mf); - /// use the fill port - void use_fill_port(mem_fetch *mf); + /// called every cache cycle to free up the ports + void replenish_port_bandwidth(); - /// called every cache cycle to free up the ports - void replenish_port_bandwidth(); + /// query for data port availability + bool data_port_free() const; + /// query for fill port availability + bool fill_port_free() const; - /// query for data port availability - bool data_port_free() const; - /// query for fill port availability - bool fill_port_free() const; - protected: - const cache_config &m_config; + protected: + const cache_config &m_config; - int m_data_port_occupied_cycles; //< Number of cycle that the data port remains used - int m_fill_port_occupied_cycles; //< Number of cycle that the fill port remains used - }; + int m_data_port_occupied_cycles; //< Number of cycle that the data port + //remains used + int m_fill_port_occupied_cycles; //< Number of cycle that the fill port + //remains used + }; - bandwidth_management m_bandwidth_management; + bandwidth_management m_bandwidth_management; }; /// Read only cache class read_only_cache : public baseline_cache { -public: - read_only_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, enum mem_fetch_status status ) - : baseline_cache(name,config,core_id,type_id,memport,status){} + public: + read_only_cache(const char *name, cache_config &config, int core_id, + int type_id, mem_fetch_interface *memport, + enum mem_fetch_status status) + : baseline_cache(name, config, core_id, type_id, memport, status) {} - /// Access cache for read_only_cache: returns RESERVATION_FAIL if request could not be accepted (for any reason) - virtual enum cache_request_status access( new_addr_type addr, mem_fetch *mf, unsigned time, std::list<cache_event> &events ); + /// Access cache for read_only_cache: returns RESERVATION_FAIL if request + /// could not be accepted (for any reason) + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events); - virtual ~read_only_cache(){} + virtual ~read_only_cache() {} -protected: - read_only_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, enum mem_fetch_status status, tag_array* new_tag_array ) - : baseline_cache(name,config,core_id,type_id,memport,status, new_tag_array){} + protected: + read_only_cache(const char *name, cache_config &config, int core_id, + int type_id, mem_fetch_interface *memport, + enum mem_fetch_status status, tag_array *new_tag_array) + : baseline_cache(name, config, core_id, type_id, memport, status, + new_tag_array) {} }; /// Data cache - Implements common functions for L1 and L2 data cache class data_cache : public baseline_cache { -public: - data_cache( const char *name, cache_config &config, - int core_id, int type_id, mem_fetch_interface *memport, - mem_fetch_allocator *mfcreator, enum mem_fetch_status status, - mem_access_type wr_alloc_type, mem_access_type wrbk_type, class gpgpu_sim* gpu ) - : baseline_cache(name,config,core_id,type_id,memport,status) - { - init( mfcreator ); - m_wr_alloc_type = wr_alloc_type; - m_wrbk_type = wrbk_type; - m_gpu=gpu; - } + public: + data_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, mem_fetch_allocator *mfcreator, + enum mem_fetch_status status, mem_access_type wr_alloc_type, + mem_access_type wrbk_type, class gpgpu_sim *gpu) + : baseline_cache(name, config, core_id, type_id, memport, status) { + init(mfcreator); + m_wr_alloc_type = wr_alloc_type; + m_wrbk_type = wrbk_type; + m_gpu = gpu; + } - virtual ~data_cache() {} + virtual ~data_cache() {} - virtual void init( mem_fetch_allocator *mfcreator ) - { - m_memfetch_creator=mfcreator; + virtual void init(mem_fetch_allocator *mfcreator) { + m_memfetch_creator = mfcreator; - // Set read hit function - m_rd_hit = &data_cache::rd_hit_base; + // Set read hit function + m_rd_hit = &data_cache::rd_hit_base; - // Set read miss function - m_rd_miss = &data_cache::rd_miss_base; + // Set read miss function + m_rd_miss = &data_cache::rd_miss_base; - // Set write hit function - switch(m_config.m_write_policy){ - // READ_ONLY is now a separate cache class, config is deprecated - case READ_ONLY: - assert(0 && "Error: Writable Data_cache set as READ_ONLY\n"); - break; - case WRITE_BACK: m_wr_hit = &data_cache::wr_hit_wb; break; - case WRITE_THROUGH: m_wr_hit = &data_cache::wr_hit_wt; break; - case WRITE_EVICT: m_wr_hit = &data_cache::wr_hit_we; break; - case LOCAL_WB_GLOBAL_WT: - m_wr_hit = &data_cache::wr_hit_global_we_local_wb; - break; - default: - assert(0 && "Error: Must set valid cache write policy\n"); - break; // Need to set a write hit function - } - - // Set write miss function - switch(m_config.m_write_alloc_policy){ - case NO_WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_no_wa; break; - case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa_naive; break; - case FETCH_ON_WRITE: m_wr_miss = &data_cache::wr_miss_wa_fetch_on_write; break; - case LAZY_FETCH_ON_READ: m_wr_miss = &data_cache::wr_miss_wa_lazy_fetch_on_read; break; - default: - assert(0 && "Error: Must set valid cache write miss policy\n"); - break; // Need to set a write miss function - } + // Set write hit function + switch (m_config.m_write_policy) { + // READ_ONLY is now a separate cache class, config is deprecated + case READ_ONLY: + assert(0 && "Error: Writable Data_cache set as READ_ONLY\n"); + break; + case WRITE_BACK: + m_wr_hit = &data_cache::wr_hit_wb; + break; + case WRITE_THROUGH: + m_wr_hit = &data_cache::wr_hit_wt; + break; + case WRITE_EVICT: + m_wr_hit = &data_cache::wr_hit_we; + break; + case LOCAL_WB_GLOBAL_WT: + m_wr_hit = &data_cache::wr_hit_global_we_local_wb; + break; + default: + assert(0 && "Error: Must set valid cache write policy\n"); + break; // Need to set a write hit function } - virtual enum cache_request_status access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ); -protected: - data_cache( const char *name, - cache_config &config, - int core_id, - int type_id, - mem_fetch_interface *memport, - mem_fetch_allocator *mfcreator, - enum mem_fetch_status status, - tag_array* new_tag_array, - mem_access_type wr_alloc_type, - mem_access_type wrbk_type, - class gpgpu_sim* gpu ) - : baseline_cache(name, config, core_id, type_id, memport,status, new_tag_array) - { - init( mfcreator ); - m_wr_alloc_type = wr_alloc_type; - m_wrbk_type = wrbk_type; - m_gpu=gpu; + // Set write miss function + switch (m_config.m_write_alloc_policy) { + case NO_WRITE_ALLOCATE: + m_wr_miss = &data_cache::wr_miss_no_wa; + break; + case WRITE_ALLOCATE: + m_wr_miss = &data_cache::wr_miss_wa_naive; + break; + case FETCH_ON_WRITE: + m_wr_miss = &data_cache::wr_miss_wa_fetch_on_write; + break; + case LAZY_FETCH_ON_READ: + m_wr_miss = &data_cache::wr_miss_wa_lazy_fetch_on_read; + break; + default: + assert(0 && "Error: Must set valid cache write miss policy\n"); + break; // Need to set a write miss function } + } - mem_access_type m_wr_alloc_type; // Specifies type of write allocate request (e.g., L1 or L2) - mem_access_type m_wrbk_type; // Specifies type of writeback request (e.g., L1 or L2) - class gpgpu_sim* m_gpu; + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events); - //! A general function that takes the result of a tag_array probe - // and performs the correspding functions based on the cache configuration - // The access fucntion calls this function - enum cache_request_status - process_tag_probe( bool wr, - enum cache_request_status status, - new_addr_type addr, - unsigned cache_index, - mem_fetch* mf, - unsigned time, - std::list<cache_event>& events ); + protected: + data_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, mem_fetch_allocator *mfcreator, + enum mem_fetch_status status, tag_array *new_tag_array, + mem_access_type wr_alloc_type, mem_access_type wrbk_type, + class gpgpu_sim *gpu) + : baseline_cache(name, config, core_id, type_id, memport, status, + new_tag_array) { + init(mfcreator); + m_wr_alloc_type = wr_alloc_type; + m_wrbk_type = wrbk_type; + m_gpu = gpu; + } -protected: - mem_fetch_allocator *m_memfetch_creator; + mem_access_type m_wr_alloc_type; // Specifies type of write allocate request + // (e.g., L1 or L2) + mem_access_type + m_wrbk_type; // Specifies type of writeback request (e.g., L1 or L2) + class gpgpu_sim *m_gpu; - // Functions for data cache access - /// Sends write request to lower level memory (write or writeback) - void send_write_request( mem_fetch *mf, - cache_event request, - unsigned time, - std::list<cache_event> &events); + //! A general function that takes the result of a tag_array probe + // and performs the correspding functions based on the cache configuration + // The access fucntion calls this function + enum cache_request_status process_tag_probe(bool wr, + enum cache_request_status status, + new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, unsigned time, + std::list<cache_event> &events); - // Member Function pointers - Set by configuration options - // to the functions below each grouping - /******* Write-hit configs *******/ - enum cache_request_status - (data_cache::*m_wr_hit)( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); - /// Marks block as MODIFIED and updates block LRU - enum cache_request_status - wr_hit_wb( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-back - enum cache_request_status - wr_hit_wt( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-through + protected: + mem_fetch_allocator *m_memfetch_creator; - /// Marks block as INVALID and sends write request to lower level memory - enum cache_request_status - wr_hit_we( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-evict - enum cache_request_status - wr_hit_global_we_local_wb( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); - // global write-evict, local write-back + // Functions for data cache access + /// Sends write request to lower level memory (write or writeback) + void send_write_request(mem_fetch *mf, cache_event request, unsigned time, + std::list<cache_event> &events); + // Member Function pointers - Set by configuration options + // to the functions below each grouping + /******* Write-hit configs *******/ + enum cache_request_status (data_cache::*m_wr_hit)( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status); + /// Marks block as MODIFIED and updates block LRU + enum cache_request_status wr_hit_wb( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); // write-back + enum cache_request_status wr_hit_wt( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); // write-through - /******* Write-miss configs *******/ - enum cache_request_status - (data_cache::*m_wr_miss)( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); - /// Sends read request, and possible write-back request, - // to lower level memory for a write miss with write-allocate - enum cache_request_status - wr_miss_wa_naive( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-allocate-send-write-and-read-request - enum cache_request_status - wr_miss_wa_fetch_on_write( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-allocate with fetch-on-every-write - enum cache_request_status - wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-allocate with read-fetch-only - enum cache_request_status - wr_miss_wa_write_validate( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // write-allocate that writes with no read fetch - enum cache_request_status - wr_miss_no_wa( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); // no write-allocate + /// Marks block as INVALID and sends write request to lower level memory + enum cache_request_status wr_hit_we( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); // write-evict + enum cache_request_status wr_hit_global_we_local_wb( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status); + // global write-evict, local write-back - // Currently no separate functions for reads - /******* Read-hit configs *******/ - enum cache_request_status - (data_cache::*m_rd_hit)( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); - enum cache_request_status - rd_hit_base( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); + /******* Write-miss configs *******/ + enum cache_request_status (data_cache::*m_wr_miss)( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status); + /// Sends read request, and possible write-back request, + // to lower level memory for a write miss with write-allocate + enum cache_request_status wr_miss_wa_naive( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status + status); // write-allocate-send-write-and-read-request + enum cache_request_status wr_miss_wa_fetch_on_write( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status + status); // write-allocate with fetch-on-every-write + enum cache_request_status wr_miss_wa_lazy_fetch_on_read( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); // write-allocate with read-fetch-only + enum cache_request_status wr_miss_wa_write_validate( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status + status); // write-allocate that writes with no read fetch + enum cache_request_status wr_miss_no_wa( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); // no write-allocate - /******* Read-miss configs *******/ - enum cache_request_status - (data_cache::*m_rd_miss)( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); - enum cache_request_status - rd_miss_base( new_addr_type addr, - unsigned cache_index, - mem_fetch*mf, - unsigned time, - std::list<cache_event> &events, - enum cache_request_status status ); + // Currently no separate functions for reads + /******* Read-hit configs *******/ + enum cache_request_status (data_cache::*m_rd_hit)( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status); + enum cache_request_status rd_hit_base(new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); + /******* Read-miss configs *******/ + enum cache_request_status (data_cache::*m_rd_miss)( + new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, + std::list<cache_event> &events, enum cache_request_status status); + enum cache_request_status rd_miss_base(new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events, + enum cache_request_status status); }; /// This is meant to model the first level data cache in Fermi. @@ -1524,243 +1528,242 @@ protected: /// the granularity of individual blocks /// (the policy used in fermi according to the CUDA manual) class l1_cache : public data_cache { -public: - l1_cache(const char *name, cache_config &config, - int core_id, int type_id, mem_fetch_interface *memport, - mem_fetch_allocator *mfcreator, enum mem_fetch_status status, class gpgpu_sim* gpu ) - : data_cache(name,config,core_id,type_id,memport,mfcreator,status, L1_WR_ALLOC_R, L1_WRBK_ACC, gpu){} + public: + l1_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, mem_fetch_allocator *mfcreator, + enum mem_fetch_status status, class gpgpu_sim *gpu) + : data_cache(name, config, core_id, type_id, memport, mfcreator, status, + L1_WR_ALLOC_R, L1_WRBK_ACC, gpu) {} - virtual ~l1_cache(){} + virtual ~l1_cache() {} - virtual enum cache_request_status - access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ); - -protected: - l1_cache( const char *name, - cache_config &config, - int core_id, - int type_id, - mem_fetch_interface *memport, - mem_fetch_allocator *mfcreator, - enum mem_fetch_status status, - tag_array* new_tag_array, - class gpgpu_sim* gpu) - : data_cache( name, - config, - core_id,type_id,memport,mfcreator,status, new_tag_array, L1_WR_ALLOC_R, L1_WRBK_ACC, gpu ){} + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events); + protected: + l1_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, mem_fetch_allocator *mfcreator, + enum mem_fetch_status status, tag_array *new_tag_array, + class gpgpu_sim *gpu) + : data_cache(name, config, core_id, type_id, memport, mfcreator, status, + new_tag_array, L1_WR_ALLOC_R, L1_WRBK_ACC, gpu) {} }; /// Models second level shared cache with global write-back /// and write-allocate policies class l2_cache : public data_cache { -public: - l2_cache(const char *name, cache_config &config, - int core_id, int type_id, mem_fetch_interface *memport, - mem_fetch_allocator *mfcreator, enum mem_fetch_status status, class gpgpu_sim* gpu ) - : data_cache(name,config,core_id,type_id,memport,mfcreator,status, L2_WR_ALLOC_R, L2_WRBK_ACC, gpu){} + public: + l2_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, mem_fetch_allocator *mfcreator, + enum mem_fetch_status status, class gpgpu_sim *gpu) + : data_cache(name, config, core_id, type_id, memport, mfcreator, status, + L2_WR_ALLOC_R, L2_WRBK_ACC, gpu) {} - virtual ~l2_cache() {} + virtual ~l2_cache() {} - virtual enum cache_request_status - access( new_addr_type addr, - mem_fetch *mf, - unsigned time, - std::list<cache_event> &events ); + virtual enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events); }; /*****************************************************************************/ // See the following paper to understand this cache model: -// -// Igehy, et al., Prefetching in a Texture Cache Architecture, +// +// Igehy, et al., Prefetching in a Texture Cache Architecture, // Proceedings of the 1998 Eurographics/SIGGRAPH Workshop on Graphics Hardware // http://www-graphics.stanford.edu/papers/texture_prefetch/ class tex_cache : public cache_t { -public: - tex_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, - enum mem_fetch_status request_status, - enum mem_fetch_status rob_status ) - : m_config(config), - m_tags(config,core_id,type_id), - m_fragment_fifo(config.m_fragment_fifo_entries), - m_request_fifo(config.m_request_fifo_entries), - m_rob(config.m_rob_entries), - m_result_fifo(config.m_result_fifo_entries) - { - m_name = name; - assert(config.m_mshr_type == TEX_FIFO || config.m_mshr_type == SECTOR_TEX_FIFO ); - assert(config.m_write_policy == READ_ONLY); - assert(config.m_alloc_policy == ON_MISS); - m_memport=memport; - m_cache = new data_block[ config.get_num_lines() ]; - m_request_queue_status = request_status; - m_rob_status = rob_status; - } + public: + tex_cache(const char *name, cache_config &config, int core_id, int type_id, + mem_fetch_interface *memport, enum mem_fetch_status request_status, + enum mem_fetch_status rob_status) + : m_config(config), + m_tags(config, core_id, type_id), + m_fragment_fifo(config.m_fragment_fifo_entries), + m_request_fifo(config.m_request_fifo_entries), + m_rob(config.m_rob_entries), + m_result_fifo(config.m_result_fifo_entries) { + m_name = name; + assert(config.m_mshr_type == TEX_FIFO || + config.m_mshr_type == SECTOR_TEX_FIFO); + assert(config.m_write_policy == READ_ONLY); + assert(config.m_alloc_policy == ON_MISS); + m_memport = memport; + m_cache = new data_block[config.get_num_lines()]; + m_request_queue_status = request_status; + m_rob_status = rob_status; + } - /// Access function for tex_cache - /// return values: RESERVATION_FAIL if request could not be accepted - /// otherwise returns HIT_RESERVED or MISS; NOTE: *never* returns HIT - /// since unlike a normal CPU cache, a "HIT" in texture cache does not - /// mean the data is ready (still need to get through fragment fifo) - enum cache_request_status access( new_addr_type addr, mem_fetch *mf, unsigned time, std::list<cache_event> &events ); - void cycle(); - /// Place returning cache block into reorder buffer - void fill( mem_fetch *mf, unsigned time ); - /// Are any (accepted) accesses that had to wait for memory now ready? (does not include accesses that "HIT") - bool access_ready() const{return !m_result_fifo.empty();} - /// Pop next ready access (includes both accesses that "HIT" and those that "MISS") - mem_fetch *next_access(){return m_result_fifo.pop();} - void display_state( FILE *fp ) const; + /// Access function for tex_cache + /// return values: RESERVATION_FAIL if request could not be accepted + /// otherwise returns HIT_RESERVED or MISS; NOTE: *never* returns HIT + /// since unlike a normal CPU cache, a "HIT" in texture cache does not + /// mean the data is ready (still need to get through fragment fifo) + enum cache_request_status access(new_addr_type addr, mem_fetch *mf, + unsigned time, + std::list<cache_event> &events); + void cycle(); + /// Place returning cache block into reorder buffer + void fill(mem_fetch *mf, unsigned time); + /// Are any (accepted) accesses that had to wait for memory now ready? (does + /// not include accesses that "HIT") + bool access_ready() const { return !m_result_fifo.empty(); } + /// Pop next ready access (includes both accesses that "HIT" and those that + /// "MISS") + mem_fetch *next_access() { return m_result_fifo.pop(); } + void display_state(FILE *fp) const; - // accessors for cache bandwidth availability - stubs for now - bool data_port_free() const { return true; } - bool fill_port_free() const { return true; } + // accessors for cache bandwidth availability - stubs for now + bool data_port_free() const { return true; } + bool fill_port_free() const { return true; } - // Stat collection - const cache_stats &get_stats() const { - return m_stats; - } - unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const{ - return m_stats.get_stats(access_type, num_access_type, access_status, num_access_status); - } + // Stat collection + const cache_stats &get_stats() const { return m_stats; } + unsigned get_stats(enum mem_access_type *access_type, + unsigned num_access_type, + enum cache_request_status *access_status, + unsigned num_access_status) const { + return m_stats.get_stats(access_type, num_access_type, access_status, + num_access_status); + } - void get_sub_stats(struct cache_sub_stats &css) const{ - m_stats.get_sub_stats(css); + void get_sub_stats(struct cache_sub_stats &css) const { + m_stats.get_sub_stats(css); + } + + private: + std::string m_name; + const cache_config &m_config; + + struct fragment_entry { + fragment_entry() {} + fragment_entry(mem_fetch *mf, unsigned idx, bool m, unsigned d) { + m_request = mf; + m_cache_index = idx; + m_miss = m; + m_data_size = d; } -private: - std::string m_name; - const cache_config &m_config; + mem_fetch *m_request; // request information + unsigned m_cache_index; // where to look for data + bool m_miss; // true if sent memory request + unsigned m_data_size; + }; - struct fragment_entry { - fragment_entry() {} - fragment_entry( mem_fetch *mf, unsigned idx, bool m, unsigned d ) - { - m_request=mf; - m_cache_index=idx; - m_miss=m; - m_data_size=d; - } - mem_fetch *m_request; // request information - unsigned m_cache_index; // where to look for data - bool m_miss; // true if sent memory request - unsigned m_data_size; - }; + struct rob_entry { + rob_entry() { + m_ready = false; + m_time = 0; + m_request = NULL; + } + rob_entry(unsigned i, mem_fetch *mf, new_addr_type a) { + m_ready = false; + m_index = i; + m_time = 0; + m_request = mf; + m_block_addr = a; + } + bool m_ready; + unsigned m_time; // which cycle did this entry become ready? + unsigned m_index; // where in cache should block be placed? + mem_fetch *m_request; + new_addr_type m_block_addr; + }; - struct rob_entry { - rob_entry() { m_ready = false; m_time=0; m_request=NULL;} - rob_entry( unsigned i, mem_fetch *mf, new_addr_type a ) - { - m_ready=false; - m_index=i; - m_time=0; - m_request=mf; - m_block_addr=a; - } - bool m_ready; - unsigned m_time; // which cycle did this entry become ready? - unsigned m_index; // where in cache should block be placed? - mem_fetch *m_request; - new_addr_type m_block_addr; - }; + struct data_block { + data_block() { m_valid = false; } + bool m_valid; + new_addr_type m_block_addr; + }; - struct data_block { - data_block() { m_valid = false;} - bool m_valid; - new_addr_type m_block_addr; - }; + // TODO: replace fifo_pipeline with this? + template <class T> + class fifo { + public: + fifo(unsigned size) { + m_size = size; + m_num = 0; + m_head = 0; + m_tail = 0; + m_data = new T[size]; + } + bool full() const { return m_num == m_size; } + bool empty() const { return m_num == 0; } + unsigned size() const { return m_num; } + unsigned capacity() const { return m_size; } + unsigned push(const T &e) { + assert(!full()); + m_data[m_head] = e; + unsigned result = m_head; + inc_head(); + return result; + } + T pop() { + assert(!empty()); + T result = m_data[m_tail]; + inc_tail(); + return result; + } + const T &peek(unsigned index) const { + assert(index < m_size); + return m_data[index]; + } + T &peek(unsigned index) { + assert(index < m_size); + return m_data[index]; + } + T &peek() const { return m_data[m_tail]; } + unsigned next_pop_index() const { return m_tail; } - // TODO: replace fifo_pipeline with this? - template<class T> class fifo { - public: - fifo( unsigned size ) - { - m_size=size; - m_num=0; - m_head=0; - m_tail=0; - m_data = new T[size]; - } - bool full() const { return m_num == m_size;} - bool empty() const { return m_num == 0;} - unsigned size() const { return m_num;} - unsigned capacity() const { return m_size;} - unsigned push( const T &e ) - { - assert(!full()); - m_data[m_head] = e; - unsigned result = m_head; - inc_head(); - return result; - } - T pop() - { - assert(!empty()); - T result = m_data[m_tail]; - inc_tail(); - return result; - } - const T &peek( unsigned index ) const - { - assert( index < m_size ); - return m_data[index]; - } - T &peek( unsigned index ) - { - assert( index < m_size ); - return m_data[index]; - } - T &peek() const - { - return m_data[m_tail]; - } - unsigned next_pop_index() const - { - return m_tail; - } - private: - void inc_head() { m_head = (m_head+1)%m_size; m_num++;} - void inc_tail() { assert(m_num>0); m_tail = (m_tail+1)%m_size; m_num--;} + private: + void inc_head() { + m_head = (m_head + 1) % m_size; + m_num++; + } + void inc_tail() { + assert(m_num > 0); + m_tail = (m_tail + 1) % m_size; + m_num--; + } - unsigned m_head; // next entry goes here - unsigned m_tail; // oldest entry found here - unsigned m_num; // how many in fifo? - unsigned m_size; // maximum number of entries in fifo - T *m_data; - }; + unsigned m_head; // next entry goes here + unsigned m_tail; // oldest entry found here + unsigned m_num; // how many in fifo? + unsigned m_size; // maximum number of entries in fifo + T *m_data; + }; - tag_array m_tags; - fifo<fragment_entry> m_fragment_fifo; - fifo<mem_fetch*> m_request_fifo; - fifo<rob_entry> m_rob; - data_block *m_cache; - fifo<mem_fetch*> m_result_fifo; // next completed texture fetch + tag_array m_tags; + fifo<fragment_entry> m_fragment_fifo; + fifo<mem_fetch *> m_request_fifo; + fifo<rob_entry> m_rob; + data_block *m_cache; + fifo<mem_fetch *> m_result_fifo; // next completed texture fetch - mem_fetch_interface *m_memport; - enum mem_fetch_status m_request_queue_status; - enum mem_fetch_status m_rob_status; + mem_fetch_interface *m_memport; + enum mem_fetch_status m_request_queue_status; + enum mem_fetch_status m_rob_status; - struct extra_mf_fields { - extra_mf_fields() { m_valid = false;} - extra_mf_fields( unsigned i, const cache_config &m_config ) - { - m_valid = true; - m_rob_index = i; - pending_read = m_config.m_mshr_type == SECTOR_TEX_FIFO? m_config.m_line_sz/SECTOR_SIZE : 0; - } - bool m_valid; - unsigned m_rob_index; - unsigned pending_read; - }; + struct extra_mf_fields { + extra_mf_fields() { m_valid = false; } + extra_mf_fields(unsigned i, const cache_config &m_config) { + m_valid = true; + m_rob_index = i; + pending_read = m_config.m_mshr_type == SECTOR_TEX_FIFO + ? m_config.m_line_sz / SECTOR_SIZE + : 0; + } + bool m_valid; + unsigned m_rob_index; + unsigned pending_read; + }; - cache_stats m_stats; + cache_stats m_stats; - typedef std::map<mem_fetch*,extra_mf_fields> extra_mf_fields_lookup; + typedef std::map<mem_fetch *, extra_mf_fields> extra_mf_fields_lookup; - extra_mf_fields_lookup m_extra_mf_fields; + extra_mf_fields_lookup m_extra_mf_fields; }; #endif diff --git a/src/gpgpu-sim/gpu-misc.cc b/src/gpgpu-sim/gpu-misc.cc index df042b1..020443a 100644 --- a/src/gpgpu-sim/gpu-misc.cc +++ b/src/gpgpu-sim/gpu-misc.cc @@ -7,37 +7,48 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "gpu-misc.h" -unsigned int LOGB2( unsigned int v ) { - unsigned int shift; - unsigned int r; +unsigned int LOGB2(unsigned int v) { + unsigned int shift; + unsigned int r; - r = 0; + r = 0; - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; - return r; + return r; } diff --git a/src/gpgpu-sim/gpu-misc.h b/src/gpgpu-sim/gpu-misc.h index 509cc3a..117eb78 100644 --- a/src/gpgpu-sim/gpu-misc.h +++ b/src/gpgpu-sim/gpu-misc.h @@ -8,35 +8,35 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef GPU_MISC_H #define GPU_MISC_H -//enables a verbose printout of all L1 cache misses and all MSHR status changes -//good for a single shader configuration +// enables a verbose printout of all L1 cache misses and all MSHR status changes +// good for a single shader configuration #define DEBUGL1MISS 0 -unsigned int LOGB2( unsigned int v ); +unsigned int LOGB2(unsigned int v); -#define gs_min2(a,b) (((a)<(b))?(a):(b)) -#define min3(x,y,z) (((x)<(y) && (x)<(z))?(x):(gs_min2((y),(z)))) +#define gs_min2(a, b) (((a) < (b)) ? (a) : (b)) +#define min3(x, y, z) (((x) < (y) && (x) < (z)) ? (x) : (gs_min2((y), (z)))) #endif - diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 56ea8c4..e44ab95 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -8,69 +8,68 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "gpu-sim.h" -#include <stdio.h> -#include <stdlib.h> #include <math.h> #include <signal.h> +#include <stdio.h> +#include <stdlib.h> #include "zlib.h" - -#include "shader.h" -#include "shader_trace.h" #include "dram.h" #include "mem_fetch.h" +#include "shader.h" +#include "shader_trace.h" #include <time.h> +#include "addrdec.h" +#include "delayqueue.h" +#include "dram.h" #include "gpu-cache.h" #include "gpu-misc.h" -#include "delayqueue.h" -#include "shader.h" #include "icnt_wrapper.h" -#include "dram.h" -#include "addrdec.h" -#include "stat-tool.h" #include "l2cache.h" +#include "shader.h" +#include "stat-tool.h" -#include "../cuda-sim/ptx-stats.h" -#include "../statwrapper.h" +#include "../../libcuda/gpgpu_context.h" #include "../abstract_hardware_model.h" -#include "../debug.h" -#include "../gpgpusim_entrypoint.h" #include "../cuda-sim/cuda-sim.h" +#include "../cuda-sim/cuda_device_runtime.h" +#include "../cuda-sim/ptx-stats.h" #include "../cuda-sim/ptx_ir.h" +#include "../debug.h" +#include "../gpgpusim_entrypoint.h" +#include "../statwrapper.h" #include "../trace.h" #include "mem_latency_stat.h" #include "power_stat.h" -#include "visualizer.h" #include "stats.h" -#include "../cuda-sim/cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" +#include "visualizer.h" #ifdef GPGPUSIM_POWER_MODEL #include "power_interface.h" #else -class gpgpu_sim_wrapper {}; +class gpgpu_sim_wrapper {}; #endif #include <stdio.h> @@ -79,1781 +78,1910 @@ class gpgpu_sim_wrapper {}; #include <sstream> #include <string> -#define MAX(a,b) (((a)>(b))?(a):(b)) - +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -bool g_interactive_debugger_enabled=false; +bool g_interactive_debugger_enabled = false; -tr1_hash_map<new_addr_type,unsigned> address_random_interleaving; +tr1_hash_map<new_addr_type, unsigned> address_random_interleaving; /* Clock Domains */ -#define CORE 0x01 -#define L2 0x02 -#define DRAM 0x04 -#define ICNT 0x08 - +#define CORE 0x01 +#define L2 0x02 +#define DRAM 0x04 +#define ICNT 0x08 #define MEM_LATENCY_STAT_IMPL - #include "mem_latency_stat.h" -void power_config::reg_options(class OptionParser * opp) -{ - - - option_parser_register(opp, "-gpuwattch_xml_file", OPT_CSTR, - &g_power_config_name,"GPUWattch XML file", - "gpuwattch.xml"); +void power_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-gpuwattch_xml_file", OPT_CSTR, + &g_power_config_name, "GPUWattch XML file", + "gpuwattch.xml"); - option_parser_register(opp, "-power_simulation_enabled", OPT_BOOL, - &g_power_simulation_enabled, "Turn on power simulator (1=On, 0=Off)", - "0"); + option_parser_register(opp, "-power_simulation_enabled", OPT_BOOL, + &g_power_simulation_enabled, + "Turn on power simulator (1=On, 0=Off)", "0"); - option_parser_register(opp, "-power_per_cycle_dump", OPT_BOOL, - &g_power_per_cycle_dump, "Dump detailed power output each cycle", - "0"); + option_parser_register(opp, "-power_per_cycle_dump", OPT_BOOL, + &g_power_per_cycle_dump, + "Dump detailed power output each cycle", "0"); - // Output Data Formats - option_parser_register(opp, "-power_trace_enabled", OPT_BOOL, - &g_power_trace_enabled, "produce a file for the power trace (1=On, 0=Off)", - "0"); + // Output Data Formats + option_parser_register( + opp, "-power_trace_enabled", OPT_BOOL, &g_power_trace_enabled, + "produce a file for the power trace (1=On, 0=Off)", "0"); - option_parser_register(opp, "-power_trace_zlevel", OPT_INT32, - &g_power_trace_zlevel, "Compression level of the power trace output log (0=no comp, 9=highest)", - "6"); + option_parser_register( + opp, "-power_trace_zlevel", OPT_INT32, &g_power_trace_zlevel, + "Compression level of the power trace output log (0=no comp, 9=highest)", + "6"); - option_parser_register(opp, "-steady_power_levels_enabled", OPT_BOOL, - &g_steady_power_levels_enabled, "produce a file for the steady power levels (1=On, 0=Off)", - "0"); - - option_parser_register(opp, "-steady_state_definition", OPT_CSTR, - &gpu_steady_state_definition, "allowed deviation:number of samples", - "8:4"); + option_parser_register( + opp, "-steady_power_levels_enabled", OPT_BOOL, + &g_steady_power_levels_enabled, + "produce a file for the steady power levels (1=On, 0=Off)", "0"); + option_parser_register(opp, "-steady_state_definition", OPT_CSTR, + &gpu_steady_state_definition, + "allowed deviation:number of samples", "8:4"); } -void memory_config::reg_options(class OptionParser * opp) -{ - option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, - "Fill the L2 cache on memcpy", "1"); - option_parser_register(opp, "-simple_dram_model", OPT_BOOL, &simple_dram_model, - "simple_dram_model with fixed latency and BW", "0"); - option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, - "0 = fifo, 1 = FR-FCFS (defaul)", "1"); - option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, - "i2$:$2d:d2$:$2i", - "8:8:8:8"); +void memory_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, + "Fill the L2 cache on memcpy", "1"); + option_parser_register(opp, "-simple_dram_model", OPT_BOOL, + &simple_dram_model, + "simple_dram_model with fixed latency and BW", "0"); + option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, + &scheduler_type, "0 = fifo, 1 = FR-FCFS (defaul)", + "1"); + option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, + &gpgpu_L2_queue_config, "i2$:$2d:d2$:$2i", "8:8:8:8"); - option_parser_register(opp, "-l2_ideal", OPT_BOOL, &l2_ideal, - "Use a ideal L2 cache that always hit", - "0"); - option_parser_register(opp, "-gpgpu_cache:dl2", OPT_CSTR, &m_L2_config.m_config_string, - "unified banked L2 data cache config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>}", - "64:128:8,L:B:m:N,A:16:4,4"); - option_parser_register(opp, "-gpgpu_cache:dl2_texture_only", OPT_BOOL, &m_L2_texure_only, - "L2 cache used for texture only", - "1"); - option_parser_register(opp, "-gpgpu_n_mem", OPT_UINT32, &m_n_mem, - "number of memory modules (e.g. memory controllers) in gpu", - "8"); - option_parser_register(opp, "-gpgpu_n_sub_partition_per_mchannel", OPT_UINT32, &m_n_sub_partition_per_memory_channel, - "number of memory subpartition in each memory module", - "1"); - option_parser_register(opp, "-gpgpu_n_mem_per_ctrlr", OPT_UINT32, &gpu_n_mem_per_ctrlr, - "number of memory chips per memory controller", - "1"); - option_parser_register(opp, "-gpgpu_memlatency_stat", OPT_INT32, &gpgpu_memlatency_stat, - "track and display latency statistics 0x2 enables MC, 0x4 enables queue logs", - "0"); - option_parser_register(opp, "-gpgpu_frfcfs_dram_sched_queue_size", OPT_INT32, &gpgpu_frfcfs_dram_sched_queue_size, - "0 = unlimited (default); # entries per chip", - "0"); - option_parser_register(opp, "-gpgpu_dram_return_queue_size", OPT_INT32, &gpgpu_dram_return_queue_size, - "0 = unlimited (default); # entries per chip", - "0"); - option_parser_register(opp, "-gpgpu_dram_buswidth", OPT_UINT32, &busW, - "default = 4 bytes (8 bytes per cycle at DDR)", - "4"); - option_parser_register(opp, "-gpgpu_dram_burst_length", OPT_UINT32, &BL, - "Burst length of each DRAM request (default = 4 data bus cycle)", - "4"); - option_parser_register(opp, "-dram_data_command_freq_ratio", OPT_UINT32, &data_command_freq_ratio, - "Frequency ratio between DRAM data bus and command bus (default = 2 times, i.e. DDR)", - "2"); - option_parser_register(opp, "-gpgpu_dram_timing_opt", OPT_CSTR, &gpgpu_dram_timing_opt, - "DRAM timing parameters = {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tCDLR:tWR:nbkgrp:tCCDL:tRTPL}", - "4:2:8:12:21:13:34:9:4:5:13:1:0:0"); - option_parser_register(opp, "-rop_latency", OPT_UINT32, &rop_latency, - "ROP queue latency (default 85)", - "85"); - option_parser_register(opp, "-dram_latency", OPT_UINT32, &dram_latency, - "DRAM latency (default 30)", - "30"); - option_parser_register(opp, "-dual_bus_interface", OPT_UINT32, &dual_bus_interface, - "dual_bus_interface (default = 0) ", - "0"); - option_parser_register(opp, "-dram_bnk_indexing_policy", OPT_UINT32, &dram_bnk_indexing_policy, - "dram_bnk_indexing_policy (0 = normal indexing, 1 = Xoring with the higher bits) (Default = 0)", - "0"); - option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy, - "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)", - "0"); - option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled, - "Seperate_Write_Queue_Enable", - "0"); - option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt, - "Write_Queue_Size", - "32:28:16"); - option_parser_register(opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround, - "elimnate_rw_turnaround i.e set tWTR and tRTW = 0", - "0"); - option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size, - "icnt_flit_size", - "32"); - m_address_mapping.addrdec_setoption(opp); + option_parser_register(opp, "-l2_ideal", OPT_BOOL, &l2_ideal, + "Use a ideal L2 cache that always hit", "0"); + option_parser_register(opp, "-gpgpu_cache:dl2", OPT_CSTR, + &m_L2_config.m_config_string, + "unified banked L2 data cache config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq>}", + "64:128:8,L:B:m:N,A:16:4,4"); + option_parser_register(opp, "-gpgpu_cache:dl2_texture_only", OPT_BOOL, + &m_L2_texure_only, "L2 cache used for texture only", + "1"); + option_parser_register( + opp, "-gpgpu_n_mem", OPT_UINT32, &m_n_mem, + "number of memory modules (e.g. memory controllers) in gpu", "8"); + option_parser_register(opp, "-gpgpu_n_sub_partition_per_mchannel", OPT_UINT32, + &m_n_sub_partition_per_memory_channel, + "number of memory subpartition in each memory module", + "1"); + option_parser_register(opp, "-gpgpu_n_mem_per_ctrlr", OPT_UINT32, + &gpu_n_mem_per_ctrlr, + "number of memory chips per memory controller", "1"); + option_parser_register(opp, "-gpgpu_memlatency_stat", OPT_INT32, + &gpgpu_memlatency_stat, + "track and display latency statistics 0x2 enables MC, " + "0x4 enables queue logs", + "0"); + option_parser_register(opp, "-gpgpu_frfcfs_dram_sched_queue_size", OPT_INT32, + &gpgpu_frfcfs_dram_sched_queue_size, + "0 = unlimited (default); # entries per chip", "0"); + option_parser_register(opp, "-gpgpu_dram_return_queue_size", OPT_INT32, + &gpgpu_dram_return_queue_size, + "0 = unlimited (default); # entries per chip", "0"); + option_parser_register(opp, "-gpgpu_dram_buswidth", OPT_UINT32, &busW, + "default = 4 bytes (8 bytes per cycle at DDR)", "4"); + option_parser_register( + opp, "-gpgpu_dram_burst_length", OPT_UINT32, &BL, + "Burst length of each DRAM request (default = 4 data bus cycle)", "4"); + option_parser_register(opp, "-dram_data_command_freq_ratio", OPT_UINT32, + &data_command_freq_ratio, + "Frequency ratio between DRAM data bus and command " + "bus (default = 2 times, i.e. DDR)", + "2"); + option_parser_register( + opp, "-gpgpu_dram_timing_opt", OPT_CSTR, &gpgpu_dram_timing_opt, + "DRAM timing parameters = " + "{nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tCDLR:tWR:nbkgrp:tCCDL:tRTPL}", + "4:2:8:12:21:13:34:9:4:5:13:1:0:0"); + option_parser_register(opp, "-rop_latency", OPT_UINT32, &rop_latency, + "ROP queue latency (default 85)", "85"); + option_parser_register(opp, "-dram_latency", OPT_UINT32, &dram_latency, + "DRAM latency (default 30)", "30"); + option_parser_register(opp, "-dual_bus_interface", OPT_UINT32, + &dual_bus_interface, + "dual_bus_interface (default = 0) ", "0"); + option_parser_register(opp, "-dram_bnk_indexing_policy", OPT_UINT32, + &dram_bnk_indexing_policy, + "dram_bnk_indexing_policy (0 = normal indexing, 1 = " + "Xoring with the higher bits) (Default = 0)", + "0"); + option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, + &dram_bnkgrp_indexing_policy, + "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 " + "= take lower bits) (Default = 0)", + "0"); + option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, + &seperate_write_queue_enabled, + "Seperate_Write_Queue_Enable", "0"); + option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, + &write_queue_size_opt, "Write_Queue_Size", "32:28:16"); + option_parser_register( + opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround, + "elimnate_rw_turnaround i.e set tWTR and tRTW = 0", "0"); + option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size, + "icnt_flit_size", "32"); + m_address_mapping.addrdec_setoption(opp); } -void shader_core_config::reg_options(class OptionParser * opp) -{ - option_parser_register(opp, "-gpgpu_simd_model", OPT_INT32, &model, - "1 = post-dominator", "1"); - option_parser_register(opp, "-gpgpu_shader_core_pipeline", OPT_CSTR, &gpgpu_shader_core_pipeline_opt, - "shader core pipeline config, i.e., {<nthread>:<warpsize>}", - "1024:32"); - option_parser_register(opp, "-gpgpu_tex_cache:l1", OPT_CSTR, &m_L1T_config.m_config_string, - "per-shader L1 texture cache (READ-ONLY) config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>:<rf>}", - "8:128:5,L:R:m:N,F:128:4,128:2"); - option_parser_register(opp, "-gpgpu_const_cache:l1", OPT_CSTR, &m_L1C_config.m_config_string, - "per-shader L1 constant memory cache (READ-ONLY) config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>} ", - "64:64:2,L:R:f:N,A:2:32,4" ); - option_parser_register(opp, "-gpgpu_cache:il1", OPT_CSTR, &m_L1I_config.m_config_string, - "shader L1 instruction cache config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>} ", - "4:256:4,L:R:f:N,A:2:32,4" ); - option_parser_register(opp, "-gpgpu_cache:dl1", OPT_CSTR, &m_L1D_config.m_config_string, - "per-shader L1 data cache config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}", - "none" ); - option_parser_register(opp, "-l1_banks", OPT_UINT32, &m_L1D_config.l1_banks, - "The number of L1 cache banks", - "1"); - option_parser_register(opp, "-l1_latency", OPT_UINT32, &m_L1D_config.l1_latency, - "L1 Hit Latency", - "1"); - option_parser_register(opp, "-smem_latency", OPT_UINT32, &smem_latency, - "smem Latency", - "3"); - option_parser_register(opp, "-gpgpu_cache:dl1PrefL1", OPT_CSTR, &m_L1D_config.m_config_stringPrefL1, - "per-shader L1 data cache config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}", - "none" ); - option_parser_register(opp, "-gpgpu_cache:dl1PrefShared", OPT_CSTR, &m_L1D_config.m_config_stringPrefShared, - "per-shader L1 data cache config " - " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}", - "none" ); - option_parser_register(opp, "-gmem_skip_L1D", OPT_BOOL, &gmem_skip_L1D, - "global memory access skip L1D cache (implements -Xptxas -dlcm=cg, default=no skip)", - "0"); - - option_parser_register(opp, "-gpgpu_perfect_mem", OPT_BOOL, &gpgpu_perfect_mem, - "enable perfect memory mode (no cache miss)", - "0"); - option_parser_register(opp, "-n_regfile_gating_group", OPT_UINT32, &n_regfile_gating_group, - "group of lanes that should be read/written together)", - "4"); - option_parser_register(opp, "-gpgpu_clock_gated_reg_file", OPT_BOOL, &gpgpu_clock_gated_reg_file, - "enable clock gated reg file for power calculations", - "0"); - option_parser_register(opp, "-gpgpu_clock_gated_lanes", OPT_BOOL, &gpgpu_clock_gated_lanes, - "enable clock gated lanes for power calculations", - "0"); - option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, - "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", - "8192"); - option_parser_register(opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block, - "Maximum number of registers per CTA. (default 8192)", - "8192"); - option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation, - "gpgpu_ignore_resources_limitation (default 0)", - "0"); - option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, - "Maximum number of concurrent CTAs in shader (default 8)", - "8"); - option_parser_register(opp, "-gpgpu_num_cta_barriers", OPT_UINT32, &max_barriers_per_cta, - "Maximum number of named barriers per CTA (default 16)", - "16"); - option_parser_register(opp, "-gpgpu_n_clusters", OPT_UINT32, &n_simt_clusters, - "number of processing clusters", - "10"); - option_parser_register(opp, "-gpgpu_n_cores_per_cluster", OPT_UINT32, &n_simt_cores_per_cluster, - "number of simd cores per cluster", - "3"); - option_parser_register(opp, "-gpgpu_n_cluster_ejection_buffer_size", OPT_UINT32, &n_simt_ejection_buffer_size, - "number of packets in ejection buffer", - "8"); - option_parser_register(opp, "-gpgpu_n_ldst_response_buffer_size", OPT_UINT32, &ldst_unit_response_queue_size, - "number of response packets in ld/st unit ejection buffer", - "2"); - option_parser_register(opp, "-gpgpu_shmem_per_block", OPT_UINT32, &gpgpu_shmem_per_block, - "Size of shared memory per thread block or CTA (default 48kB)", - "49152"); - option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, - "Size of shared memory per shader core (default 16kB)", - "16384"); - option_parser_register(opp, "-adaptive_cache_config", OPT_BOOL, &adaptive_volta_cache_config, - "adaptive_cache_config", - "0"); - option_parser_register(opp, "-gpgpu_shmem_sizeDefault", OPT_UINT32, &gpgpu_shmem_sizeDefault, - "Size of shared memory per shader core (default 16kB)", - "16384"); - option_parser_register(opp, "-gpgpu_shmem_size_PrefL1", OPT_UINT32, &gpgpu_shmem_sizePrefL1, - "Size of shared memory per shader core (default 16kB)", - "16384"); - option_parser_register(opp, "-gpgpu_shmem_size_PrefShared", OPT_UINT32, &gpgpu_shmem_sizePrefShared, - "Size of shared memory per shader core (default 16kB)", - "16384"); - option_parser_register(opp, "-gpgpu_shmem_num_banks", OPT_UINT32, &num_shmem_bank, - "Number of banks in the shared memory in each shader core (default 16)", - "16"); - option_parser_register(opp, "-gpgpu_shmem_limited_broadcast", OPT_BOOL, &shmem_limited_broadcast, - "Limit shared memory to do one broadcast per cycle (default on)", - "1"); - option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, - "Number of portions a warp is divided into for shared memory bank conflict check ", - "2"); - option_parser_register(opp, "-mem_unit_ports", OPT_INT32, &mem_unit_ports, - "The number of memory transactions allowed per core cycle", - "1"); - option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, - "Number of portions a warp is divided into for shared memory bank conflict check ", - "2"); - option_parser_register(opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, - "Specify which shader core to collect the warp size distribution from", - "-1"); - option_parser_register(opp, "-gpgpu_warp_issue_shader", OPT_INT32, &gpgpu_warp_issue_shader, - "Specify which shader core to collect the warp issue distribution from", - "0"); - option_parser_register(opp, "-gpgpu_local_mem_map", OPT_BOOL, &gpgpu_local_mem_map, - "Mapping from local memory space address to simulated GPU physical address space (default = enabled)", - "1"); - option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, - "Number of register banks (default = 8)", - "8"); - option_parser_register(opp, "-gpgpu_reg_bank_use_warp_id", OPT_BOOL, &gpgpu_reg_bank_use_warp_id, - "Use warp ID in mapping registers to banks (default = off)", - "0"); - option_parser_register(opp, "-sub_core_model", OPT_BOOL, &sub_core_model, - "Sub Core Volta/Pascal model (default = off)", - "0"); - option_parser_register(opp, "-enable_specialized_operand_collector", OPT_BOOL, &enable_specialized_operand_collector, - "enable_specialized_operand_collector", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", OPT_INT32, &gpgpu_operand_collector_num_units_sp, - "number of collector units (default = 4)", - "4"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", OPT_INT32, &gpgpu_operand_collector_num_units_dp, - "number of collector units (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu, - "number of collector units (default = 4)", - "4"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_int", OPT_INT32, &gpgpu_operand_collector_num_units_int, - "number of collector units (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_units_tensor_core, - "number of collector units (default = 4)", - "4"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_mem", OPT_INT32, &gpgpu_operand_collector_num_units_mem, - "number of collector units (default = 2)", - "2"); - option_parser_register(opp, "-gpgpu_operand_collector_num_units_gen", OPT_INT32, &gpgpu_operand_collector_num_units_gen, - "number of collector units (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sp, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_dp, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_int", OPT_INT32, &gpgpu_operand_collector_num_in_ports_int, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_in_ports_tensor_core, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_in_ports_mem, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_gen", OPT_INT32, &gpgpu_operand_collector_num_in_ports_gen, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sp, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_dp, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_int", OPT_INT32, &gpgpu_operand_collector_num_out_ports_int, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_out_ports_tensor_core, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_out_ports_mem, - "number of collector unit in ports (default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_gen", OPT_INT32, &gpgpu_operand_collector_num_out_ports_gen, - "number of collector unit in ports (default = 0)", - "0"); - option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, - "Coalescing arch (GT200 = 13, Fermi = 20)", - "13"); - option_parser_register(opp, "-gpgpu_num_sched_per_core", OPT_INT32, &gpgpu_num_sched_per_core, - "Number of warp schedulers per core", - "1"); - option_parser_register(opp, "-gpgpu_max_insn_issue_per_warp", OPT_INT32, &gpgpu_max_insn_issue_per_warp, - "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)", - "2"); - option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, &gpgpu_dual_issue_diff_exec_units, - "should dual issue use two different execution unit resources (Default = 1)", - "1"); - option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, &simt_core_sim_order, - "Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)", - "1"); - option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, - "Pipeline widths " - "ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE", - "1,1,1,1,1,1,1,1,1,1,1,1,1" ); - option_parser_register(opp, "-gpgpu_tensor_core_avail", OPT_INT32, &gpgpu_tensor_core_avail, - "Tensor Core Available (default=0)", - "0"); - option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units, - "Number of SP units (default=1)", - "1"); - option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, &gpgpu_num_dp_units, - "Number of DP units (default=0)", - "0"); - option_parser_register(opp, "-gpgpu_num_int_units", OPT_INT32, &gpgpu_num_int_units, - "Number of INT units (default=0)", - "0"); - option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, &gpgpu_num_sfu_units, - "Number of SF units (default=1)", - "1"); - option_parser_register(opp, "-gpgpu_num_tensor_core_units", OPT_INT32, &gpgpu_num_tensor_core_units, - "Number of tensor_core units (default=1)", - "1"); - option_parser_register(opp, "-gpgpu_num_mem_units", OPT_INT32, &gpgpu_num_mem_units, - "Number if ldst units (default=1) WARNING: not hooked up to anything", - "1"); - option_parser_register(opp, "-gpgpu_scheduler", OPT_CSTR, &gpgpu_scheduler_string, - "Scheduler configuration: < lrr | gto | two_level_active > " - "If two_level_active:<num_active_warps>:<inner_prioritization>:<outer_prioritization>" - "For complete list of prioritization values see shader.h enum scheduler_prioritization_type" - "Default: gto", - "gto"); +void shader_core_config::reg_options(class OptionParser *opp) { + option_parser_register(opp, "-gpgpu_simd_model", OPT_INT32, &model, + "1 = post-dominator", "1"); + option_parser_register( + opp, "-gpgpu_shader_core_pipeline", OPT_CSTR, + &gpgpu_shader_core_pipeline_opt, + "shader core pipeline config, i.e., {<nthread>:<warpsize>}", "1024:32"); + option_parser_register(opp, "-gpgpu_tex_cache:l1", OPT_CSTR, + &m_L1T_config.m_config_string, + "per-shader L1 texture cache (READ-ONLY) config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq>:<rf>}", + "8:128:5,L:R:m:N,F:128:4,128:2"); + option_parser_register( + opp, "-gpgpu_const_cache:l1", OPT_CSTR, &m_L1C_config.m_config_string, + "per-shader L1 constant memory cache (READ-ONLY) config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<" + "merge>,<mq>} ", + "64:64:2,L:R:f:N,A:2:32,4"); + option_parser_register(opp, "-gpgpu_cache:il1", OPT_CSTR, + &m_L1I_config.m_config_string, + "shader L1 instruction cache config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq>} ", + "4:256:4,L:R:f:N,A:2:32,4"); + option_parser_register(opp, "-gpgpu_cache:dl1", OPT_CSTR, + &m_L1D_config.m_config_string, + "per-shader L1 data cache config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq> | none}", + "none"); + option_parser_register(opp, "-l1_banks", OPT_UINT32, &m_L1D_config.l1_banks, + "The number of L1 cache banks", "1"); + option_parser_register(opp, "-l1_latency", OPT_UINT32, + &m_L1D_config.l1_latency, "L1 Hit Latency", "1"); + option_parser_register(opp, "-smem_latency", OPT_UINT32, &smem_latency, + "smem Latency", "3"); + option_parser_register(opp, "-gpgpu_cache:dl1PrefL1", OPT_CSTR, + &m_L1D_config.m_config_stringPrefL1, + "per-shader L1 data cache config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq> | none}", + "none"); + option_parser_register(opp, "-gpgpu_cache:dl1PrefShared", OPT_CSTR, + &m_L1D_config.m_config_stringPrefShared, + "per-shader L1 data cache config " + " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_" + "alloc>,<mshr>:<N>:<merge>,<mq> | none}", + "none"); + option_parser_register(opp, "-gmem_skip_L1D", OPT_BOOL, &gmem_skip_L1D, + "global memory access skip L1D cache (implements " + "-Xptxas -dlcm=cg, default=no skip)", + "0"); - option_parser_register(opp, "-gpgpu_concurrent_kernel_sm", OPT_BOOL, &gpgpu_concurrent_kernel_sm, - "Support concurrent kernels on a SM (default = disabled)", - "0"); + option_parser_register(opp, "-gpgpu_perfect_mem", OPT_BOOL, + &gpgpu_perfect_mem, + "enable perfect memory mode (no cache miss)", "0"); + option_parser_register( + opp, "-n_regfile_gating_group", OPT_UINT32, &n_regfile_gating_group, + "group of lanes that should be read/written together)", "4"); + option_parser_register( + opp, "-gpgpu_clock_gated_reg_file", OPT_BOOL, &gpgpu_clock_gated_reg_file, + "enable clock gated reg file for power calculations", "0"); + option_parser_register( + opp, "-gpgpu_clock_gated_lanes", OPT_BOOL, &gpgpu_clock_gated_lanes, + "enable clock gated lanes for power calculations", "0"); + option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, + &gpgpu_shader_registers, + "Number of registers per shader core. Limits number " + "of concurrent CTAs. (default 8192)", + "8192"); + option_parser_register( + opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block, + "Maximum number of registers per CTA. (default 8192)", "8192"); + option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, + &gpgpu_ignore_resources_limitation, + "gpgpu_ignore_resources_limitation (default 0)", "0"); + option_parser_register( + opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, + "Maximum number of concurrent CTAs in shader (default 8)", "8"); + option_parser_register( + opp, "-gpgpu_num_cta_barriers", OPT_UINT32, &max_barriers_per_cta, + "Maximum number of named barriers per CTA (default 16)", "16"); + option_parser_register(opp, "-gpgpu_n_clusters", OPT_UINT32, &n_simt_clusters, + "number of processing clusters", "10"); + option_parser_register(opp, "-gpgpu_n_cores_per_cluster", OPT_UINT32, + &n_simt_cores_per_cluster, + "number of simd cores per cluster", "3"); + option_parser_register(opp, "-gpgpu_n_cluster_ejection_buffer_size", + OPT_UINT32, &n_simt_ejection_buffer_size, + "number of packets in ejection buffer", "8"); + option_parser_register( + opp, "-gpgpu_n_ldst_response_buffer_size", OPT_UINT32, + &ldst_unit_response_queue_size, + "number of response packets in ld/st unit ejection buffer", "2"); + option_parser_register( + opp, "-gpgpu_shmem_per_block", OPT_UINT32, &gpgpu_shmem_per_block, + "Size of shared memory per thread block or CTA (default 48kB)", "49152"); + option_parser_register( + opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, + "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register(opp, "-adaptive_cache_config", OPT_BOOL, + &adaptive_volta_cache_config, "adaptive_cache_config", + "0"); + option_parser_register( + opp, "-gpgpu_shmem_sizeDefault", OPT_UINT32, &gpgpu_shmem_sizeDefault, + "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register( + opp, "-gpgpu_shmem_size_PrefL1", OPT_UINT32, &gpgpu_shmem_sizePrefL1, + "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register(opp, "-gpgpu_shmem_size_PrefShared", OPT_UINT32, + &gpgpu_shmem_sizePrefShared, + "Size of shared memory per shader core (default 16kB)", + "16384"); + option_parser_register( + opp, "-gpgpu_shmem_num_banks", OPT_UINT32, &num_shmem_bank, + "Number of banks in the shared memory in each shader core (default 16)", + "16"); + option_parser_register( + opp, "-gpgpu_shmem_limited_broadcast", OPT_BOOL, &shmem_limited_broadcast, + "Limit shared memory to do one broadcast per cycle (default on)", "1"); + option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, + &mem_warp_parts, + "Number of portions a warp is divided into for shared " + "memory bank conflict check ", + "2"); + option_parser_register( + opp, "-mem_unit_ports", OPT_INT32, &mem_unit_ports, + "The number of memory transactions allowed per core cycle", "1"); + option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, + &mem_warp_parts, + "Number of portions a warp is divided into for shared " + "memory bank conflict check ", + "2"); + option_parser_register( + opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, + "Specify which shader core to collect the warp size distribution from", + "-1"); + option_parser_register( + opp, "-gpgpu_warp_issue_shader", OPT_INT32, &gpgpu_warp_issue_shader, + "Specify which shader core to collect the warp issue distribution from", + "0"); + option_parser_register(opp, "-gpgpu_local_mem_map", OPT_BOOL, + &gpgpu_local_mem_map, + "Mapping from local memory space address to simulated " + "GPU physical address space (default = enabled)", + "1"); + option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, + &gpgpu_num_reg_banks, + "Number of register banks (default = 8)", "8"); + option_parser_register( + opp, "-gpgpu_reg_bank_use_warp_id", OPT_BOOL, &gpgpu_reg_bank_use_warp_id, + "Use warp ID in mapping registers to banks (default = off)", "0"); + option_parser_register(opp, "-sub_core_model", OPT_BOOL, &sub_core_model, + "Sub Core Volta/Pascal model (default = off)", "0"); + option_parser_register(opp, "-enable_specialized_operand_collector", OPT_BOOL, + &enable_specialized_operand_collector, + "enable_specialized_operand_collector", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", + OPT_INT32, &gpgpu_operand_collector_num_units_sp, + "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", + OPT_INT32, &gpgpu_operand_collector_num_units_dp, + "number of collector units (default = 0)", "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", + OPT_INT32, &gpgpu_operand_collector_num_units_sfu, + "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_int", + OPT_INT32, &gpgpu_operand_collector_num_units_int, + "number of collector units (default = 0)", "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_tensor_core", + OPT_INT32, + &gpgpu_operand_collector_num_units_tensor_core, + "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_mem", + OPT_INT32, &gpgpu_operand_collector_num_units_mem, + "number of collector units (default = 2)", "2"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_gen", + OPT_INT32, &gpgpu_operand_collector_num_units_gen, + "number of collector units (default = 0)", "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sp", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_sp, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_dp", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_int", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_int, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register( + opp, "-gpgpu_operand_collector_num_in_ports_tensor_core", OPT_INT32, + &gpgpu_operand_collector_num_in_ports_tensor_core, + "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_mem", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_mem, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_gen", + OPT_INT32, &gpgpu_operand_collector_num_in_ports_gen, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sp", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_sp, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_dp", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_int", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_int, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register( + opp, "-gpgpu_operand_collector_num_out_ports_tensor_core", OPT_INT32, + &gpgpu_operand_collector_num_out_ports_tensor_core, + "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_mem", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_mem, + "number of collector unit in ports (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_gen", + OPT_INT32, &gpgpu_operand_collector_num_out_ports_gen, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, + &gpgpu_coalesce_arch, + "Coalescing arch (GT200 = 13, Fermi = 20)", "13"); + option_parser_register(opp, "-gpgpu_num_sched_per_core", OPT_INT32, + &gpgpu_num_sched_per_core, + "Number of warp schedulers per core", "1"); + option_parser_register(opp, "-gpgpu_max_insn_issue_per_warp", OPT_INT32, + &gpgpu_max_insn_issue_per_warp, + "Max number of instructions that can be issued per " + "warp in one cycle by scheduler (either 1 or 2)", + "2"); + option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, + &gpgpu_dual_issue_diff_exec_units, + "should dual issue use two different execution unit " + "resources (Default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, + &simt_core_sim_order, + "Select the simulation order of cores in a cluster " + "(0=Fix, 1=Round-Robin)", + "1"); + option_parser_register( + opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, + "Pipeline widths " + "ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_" + "INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE", + "1,1,1,1,1,1,1,1,1,1,1,1,1"); + option_parser_register(opp, "-gpgpu_tensor_core_avail", OPT_INT32, + &gpgpu_tensor_core_avail, + "Tensor Core Available (default=0)", "0"); + option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, + &gpgpu_num_sp_units, "Number of SP units (default=1)", + "1"); + option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, + &gpgpu_num_dp_units, "Number of DP units (default=0)", + "0"); + option_parser_register(opp, "-gpgpu_num_int_units", OPT_INT32, + &gpgpu_num_int_units, + "Number of INT units (default=0)", "0"); + option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, + &gpgpu_num_sfu_units, "Number of SF units (default=1)", + "1"); + option_parser_register(opp, "-gpgpu_num_tensor_core_units", OPT_INT32, + &gpgpu_num_tensor_core_units, + "Number of tensor_core units (default=1)", "1"); + option_parser_register( + opp, "-gpgpu_num_mem_units", OPT_INT32, &gpgpu_num_mem_units, + "Number if ldst units (default=1) WARNING: not hooked up to anything", + "1"); + option_parser_register( + opp, "-gpgpu_scheduler", OPT_CSTR, &gpgpu_scheduler_string, + "Scheduler configuration: < lrr | gto | two_level_active > " + "If " + "two_level_active:<num_active_warps>:<inner_prioritization>:<outer_" + "prioritization>" + "For complete list of prioritization values see shader.h enum " + "scheduler_prioritization_type" + "Default: gto", + "gto"); + option_parser_register( + opp, "-gpgpu_concurrent_kernel_sm", OPT_BOOL, &gpgpu_concurrent_kernel_sm, + "Support concurrent kernels on a SM (default = disabled)", "0"); } -void gpgpu_sim_config::reg_options(option_parser_t opp) -{ - gpgpu_functional_sim_config::reg_options(opp); - m_shader_config.reg_options(opp); - m_memory_config.reg_options(opp); - power_config::reg_options(opp); - option_parser_register(opp, "-gpgpu_max_cycle", OPT_INT64, &gpu_max_cycle_opt, - "terminates gpu simulation early (0 = no limit)", - "0"); - option_parser_register(opp, "-gpgpu_max_insn", OPT_INT64, &gpu_max_insn_opt, - "terminates gpu simulation early (0 = no limit)", - "0"); - option_parser_register(opp, "-gpgpu_max_cta", OPT_INT32, &gpu_max_cta_opt, - "terminates gpu simulation early (0 = no limit)", - "0"); - option_parser_register(opp, "-gpgpu_runtime_stat", OPT_CSTR, &gpgpu_runtime_stat, - "display runtime statistics such as dram utilization {<freq>:<flag>}", - "10000:0"); - option_parser_register(opp, "-liveness_message_freq", OPT_INT64, &liveness_message_freq, - "Minimum number of seconds between simulation liveness messages (0 = always print)", - "1"); - option_parser_register(opp, "-gpgpu_compute_capability_major", OPT_UINT32, &gpgpu_compute_capability_major, - "Major compute capability version number", - "7"); - option_parser_register(opp, "-gpgpu_compute_capability_minor", OPT_UINT32, &gpgpu_compute_capability_minor, - "Minor compute capability version number", - "0"); - option_parser_register(opp, "-gpgpu_flush_l1_cache", OPT_BOOL, &gpgpu_flush_l1_cache, - "Flush L1 cache at the end of each kernel call", - "0"); - option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, &gpgpu_flush_l2_cache, - "Flush L2 cache at the end of each kernel call", - "0"); - option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, - "Stop the simulation at deadlock (1=on (default), 0=off)", - "1"); - option_parser_register(opp, "-gpgpu_ptx_instruction_classification", OPT_INT32, - &(gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification), - "if enabled will classify ptx instruction types per kernel (Max 255 kernels now)", - "0"); - option_parser_register(opp, "-gpgpu_ptx_sim_mode", OPT_INT32, &(gpgpu_ctx->func_sim->g_ptx_sim_mode), - "Select between Performance (default) or Functional simulation (1)", - "0"); - option_parser_register(opp, "-gpgpu_clock_domains", OPT_CSTR, &gpgpu_clock_domains, - "Clock Domain Frequencies in MhZ {<Core Clock>:<ICNT Clock>:<L2 Clock>:<DRAM Clock>}", - "500.0:2000.0:2000.0:2000.0"); - option_parser_register(opp, "-gpgpu_max_concurrent_kernel", OPT_INT32, &max_concurrent_kernel, - "maximum kernels that can run concurrently on GPU", "8" ); - option_parser_register(opp, "-gpgpu_cflog_interval", OPT_INT32, &gpgpu_cflog_interval, - "Interval between each snapshot in control flow logger", - "0"); - option_parser_register(opp, "-visualizer_enabled", OPT_BOOL, - &g_visualizer_enabled, "Turn on visualizer output (1=On, 0=Off)", - "1"); - option_parser_register(opp, "-visualizer_outputfile", OPT_CSTR, - &g_visualizer_filename, "Specifies the output log file for visualizer", - NULL); - option_parser_register(opp, "-visualizer_zlevel", OPT_INT32, - &g_visualizer_zlevel, "Compression level of the visualizer output log (0=no comp, 9=highest)", - "6"); - option_parser_register(opp, "-gpgpu_stack_size_limit", OPT_INT32, &stack_size_limit, - "GPU thread stack size", "1024" ); - option_parser_register(opp, "-gpgpu_heap_size_limit", OPT_INT32, &heap_size_limit, - "GPU malloc heap size ", "8388608" ); - option_parser_register(opp, "-gpgpu_runtime_sync_depth_limit", OPT_INT32, &runtime_sync_depth_limit, - "GPU device runtime synchronize depth", "2" ); - option_parser_register(opp, "-gpgpu_runtime_pending_launch_count_limit", OPT_INT32, &runtime_pending_launch_count_limit, - "GPU device runtime pending launch count", "2048" ); - option_parser_register(opp, "-trace_enabled", OPT_BOOL, - &Trace::enabled, "Turn on traces", - "0"); - option_parser_register(opp, "-trace_components", OPT_CSTR, - &Trace::config_str, "comma seperated list of traces to enable. " - "Complete list found in trace_streams.tup. " - "Default none", - "none"); - option_parser_register(opp, "-trace_sampling_core", OPT_INT32, - &Trace::sampling_core, "The core which is printed using CORE_DPRINTF. Default 0", - "0"); - option_parser_register(opp, "-trace_sampling_memory_partition", OPT_INT32, - &Trace::sampling_memory_partition, "The memory partition which is printed using MEMPART_DPRINTF. Default -1 (i.e. all)", - "-1"); - gpgpu_ctx->stats->ptx_file_line_stats_options(opp); +void gpgpu_sim_config::reg_options(option_parser_t opp) { + gpgpu_functional_sim_config::reg_options(opp); + m_shader_config.reg_options(opp); + m_memory_config.reg_options(opp); + power_config::reg_options(opp); + option_parser_register(opp, "-gpgpu_max_cycle", OPT_INT64, &gpu_max_cycle_opt, + "terminates gpu simulation early (0 = no limit)", "0"); + option_parser_register(opp, "-gpgpu_max_insn", OPT_INT64, &gpu_max_insn_opt, + "terminates gpu simulation early (0 = no limit)", "0"); + option_parser_register(opp, "-gpgpu_max_cta", OPT_INT32, &gpu_max_cta_opt, + "terminates gpu simulation early (0 = no limit)", "0"); + option_parser_register( + opp, "-gpgpu_runtime_stat", OPT_CSTR, &gpgpu_runtime_stat, + "display runtime statistics such as dram utilization {<freq>:<flag>}", + "10000:0"); + option_parser_register(opp, "-liveness_message_freq", OPT_INT64, + &liveness_message_freq, + "Minimum number of seconds between simulation " + "liveness messages (0 = always print)", + "1"); + option_parser_register(opp, "-gpgpu_compute_capability_major", OPT_UINT32, + &gpgpu_compute_capability_major, + "Major compute capability version number", "7"); + option_parser_register(opp, "-gpgpu_compute_capability_minor", OPT_UINT32, + &gpgpu_compute_capability_minor, + "Minor compute capability version number", "0"); + option_parser_register(opp, "-gpgpu_flush_l1_cache", OPT_BOOL, + &gpgpu_flush_l1_cache, + "Flush L1 cache at the end of each kernel call", "0"); + option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, + &gpgpu_flush_l2_cache, + "Flush L2 cache at the end of each kernel call", "0"); + option_parser_register( + opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, + "Stop the simulation at deadlock (1=on (default), 0=off)", "1"); + option_parser_register( + opp, "-gpgpu_ptx_instruction_classification", OPT_INT32, + &(gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification), + "if enabled will classify ptx instruction types per kernel (Max 255 " + "kernels now)", + "0"); + option_parser_register( + opp, "-gpgpu_ptx_sim_mode", OPT_INT32, + &(gpgpu_ctx->func_sim->g_ptx_sim_mode), + "Select between Performance (default) or Functional simulation (1)", "0"); + option_parser_register(opp, "-gpgpu_clock_domains", OPT_CSTR, + &gpgpu_clock_domains, + "Clock Domain Frequencies in MhZ {<Core Clock>:<ICNT " + "Clock>:<L2 Clock>:<DRAM Clock>}", + "500.0:2000.0:2000.0:2000.0"); + option_parser_register( + opp, "-gpgpu_max_concurrent_kernel", OPT_INT32, &max_concurrent_kernel, + "maximum kernels that can run concurrently on GPU", "8"); + option_parser_register( + opp, "-gpgpu_cflog_interval", OPT_INT32, &gpgpu_cflog_interval, + "Interval between each snapshot in control flow logger", "0"); + option_parser_register(opp, "-visualizer_enabled", OPT_BOOL, + &g_visualizer_enabled, + "Turn on visualizer output (1=On, 0=Off)", "1"); + option_parser_register(opp, "-visualizer_outputfile", OPT_CSTR, + &g_visualizer_filename, + "Specifies the output log file for visualizer", NULL); + option_parser_register( + opp, "-visualizer_zlevel", OPT_INT32, &g_visualizer_zlevel, + "Compression level of the visualizer output log (0=no comp, 9=highest)", + "6"); + option_parser_register(opp, "-gpgpu_stack_size_limit", OPT_INT32, + &stack_size_limit, "GPU thread stack size", "1024"); + option_parser_register(opp, "-gpgpu_heap_size_limit", OPT_INT32, + &heap_size_limit, "GPU malloc heap size ", "8388608"); + option_parser_register(opp, "-gpgpu_runtime_sync_depth_limit", OPT_INT32, + &runtime_sync_depth_limit, + "GPU device runtime synchronize depth", "2"); + option_parser_register(opp, "-gpgpu_runtime_pending_launch_count_limit", + OPT_INT32, &runtime_pending_launch_count_limit, + "GPU device runtime pending launch count", "2048"); + option_parser_register(opp, "-trace_enabled", OPT_BOOL, &Trace::enabled, + "Turn on traces", "0"); + option_parser_register(opp, "-trace_components", OPT_CSTR, &Trace::config_str, + "comma seperated list of traces to enable. " + "Complete list found in trace_streams.tup. " + "Default none", + "none"); + option_parser_register( + opp, "-trace_sampling_core", OPT_INT32, &Trace::sampling_core, + "The core which is printed using CORE_DPRINTF. Default 0", "0"); + option_parser_register(opp, "-trace_sampling_memory_partition", OPT_INT32, + &Trace::sampling_memory_partition, + "The memory partition which is printed using " + "MEMPART_DPRINTF. Default -1 (i.e. all)", + "-1"); + gpgpu_ctx->stats->ptx_file_line_stats_options(opp); - //Jin: kernel launch latency - option_parser_register(opp, "-gpgpu_kernel_launch_latency", OPT_INT32, - &(gpgpu_ctx->device_runtime->g_kernel_launch_latency), "Kernel launch latency in cycles. Default: 0", - "0"); - option_parser_register(opp, "-gpgpu_cdp_enabled", OPT_BOOL, - &(gpgpu_ctx->device_runtime->g_cdp_enabled), "Turn on CDP", - "0"); + // Jin: kernel launch latency + option_parser_register(opp, "-gpgpu_kernel_launch_latency", OPT_INT32, + &(gpgpu_ctx->device_runtime->g_kernel_launch_latency), + "Kernel launch latency in cycles. Default: 0", "0"); + option_parser_register(opp, "-gpgpu_cdp_enabled", OPT_BOOL, + &(gpgpu_ctx->device_runtime->g_cdp_enabled), + "Turn on CDP", "0"); } ///////////////////////////////////////////////////////////////////////////// -void increment_x_then_y_then_z( dim3 &i, const dim3 &bound) -{ - i.x++; - if ( i.x >= bound.x ) { - i.x = 0; - i.y++; - if ( i.y >= bound.y ) { - i.y = 0; - if( i.z < bound.z ) - i.z++; - } - } +void increment_x_then_y_then_z(dim3 &i, const dim3 &bound) { + i.x++; + if (i.x >= bound.x) { + i.x = 0; + i.y++; + if (i.y >= bound.y) { + i.y = 0; + if (i.z < bound.z) i.z++; + } + } } -void gpgpu_sim::launch( kernel_info_t *kinfo ) -{ - unsigned cta_size = kinfo->threads_per_cta(); - if ( cta_size > m_shader_config->n_thread_per_shader ) { - printf("Execution error: Shader kernel CTA (block) size is too large for microarch config.\n"); - printf(" CTA size (x*y*z) = %u, max supported = %u\n", cta_size, - m_shader_config->n_thread_per_shader ); - printf(" => either change -gpgpu_shader argument in gpgpusim.config file or\n"); - printf(" modify the CUDA source to decrease the kernel block size.\n"); - abort(); - } - unsigned n=0; - for(n=0; n < m_running_kernels.size(); n++ ) { - if( (NULL==m_running_kernels[n]) || m_running_kernels[n]->done() ) { - m_running_kernels[n] = kinfo; - break; - } - } - assert(n < m_running_kernels.size()); +void gpgpu_sim::launch(kernel_info_t *kinfo) { + unsigned cta_size = kinfo->threads_per_cta(); + if (cta_size > m_shader_config->n_thread_per_shader) { + printf( + "Execution error: Shader kernel CTA (block) size is too large for " + "microarch config.\n"); + printf(" CTA size (x*y*z) = %u, max supported = %u\n", + cta_size, m_shader_config->n_thread_per_shader); + printf( + " => either change -gpgpu_shader argument in " + "gpgpusim.config file or\n"); + printf( + " modify the CUDA source to decrease the kernel block " + "size.\n"); + abort(); + } + unsigned n = 0; + for (n = 0; n < m_running_kernels.size(); n++) { + if ((NULL == m_running_kernels[n]) || m_running_kernels[n]->done()) { + m_running_kernels[n] = kinfo; + break; + } + } + assert(n < m_running_kernels.size()); } -bool gpgpu_sim::can_start_kernel() -{ - for(unsigned n=0; n < m_running_kernels.size(); n++ ) { - if( (NULL==m_running_kernels[n]) || m_running_kernels[n]->done() ) - return true; - } - return false; +bool gpgpu_sim::can_start_kernel() { + for (unsigned n = 0; n < m_running_kernels.size(); n++) { + if ((NULL == m_running_kernels[n]) || m_running_kernels[n]->done()) + return true; + } + return false; } bool gpgpu_sim::hit_max_cta_count() const { - if (m_config.gpu_max_cta_opt != 0) { - if( (gpu_tot_issued_cta + m_total_cta_launched) >= m_config.gpu_max_cta_opt ) - return true; - } - return false; + if (m_config.gpu_max_cta_opt != 0) { + if ((gpu_tot_issued_cta + m_total_cta_launched) >= m_config.gpu_max_cta_opt) + return true; + } + return false; } bool gpgpu_sim::kernel_more_cta_left(kernel_info_t *kernel) const { - if(hit_max_cta_count()) - return false; + if (hit_max_cta_count()) return false; - if(kernel && !kernel->no_more_ctas_to_run()) - return true; + if (kernel && !kernel->no_more_ctas_to_run()) return true; - return false; + return false; } -bool gpgpu_sim::get_more_cta_left() const -{ - if(hit_max_cta_count()) - return false; +bool gpgpu_sim::get_more_cta_left() const { + if (hit_max_cta_count()) return false; - for(unsigned n=0; n < m_running_kernels.size(); n++ ) { - if( m_running_kernels[n] && !m_running_kernels[n]->no_more_ctas_to_run() ) - return true; - } - return false; + for (unsigned n = 0; n < m_running_kernels.size(); n++) { + if (m_running_kernels[n] && !m_running_kernels[n]->no_more_ctas_to_run()) + return true; + } + return false; } -kernel_info_t *gpgpu_sim::select_kernel() -{ - if(m_running_kernels[m_last_issued_kernel] && - !m_running_kernels[m_last_issued_kernel]->no_more_ctas_to_run()) { - unsigned launch_uid = m_running_kernels[m_last_issued_kernel]->get_uid(); - if(std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), launch_uid) == m_executed_kernel_uids.end()) { - m_running_kernels[m_last_issued_kernel]->start_cycle = gpu_sim_cycle + gpu_tot_sim_cycle; - m_executed_kernel_uids.push_back(launch_uid); - m_executed_kernel_names.push_back(m_running_kernels[m_last_issued_kernel]->name()); - } - return m_running_kernels[m_last_issued_kernel]; +kernel_info_t *gpgpu_sim::select_kernel() { + if (m_running_kernels[m_last_issued_kernel] && + !m_running_kernels[m_last_issued_kernel]->no_more_ctas_to_run()) { + unsigned launch_uid = m_running_kernels[m_last_issued_kernel]->get_uid(); + if (std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), + launch_uid) == m_executed_kernel_uids.end()) { + m_running_kernels[m_last_issued_kernel]->start_cycle = + gpu_sim_cycle + gpu_tot_sim_cycle; + m_executed_kernel_uids.push_back(launch_uid); + m_executed_kernel_names.push_back( + m_running_kernels[m_last_issued_kernel]->name()); } + return m_running_kernels[m_last_issued_kernel]; + } - for(unsigned n=0; n < m_running_kernels.size(); n++ ) { - unsigned idx = (n+m_last_issued_kernel+1)%m_config.max_concurrent_kernel; - if( kernel_more_cta_left(m_running_kernels[idx]) ){ - m_last_issued_kernel=idx; - m_running_kernels[idx]->start_cycle = gpu_sim_cycle + gpu_tot_sim_cycle; - // record this kernel for stat print if it is the first time this kernel is selected for execution - unsigned launch_uid = m_running_kernels[idx]->get_uid(); - assert(std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), launch_uid) == m_executed_kernel_uids.end()); - m_executed_kernel_uids.push_back(launch_uid); - m_executed_kernel_names.push_back(m_running_kernels[idx]->name()); + for (unsigned n = 0; n < m_running_kernels.size(); n++) { + unsigned idx = + (n + m_last_issued_kernel + 1) % m_config.max_concurrent_kernel; + if (kernel_more_cta_left(m_running_kernels[idx])) { + m_last_issued_kernel = idx; + m_running_kernels[idx]->start_cycle = gpu_sim_cycle + gpu_tot_sim_cycle; + // record this kernel for stat print if it is the first time this kernel + // is selected for execution + unsigned launch_uid = m_running_kernels[idx]->get_uid(); + assert(std::find(m_executed_kernel_uids.begin(), + m_executed_kernel_uids.end(), + launch_uid) == m_executed_kernel_uids.end()); + m_executed_kernel_uids.push_back(launch_uid); + m_executed_kernel_names.push_back(m_running_kernels[idx]->name()); - return m_running_kernels[idx]; - } + return m_running_kernels[idx]; } - return NULL; + } + return NULL; } -unsigned gpgpu_sim::finished_kernel() -{ - if( m_finished_kernel.empty() ) - return 0; - unsigned result = m_finished_kernel.front(); - m_finished_kernel.pop_front(); - return result; +unsigned gpgpu_sim::finished_kernel() { + if (m_finished_kernel.empty()) return 0; + unsigned result = m_finished_kernel.front(); + m_finished_kernel.pop_front(); + return result; } -void gpgpu_sim::set_kernel_done( kernel_info_t *kernel ) -{ - unsigned uid = kernel->get_uid(); - m_finished_kernel.push_back(uid); - std::vector<kernel_info_t*>::iterator k; - for( k=m_running_kernels.begin(); k!=m_running_kernels.end(); k++ ) { - if( *k == kernel ) { - kernel->end_cycle = gpu_sim_cycle + gpu_tot_sim_cycle; - *k = NULL; - break; - } +void gpgpu_sim::set_kernel_done(kernel_info_t *kernel) { + unsigned uid = kernel->get_uid(); + m_finished_kernel.push_back(uid); + std::vector<kernel_info_t *>::iterator k; + for (k = m_running_kernels.begin(); k != m_running_kernels.end(); k++) { + if (*k == kernel) { + kernel->end_cycle = gpu_sim_cycle + gpu_tot_sim_cycle; + *k = NULL; + break; } - assert( k != m_running_kernels.end() ); + } + assert(k != m_running_kernels.end()); } -void gpgpu_sim::stop_all_running_kernels(){ - std::vector<kernel_info_t *>::iterator k; - for(k = m_running_kernels.begin(); k != m_running_kernels.end(); ++k){ - if(*k != NULL){ // If a kernel is active - set_kernel_done(*k); // Stop the kernel - assert(*k==NULL); - } +void gpgpu_sim::stop_all_running_kernels() { + std::vector<kernel_info_t *>::iterator k; + for (k = m_running_kernels.begin(); k != m_running_kernels.end(); ++k) { + if (*k != NULL) { // If a kernel is active + set_kernel_done(*k); // Stop the kernel + assert(*k == NULL); } + } } -gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config, gpgpu_context* ctx ) - : gpgpu_t(config, ctx), m_config(config) -{ - gpgpu_ctx = ctx; - m_shader_config = &m_config.m_shader_config; - m_memory_config = &m_config.m_memory_config; - ctx->ptx_parser->set_ptx_warp_size(m_shader_config); - ptx_file_line_stats_create_exposed_latency_tracker(m_config.num_shader()); +gpgpu_sim::gpgpu_sim(const gpgpu_sim_config &config, gpgpu_context *ctx) + : gpgpu_t(config, ctx), m_config(config) { + gpgpu_ctx = ctx; + m_shader_config = &m_config.m_shader_config; + m_memory_config = &m_config.m_memory_config; + ctx->ptx_parser->set_ptx_warp_size(m_shader_config); + ptx_file_line_stats_create_exposed_latency_tracker(m_config.num_shader()); #ifdef GPGPUSIM_POWER_MODEL - m_gpgpusim_wrapper = new gpgpu_sim_wrapper(config.g_power_simulation_enabled,config.g_power_config_name); + m_gpgpusim_wrapper = new gpgpu_sim_wrapper(config.g_power_simulation_enabled, + config.g_power_config_name); #endif - m_shader_stats = new shader_core_stats(m_shader_config); - m_memory_stats = new memory_stats_t(m_config.num_shader(),m_shader_config,m_memory_config,this); - average_pipeline_duty_cycle = (float *)malloc(sizeof(float)); - active_sms=(float *)malloc(sizeof(float)); - m_power_stats = new power_stat_t(m_shader_config,average_pipeline_duty_cycle,active_sms,m_shader_stats,m_memory_config,m_memory_stats); + m_shader_stats = new shader_core_stats(m_shader_config); + m_memory_stats = new memory_stats_t(m_config.num_shader(), m_shader_config, + m_memory_config, this); + average_pipeline_duty_cycle = (float *)malloc(sizeof(float)); + active_sms = (float *)malloc(sizeof(float)); + m_power_stats = + new power_stat_t(m_shader_config, average_pipeline_duty_cycle, active_sms, + m_shader_stats, m_memory_config, m_memory_stats); - gpu_sim_insn = 0; - gpu_tot_sim_insn = 0; - gpu_tot_issued_cta = 0; - m_total_cta_launched = 0; - gpu_deadlock = false; + gpu_sim_insn = 0; + gpu_tot_sim_insn = 0; + gpu_tot_issued_cta = 0; + m_total_cta_launched = 0; + gpu_deadlock = false; - gpu_stall_dramfull = 0; - gpu_stall_icnt2sh = 0; - partiton_reqs_in_parallel = 0; - partiton_reqs_in_parallel_total = 0; - partiton_reqs_in_parallel_util = 0; - partiton_reqs_in_parallel_util_total = 0; - gpu_sim_cycle_parition_util = 0; - gpu_tot_sim_cycle_parition_util = 0; - partiton_replys_in_parallel = 0; - partiton_replys_in_parallel_total = 0; + gpu_stall_dramfull = 0; + gpu_stall_icnt2sh = 0; + partiton_reqs_in_parallel = 0; + partiton_reqs_in_parallel_total = 0; + partiton_reqs_in_parallel_util = 0; + partiton_reqs_in_parallel_util_total = 0; + gpu_sim_cycle_parition_util = 0; + gpu_tot_sim_cycle_parition_util = 0; + partiton_replys_in_parallel = 0; + partiton_replys_in_parallel_total = 0; - m_cluster = new simt_core_cluster*[m_shader_config->n_simt_clusters]; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) - m_cluster[i] = new simt_core_cluster(this,i,m_shader_config,m_memory_config,m_shader_stats,m_memory_stats); + m_cluster = new simt_core_cluster *[m_shader_config->n_simt_clusters]; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) + m_cluster[i] = + new simt_core_cluster(this, i, m_shader_config, m_memory_config, + m_shader_stats, m_memory_stats); - m_memory_partition_unit = new memory_partition_unit*[m_memory_config->m_n_mem]; - m_memory_sub_partition = new memory_sub_partition*[m_memory_config->m_n_mem_sub_partition]; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - m_memory_partition_unit[i] = new memory_partition_unit(i, m_memory_config, m_memory_stats, this); - for (unsigned p = 0; p < m_memory_config->m_n_sub_partition_per_memory_channel; p++) { - unsigned submpid = i * m_memory_config->m_n_sub_partition_per_memory_channel + p; - m_memory_sub_partition[submpid] = m_memory_partition_unit[i]->get_sub_partition(p); - } + m_memory_partition_unit = + new memory_partition_unit *[m_memory_config->m_n_mem]; + m_memory_sub_partition = + new memory_sub_partition *[m_memory_config->m_n_mem_sub_partition]; + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + m_memory_partition_unit[i] = + new memory_partition_unit(i, m_memory_config, m_memory_stats, this); + for (unsigned p = 0; + p < m_memory_config->m_n_sub_partition_per_memory_channel; p++) { + unsigned submpid = + i * m_memory_config->m_n_sub_partition_per_memory_channel + p; + m_memory_sub_partition[submpid] = + m_memory_partition_unit[i]->get_sub_partition(p); } + } - icnt_wrapper_init(); - icnt_create(m_shader_config->n_simt_clusters,m_memory_config->m_n_mem_sub_partition); + icnt_wrapper_init(); + icnt_create(m_shader_config->n_simt_clusters, + m_memory_config->m_n_mem_sub_partition); - time_vector_create(NUM_MEM_REQ_STAT); - fprintf(stdout, "GPGPU-Sim uArch: performance model initialization complete.\n"); + time_vector_create(NUM_MEM_REQ_STAT); + fprintf(stdout, + "GPGPU-Sim uArch: performance model initialization complete.\n"); - m_running_kernels.resize( config.max_concurrent_kernel, NULL ); - m_last_issued_kernel = 0; - m_last_cluster_issue = m_shader_config->n_simt_clusters-1; // this causes first launch to use simt cluster 0 - *average_pipeline_duty_cycle=0; - *active_sms=0; + m_running_kernels.resize(config.max_concurrent_kernel, NULL); + m_last_issued_kernel = 0; + m_last_cluster_issue = m_shader_config->n_simt_clusters - + 1; // this causes first launch to use simt cluster 0 + *average_pipeline_duty_cycle = 0; + *active_sms = 0; - last_liveness_message_time = 0; - - //Jin: functional simulation for CDP - m_functional_sim = false; - m_functional_sim_kernel = NULL; -} + last_liveness_message_time = 0; -int gpgpu_sim::shared_mem_size() const -{ - return m_shader_config->gpgpu_shmem_size; + // Jin: functional simulation for CDP + m_functional_sim = false; + m_functional_sim_kernel = NULL; } -int gpgpu_sim::shared_mem_per_block() const -{ - return m_shader_config->gpgpu_shmem_per_block; +int gpgpu_sim::shared_mem_size() const { + return m_shader_config->gpgpu_shmem_size; } -int gpgpu_sim::num_registers_per_core() const -{ - return m_shader_config->gpgpu_shader_registers; +int gpgpu_sim::shared_mem_per_block() const { + return m_shader_config->gpgpu_shmem_per_block; } -int gpgpu_sim::num_registers_per_block() const -{ - return m_shader_config->gpgpu_registers_per_block; +int gpgpu_sim::num_registers_per_core() const { + return m_shader_config->gpgpu_shader_registers; } -int gpgpu_sim::wrp_size() const -{ - return m_shader_config->warp_size; +int gpgpu_sim::num_registers_per_block() const { + return m_shader_config->gpgpu_registers_per_block; } -int gpgpu_sim::shader_clock() const -{ - return m_config.core_freq/1000; -} +int gpgpu_sim::wrp_size() const { return m_shader_config->warp_size; } -int gpgpu_sim::max_cta_per_core() const -{ - return m_shader_config->max_cta_per_core; -} +int gpgpu_sim::shader_clock() const { return m_config.core_freq / 1000; } -int gpgpu_sim::get_max_cta( const kernel_info_t &k ) const -{ - return m_shader_config->max_cta(k); +int gpgpu_sim::max_cta_per_core() const { + return m_shader_config->max_cta_per_core; } -void gpgpu_sim::set_prop( cudaDeviceProp *prop ) -{ - m_cuda_properties = prop; +int gpgpu_sim::get_max_cta(const kernel_info_t &k) const { + return m_shader_config->max_cta(k); } -int gpgpu_sim::compute_capability_major() const -{ - return m_config.gpgpu_compute_capability_major; +void gpgpu_sim::set_prop(cudaDeviceProp *prop) { m_cuda_properties = prop; } + +int gpgpu_sim::compute_capability_major() const { + return m_config.gpgpu_compute_capability_major; } -int gpgpu_sim::compute_capability_minor() const -{ - return m_config.gpgpu_compute_capability_minor; +int gpgpu_sim::compute_capability_minor() const { + return m_config.gpgpu_compute_capability_minor; } -const struct cudaDeviceProp *gpgpu_sim::get_prop() const -{ - return m_cuda_properties; +const struct cudaDeviceProp *gpgpu_sim::get_prop() const { + return m_cuda_properties; } -enum divergence_support_t gpgpu_sim::simd_model() const -{ - return m_shader_config->model; +enum divergence_support_t gpgpu_sim::simd_model() const { + return m_shader_config->model; } -void gpgpu_sim_config::init_clock_domains(void ) -{ - sscanf(gpgpu_clock_domains,"%lf:%lf:%lf:%lf", - &core_freq, &icnt_freq, &l2_freq, &dram_freq); - core_freq = core_freq MhZ; - icnt_freq = icnt_freq MhZ; - l2_freq = l2_freq MhZ; - dram_freq = dram_freq MhZ; - core_period = 1/core_freq; - icnt_period = 1/icnt_freq; - dram_period = 1/dram_freq; - l2_period = 1/l2_freq; - printf("GPGPU-Sim uArch: clock freqs: %lf:%lf:%lf:%lf\n",core_freq,icnt_freq,l2_freq,dram_freq); - printf("GPGPU-Sim uArch: clock periods: %.20lf:%.20lf:%.20lf:%.20lf\n",core_period,icnt_period,l2_period,dram_period); +void gpgpu_sim_config::init_clock_domains(void) { + sscanf(gpgpu_clock_domains, "%lf:%lf:%lf:%lf", &core_freq, &icnt_freq, + &l2_freq, &dram_freq); + core_freq = core_freq MhZ; + icnt_freq = icnt_freq MhZ; + l2_freq = l2_freq MhZ; + dram_freq = dram_freq MhZ; + core_period = 1 / core_freq; + icnt_period = 1 / icnt_freq; + dram_period = 1 / dram_freq; + l2_period = 1 / l2_freq; + printf("GPGPU-Sim uArch: clock freqs: %lf:%lf:%lf:%lf\n", core_freq, + icnt_freq, l2_freq, dram_freq); + printf("GPGPU-Sim uArch: clock periods: %.20lf:%.20lf:%.20lf:%.20lf\n", + core_period, icnt_period, l2_period, dram_period); } -void gpgpu_sim::reinit_clock_domains(void) -{ - core_time = 0; - dram_time = 0; - icnt_time = 0; - l2_time = 0; +void gpgpu_sim::reinit_clock_domains(void) { + core_time = 0; + dram_time = 0; + icnt_time = 0; + l2_time = 0; } -bool gpgpu_sim::active() -{ - if (m_config.gpu_max_cycle_opt && (gpu_tot_sim_cycle + gpu_sim_cycle) >= m_config.gpu_max_cycle_opt) - return false; - if (m_config.gpu_max_insn_opt && (gpu_tot_sim_insn + gpu_sim_insn) >= m_config.gpu_max_insn_opt) - return false; - if (m_config.gpu_max_cta_opt && (gpu_tot_issued_cta >= m_config.gpu_max_cta_opt) ) - return false; - if (m_config.gpu_deadlock_detect && gpu_deadlock) - return false; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) - if( m_cluster[i]->get_not_completed()>0 ) - return true;; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) - if( m_memory_partition_unit[i]->busy()>0 ) - return true;; - if( icnt_busy() ) - return true; - if( get_more_cta_left() ) - return true; +bool gpgpu_sim::active() { + if (m_config.gpu_max_cycle_opt && + (gpu_tot_sim_cycle + gpu_sim_cycle) >= m_config.gpu_max_cycle_opt) + return false; + if (m_config.gpu_max_insn_opt && + (gpu_tot_sim_insn + gpu_sim_insn) >= m_config.gpu_max_insn_opt) return false; + if (m_config.gpu_max_cta_opt && + (gpu_tot_issued_cta >= m_config.gpu_max_cta_opt)) + return false; + if (m_config.gpu_deadlock_detect && gpu_deadlock) return false; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) + if (m_cluster[i]->get_not_completed() > 0) return true; + ; + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) + if (m_memory_partition_unit[i]->busy() > 0) return true; + ; + if (icnt_busy()) return true; + if (get_more_cta_left()) return true; + return false; } -void gpgpu_sim::init() -{ - // run a CUDA grid on the GPU microarchitecture simulator - gpu_sim_cycle = 0; - gpu_sim_insn = 0; - last_gpu_sim_insn = 0; - m_total_cta_launched=0; - partiton_reqs_in_parallel = 0; - partiton_replys_in_parallel = 0; - partiton_reqs_in_parallel_util = 0; - gpu_sim_cycle_parition_util = 0; +void gpgpu_sim::init() { + // run a CUDA grid on the GPU microarchitecture simulator + gpu_sim_cycle = 0; + gpu_sim_insn = 0; + last_gpu_sim_insn = 0; + m_total_cta_launched = 0; + partiton_reqs_in_parallel = 0; + partiton_replys_in_parallel = 0; + partiton_reqs_in_parallel_util = 0; + gpu_sim_cycle_parition_util = 0; - reinit_clock_domains(); - gpgpu_ctx->func_sim->set_param_gpgpu_num_shaders(m_config.num_shader()); - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) - m_cluster[i]->reinit(); - m_shader_stats->new_grid(); - // initialize the control-flow, memory access, memory latency logger - if (m_config.g_visualizer_enabled) { - create_thread_CFlogger( gpgpu_ctx, m_config.num_shader(), m_shader_config->n_thread_per_shader, 0, m_config.gpgpu_cflog_interval ); - } - shader_CTA_count_create( m_config.num_shader(), m_config.gpgpu_cflog_interval); - if (m_config.gpgpu_cflog_interval != 0) { - insn_warp_occ_create( m_config.num_shader(), m_shader_config->warp_size ); - shader_warp_occ_create( m_config.num_shader(), m_shader_config->warp_size, m_config.gpgpu_cflog_interval); - shader_mem_acc_create( m_config.num_shader(), m_memory_config->m_n_mem, 4, m_config.gpgpu_cflog_interval); - shader_mem_lat_create( m_config.num_shader(), m_config.gpgpu_cflog_interval); - shader_cache_access_create( m_config.num_shader(), 3, m_config.gpgpu_cflog_interval); - set_spill_interval (m_config.gpgpu_cflog_interval * 40); - } + reinit_clock_domains(); + gpgpu_ctx->func_sim->set_param_gpgpu_num_shaders(m_config.num_shader()); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) + m_cluster[i]->reinit(); + m_shader_stats->new_grid(); + // initialize the control-flow, memory access, memory latency logger + if (m_config.g_visualizer_enabled) { + create_thread_CFlogger(gpgpu_ctx, m_config.num_shader(), + m_shader_config->n_thread_per_shader, 0, + m_config.gpgpu_cflog_interval); + } + shader_CTA_count_create(m_config.num_shader(), m_config.gpgpu_cflog_interval); + if (m_config.gpgpu_cflog_interval != 0) { + insn_warp_occ_create(m_config.num_shader(), m_shader_config->warp_size); + shader_warp_occ_create(m_config.num_shader(), m_shader_config->warp_size, + m_config.gpgpu_cflog_interval); + shader_mem_acc_create(m_config.num_shader(), m_memory_config->m_n_mem, 4, + m_config.gpgpu_cflog_interval); + shader_mem_lat_create(m_config.num_shader(), m_config.gpgpu_cflog_interval); + shader_cache_access_create(m_config.num_shader(), 3, + m_config.gpgpu_cflog_interval); + set_spill_interval(m_config.gpgpu_cflog_interval * 40); + } - if (g_network_mode) - icnt_init(); + if (g_network_mode) icnt_init(); // McPAT initialization function. Called on first launch of GPU #ifdef GPGPUSIM_POWER_MODEL - if(m_config.g_power_simulation_enabled){ - init_mcpat(m_config, m_gpgpusim_wrapper, m_config.gpu_stat_sample_freq, gpu_tot_sim_insn, gpu_sim_insn); - } + if (m_config.g_power_simulation_enabled) { + init_mcpat(m_config, m_gpgpusim_wrapper, m_config.gpu_stat_sample_freq, + gpu_tot_sim_insn, gpu_sim_insn); + } #endif } void gpgpu_sim::update_stats() { - m_memory_stats->memlatstat_lat_pw(); - gpu_tot_sim_cycle += gpu_sim_cycle; - gpu_tot_sim_insn += gpu_sim_insn; - gpu_tot_issued_cta += m_total_cta_launched; - partiton_reqs_in_parallel_total += partiton_reqs_in_parallel; - partiton_replys_in_parallel_total += partiton_replys_in_parallel; - partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util; - gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ; - gpu_tot_occupancy += gpu_occupancy; + m_memory_stats->memlatstat_lat_pw(); + gpu_tot_sim_cycle += gpu_sim_cycle; + gpu_tot_sim_insn += gpu_sim_insn; + gpu_tot_issued_cta += m_total_cta_launched; + partiton_reqs_in_parallel_total += partiton_reqs_in_parallel; + partiton_replys_in_parallel_total += partiton_replys_in_parallel; + partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util; + gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util; + gpu_tot_occupancy += gpu_occupancy; - gpu_sim_cycle = 0; - partiton_reqs_in_parallel = 0; - partiton_replys_in_parallel = 0; - partiton_reqs_in_parallel_util = 0; - gpu_sim_cycle_parition_util = 0; - gpu_sim_insn = 0; - m_total_cta_launched = 0; - gpu_occupancy = occupancy_stats(); + gpu_sim_cycle = 0; + partiton_reqs_in_parallel = 0; + partiton_replys_in_parallel = 0; + partiton_reqs_in_parallel_util = 0; + gpu_sim_cycle_parition_util = 0; + gpu_sim_insn = 0; + m_total_cta_launched = 0; + gpu_occupancy = occupancy_stats(); } -void gpgpu_sim::print_stats() -{ - gpgpu_ctx->stats->ptx_file_line_stats_write_file(); - gpu_print_stat(); +void gpgpu_sim::print_stats() { + gpgpu_ctx->stats->ptx_file_line_stats_write_file(); + gpu_print_stat(); - if (g_network_mode) { - printf("----------------------------Interconnect-DETAILS--------------------------------\n" ); - icnt_display_stats(); - icnt_display_overall_stats(); - printf("----------------------------END-of-Interconnect-DETAILS-------------------------\n" ); - } + if (g_network_mode) { + printf( + "----------------------------Interconnect-DETAILS----------------------" + "----------\n"); + icnt_display_stats(); + icnt_display_overall_stats(); + printf( + "----------------------------END-of-Interconnect-DETAILS---------------" + "----------\n"); + } } -void gpgpu_sim::deadlock_check() -{ - if (m_config.gpu_deadlock_detect && gpu_deadlock) { - fflush(stdout); - printf("\n\nGPGPU-Sim uArch: ERROR ** deadlock detected: last writeback core %u @ gpu_sim_cycle %u (+ gpu_tot_sim_cycle %u) (%u cycles ago)\n", - gpu_sim_insn_last_update_sid, - (unsigned) gpu_sim_insn_last_update, (unsigned) (gpu_tot_sim_cycle-gpu_sim_cycle), - (unsigned) (gpu_sim_cycle - gpu_sim_insn_last_update )); - unsigned num_cores=0; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - unsigned not_completed = m_cluster[i]->get_not_completed(); - if( not_completed ) { - if ( !num_cores ) { - printf("GPGPU-Sim uArch: DEADLOCK shader cores no longer committing instructions [core(# threads)]:\n" ); - printf("GPGPU-Sim uArch: DEADLOCK "); - m_cluster[i]->print_not_completed(stdout); - } else if (num_cores < 8 ) { - m_cluster[i]->print_not_completed(stdout); - } else if (num_cores >= 8 ) { - printf(" + others ... "); - } - num_cores+=m_shader_config->n_simt_cores_per_cluster; - } - } - printf("\n"); - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - bool busy = m_memory_partition_unit[i]->busy(); - if( busy ) - printf("GPGPU-Sim uArch DEADLOCK: memory partition %u busy\n", i ); - } - if( icnt_busy() ) { - printf("GPGPU-Sim uArch DEADLOCK: iterconnect contains traffic\n"); - icnt_display_state( stdout ); +void gpgpu_sim::deadlock_check() { + if (m_config.gpu_deadlock_detect && gpu_deadlock) { + fflush(stdout); + printf( + "\n\nGPGPU-Sim uArch: ERROR ** deadlock detected: last writeback core " + "%u @ gpu_sim_cycle %u (+ gpu_tot_sim_cycle %u) (%u cycles ago)\n", + gpu_sim_insn_last_update_sid, (unsigned)gpu_sim_insn_last_update, + (unsigned)(gpu_tot_sim_cycle - gpu_sim_cycle), + (unsigned)(gpu_sim_cycle - gpu_sim_insn_last_update)); + unsigned num_cores = 0; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + unsigned not_completed = m_cluster[i]->get_not_completed(); + if (not_completed) { + if (!num_cores) { + printf( + "GPGPU-Sim uArch: DEADLOCK shader cores no longer committing " + "instructions [core(# threads)]:\n"); + printf("GPGPU-Sim uArch: DEADLOCK "); + m_cluster[i]->print_not_completed(stdout); + } else if (num_cores < 8) { + m_cluster[i]->print_not_completed(stdout); + } else if (num_cores >= 8) { + printf(" + others ... "); + } + num_cores += m_shader_config->n_simt_cores_per_cluster; } - printf("\nRe-run the simulator in gdb and use debug routines in .gdbinit to debug this\n"); - fflush(stdout); - abort(); - } + } + printf("\n"); + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + bool busy = m_memory_partition_unit[i]->busy(); + if (busy) + printf("GPGPU-Sim uArch DEADLOCK: memory partition %u busy\n", i); + } + if (icnt_busy()) { + printf("GPGPU-Sim uArch DEADLOCK: iterconnect contains traffic\n"); + icnt_display_state(stdout); + } + printf( + "\nRe-run the simulator in gdb and use debug routines in .gdbinit to " + "debug this\n"); + fflush(stdout); + abort(); + } } -/// printing the names and uids of a set of executed kernels (usually there is only one) -std::string gpgpu_sim::executed_kernel_info_string() -{ - std::stringstream statout; +/// printing the names and uids of a set of executed kernels (usually there is +/// only one) +std::string gpgpu_sim::executed_kernel_info_string() { + std::stringstream statout; - statout << "kernel_name = "; - for (unsigned int k = 0; k < m_executed_kernel_names.size(); k++) { - statout << m_executed_kernel_names[k] << " "; - } - statout << std::endl; - statout << "kernel_launch_uid = "; - for (unsigned int k = 0; k < m_executed_kernel_uids.size(); k++) { - statout << m_executed_kernel_uids[k] << " "; - } - statout << std::endl; + statout << "kernel_name = "; + for (unsigned int k = 0; k < m_executed_kernel_names.size(); k++) { + statout << m_executed_kernel_names[k] << " "; + } + statout << std::endl; + statout << "kernel_launch_uid = "; + for (unsigned int k = 0; k < m_executed_kernel_uids.size(); k++) { + statout << m_executed_kernel_uids[k] << " "; + } + statout << std::endl; - return statout.str(); + return statout.str(); } -void gpgpu_sim::set_cache_config(std::string kernel_name, FuncCache cacheConfig ) -{ - m_special_cache_config[kernel_name]=cacheConfig ; +void gpgpu_sim::set_cache_config(std::string kernel_name, + FuncCache cacheConfig) { + m_special_cache_config[kernel_name] = cacheConfig; } -FuncCache gpgpu_sim::get_cache_config(std::string kernel_name) -{ - for ( std::map<std::string, FuncCache>::iterator iter = m_special_cache_config.begin(); iter != m_special_cache_config.end(); iter++){ - std::string kernel= iter->first; - if (kernel_name.compare(kernel) == 0){ - return iter->second; - } - } - return (FuncCache)0; +FuncCache gpgpu_sim::get_cache_config(std::string kernel_name) { + for (std::map<std::string, FuncCache>::iterator iter = + m_special_cache_config.begin(); + iter != m_special_cache_config.end(); iter++) { + std::string kernel = iter->first; + if (kernel_name.compare(kernel) == 0) { + return iter->second; + } + } + return (FuncCache)0; } -bool gpgpu_sim::has_special_cache_config(std::string kernel_name) -{ - for ( std::map<std::string, FuncCache>::iterator iter = m_special_cache_config.begin(); iter != m_special_cache_config.end(); iter++){ - std::string kernel= iter->first; - if (kernel_name.compare(kernel) == 0){ - return true; - } - } - return false; +bool gpgpu_sim::has_special_cache_config(std::string kernel_name) { + for (std::map<std::string, FuncCache>::iterator iter = + m_special_cache_config.begin(); + iter != m_special_cache_config.end(); iter++) { + std::string kernel = iter->first; + if (kernel_name.compare(kernel) == 0) { + return true; + } + } + return false; } - -void gpgpu_sim::set_cache_config(std::string kernel_name) -{ - if(has_special_cache_config(kernel_name)){ - change_cache_config(get_cache_config(kernel_name)); - }else{ - change_cache_config(FuncCachePreferNone); - } +void gpgpu_sim::set_cache_config(std::string kernel_name) { + if (has_special_cache_config(kernel_name)) { + change_cache_config(get_cache_config(kernel_name)); + } else { + change_cache_config(FuncCachePreferNone); + } } +void gpgpu_sim::change_cache_config(FuncCache cache_config) { + if (cache_config != m_shader_config->m_L1D_config.get_cache_status()) { + printf("FLUSH L1 Cache at configuration change between kernels\n"); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + m_cluster[i]->cache_invalidate(); + } + } -void gpgpu_sim::change_cache_config(FuncCache cache_config) -{ - if(cache_config != m_shader_config->m_L1D_config.get_cache_status()){ - printf("FLUSH L1 Cache at configuration change between kernels\n"); - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - m_cluster[i]->cache_invalidate(); - } - } - - switch(cache_config){ - case FuncCachePreferNone: - m_shader_config->m_L1D_config.init(m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); - m_shader_config->gpgpu_shmem_size=m_shader_config->gpgpu_shmem_sizeDefault; - break; - case FuncCachePreferL1: - if((m_shader_config->m_L1D_config.m_config_stringPrefL1 == NULL) || (m_shader_config->gpgpu_shmem_sizePrefL1 == (unsigned)-1)) - { - printf("WARNING: missing Preferred L1 configuration\n"); - m_shader_config->m_L1D_config.init(m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); - m_shader_config->gpgpu_shmem_size=m_shader_config->gpgpu_shmem_sizeDefault; + switch (cache_config) { + case FuncCachePreferNone: + m_shader_config->m_L1D_config.init( + m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); + m_shader_config->gpgpu_shmem_size = + m_shader_config->gpgpu_shmem_sizeDefault; + break; + case FuncCachePreferL1: + if ((m_shader_config->m_L1D_config.m_config_stringPrefL1 == NULL) || + (m_shader_config->gpgpu_shmem_sizePrefL1 == (unsigned)-1)) { + printf("WARNING: missing Preferred L1 configuration\n"); + m_shader_config->m_L1D_config.init( + m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); + m_shader_config->gpgpu_shmem_size = + m_shader_config->gpgpu_shmem_sizeDefault; - }else{ - m_shader_config->m_L1D_config.init(m_shader_config->m_L1D_config.m_config_stringPrefL1, FuncCachePreferL1); - m_shader_config->gpgpu_shmem_size=m_shader_config->gpgpu_shmem_sizePrefL1; - } - break; - case FuncCachePreferShared: - if((m_shader_config->m_L1D_config.m_config_stringPrefShared == NULL) || (m_shader_config->gpgpu_shmem_sizePrefShared == (unsigned)-1)) - { - printf("WARNING: missing Preferred L1 configuration\n"); - m_shader_config->m_L1D_config.init(m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); - m_shader_config->gpgpu_shmem_size=m_shader_config->gpgpu_shmem_sizeDefault; - }else{ - m_shader_config->m_L1D_config.init(m_shader_config->m_L1D_config.m_config_stringPrefShared, FuncCachePreferShared); - m_shader_config->gpgpu_shmem_size=m_shader_config->gpgpu_shmem_sizePrefShared; - } - break; - default: - break; - } + } else { + m_shader_config->m_L1D_config.init( + m_shader_config->m_L1D_config.m_config_stringPrefL1, + FuncCachePreferL1); + m_shader_config->gpgpu_shmem_size = + m_shader_config->gpgpu_shmem_sizePrefL1; + } + break; + case FuncCachePreferShared: + if ((m_shader_config->m_L1D_config.m_config_stringPrefShared == NULL) || + (m_shader_config->gpgpu_shmem_sizePrefShared == (unsigned)-1)) { + printf("WARNING: missing Preferred L1 configuration\n"); + m_shader_config->m_L1D_config.init( + m_shader_config->m_L1D_config.m_config_string, FuncCachePreferNone); + m_shader_config->gpgpu_shmem_size = + m_shader_config->gpgpu_shmem_sizeDefault; + } else { + m_shader_config->m_L1D_config.init( + m_shader_config->m_L1D_config.m_config_stringPrefShared, + FuncCachePreferShared); + m_shader_config->gpgpu_shmem_size = + m_shader_config->gpgpu_shmem_sizePrefShared; + } + break; + default: + break; + } } - -void gpgpu_sim::clear_executed_kernel_info() -{ - m_executed_kernel_names.clear(); - m_executed_kernel_uids.clear(); +void gpgpu_sim::clear_executed_kernel_info() { + m_executed_kernel_names.clear(); + m_executed_kernel_uids.clear(); } -void gpgpu_sim::gpu_print_stat() -{ - FILE *statfout = stdout; +void gpgpu_sim::gpu_print_stat() { + FILE *statfout = stdout; - std::string kernel_info_str = executed_kernel_info_string(); - fprintf(statfout, "%s", kernel_info_str.c_str()); + std::string kernel_info_str = executed_kernel_info_string(); + fprintf(statfout, "%s", kernel_info_str.c_str()); - printf("gpu_sim_cycle = %lld\n", gpu_sim_cycle); - printf("gpu_sim_insn = %lld\n", gpu_sim_insn); - printf("gpu_ipc = %12.4f\n", (float)gpu_sim_insn / gpu_sim_cycle); - printf("gpu_tot_sim_cycle = %lld\n", gpu_tot_sim_cycle+gpu_sim_cycle); - printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn+gpu_sim_insn); - printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn+gpu_sim_insn) / (gpu_tot_sim_cycle+gpu_sim_cycle)); - printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta + m_total_cta_launched); - printf("gpu_occupancy = %.4f%% \n", gpu_occupancy.get_occ_fraction() * 100); - printf("gpu_tot_occupancy = %.4f%% \n", (gpu_occupancy + gpu_tot_occupancy).get_occ_fraction() * 100); + printf("gpu_sim_cycle = %lld\n", gpu_sim_cycle); + printf("gpu_sim_insn = %lld\n", gpu_sim_insn); + printf("gpu_ipc = %12.4f\n", (float)gpu_sim_insn / gpu_sim_cycle); + printf("gpu_tot_sim_cycle = %lld\n", gpu_tot_sim_cycle + gpu_sim_cycle); + printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn + gpu_sim_insn); + printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn + gpu_sim_insn) / + (gpu_tot_sim_cycle + gpu_sim_cycle)); + printf("gpu_tot_issued_cta = %lld\n", + gpu_tot_issued_cta + m_total_cta_launched); + printf("gpu_occupancy = %.4f%% \n", gpu_occupancy.get_occ_fraction() * 100); + printf("gpu_tot_occupancy = %.4f%% \n", + (gpu_occupancy + gpu_tot_occupancy).get_occ_fraction() * 100); + fprintf(statfout, "max_total_param_size = %llu\n", + gpgpu_ctx->device_runtime->g_max_total_param_size); - fprintf(statfout, "max_total_param_size = %llu\n", gpgpu_ctx->device_runtime->g_max_total_param_size); + // performance counter for stalls due to congestion. + printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull); + printf("gpu_stall_icnt2sh = %d\n", gpu_stall_icnt2sh); - // performance counter for stalls due to congestion. - printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull); - printf("gpu_stall_icnt2sh = %d\n", gpu_stall_icnt2sh ); + // printf("partiton_reqs_in_parallel = %lld\n", partiton_reqs_in_parallel); + // printf("partiton_reqs_in_parallel_total = %lld\n", + // partiton_reqs_in_parallel_total ); + printf("partiton_level_parallism = %12.4f\n", + (float)partiton_reqs_in_parallel / gpu_sim_cycle); + printf("partiton_level_parallism_total = %12.4f\n", + (float)(partiton_reqs_in_parallel + partiton_reqs_in_parallel_total) / + (gpu_tot_sim_cycle + gpu_sim_cycle)); + // printf("partiton_reqs_in_parallel_util = %lld\n", + // partiton_reqs_in_parallel_util); + // printf("partiton_reqs_in_parallel_util_total = %lld\n", + // partiton_reqs_in_parallel_util_total ); printf("gpu_sim_cycle_parition_util + // = %lld\n", gpu_sim_cycle_parition_util); + // printf("gpu_tot_sim_cycle_parition_util = %lld\n", + // gpu_tot_sim_cycle_parition_util ); + printf("partiton_level_parallism_util = %12.4f\n", + (float)partiton_reqs_in_parallel_util / gpu_sim_cycle_parition_util); + printf("partiton_level_parallism_util_total = %12.4f\n", + (float)(partiton_reqs_in_parallel_util + + partiton_reqs_in_parallel_util_total) / + (gpu_sim_cycle_parition_util + gpu_tot_sim_cycle_parition_util)); + // printf("partiton_replys_in_parallel = %lld\n", + // partiton_replys_in_parallel); printf("partiton_replys_in_parallel_total = + // %lld\n", partiton_replys_in_parallel_total ); + printf("L2_BW = %12.4f GB/Sec\n", + ((float)(partiton_replys_in_parallel * 32) / + (gpu_sim_cycle * m_config.icnt_period)) / + 1000000000); + printf("L2_BW_total = %12.4f GB/Sec\n", + ((float)((partiton_replys_in_parallel + + partiton_replys_in_parallel_total) * + 32) / + ((gpu_tot_sim_cycle + gpu_sim_cycle) * m_config.icnt_period)) / + 1000000000); - //printf("partiton_reqs_in_parallel = %lld\n", partiton_reqs_in_parallel); - //printf("partiton_reqs_in_parallel_total = %lld\n", partiton_reqs_in_parallel_total ); - printf("partiton_level_parallism = %12.4f\n", (float)partiton_reqs_in_parallel / gpu_sim_cycle); - printf("partiton_level_parallism_total = %12.4f\n", (float)(partiton_reqs_in_parallel+partiton_reqs_in_parallel_total) / (gpu_tot_sim_cycle+gpu_sim_cycle) ); - //printf("partiton_reqs_in_parallel_util = %lld\n", partiton_reqs_in_parallel_util); - //printf("partiton_reqs_in_parallel_util_total = %lld\n", partiton_reqs_in_parallel_util_total ); - //printf("gpu_sim_cycle_parition_util = %lld\n", gpu_sim_cycle_parition_util); - // printf("gpu_tot_sim_cycle_parition_util = %lld\n", gpu_tot_sim_cycle_parition_util ); - printf("partiton_level_parallism_util = %12.4f\n", (float)partiton_reqs_in_parallel_util / gpu_sim_cycle_parition_util); - printf("partiton_level_parallism_util_total = %12.4f\n", (float)(partiton_reqs_in_parallel_util+partiton_reqs_in_parallel_util_total) / (gpu_sim_cycle_parition_util+gpu_tot_sim_cycle_parition_util) ); - //printf("partiton_replys_in_parallel = %lld\n", partiton_replys_in_parallel); - //printf("partiton_replys_in_parallel_total = %lld\n", partiton_replys_in_parallel_total ); - printf("L2_BW = %12.4f GB/Sec\n", ((float)(partiton_replys_in_parallel * 32) / (gpu_sim_cycle * m_config.icnt_period)) / 1000000000); - printf("L2_BW_total = %12.4f GB/Sec\n", ((float)((partiton_replys_in_parallel+partiton_replys_in_parallel_total) * 32) / ((gpu_tot_sim_cycle+gpu_sim_cycle) * m_config.icnt_period)) / 1000000000 ); + time_t curr_time; + time(&curr_time); + unsigned long long elapsed_time = + MAX(curr_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + printf("gpu_total_sim_rate=%u\n", + (unsigned)((gpu_tot_sim_insn + gpu_sim_insn) / elapsed_time)); - time_t curr_time; - time(&curr_time); - unsigned long long elapsed_time = MAX( curr_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1 ); - printf( "gpu_total_sim_rate=%u\n", (unsigned)( ( gpu_tot_sim_insn + gpu_sim_insn ) / elapsed_time ) ); + // shader_print_l1_miss_stat( stdout ); + shader_print_cache_stats(stdout); - //shader_print_l1_miss_stat( stdout ); - shader_print_cache_stats(stdout); + cache_stats core_cache_stats; + core_cache_stats.clear(); + for (unsigned i = 0; i < m_config.num_cluster(); i++) { + m_cluster[i]->get_cache_stats(core_cache_stats); + } + printf("\nTotal_core_cache_stats:\n"); + core_cache_stats.print_stats(stdout, "Total_core_cache_stats_breakdown"); + printf("\nTotal_core_cache_fail_stats:\n"); + core_cache_stats.print_fail_stats(stdout, + "Total_core_cache_fail_stats_breakdown"); + shader_print_scheduler_stat(stdout, false); - cache_stats core_cache_stats; - core_cache_stats.clear(); - for(unsigned i=0; i<m_config.num_cluster(); i++){ - m_cluster[i]->get_cache_stats(core_cache_stats); - } - printf("\nTotal_core_cache_stats:\n"); - core_cache_stats.print_stats(stdout, "Total_core_cache_stats_breakdown"); - printf("\nTotal_core_cache_fail_stats:\n"); - core_cache_stats.print_fail_stats(stdout, "Total_core_cache_fail_stats_breakdown"); - shader_print_scheduler_stat( stdout, false ); - - m_shader_stats->print(stdout); + m_shader_stats->print(stdout); #ifdef GPGPUSIM_POWER_MODEL - if(m_config.g_power_simulation_enabled){ - m_gpgpusim_wrapper->print_power_kernel_stats(gpu_sim_cycle, gpu_tot_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, kernel_info_str, true ); - mcpat_reset_perf_count(m_gpgpusim_wrapper); - } + if (m_config.g_power_simulation_enabled) { + m_gpgpusim_wrapper->print_power_kernel_stats( + gpu_sim_cycle, gpu_tot_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, + kernel_info_str, true); + mcpat_reset_perf_count(m_gpgpusim_wrapper); + } #endif - // performance counter that are not local to one shader - m_memory_stats->memlatstat_print(m_memory_config->m_n_mem,m_memory_config->nbk); - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) - m_memory_partition_unit[i]->print(stdout); + // performance counter that are not local to one shader + m_memory_stats->memlatstat_print(m_memory_config->m_n_mem, + m_memory_config->nbk); + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) + m_memory_partition_unit[i]->print(stdout); - // L2 cache stats - if(!m_memory_config->m_L2_config.disabled()){ - cache_stats l2_stats; - struct cache_sub_stats l2_css; - struct cache_sub_stats total_l2_css; - l2_stats.clear(); - l2_css.clear(); - total_l2_css.clear(); + // L2 cache stats + if (!m_memory_config->m_L2_config.disabled()) { + cache_stats l2_stats; + struct cache_sub_stats l2_css; + struct cache_sub_stats total_l2_css; + l2_stats.clear(); + l2_css.clear(); + total_l2_css.clear(); - printf("\n========= L2 cache stats =========\n"); - for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++){ - m_memory_sub_partition[i]->accumulate_L2cache_stats(l2_stats); - m_memory_sub_partition[i]->get_L2cache_sub_stats(l2_css); + printf("\n========= L2 cache stats =========\n"); + for (unsigned i = 0; i < m_memory_config->m_n_mem_sub_partition; i++) { + m_memory_sub_partition[i]->accumulate_L2cache_stats(l2_stats); + m_memory_sub_partition[i]->get_L2cache_sub_stats(l2_css); - fprintf( stdout, "L2_cache_bank[%d]: Access = %llu, Miss = %llu, Miss_rate = %.3lf, Pending_hits = %llu, Reservation_fails = %llu\n", - i, l2_css.accesses, l2_css.misses, (double)l2_css.misses / (double)l2_css.accesses, l2_css.pending_hits, l2_css.res_fails); + fprintf(stdout, + "L2_cache_bank[%d]: Access = %llu, Miss = %llu, Miss_rate = " + "%.3lf, Pending_hits = %llu, Reservation_fails = %llu\n", + i, l2_css.accesses, l2_css.misses, + (double)l2_css.misses / (double)l2_css.accesses, + l2_css.pending_hits, l2_css.res_fails); - total_l2_css += l2_css; - } - if (!m_memory_config->m_L2_config.disabled() && m_memory_config->m_L2_config.get_num_lines()) { - //L2c_print_cache_stat(); - printf("L2_total_cache_accesses = %llu\n", total_l2_css.accesses); - printf("L2_total_cache_misses = %llu\n", total_l2_css.misses); - if(total_l2_css.accesses > 0) - printf("L2_total_cache_miss_rate = %.4lf\n", (double)total_l2_css.misses/(double)total_l2_css.accesses); - printf("L2_total_cache_pending_hits = %llu\n", total_l2_css.pending_hits); - printf("L2_total_cache_reservation_fails = %llu\n", total_l2_css.res_fails); - printf("L2_total_cache_breakdown:\n"); - l2_stats.print_stats(stdout, "L2_cache_stats_breakdown"); - printf("L2_total_cache_reservation_fail_breakdown:\n"); - l2_stats.print_fail_stats(stdout, "L2_cache_stats_fail_breakdown"); - total_l2_css.print_port_stats(stdout, "L2_cache"); - } - } + total_l2_css += l2_css; + } + if (!m_memory_config->m_L2_config.disabled() && + m_memory_config->m_L2_config.get_num_lines()) { + // L2c_print_cache_stat(); + printf("L2_total_cache_accesses = %llu\n", total_l2_css.accesses); + printf("L2_total_cache_misses = %llu\n", total_l2_css.misses); + if (total_l2_css.accesses > 0) + printf("L2_total_cache_miss_rate = %.4lf\n", + (double)total_l2_css.misses / (double)total_l2_css.accesses); + printf("L2_total_cache_pending_hits = %llu\n", total_l2_css.pending_hits); + printf("L2_total_cache_reservation_fails = %llu\n", + total_l2_css.res_fails); + printf("L2_total_cache_breakdown:\n"); + l2_stats.print_stats(stdout, "L2_cache_stats_breakdown"); + printf("L2_total_cache_reservation_fail_breakdown:\n"); + l2_stats.print_fail_stats(stdout, "L2_cache_stats_fail_breakdown"); + total_l2_css.print_port_stats(stdout, "L2_cache"); + } + } - if (m_config.gpgpu_cflog_interval != 0) { - spill_log_to_file (stdout, 1, gpu_sim_cycle); - insn_warp_occ_print(stdout); - } - if ( gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { - StatDisp( gpgpu_ctx->func_sim->g_inst_classification_stat[gpgpu_ctx->func_sim->g_ptx_kernel_count]); - StatDisp( gpgpu_ctx->func_sim->g_inst_op_classification_stat[gpgpu_ctx->func_sim->g_ptx_kernel_count]); - } + if (m_config.gpgpu_cflog_interval != 0) { + spill_log_to_file(stdout, 1, gpu_sim_cycle); + insn_warp_occ_print(stdout); + } + if (gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { + StatDisp(gpgpu_ctx->func_sim->g_inst_classification_stat + [gpgpu_ctx->func_sim->g_ptx_kernel_count]); + StatDisp(gpgpu_ctx->func_sim->g_inst_op_classification_stat + [gpgpu_ctx->func_sim->g_ptx_kernel_count]); + } #ifdef GPGPUSIM_POWER_MODEL - if(m_config.g_power_simulation_enabled){ - m_gpgpusim_wrapper->detect_print_steady_state(1,gpu_tot_sim_insn+gpu_sim_insn); - } + if (m_config.g_power_simulation_enabled) { + m_gpgpusim_wrapper->detect_print_steady_state( + 1, gpu_tot_sim_insn + gpu_sim_insn); + } #endif + // Interconnect power stat print + long total_simt_to_mem = 0; + long total_mem_to_simt = 0; + long temp_stm = 0; + long temp_mts = 0; + for (unsigned i = 0; i < m_config.num_cluster(); i++) { + m_cluster[i]->get_icnt_stats(temp_stm, temp_mts); + total_simt_to_mem += temp_stm; + total_mem_to_simt += temp_mts; + } + printf("\nicnt_total_pkts_mem_to_simt=%ld\n", total_mem_to_simt); + printf("icnt_total_pkts_simt_to_mem=%ld\n", total_simt_to_mem); - // Interconnect power stat print - long total_simt_to_mem=0; - long total_mem_to_simt=0; - long temp_stm=0; - long temp_mts = 0; - for(unsigned i=0; i<m_config.num_cluster(); i++){ - m_cluster[i]->get_icnt_stats(temp_stm, temp_mts); - total_simt_to_mem += temp_stm; - total_mem_to_simt += temp_mts; - } - printf("\nicnt_total_pkts_mem_to_simt=%ld\n", total_mem_to_simt); - printf("icnt_total_pkts_simt_to_mem=%ld\n", total_simt_to_mem); + time_vector_print(); + fflush(stdout); - time_vector_print(); - fflush(stdout); - - clear_executed_kernel_info(); + clear_executed_kernel_info(); } - // performance counter that are not local to one shader -unsigned gpgpu_sim::threads_per_core() const -{ - return m_shader_config->n_thread_per_shader; +unsigned gpgpu_sim::threads_per_core() const { + return m_shader_config->n_thread_per_shader; } -void shader_core_ctx::mem_instruction_stats(const warp_inst_t &inst) -{ - unsigned active_count = inst.active_count(); - //this breaks some encapsulation: the is_[space] functions, if you change those, change this. - switch (inst.space.get_type()) { +void shader_core_ctx::mem_instruction_stats(const warp_inst_t &inst) { + unsigned active_count = inst.active_count(); + // this breaks some encapsulation: the is_[space] functions, if you change + // those, change this. + switch (inst.space.get_type()) { case undefined_space: case reg_space: - break; + break; case shared_space: - m_stats->gpgpu_n_shmem_insn += active_count; - break; + m_stats->gpgpu_n_shmem_insn += active_count; + break; case sstarr_space: - m_stats->gpgpu_n_sstarr_insn += active_count; - break; + m_stats->gpgpu_n_sstarr_insn += active_count; + break; case const_space: - m_stats->gpgpu_n_const_insn += active_count; - break; + m_stats->gpgpu_n_const_insn += active_count; + break; case param_space_kernel: case param_space_local: - m_stats->gpgpu_n_param_insn += active_count; - break; + m_stats->gpgpu_n_param_insn += active_count; + break; case tex_space: - m_stats->gpgpu_n_tex_insn += active_count; - break; + m_stats->gpgpu_n_tex_insn += active_count; + break; case global_space: case local_space: - if( inst.is_store() ) - m_stats->gpgpu_n_store_insn += active_count; - else - m_stats->gpgpu_n_load_insn += active_count; - break; + if (inst.is_store()) + m_stats->gpgpu_n_store_insn += active_count; + else + m_stats->gpgpu_n_load_insn += active_count; + break; default: - abort(); - } + abort(); + } } -bool shader_core_ctx::can_issue_1block(kernel_info_t & kernel) { +bool shader_core_ctx::can_issue_1block(kernel_info_t &kernel) { + // Jin: concurrent kernels on one SM + if (m_config->gpgpu_concurrent_kernel_sm) { + if (m_config->max_cta(kernel) < 1) return false; - //Jin: concurrent kernels on one SM - if(m_config->gpgpu_concurrent_kernel_sm) { - if(m_config->max_cta(kernel) < 1) - return false; - - return occupy_shader_resource_1block(kernel, false); - } - else { - return (get_n_active_cta() < m_config->max_cta(kernel)); - } + return occupy_shader_resource_1block(kernel, false); + } else { + return (get_n_active_cta() < m_config->max_cta(kernel)); + } } int shader_core_ctx::find_available_hwtid(unsigned int cta_size, bool occupy) { - - unsigned int step; - for(step = 0; step < m_config->n_thread_per_shader; - step += cta_size) { - - unsigned int hw_tid; - for(hw_tid = step; hw_tid < step + cta_size; - hw_tid++) { - if(m_occupied_hwtid.test(hw_tid)) - break; - } - if(hw_tid == step + cta_size) //consecutive non-active - break; - } - if(step >= m_config->n_thread_per_shader) //didn't find - return -1; - else { - if(occupy) { - for(unsigned hw_tid = step; hw_tid < step + cta_size; - hw_tid++) - m_occupied_hwtid.set(hw_tid); - } - return step; - } + unsigned int step; + for (step = 0; step < m_config->n_thread_per_shader; step += cta_size) { + unsigned int hw_tid; + for (hw_tid = step; hw_tid < step + cta_size; hw_tid++) { + if (m_occupied_hwtid.test(hw_tid)) break; + } + if (hw_tid == step + cta_size) // consecutive non-active + break; + } + if (step >= m_config->n_thread_per_shader) // didn't find + return -1; + else { + if (occupy) { + for (unsigned hw_tid = step; hw_tid < step + cta_size; hw_tid++) + m_occupied_hwtid.set(hw_tid); + } + return step; + } } -bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t & k, bool occupy) { - unsigned threads_per_cta = k.threads_per_cta(); - const class function_info *kernel = k.entry(); - unsigned int padded_cta_size = threads_per_cta; - unsigned int warp_size = m_config->warp_size; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); +bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t &k, + bool occupy) { + unsigned threads_per_cta = k.threads_per_cta(); + const class function_info *kernel = k.entry(); + unsigned int padded_cta_size = threads_per_cta; + unsigned int warp_size = m_config->warp_size; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); - if(m_occupied_n_threads + padded_cta_size > m_config->n_thread_per_shader) - return false; + if (m_occupied_n_threads + padded_cta_size > m_config->n_thread_per_shader) + return false; - if(find_available_hwtid(padded_cta_size, false) == -1) - return false; + if (find_available_hwtid(padded_cta_size, false) == -1) return false; - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); - if(m_occupied_shmem + kernel_info->smem > m_config->gpgpu_shmem_size) - return false; + if (m_occupied_shmem + kernel_info->smem > m_config->gpgpu_shmem_size) + return false; + + unsigned int used_regs = padded_cta_size * ((kernel_info->regs + 3) & ~3); + if (m_occupied_regs + used_regs > m_config->gpgpu_shader_registers) + return false; - unsigned int used_regs = padded_cta_size * ((kernel_info->regs+3)&~3); - if(m_occupied_regs + used_regs > m_config->gpgpu_shader_registers) - return false; + if (m_occupied_ctas + 1 > m_config->max_cta_per_core) return false; - if(m_occupied_ctas +1 > m_config->max_cta_per_core) - return false; - - if(occupy) { - m_occupied_n_threads += padded_cta_size; - m_occupied_shmem += kernel_info->smem; - m_occupied_regs += (padded_cta_size * ((kernel_info->regs+3)&~3)); - m_occupied_ctas++; + if (occupy) { + m_occupied_n_threads += padded_cta_size; + m_occupied_shmem += kernel_info->smem; + m_occupied_regs += (padded_cta_size * ((kernel_info->regs + 3) & ~3)); + m_occupied_ctas++; - SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Occupied %u threads, %u shared mem, %u registers, %u ctas\n", - m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas); - } + SHADER_DPRINTF(LIVENESS, + "GPGPU-Sim uArch: Occupied %u threads, %u shared mem, %u " + "registers, %u ctas\n", + m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, + m_occupied_ctas); + } - return true; + return true; } -void shader_core_ctx::release_shader_resource_1block(unsigned hw_ctaid, kernel_info_t & k) { +void shader_core_ctx::release_shader_resource_1block(unsigned hw_ctaid, + kernel_info_t &k) { + if (m_config->gpgpu_concurrent_kernel_sm) { + unsigned threads_per_cta = k.threads_per_cta(); + const class function_info *kernel = k.entry(); + unsigned int padded_cta_size = threads_per_cta; + unsigned int warp_size = m_config->warp_size; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); + + assert(m_occupied_n_threads >= padded_cta_size); + m_occupied_n_threads -= padded_cta_size; + + int start_thread = m_occupied_cta_to_hwtid[hw_ctaid]; - if(m_config->gpgpu_concurrent_kernel_sm) { - unsigned threads_per_cta = k.threads_per_cta(); - const class function_info *kernel = k.entry(); - unsigned int padded_cta_size = threads_per_cta; - unsigned int warp_size = m_config->warp_size; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); - - assert(m_occupied_n_threads >= padded_cta_size); - m_occupied_n_threads -= padded_cta_size; - - int start_thread = m_occupied_cta_to_hwtid[hw_ctaid]; - - for(unsigned hwtid = start_thread; hwtid < start_thread + padded_cta_size; - hwtid++) - m_occupied_hwtid.reset(hwtid); - m_occupied_cta_to_hwtid.erase(hw_ctaid); - - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); - - assert(m_occupied_shmem >= (unsigned int)kernel_info->smem); - m_occupied_shmem -= kernel_info->smem; - - unsigned int used_regs = padded_cta_size * ((kernel_info->regs+3)&~3); - assert(m_occupied_regs >= used_regs); - m_occupied_regs -= used_regs; - - assert(m_occupied_ctas >= 1); - m_occupied_ctas--; - } + for (unsigned hwtid = start_thread; hwtid < start_thread + padded_cta_size; + hwtid++) + m_occupied_hwtid.reset(hwtid); + m_occupied_cta_to_hwtid.erase(hw_ctaid); + + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); + + assert(m_occupied_shmem >= (unsigned int)kernel_info->smem); + m_occupied_shmem -= kernel_info->smem; + + unsigned int used_regs = padded_cta_size * ((kernel_info->regs + 3) & ~3); + assert(m_occupied_regs >= used_regs); + m_occupied_regs -= used_regs; + + assert(m_occupied_ctas >= 1); + m_occupied_ctas--; + } } //////////////////////////////////////////////////////////////////////////////////////////////// /** - * Launches a cooperative thread array (CTA). - * - * @param kernel - * object that tells us which kernel to ask for a CTA from + * Launches a cooperative thread array (CTA). + * + * @param kernel + * object that tells us which kernel to ask for a CTA from */ -void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) -{ - - if(!m_config->gpgpu_concurrent_kernel_sm) - set_max_cta(kernel); - else - assert(occupy_shader_resource_1block(kernel, true)); +void shader_core_ctx::issue_block2core(kernel_info_t &kernel) { + if (!m_config->gpgpu_concurrent_kernel_sm) + set_max_cta(kernel); + else + assert(occupy_shader_resource_1block(kernel, true)); - kernel.inc_running(); + kernel.inc_running(); - // find a free CTA context - unsigned free_cta_hw_id=(unsigned)-1; + // find a free CTA context + unsigned free_cta_hw_id = (unsigned)-1; - unsigned max_cta_per_core; - if(!m_config->gpgpu_concurrent_kernel_sm) - max_cta_per_core = kernel_max_cta_per_shader; - else - max_cta_per_core = m_config->max_cta_per_core; - for (unsigned i=0;i<max_cta_per_core;i++ ) { - if( m_cta_status[i]==0 ) { - free_cta_hw_id=i; - break; - } + unsigned max_cta_per_core; + if (!m_config->gpgpu_concurrent_kernel_sm) + max_cta_per_core = kernel_max_cta_per_shader; + else + max_cta_per_core = m_config->max_cta_per_core; + for (unsigned i = 0; i < max_cta_per_core; i++) { + if (m_cta_status[i] == 0) { + free_cta_hw_id = i; + break; } - assert( free_cta_hw_id!=(unsigned)-1 ); + } + assert(free_cta_hw_id != (unsigned)-1); - // determine hardware threads and warps that will be used for this CTA - int cta_size = kernel.threads_per_cta(); + // determine hardware threads and warps that will be used for this CTA + int cta_size = kernel.threads_per_cta(); - // hw warp id = hw thread id mod warp size, so we need to find a range - // of hardware thread ids corresponding to an integral number of hardware - // thread ids - int padded_cta_size = cta_size; - if (cta_size%m_config->warp_size) - padded_cta_size = ((cta_size/m_config->warp_size)+1)*(m_config->warp_size); + // hw warp id = hw thread id mod warp size, so we need to find a range + // of hardware thread ids corresponding to an integral number of hardware + // thread ids + int padded_cta_size = cta_size; + if (cta_size % m_config->warp_size) + padded_cta_size = + ((cta_size / m_config->warp_size) + 1) * (m_config->warp_size); - unsigned int start_thread, end_thread; + unsigned int start_thread, end_thread; - if(!m_config->gpgpu_concurrent_kernel_sm) { - start_thread = free_cta_hw_id * padded_cta_size; - end_thread = start_thread + cta_size; - } - else { - start_thread = find_available_hwtid(padded_cta_size, true); - assert((int)start_thread != -1); - end_thread = start_thread + cta_size; - assert(m_occupied_cta_to_hwtid.find(free_cta_hw_id) == m_occupied_cta_to_hwtid.end()); - m_occupied_cta_to_hwtid[free_cta_hw_id]= start_thread; - } + if (!m_config->gpgpu_concurrent_kernel_sm) { + start_thread = free_cta_hw_id * padded_cta_size; + end_thread = start_thread + cta_size; + } else { + start_thread = find_available_hwtid(padded_cta_size, true); + assert((int)start_thread != -1); + end_thread = start_thread + cta_size; + assert(m_occupied_cta_to_hwtid.find(free_cta_hw_id) == + m_occupied_cta_to_hwtid.end()); + m_occupied_cta_to_hwtid[free_cta_hw_id] = start_thread; + } - // reset the microarchitecture state of the selected hardware thread and warp contexts - reinit(start_thread, end_thread,false); - - // initalize scalar threads and determine which hardware warps they are allocated to - // bind functional simulation state of threads to hardware resources (simulation) - warp_set_t warps; - unsigned nthreads_in_block= 0; - function_info *kernel_func_info = kernel.entry(); - symbol_table * symtab= kernel_func_info->get_symtab(); - unsigned ctaid= kernel.get_next_cta_id_single(); - checkpoint *g_checkpoint= new checkpoint(); - for (unsigned i = start_thread; i<end_thread; i++) { - m_threadState[i].m_cta_id = free_cta_hw_id; - unsigned warp_id = i/m_config->warp_size; - nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); - m_threadState[i].m_active = true; - // load thread local memory and register file - if(m_gpu->resume_option == 1 && kernel.get_uid() == m_gpu->resume_kernel && ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t ) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); - m_thread[i]->resume_reg_thread(fname,symtab); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); - g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); - } - // - warps.set( warp_id ); - } - assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max - m_cta_status[free_cta_hw_id]=nthreads_in_block; + // reset the microarchitecture state of the selected hardware thread and warp + // contexts + reinit(start_thread, end_thread, false); - if(m_gpu->resume_option == 1 && kernel.get_uid() == m_gpu->resume_kernel && ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t ) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); - - g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); + // initalize scalar threads and determine which hardware warps they are + // allocated to bind functional simulation state of threads to hardware + // resources (simulation) + warp_set_t warps; + unsigned nthreads_in_block = 0; + function_info *kernel_func_info = kernel.entry(); + symbol_table *symtab = kernel_func_info->get_symtab(); + unsigned ctaid = kernel.get_next_cta_id_single(); + checkpoint *g_checkpoint = new checkpoint(); + for (unsigned i = start_thread; i < end_thread; i++) { + m_threadState[i].m_cta_id = free_cta_hw_id; + unsigned warp_id = i / m_config->warp_size; + nthreads_in_block += ptx_sim_init_thread( + kernel, &m_thread[i], m_sid, i, cta_size - (i - start_thread), + m_config->n_thread_per_shader, this, free_cta_hw_id, warp_id, + m_cluster->get_gpu()); + m_threadState[i].m_active = true; + // load thread local memory and register file + if (m_gpu->resume_option == 1 && kernel.get_uid() == m_gpu->resume_kernel && + ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", + i % cta_size, ctaid); + m_thread[i]->resume_reg_thread(fname, symtab); + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", + i % cta_size, ctaid); + g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); } - // now that we know which warps are used in this CTA, we can allocate - // resources for use in CTA-wide barrier operations - m_barriers.allocate_barrier(free_cta_hw_id,warps); + // + warps.set(warp_id); + } + assert(nthreads_in_block > 0 && + nthreads_in_block <= + m_config->n_thread_per_shader); // should be at least one, but + // less than max + m_cta_status[free_cta_hw_id] = nthreads_in_block; - // initialize the SIMT stacks and fetch hardware - init_warps( free_cta_hw_id, start_thread, end_thread, ctaid, cta_size, kernel.get_uid()); - m_n_active_cta++; + if (m_gpu->resume_option == 1 && kernel.get_uid() == m_gpu->resume_kernel && + ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t) { + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid); - shader_CTA_count_log(m_sid, 1); - SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", - free_cta_hw_id, start_thread, end_thread, m_gpu->gpu_sim_cycle, m_gpu->gpu_tot_sim_cycle ); + g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); + } + // now that we know which warps are used in this CTA, we can allocate + // resources for use in CTA-wide barrier operations + m_barriers.allocate_barrier(free_cta_hw_id, warps); + // initialize the SIMT stacks and fetch hardware + init_warps(free_cta_hw_id, start_thread, end_thread, ctaid, cta_size, + kernel.get_uid()); + m_n_active_cta++; + + shader_CTA_count_log(m_sid, 1); + SHADER_DPRINTF(LIVENESS, + "GPGPU-Sim uArch: cta:%2u, start_tid:%4u, end_tid:%4u, " + "initialized @(%lld,%lld)\n", + free_cta_hw_id, start_thread, end_thread, m_gpu->gpu_sim_cycle, + m_gpu->gpu_tot_sim_cycle); } /////////////////////////////////////////////////////////////////////////////////////////// -void dram_t::dram_log( int task ) -{ - if (task == SAMPLELOG) { - StatAddSample(mrqq_Dist, que_length()); - } else if (task == DUMPLOG) { - printf ("Queue Length DRAM[%d] ",id);StatDisp(mrqq_Dist); - } +void dram_t::dram_log(int task) { + if (task == SAMPLELOG) { + StatAddSample(mrqq_Dist, que_length()); + } else if (task == DUMPLOG) { + printf("Queue Length DRAM[%d] ", id); + StatDisp(mrqq_Dist); + } } -//Find next clock domain and increment its time -int gpgpu_sim::next_clock_domain(void) -{ - double smallest = min3(core_time,icnt_time,dram_time); - int mask = 0x00; - if ( l2_time <= smallest ) { - smallest = l2_time; - mask |= L2 ; - l2_time += m_config.l2_period; - } - if ( icnt_time <= smallest ) { - mask |= ICNT; - icnt_time += m_config.icnt_period; - } - if ( dram_time <= smallest ) { - mask |= DRAM; - dram_time += m_config.dram_period; - } - if ( core_time <= smallest ) { - mask |= CORE; - core_time += m_config.core_period; - } - return mask; +// Find next clock domain and increment its time +int gpgpu_sim::next_clock_domain(void) { + double smallest = min3(core_time, icnt_time, dram_time); + int mask = 0x00; + if (l2_time <= smallest) { + smallest = l2_time; + mask |= L2; + l2_time += m_config.l2_period; + } + if (icnt_time <= smallest) { + mask |= ICNT; + icnt_time += m_config.icnt_period; + } + if (dram_time <= smallest) { + mask |= DRAM; + dram_time += m_config.dram_period; + } + if (core_time <= smallest) { + mask |= CORE; + core_time += m_config.core_period; + } + return mask; } -void gpgpu_sim::issue_block2core() -{ - unsigned last_issued = m_last_cluster_issue; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - unsigned idx = (i + last_issued + 1) % m_shader_config->n_simt_clusters; - unsigned num = m_cluster[idx]->issue_block2core(); - if( num ) { - m_last_cluster_issue=idx; - m_total_cta_launched += num; - } +void gpgpu_sim::issue_block2core() { + unsigned last_issued = m_last_cluster_issue; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + unsigned idx = (i + last_issued + 1) % m_shader_config->n_simt_clusters; + unsigned num = m_cluster[idx]->issue_block2core(); + if (num) { + m_last_cluster_issue = idx; + m_total_cta_launched += num; } + } } -unsigned long long g_single_step=0; // set this in gdb to single step the pipeline +unsigned long long g_single_step = + 0; // set this in gdb to single step the pipeline -void gpgpu_sim::cycle() -{ - int clock_mask = next_clock_domain(); +void gpgpu_sim::cycle() { + int clock_mask = next_clock_domain(); - if (clock_mask & CORE ) { - // shader core loading (pop from ICNT into core) follows CORE clock - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) - m_cluster[i]->icnt_cycle(); - } - unsigned partiton_replys_in_parallel_per_cycle = 0; - if (clock_mask & ICNT) { - // pop from memory controller to interconnect - for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) { - mem_fetch* mf = m_memory_sub_partition[i]->top(); - if (mf) { - unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size(); - if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) { - //if (!mf->get_is_write()) - mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle); - mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle); - ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size ); - m_memory_sub_partition[i]->pop(); - partiton_replys_in_parallel_per_cycle++; - } else { - gpu_stall_icnt2sh++; - } - } else { - m_memory_sub_partition[i]->pop(); - } + if (clock_mask & CORE) { + // shader core loading (pop from ICNT into core) follows CORE clock + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) + m_cluster[i]->icnt_cycle(); + } + unsigned partiton_replys_in_parallel_per_cycle = 0; + if (clock_mask & ICNT) { + // pop from memory controller to interconnect + for (unsigned i = 0; i < m_memory_config->m_n_mem_sub_partition; i++) { + mem_fetch *mf = m_memory_sub_partition[i]->top(); + if (mf) { + unsigned response_size = + mf->get_is_write() ? mf->get_ctrl_size() : mf->size(); + if (::icnt_has_buffer(m_shader_config->mem2device(i), response_size)) { + // if (!mf->get_is_write()) + mf->set_return_timestamp(gpu_sim_cycle + gpu_tot_sim_cycle); + mf->set_status(IN_ICNT_TO_SHADER, gpu_sim_cycle + gpu_tot_sim_cycle); + ::icnt_push(m_shader_config->mem2device(i), mf->get_tpc(), mf, + response_size); + m_memory_sub_partition[i]->pop(); + partiton_replys_in_parallel_per_cycle++; + } else { + gpu_stall_icnt2sh++; } - } - partiton_replys_in_parallel += partiton_replys_in_parallel_per_cycle; - - if (clock_mask & DRAM) { - for (unsigned i=0;i<m_memory_config->m_n_mem;i++){ - if(m_memory_config->simple_dram_model) - m_memory_partition_unit[i]->simple_dram_model_cycle(); - else - m_memory_partition_unit[i]->dram_cycle(); // Issue the dram command (scheduler + delay model) - // Update performance counters for DRAM - m_memory_partition_unit[i]->set_dram_power_stats(m_power_stats->pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i], - m_power_stats->pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_act[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i], - m_power_stats->pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_req[CURRENT_STAT_IDX][i]); + } else { + m_memory_sub_partition[i]->pop(); } - } + } + } + partiton_replys_in_parallel += partiton_replys_in_parallel_per_cycle; - // L2 operations follow L2 clock domain - unsigned partiton_reqs_in_parallel_per_cycle = 0; - if (clock_mask & L2) { - m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].clear(); - for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) { - //move memory request from interconnect into memory partition (if not backed up) - //Note:This needs to be called in DRAM clock domain if there is no L2 cache in the system - //In the worst case, we may need to push SECTOR_CHUNCK_SIZE requests, so ensure you have enough buffer for them - if ( m_memory_sub_partition[i]->full(SECTOR_CHUNCK_SIZE) ) { - gpu_stall_dramfull++; - } else { - mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) ); - m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle ); - if(mf) - partiton_reqs_in_parallel_per_cycle++; - } - m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle); - m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); - } - } - partiton_reqs_in_parallel += partiton_reqs_in_parallel_per_cycle; - if(partiton_reqs_in_parallel_per_cycle > 0){ - partiton_reqs_in_parallel_util += partiton_reqs_in_parallel_per_cycle; - gpu_sim_cycle_parition_util++; - } + if (clock_mask & DRAM) { + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + if (m_memory_config->simple_dram_model) + m_memory_partition_unit[i]->simple_dram_model_cycle(); + else + m_memory_partition_unit[i] + ->dram_cycle(); // Issue the dram command (scheduler + delay model) + // Update performance counters for DRAM + m_memory_partition_unit[i]->set_dram_power_stats( + m_power_stats->pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_act[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_req[CURRENT_STAT_IDX][i]); + } + } - if (clock_mask & ICNT) { - icnt_transfer(); - } + // L2 operations follow L2 clock domain + unsigned partiton_reqs_in_parallel_per_cycle = 0; + if (clock_mask & L2) { + m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].clear(); + for (unsigned i = 0; i < m_memory_config->m_n_mem_sub_partition; i++) { + // move memory request from interconnect into memory partition (if not + // backed up) Note:This needs to be called in DRAM clock domain if there is + // no L2 cache in the system In the worst case, we may need to push + // SECTOR_CHUNCK_SIZE requests, so ensure you have enough buffer for them + if (m_memory_sub_partition[i]->full(SECTOR_CHUNCK_SIZE)) { + gpu_stall_dramfull++; + } else { + mem_fetch *mf = (mem_fetch *)icnt_pop(m_shader_config->mem2device(i)); + m_memory_sub_partition[i]->push(mf, gpu_sim_cycle + gpu_tot_sim_cycle); + if (mf) partiton_reqs_in_parallel_per_cycle++; + } + m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle + gpu_tot_sim_cycle); + m_memory_sub_partition[i]->accumulate_L2cache_stats( + m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); + } + } + partiton_reqs_in_parallel += partiton_reqs_in_parallel_per_cycle; + if (partiton_reqs_in_parallel_per_cycle > 0) { + partiton_reqs_in_parallel_util += partiton_reqs_in_parallel_per_cycle; + gpu_sim_cycle_parition_util++; + } - if (clock_mask & CORE) { - // L1 cache + shader core pipeline stages - m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].clear(); - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - if (m_cluster[i]->get_not_completed() || get_more_cta_left() ) { - m_cluster[i]->core_cycle(); - *active_sms+=m_cluster[i]->get_n_active_sms(); - } - // Update core icnt/cache stats for GPUWattch - m_cluster[i]->get_icnt_stats(m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]); - m_cluster[i]->get_cache_stats(m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX]); - m_cluster[i]->get_current_occupancy(gpu_occupancy.aggregate_warp_slot_filled, gpu_occupancy.aggregate_theoretical_warp_slots); + if (clock_mask & ICNT) { + icnt_transfer(); + } + if (clock_mask & CORE) { + // L1 cache + shader core pipeline stages + m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].clear(); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (m_cluster[i]->get_not_completed() || get_more_cta_left()) { + m_cluster[i]->core_cycle(); + *active_sms += m_cluster[i]->get_n_active_sms(); } - float temp=0; - for (unsigned i=0;i<m_shader_config->num_shader();i++){ - temp+=m_shader_stats->m_pipeline_duty_cycle[i]; - } - temp=temp/m_shader_config->num_shader(); - *average_pipeline_duty_cycle=((*average_pipeline_duty_cycle)+temp); - //cout<<"Average pipeline duty cycle: "<<*average_pipeline_duty_cycle<<endl; + // Update core icnt/cache stats for GPUWattch + m_cluster[i]->get_icnt_stats( + m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i], + m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]); + m_cluster[i]->get_cache_stats( + m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX]); + m_cluster[i]->get_current_occupancy( + gpu_occupancy.aggregate_warp_slot_filled, + gpu_occupancy.aggregate_theoretical_warp_slots); + } + float temp = 0; + for (unsigned i = 0; i < m_shader_config->num_shader(); i++) { + temp += m_shader_stats->m_pipeline_duty_cycle[i]; + } + temp = temp / m_shader_config->num_shader(); + *average_pipeline_duty_cycle = ((*average_pipeline_duty_cycle) + temp); + // cout<<"Average pipeline duty cycle: + // "<<*average_pipeline_duty_cycle<<endl; + if (g_single_step && + ((gpu_sim_cycle + gpu_tot_sim_cycle) >= g_single_step)) { + raise(SIGTRAP); // Debug breakpoint + } + gpu_sim_cycle++; - if( g_single_step && ((gpu_sim_cycle+gpu_tot_sim_cycle) >= g_single_step) ) { - raise(SIGTRAP); // Debug breakpoint - } - gpu_sim_cycle++; - - if( g_interactive_debugger_enabled ) - gpgpu_debug(); + if (g_interactive_debugger_enabled) gpgpu_debug(); // McPAT main cycle (interface with McPAT) #ifdef GPGPUSIM_POWER_MODEL - if(m_config.g_power_simulation_enabled){ - mcpat_cycle(m_config, getShaderCoreConfig(), m_gpgpusim_wrapper, m_power_stats, m_config.gpu_stat_sample_freq, gpu_tot_sim_cycle, gpu_sim_cycle, gpu_tot_sim_insn, gpu_sim_insn); - } + if (m_config.g_power_simulation_enabled) { + mcpat_cycle(m_config, getShaderCoreConfig(), m_gpgpusim_wrapper, + m_power_stats, m_config.gpu_stat_sample_freq, + gpu_tot_sim_cycle, gpu_sim_cycle, gpu_tot_sim_insn, + gpu_sim_insn); + } #endif - issue_block2core(); - - // Depending on configuration, invalidate the caches once all of threads are completed. - int all_threads_complete = 1; - if (m_config.gpgpu_flush_l1_cache) { - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - if (m_cluster[i]->get_not_completed() == 0) - m_cluster[i]->cache_invalidate(); - else - all_threads_complete = 0 ; - } + issue_block2core(); + + // Depending on configuration, invalidate the caches once all of threads are + // completed. + int all_threads_complete = 1; + if (m_config.gpgpu_flush_l1_cache) { + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (m_cluster[i]->get_not_completed() == 0) + m_cluster[i]->cache_invalidate(); + else + all_threads_complete = 0; } + } - if(m_config.gpgpu_flush_l2_cache){ - if(!m_config.gpgpu_flush_l1_cache){ - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - if (m_cluster[i]->get_not_completed() != 0){ - all_threads_complete = 0 ; - break; - } - } + if (m_config.gpgpu_flush_l2_cache) { + if (!m_config.gpgpu_flush_l1_cache) { + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (m_cluster[i]->get_not_completed() != 0) { + all_threads_complete = 0; + break; } + } + } - if (all_threads_complete && !m_memory_config->m_L2_config.disabled() ) { - printf("Flushed L2 caches...\n"); - if (m_memory_config->m_L2_config.get_num_lines()) { - int dlc = 0; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - dlc = m_memory_sub_partition[i]->flushL2(); - assert (dlc == 0); // TODO: need to model actual writes to DRAM here - printf("Dirty lines flushed from L2 %d is %d\n", i, dlc ); - } - } - } + if (all_threads_complete && !m_memory_config->m_L2_config.disabled()) { + printf("Flushed L2 caches...\n"); + if (m_memory_config->m_L2_config.get_num_lines()) { + int dlc = 0; + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + dlc = m_memory_sub_partition[i]->flushL2(); + assert(dlc == 0); // TODO: need to model actual writes to DRAM here + printf("Dirty lines flushed from L2 %d is %d\n", i, dlc); + } + } } + } - if (!(gpu_sim_cycle % m_config.gpu_stat_sample_freq)) { - time_t days, hrs, minutes, sec; - time_t curr_time; - time(&curr_time); - unsigned long long elapsed_time = MAX(curr_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq && DTRACE(LIVENESS) ) { - days = elapsed_time/(3600*24); - hrs = elapsed_time/3600 - 24*days; - minutes = elapsed_time/60 - 60*(hrs + 24*days); - sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - unsigned long long active = 0, total = 0; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - m_cluster[i]->get_current_occupancy(active, total); - } - DPRINTFG(LIVENESS, "uArch: inst.: %lld (ipc=%4.1f, occ=%0.4f\% [%llu / %llu]) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", - gpu_tot_sim_insn + gpu_sim_insn, - (double)gpu_sim_insn/(double)gpu_sim_cycle, - float(active)/float(total) * 100, active, total, - (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time), - (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec, - ctime(&curr_time)); - fflush(stdout); - last_liveness_message_time = elapsed_time; - } - visualizer_printstat(); - m_memory_stats->memlatstat_lat_pw(); - if (m_config.gpgpu_runtime_stat && (m_config.gpu_runtime_stat_flag != 0) ) { - if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_BW_STAT) { - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) - m_memory_partition_unit[i]->print_stat(stdout); - printf("maxmrqlatency = %d \n", m_memory_stats->max_mrq_latency); - printf("maxmflatency = %d \n", m_memory_stats->max_mf_latency); - } - if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_SHD_INFO) - shader_print_runtime_stat( stdout ); - if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_L1MISS) - shader_print_l1_miss_stat( stdout ); - if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_SCHED) - shader_print_scheduler_stat( stdout, false ); - } + if (!(gpu_sim_cycle % m_config.gpu_stat_sample_freq)) { + time_t days, hrs, minutes, sec; + time_t curr_time; + time(&curr_time); + unsigned long long elapsed_time = + MAX(curr_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + if ((elapsed_time - last_liveness_message_time) >= + m_config.liveness_message_freq && + DTRACE(LIVENESS)) { + days = elapsed_time / (3600 * 24); + hrs = elapsed_time / 3600 - 24 * days; + minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); + sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); + + unsigned long long active = 0, total = 0; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + m_cluster[i]->get_current_occupancy(active, total); + } + DPRINTFG(LIVENESS, + "uArch: inst.: %lld (ipc=%4.1f, occ=%0.4f\% [%llu / %llu]) " + "sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", + gpu_tot_sim_insn + gpu_sim_insn, + (double)gpu_sim_insn / (double)gpu_sim_cycle, + float(active) / float(total) * 100, active, total, + (unsigned)((gpu_tot_sim_insn + gpu_sim_insn) / elapsed_time), + (unsigned)days, (unsigned)hrs, (unsigned)minutes, + (unsigned)sec, ctime(&curr_time)); + fflush(stdout); + last_liveness_message_time = elapsed_time; } + visualizer_printstat(); + m_memory_stats->memlatstat_lat_pw(); + if (m_config.gpgpu_runtime_stat && + (m_config.gpu_runtime_stat_flag != 0)) { + if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_BW_STAT) { + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) + m_memory_partition_unit[i]->print_stat(stdout); + printf("maxmrqlatency = %d \n", m_memory_stats->max_mrq_latency); + printf("maxmflatency = %d \n", m_memory_stats->max_mf_latency); + } + if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_SHD_INFO) + shader_print_runtime_stat(stdout); + if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_L1MISS) + shader_print_l1_miss_stat(stdout); + if (m_config.gpu_runtime_stat_flag & GPU_RSTAT_SCHED) + shader_print_scheduler_stat(stdout, false); + } + } - if (!(gpu_sim_cycle % 50000)) { - // deadlock detection - if (m_config.gpu_deadlock_detect && gpu_sim_insn == last_gpu_sim_insn) { - gpu_deadlock = true; - } else { - last_gpu_sim_insn = gpu_sim_insn; - } + if (!(gpu_sim_cycle % 50000)) { + // deadlock detection + if (m_config.gpu_deadlock_detect && gpu_sim_insn == last_gpu_sim_insn) { + gpu_deadlock = true; + } else { + last_gpu_sim_insn = gpu_sim_insn; } - try_snap_shot(gpu_sim_cycle); - spill_log_to_file (stdout, 0, gpu_sim_cycle); + } + try_snap_shot(gpu_sim_cycle); + spill_log_to_file(stdout, 0, gpu_sim_cycle); #if (CUDART_VERSION >= 5000) - //launch device kernel - gpgpu_ctx->device_runtime->launch_one_device_kernel(); + // launch device kernel + gpgpu_ctx->device_runtime->launch_one_device_kernel(); #endif - } + } } - -void shader_core_ctx::dump_warp_state( FILE *fout ) const -{ - fprintf(fout, "\n"); - fprintf(fout, "per warp functional simulation status:\n"); - for (unsigned w=0; w < m_config->max_warps_per_shader; w++ ) - m_warp[w].print(fout); +void shader_core_ctx::dump_warp_state(FILE *fout) const { + fprintf(fout, "\n"); + fprintf(fout, "per warp functional simulation status:\n"); + for (unsigned w = 0; w < m_config->max_warps_per_shader; w++) + m_warp[w].print(fout); } +void gpgpu_sim::perf_memcpy_to_gpu(size_t dst_start_addr, size_t count) { + if (m_memory_config->m_perf_sim_memcpy) { + assert(dst_start_addr % 32 == 0); -void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ) -{ - if (m_memory_config->m_perf_sim_memcpy) { - assert (dst_start_addr % 32 == 0); - - for ( unsigned counter = 0; counter < count; counter += 32 ) { - const unsigned wr_addr = dst_start_addr + counter; - addrdec_t raw_addr; - mem_access_sector_mask_t mask; - mask.set(wr_addr % 128 / 32); - m_memory_config->m_address_mapping.addrdec_tlx( wr_addr, &raw_addr ); - const unsigned partition_id = raw_addr.sub_partition / m_memory_config->m_n_sub_partition_per_memory_channel; - m_memory_partition_unit[ partition_id ]->handle_memcpy_to_gpu( wr_addr, raw_addr.sub_partition, mask ); - } + for (unsigned counter = 0; counter < count; counter += 32) { + const unsigned wr_addr = dst_start_addr + counter; + addrdec_t raw_addr; + mem_access_sector_mask_t mask; + mask.set(wr_addr % 128 / 32); + m_memory_config->m_address_mapping.addrdec_tlx(wr_addr, &raw_addr); + const unsigned partition_id = + raw_addr.sub_partition / + m_memory_config->m_n_sub_partition_per_memory_channel; + m_memory_partition_unit[partition_id]->handle_memcpy_to_gpu( + wr_addr, raw_addr.sub_partition, mask); } + } } -void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const -{ -/* - You may want to use this function while running GPGPU-Sim in gdb. - One way to do that is add the following to your .gdbinit file: - - define dp - call g_the_gpu.dump_pipeline_impl((0x40|0x4|0x1),$arg0,0) - end - - Then, typing "dp 3" will show the contents of the pipeline for shader core 3. -*/ +void gpgpu_sim::dump_pipeline(int mask, int s, int m) const { + /* + You may want to use this function while running GPGPU-Sim in gdb. + One way to do that is add the following to your .gdbinit file: - printf("Dumping pipeline state...\n"); - if(!mask) mask = 0xFFFFFFFF; - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) { - if(s != -1) { - i = s; - } - if(mask&1) m_cluster[m_shader_config->sid_to_cluster(i)]->display_pipeline(i,stdout,1,mask & 0x2E); - if(s != -1) { - break; + define dp + call g_the_gpu.dump_pipeline_impl((0x40|0x4|0x1),$arg0,0) + end + + Then, typing "dp 3" will show the contents of the pipeline for shader + core 3. + */ + + printf("Dumping pipeline state...\n"); + if (!mask) mask = 0xFFFFFFFF; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (s != -1) { + i = s; + } + if (mask & 1) + m_cluster[m_shader_config->sid_to_cluster(i)]->display_pipeline( + i, stdout, 1, mask & 0x2E); + if (s != -1) { + break; + } + } + if (mask & 0x10000) { + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + if (m != -1) { + i = m; } - } - if(mask&0x10000) { - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - if(m != -1) { - i=m; - } - printf("DRAM / memory controller %u:\n", i); - if(mask&0x100000) m_memory_partition_unit[i]->print_stat(stdout); - if(mask&0x1000000) m_memory_partition_unit[i]->visualize(); - if(mask&0x10000000) m_memory_partition_unit[i]->print(stdout); - if(m != -1) { - break; - } + printf("DRAM / memory controller %u:\n", i); + if (mask & 0x100000) m_memory_partition_unit[i]->print_stat(stdout); + if (mask & 0x1000000) m_memory_partition_unit[i]->visualize(); + if (mask & 0x10000000) m_memory_partition_unit[i]->print(stdout); + if (m != -1) { + break; } - } - fflush(stdout); -} - -const shader_core_config * gpgpu_sim::getShaderCoreConfig() -{ - return m_shader_config; + } + } + fflush(stdout); } -const memory_config * gpgpu_sim::getMemoryConfig() -{ - return m_memory_config; +const shader_core_config *gpgpu_sim::getShaderCoreConfig() { + return m_shader_config; } -simt_core_cluster * gpgpu_sim::getSIMTCluster() -{ - return *m_cluster; -} +const memory_config *gpgpu_sim::getMemoryConfig() { return m_memory_config; } +simt_core_cluster *gpgpu_sim::getSIMTCluster() { return *m_cluster; } diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index fba770d..9765b38 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -7,45 +7,44 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef GPU_SIM_H #define GPU_SIM_H -#include "../option_parser.h" +#include <stdio.h> +#include <fstream> +#include <iostream> +#include <list> #include "../abstract_hardware_model.h" +#include "../option_parser.h" #include "../trace.h" #include "addrdec.h" -#include "shader.h" #include "gpu-cache.h" -#include <iostream> -#include <fstream> -#include <list> -#include <stdio.h> - - +#include "shader.h" // constants for statistics printouts #define GPU_RSTAT_SHD_INFO 0x1 -#define GPU_RSTAT_BW_STAT 0x2 +#define GPU_RSTAT_BW_STAT 0x2 #define GPU_RSTAT_WARP_DIS 0x4 -#define GPU_RSTAT_DWF_MAP 0x8 +#define GPU_RSTAT_DWF_MAP 0x8 #define GPU_RSTAT_L1MISS 0x10 #define GPU_RSTAT_PDOM 0x20 #define GPU_RSTAT_SCHED 0x40 @@ -65,584 +64,617 @@ class gpgpu_context; -extern tr1_hash_map<new_addr_type,unsigned> address_random_interleaving; - -enum dram_ctrl_t { - DRAM_FIFO=0, - DRAM_FRFCFS=1 -}; - +extern tr1_hash_map<new_addr_type, unsigned> address_random_interleaving; +enum dram_ctrl_t { DRAM_FIFO = 0, DRAM_FRFCFS = 1 }; struct power_config { - power_config() - { - m_valid = true; - } - void init() - { - - // initialize file name if it is not set - time_t curr_time; - time(&curr_time); - char *date = ctime(&curr_time); - char *s = date; - while (*s) { - if (*s == ' ' || *s == '\t' || *s == ':') *s = '-'; - if (*s == '\n' || *s == '\r' ) *s = 0; - s++; - } - char buf1[1024]; - snprintf(buf1,1024,"gpgpusim_power_report__%s.log",date); - g_power_filename = strdup(buf1); - char buf2[1024]; - snprintf(buf2,1024,"gpgpusim_power_trace_report__%s.log.gz",date); - g_power_trace_filename = strdup(buf2); - char buf3[1024]; - snprintf(buf3,1024,"gpgpusim_metric_trace_report__%s.log.gz",date); - g_metric_trace_filename = strdup(buf3); - char buf4[1024]; - snprintf(buf4,1024,"gpgpusim_steady_state_tracking_report__%s.log.gz",date); - g_steady_state_tracking_filename = strdup(buf4); - - if(g_steady_power_levels_enabled){ - sscanf(gpu_steady_state_definition,"%lf:%lf", &gpu_steady_power_deviation,&gpu_steady_min_period); - } - - //NOTE: After changing the nonlinear model to only scaling idle core, - //NOTE: The min_inc_per_active_sm is not used any more - if (g_use_nonlinear_model) - sscanf(gpu_nonlinear_model_config,"%lf:%lf", &gpu_idle_core_power,&gpu_min_inc_per_active_sm); - - } - void reg_options(class OptionParser * opp); + power_config() { m_valid = true; } + void init() { + // initialize file name if it is not set + time_t curr_time; + time(&curr_time); + char *date = ctime(&curr_time); + char *s = date; + while (*s) { + if (*s == ' ' || *s == '\t' || *s == ':') *s = '-'; + if (*s == '\n' || *s == '\r') *s = 0; + s++; + } + char buf1[1024]; + snprintf(buf1, 1024, "gpgpusim_power_report__%s.log", date); + g_power_filename = strdup(buf1); + char buf2[1024]; + snprintf(buf2, 1024, "gpgpusim_power_trace_report__%s.log.gz", date); + g_power_trace_filename = strdup(buf2); + char buf3[1024]; + snprintf(buf3, 1024, "gpgpusim_metric_trace_report__%s.log.gz", date); + g_metric_trace_filename = strdup(buf3); + char buf4[1024]; + snprintf(buf4, 1024, "gpgpusim_steady_state_tracking_report__%s.log.gz", + date); + g_steady_state_tracking_filename = strdup(buf4); - char *g_power_config_name; + if (g_steady_power_levels_enabled) { + sscanf(gpu_steady_state_definition, "%lf:%lf", + &gpu_steady_power_deviation, &gpu_steady_min_period); + } - bool m_valid; - bool g_power_simulation_enabled; - bool g_power_trace_enabled; - bool g_steady_power_levels_enabled; - bool g_power_per_cycle_dump; - bool g_power_simulator_debug; - char *g_power_filename; - char *g_power_trace_filename; - char *g_metric_trace_filename; - char * g_steady_state_tracking_filename; - int g_power_trace_zlevel; - char * gpu_steady_state_definition; - double gpu_steady_power_deviation; - double gpu_steady_min_period; + // NOTE: After changing the nonlinear model to only scaling idle core, + // NOTE: The min_inc_per_active_sm is not used any more + if (g_use_nonlinear_model) + sscanf(gpu_nonlinear_model_config, "%lf:%lf", &gpu_idle_core_power, + &gpu_min_inc_per_active_sm); + } + void reg_options(class OptionParser *opp); - //Nonlinear power model - bool g_use_nonlinear_model; - char * gpu_nonlinear_model_config; - double gpu_idle_core_power; - double gpu_min_inc_per_active_sm; + char *g_power_config_name; + bool m_valid; + bool g_power_simulation_enabled; + bool g_power_trace_enabled; + bool g_steady_power_levels_enabled; + bool g_power_per_cycle_dump; + bool g_power_simulator_debug; + char *g_power_filename; + char *g_power_trace_filename; + char *g_metric_trace_filename; + char *g_steady_state_tracking_filename; + int g_power_trace_zlevel; + char *gpu_steady_state_definition; + double gpu_steady_power_deviation; + double gpu_steady_min_period; + // Nonlinear power model + bool g_use_nonlinear_model; + char *gpu_nonlinear_model_config; + double gpu_idle_core_power; + double gpu_min_inc_per_active_sm; }; - class memory_config { - public: - memory_config(gpgpu_context* ctx) - { - m_valid = false; - gpgpu_dram_timing_opt=NULL; - gpgpu_L2_queue_config=NULL; - gpgpu_ctx = ctx; - } - void init() - { - assert(gpgpu_dram_timing_opt); - if (strchr(gpgpu_dram_timing_opt, '=') == NULL) { - // dram timing option in ordered variables (legacy) - // Disabling bank groups if their values are not specified - nbkgrp = 1; - tCCDL = 0; - tRTPL = 0; - sscanf(gpgpu_dram_timing_opt,"%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", - &nbk,&tCCD,&tRRD,&tRCD,&tRAS,&tRP,&tRC,&CL,&WL,&tCDLR,&tWR,&nbkgrp,&tCCDL,&tRTPL); - } else { - // named dram timing options (unordered) - option_parser_t dram_opp = option_parser_create(); + public: + memory_config(gpgpu_context *ctx) { + m_valid = false; + gpgpu_dram_timing_opt = NULL; + gpgpu_L2_queue_config = NULL; + gpgpu_ctx = ctx; + } + void init() { + assert(gpgpu_dram_timing_opt); + if (strchr(gpgpu_dram_timing_opt, '=') == NULL) { + // dram timing option in ordered variables (legacy) + // Disabling bank groups if their values are not specified + nbkgrp = 1; + tCCDL = 0; + tRTPL = 0; + sscanf(gpgpu_dram_timing_opt, "%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", + &nbk, &tCCD, &tRRD, &tRCD, &tRAS, &tRP, &tRC, &CL, &WL, &tCDLR, + &tWR, &nbkgrp, &tCCDL, &tRTPL); + } else { + // named dram timing options (unordered) + option_parser_t dram_opp = option_parser_create(); - option_parser_register(dram_opp, "nbk", OPT_UINT32, &nbk, "number of banks", ""); - option_parser_register(dram_opp, "CCD", OPT_UINT32, &tCCD, "column to column delay", ""); - option_parser_register(dram_opp, "RRD", OPT_UINT32, &tRRD, "minimal delay between activation of rows in different banks", ""); - option_parser_register(dram_opp, "RCD", OPT_UINT32, &tRCD, "row to column delay", ""); - option_parser_register(dram_opp, "RAS", OPT_UINT32, &tRAS, "time needed to activate row", ""); - option_parser_register(dram_opp, "RP", OPT_UINT32, &tRP, "time needed to precharge (deactivate) row", ""); - option_parser_register(dram_opp, "RC", OPT_UINT32, &tRC, "row cycle time", ""); - option_parser_register(dram_opp, "CDLR", OPT_UINT32, &tCDLR, "switching from write to read (changes tWTR)", ""); - option_parser_register(dram_opp, "WR", OPT_UINT32, &tWR, "last data-in to row precharge", ""); + option_parser_register(dram_opp, "nbk", OPT_UINT32, &nbk, + "number of banks", ""); + option_parser_register(dram_opp, "CCD", OPT_UINT32, &tCCD, + "column to column delay", ""); + option_parser_register( + dram_opp, "RRD", OPT_UINT32, &tRRD, + "minimal delay between activation of rows in different banks", ""); + option_parser_register(dram_opp, "RCD", OPT_UINT32, &tRCD, + "row to column delay", ""); + option_parser_register(dram_opp, "RAS", OPT_UINT32, &tRAS, + "time needed to activate row", ""); + option_parser_register(dram_opp, "RP", OPT_UINT32, &tRP, + "time needed to precharge (deactivate) row", ""); + option_parser_register(dram_opp, "RC", OPT_UINT32, &tRC, "row cycle time", + ""); + option_parser_register(dram_opp, "CDLR", OPT_UINT32, &tCDLR, + "switching from write to read (changes tWTR)", ""); + option_parser_register(dram_opp, "WR", OPT_UINT32, &tWR, + "last data-in to row precharge", ""); - option_parser_register(dram_opp, "CL", OPT_UINT32, &CL, "CAS latency", ""); - option_parser_register(dram_opp, "WL", OPT_UINT32, &WL, "Write latency", ""); + option_parser_register(dram_opp, "CL", OPT_UINT32, &CL, "CAS latency", + ""); + option_parser_register(dram_opp, "WL", OPT_UINT32, &WL, "Write latency", + ""); - //Disabling bank groups if their values are not specified - option_parser_register(dram_opp, "nbkgrp", OPT_UINT32, &nbkgrp, "number of bank groups", "1"); - option_parser_register(dram_opp, "CCDL", OPT_UINT32, &tCCDL, "column to column delay between accesses to different bank groups", "0"); - option_parser_register(dram_opp, "RTPL", OPT_UINT32, &tRTPL, "read to precharge delay between accesses to different bank groups", "0"); + // Disabling bank groups if their values are not specified + option_parser_register(dram_opp, "nbkgrp", OPT_UINT32, &nbkgrp, + "number of bank groups", "1"); + option_parser_register( + dram_opp, "CCDL", OPT_UINT32, &tCCDL, + "column to column delay between accesses to different bank groups", + "0"); + option_parser_register( + dram_opp, "RTPL", OPT_UINT32, &tRTPL, + "read to precharge delay between accesses to different bank groups", + "0"); - option_parser_delimited_string(dram_opp, gpgpu_dram_timing_opt, "=:;"); - fprintf(stdout, "DRAM Timing Options:\n"); - option_parser_print(dram_opp, stdout); - option_parser_destroy(dram_opp); - } + option_parser_delimited_string(dram_opp, gpgpu_dram_timing_opt, "=:;"); + fprintf(stdout, "DRAM Timing Options:\n"); + option_parser_print(dram_opp, stdout); + option_parser_destroy(dram_opp); + } - int nbkt = nbk/nbkgrp; - unsigned i; - for (i=0; nbkt>0; i++) { - nbkt = nbkt>>1; - } - bk_tag_length = i-1; - assert(nbkgrp>0 && "Number of bank groups cannot be zero"); - tRCDWR = tRCD-(WL+1); - if(elimnate_rw_turnaround) - { - tRTW = 0; - tWTR = 0; - } else { - tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); - tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); - } - tWTP = (WL+(BL/data_command_freq_ratio)+tWR); - dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition + int nbkt = nbk / nbkgrp; + unsigned i; + for (i = 0; nbkt > 0; i++) { + nbkt = nbkt >> 1; + } + bk_tag_length = i - 1; + assert(nbkgrp > 0 && "Number of bank groups cannot be zero"); + tRCDWR = tRCD - (WL + 1); + if (elimnate_rw_turnaround) { + tRTW = 0; + tWTR = 0; + } else { + tRTW = (CL + (BL / data_command_freq_ratio) + 2 - WL); + tWTR = (WL + (BL / data_command_freq_ratio) + tCDLR); + } + tWTP = (WL + (BL / data_command_freq_ratio) + tWR); + dram_atom_size = + BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips + // per partition - assert( m_n_sub_partition_per_memory_channel > 0 ); - assert( (nbk % m_n_sub_partition_per_memory_channel == 0) - && "Number of DRAM banks must be a perfect multiple of memory sub partition"); - m_n_mem_sub_partition = m_n_mem * m_n_sub_partition_per_memory_channel; - fprintf(stdout, "Total number of memory sub partition = %u\n", m_n_mem_sub_partition); + assert(m_n_sub_partition_per_memory_channel > 0); + assert((nbk % m_n_sub_partition_per_memory_channel == 0) && + "Number of DRAM banks must be a perfect multiple of memory sub " + "partition"); + m_n_mem_sub_partition = m_n_mem * m_n_sub_partition_per_memory_channel; + fprintf(stdout, "Total number of memory sub partition = %u\n", + m_n_mem_sub_partition); - m_address_mapping.init(m_n_mem, m_n_sub_partition_per_memory_channel); - m_L2_config.init(&m_address_mapping); + m_address_mapping.init(m_n_mem, m_n_sub_partition_per_memory_channel); + m_L2_config.init(&m_address_mapping); - m_valid = true; + m_valid = true; - sscanf(write_queue_size_opt,"%d:%d:%d", - &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark); - } - void reg_options(class OptionParser * opp); + sscanf(write_queue_size_opt, "%d:%d:%d", + &gpgpu_frfcfs_dram_write_queue_size, &write_high_watermark, + &write_low_watermark); + } + void reg_options(class OptionParser *opp); - bool m_valid; - mutable l2_cache_config m_L2_config; - bool m_L2_texure_only; + bool m_valid; + mutable l2_cache_config m_L2_config; + bool m_L2_texure_only; - char *gpgpu_dram_timing_opt; - char *gpgpu_L2_queue_config; - bool l2_ideal; - unsigned gpgpu_frfcfs_dram_sched_queue_size; - unsigned gpgpu_dram_return_queue_size; - enum dram_ctrl_t scheduler_type; - bool gpgpu_memlatency_stat; - unsigned m_n_mem; - unsigned m_n_sub_partition_per_memory_channel; - unsigned m_n_mem_sub_partition; - unsigned gpu_n_mem_per_ctrlr; + char *gpgpu_dram_timing_opt; + char *gpgpu_L2_queue_config; + bool l2_ideal; + unsigned gpgpu_frfcfs_dram_sched_queue_size; + unsigned gpgpu_dram_return_queue_size; + enum dram_ctrl_t scheduler_type; + bool gpgpu_memlatency_stat; + unsigned m_n_mem; + unsigned m_n_sub_partition_per_memory_channel; + unsigned m_n_mem_sub_partition; + unsigned gpu_n_mem_per_ctrlr; - unsigned rop_latency; - unsigned dram_latency; + unsigned rop_latency; + unsigned dram_latency; - // DRAM parameters + // DRAM parameters - unsigned tCCDL; //column to column delay when bank groups are enabled - unsigned tRTPL; //read to precharge delay when bank groups are enabled for GDDR5 this is identical to RTPS, if for other DRAM this is different, you will need to split them in two + unsigned tCCDL; // column to column delay when bank groups are enabled + unsigned tRTPL; // read to precharge delay when bank groups are enabled for + // GDDR5 this is identical to RTPS, if for other DRAM this is + // different, you will need to split them in two - unsigned tCCD; //column to column delay - unsigned tRRD; //minimal time required between activation of rows in different banks - unsigned tRCD; //row to column delay - time required to activate a row before a read - unsigned tRCDWR; //row to column delay for a write command - unsigned tRAS; //time needed to activate row - unsigned tRP; //row precharge ie. deactivate row - unsigned tRC; //row cycle time ie. precharge current, then activate different row - unsigned tCDLR; //Last data-in to Read command (switching from write to read) - unsigned tWR; //Last data-in to Row precharge + unsigned tCCD; // column to column delay + unsigned tRRD; // minimal time required between activation of rows in + // different banks + unsigned tRCD; // row to column delay - time required to activate a row + // before a read + unsigned tRCDWR; // row to column delay for a write command + unsigned tRAS; // time needed to activate row + unsigned tRP; // row precharge ie. deactivate row + unsigned + tRC; // row cycle time ie. precharge current, then activate different row + unsigned tCDLR; // Last data-in to Read command (switching from write to + // read) + unsigned tWR; // Last data-in to Row precharge - unsigned CL; //CAS latency - unsigned WL; //WRITE latency - unsigned BL; //Burst Length in bytes (4 in GDDR3, 8 in GDDR5) - unsigned tRTW; //time to switch from read to write - unsigned tWTR; //time to switch from write to read - unsigned tWTP; //time to switch from write to precharge in the same bank - unsigned busW; + unsigned CL; // CAS latency + unsigned WL; // WRITE latency + unsigned BL; // Burst Length in bytes (4 in GDDR3, 8 in GDDR5) + unsigned tRTW; // time to switch from read to write + unsigned tWTR; // time to switch from write to read + unsigned tWTP; // time to switch from write to precharge in the same bank + unsigned busW; - unsigned nbkgrp; // number of bank groups (has to be power of 2) - unsigned bk_tag_length; //number of bits that define a bank inside a bank group + unsigned nbkgrp; // number of bank groups (has to be power of 2) + unsigned + bk_tag_length; // number of bits that define a bank inside a bank group - unsigned nbk; + unsigned nbk; - bool elimnate_rw_turnaround; + bool elimnate_rw_turnaround; - unsigned data_command_freq_ratio; // frequency ratio between DRAM data bus and command bus (2 for GDDR3, 4 for GDDR5) - unsigned dram_atom_size; // number of bytes transferred per read or write command + unsigned + data_command_freq_ratio; // frequency ratio between DRAM data bus and + // command bus (2 for GDDR3, 4 for GDDR5) + unsigned + dram_atom_size; // number of bytes transferred per read or write command - linear_to_raw_address_translation m_address_mapping; + linear_to_raw_address_translation m_address_mapping; - unsigned icnt_flit_size; + unsigned icnt_flit_size; - unsigned dram_bnk_indexing_policy; - unsigned dram_bnkgrp_indexing_policy; - bool dual_bus_interface; + unsigned dram_bnk_indexing_policy; + unsigned dram_bnkgrp_indexing_policy; + bool dual_bus_interface; - bool seperate_write_queue_enabled; - char *write_queue_size_opt; - unsigned gpgpu_frfcfs_dram_write_queue_size; - unsigned write_high_watermark; - unsigned write_low_watermark; - bool m_perf_sim_memcpy; - bool simple_dram_model; + bool seperate_write_queue_enabled; + char *write_queue_size_opt; + unsigned gpgpu_frfcfs_dram_write_queue_size; + unsigned write_high_watermark; + unsigned write_low_watermark; + bool m_perf_sim_memcpy; + bool simple_dram_model; - gpgpu_context* gpgpu_ctx; + gpgpu_context *gpgpu_ctx; }; - extern bool g_interactive_debugger_enabled; -class gpgpu_sim_config : public power_config, public gpgpu_functional_sim_config { -public: - gpgpu_sim_config(gpgpu_context* ctx): m_shader_config(ctx), m_memory_config(ctx) { - m_valid = false; - gpgpu_ctx = ctx; - } - void reg_options(class OptionParser * opp); - void init() - { - gpu_stat_sample_freq = 10000; - gpu_runtime_stat_flag = 0; - sscanf(gpgpu_runtime_stat, "%d:%x", &gpu_stat_sample_freq, &gpu_runtime_stat_flag); - m_shader_config.init(); - ptx_set_tex_cache_linesize(m_shader_config.m_L1T_config.get_line_sz()); - m_memory_config.init(); - init_clock_domains(); - power_config::init(); - Trace::init(); - +class gpgpu_sim_config : public power_config, + public gpgpu_functional_sim_config { + public: + gpgpu_sim_config(gpgpu_context *ctx) + : m_shader_config(ctx), m_memory_config(ctx) { + m_valid = false; + gpgpu_ctx = ctx; + } + void reg_options(class OptionParser *opp); + void init() { + gpu_stat_sample_freq = 10000; + gpu_runtime_stat_flag = 0; + sscanf(gpgpu_runtime_stat, "%d:%x", &gpu_stat_sample_freq, + &gpu_runtime_stat_flag); + m_shader_config.init(); + ptx_set_tex_cache_linesize(m_shader_config.m_L1T_config.get_line_sz()); + m_memory_config.init(); + init_clock_domains(); + power_config::init(); + Trace::init(); - // initialize file name if it is not set - time_t curr_time; - time(&curr_time); - char *date = ctime(&curr_time); - char *s = date; - while (*s) { - if (*s == ' ' || *s == '\t' || *s == ':') *s = '-'; - if (*s == '\n' || *s == '\r' ) *s = 0; - s++; - } - char buf[1024]; - snprintf(buf,1024,"gpgpusim_visualizer__%s.log.gz",date); - g_visualizer_filename = strdup(buf); - - m_valid=true; + // initialize file name if it is not set + time_t curr_time; + time(&curr_time); + char *date = ctime(&curr_time); + char *s = date; + while (*s) { + if (*s == ' ' || *s == '\t' || *s == ':') *s = '-'; + if (*s == '\n' || *s == '\r') *s = 0; + s++; } + char buf[1024]; + snprintf(buf, 1024, "gpgpusim_visualizer__%s.log.gz", date); + g_visualizer_filename = strdup(buf); - unsigned num_shader() const { return m_shader_config.num_shader(); } - unsigned num_cluster() const { return m_shader_config.n_simt_clusters; } - unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; } - unsigned checkpoint_option; - - size_t stack_limit() const {return stack_size_limit; } - size_t heap_limit() const {return heap_size_limit; } - size_t sync_depth_limit() const {return runtime_sync_depth_limit; } - size_t pending_launch_count_limit() const {return runtime_pending_launch_count_limit;} + m_valid = true; + } -private: - void init_clock_domains(void ); + unsigned num_shader() const { return m_shader_config.num_shader(); } + unsigned num_cluster() const { return m_shader_config.n_simt_clusters; } + unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; } + unsigned checkpoint_option; + size_t stack_limit() const { return stack_size_limit; } + size_t heap_limit() const { return heap_size_limit; } + size_t sync_depth_limit() const { return runtime_sync_depth_limit; } + size_t pending_launch_count_limit() const { + return runtime_pending_launch_count_limit; + } - // backward pointer - class gpgpu_context* gpgpu_ctx; - bool m_valid; - shader_core_config m_shader_config; - memory_config m_memory_config; - // clock domains - frequency - double core_freq; - double icnt_freq; - double dram_freq; - double l2_freq; - double core_period; - double icnt_period; - double dram_period; - double l2_period; + private: + void init_clock_domains(void); - // GPGPU-Sim timing model options - unsigned long long gpu_max_cycle_opt; - unsigned long long gpu_max_insn_opt; - unsigned gpu_max_cta_opt; - char *gpgpu_runtime_stat; - bool gpgpu_flush_l1_cache; - bool gpgpu_flush_l2_cache; - bool gpu_deadlock_detect; - int gpgpu_frfcfs_dram_sched_queue_size; - int gpgpu_cflog_interval; - char * gpgpu_clock_domains; - unsigned max_concurrent_kernel; + // backward pointer + class gpgpu_context *gpgpu_ctx; + bool m_valid; + shader_core_config m_shader_config; + memory_config m_memory_config; + // clock domains - frequency + double core_freq; + double icnt_freq; + double dram_freq; + double l2_freq; + double core_period; + double icnt_period; + double dram_period; + double l2_period; - // visualizer - bool g_visualizer_enabled; - char *g_visualizer_filename; - int g_visualizer_zlevel; + // GPGPU-Sim timing model options + unsigned long long gpu_max_cycle_opt; + unsigned long long gpu_max_insn_opt; + unsigned gpu_max_cta_opt; + char *gpgpu_runtime_stat; + bool gpgpu_flush_l1_cache; + bool gpgpu_flush_l2_cache; + bool gpu_deadlock_detect; + int gpgpu_frfcfs_dram_sched_queue_size; + int gpgpu_cflog_interval; + char *gpgpu_clock_domains; + unsigned max_concurrent_kernel; + // visualizer + bool g_visualizer_enabled; + char *g_visualizer_filename; + int g_visualizer_zlevel; - // statistics collection - int gpu_stat_sample_freq; - int gpu_runtime_stat_flag; + // statistics collection + int gpu_stat_sample_freq; + int gpu_runtime_stat_flag; - // Device Limits - size_t stack_size_limit; - size_t heap_size_limit; - size_t runtime_sync_depth_limit; - size_t runtime_pending_launch_count_limit; + // Device Limits + size_t stack_size_limit; + size_t heap_size_limit; + size_t runtime_sync_depth_limit; + size_t runtime_pending_launch_count_limit; - //gpu compute capability options - unsigned int gpgpu_compute_capability_major; - unsigned int gpgpu_compute_capability_minor; - unsigned long long liveness_message_freq; + // gpu compute capability options + unsigned int gpgpu_compute_capability_major; + unsigned int gpgpu_compute_capability_minor; + unsigned long long liveness_message_freq; - friend class gpgpu_sim; + friend class gpgpu_sim; }; struct occupancy_stats { - occupancy_stats() : aggregate_warp_slot_filled(0), aggregate_theoretical_warp_slots(0){} - occupancy_stats( unsigned long long wsf, unsigned long long tws ) - : aggregate_warp_slot_filled(wsf), aggregate_theoretical_warp_slots(tws){} + occupancy_stats() + : aggregate_warp_slot_filled(0), aggregate_theoretical_warp_slots(0) {} + occupancy_stats(unsigned long long wsf, unsigned long long tws) + : aggregate_warp_slot_filled(wsf), + aggregate_theoretical_warp_slots(tws) {} - unsigned long long aggregate_warp_slot_filled; - unsigned long long aggregate_theoretical_warp_slots; + unsigned long long aggregate_warp_slot_filled; + unsigned long long aggregate_theoretical_warp_slots; - float get_occ_fraction() const { - return float(aggregate_warp_slot_filled) / float(aggregate_theoretical_warp_slots); - } + float get_occ_fraction() const { + return float(aggregate_warp_slot_filled) / + float(aggregate_theoretical_warp_slots); + } - occupancy_stats& operator+=(const occupancy_stats& rhs) { - aggregate_warp_slot_filled += rhs.aggregate_warp_slot_filled; - aggregate_theoretical_warp_slots += rhs.aggregate_theoretical_warp_slots; - return *this; - } + occupancy_stats &operator+=(const occupancy_stats &rhs) { + aggregate_warp_slot_filled += rhs.aggregate_warp_slot_filled; + aggregate_theoretical_warp_slots += rhs.aggregate_theoretical_warp_slots; + return *this; + } - occupancy_stats operator+(const occupancy_stats& rhs) const{ - return occupancy_stats( aggregate_warp_slot_filled + rhs.aggregate_warp_slot_filled, - aggregate_theoretical_warp_slots + rhs.aggregate_theoretical_warp_slots - ); - } + occupancy_stats operator+(const occupancy_stats &rhs) const { + return occupancy_stats( + aggregate_warp_slot_filled + rhs.aggregate_warp_slot_filled, + aggregate_theoretical_warp_slots + + rhs.aggregate_theoretical_warp_slots); + } }; class gpgpu_context; class ptx_instruction; class watchpoint_event { -public: - watchpoint_event() - { - m_thread=NULL; - m_inst=NULL; - } - watchpoint_event(const ptx_thread_info *thd, const ptx_instruction *pI) - { - m_thread=thd; - m_inst = pI; - } - const ptx_thread_info *thread() const { return m_thread; } - const ptx_instruction *inst() const { return m_inst; } -private: - const ptx_thread_info *m_thread; - const ptx_instruction *m_inst; + public: + watchpoint_event() { + m_thread = NULL; + m_inst = NULL; + } + watchpoint_event(const ptx_thread_info *thd, const ptx_instruction *pI) { + m_thread = thd; + m_inst = pI; + } + const ptx_thread_info *thread() const { return m_thread; } + const ptx_instruction *inst() const { return m_inst; } + + private: + const ptx_thread_info *m_thread; + const ptx_instruction *m_inst; }; class gpgpu_sim : public gpgpu_t { -public: - gpgpu_sim( const gpgpu_sim_config &config, gpgpu_context* ctx ); + public: + gpgpu_sim(const gpgpu_sim_config &config, gpgpu_context *ctx); - void set_prop( struct cudaDeviceProp *prop ); + void set_prop(struct cudaDeviceProp *prop); - void launch( kernel_info_t *kinfo ); - bool can_start_kernel(); - unsigned finished_kernel(); - void set_kernel_done( kernel_info_t *kernel ); - void stop_all_running_kernels(); + void launch(kernel_info_t *kinfo); + bool can_start_kernel(); + unsigned finished_kernel(); + void set_kernel_done(kernel_info_t *kernel); + void stop_all_running_kernels(); - void init(); - void cycle(); - bool active(); - bool cycle_insn_cta_max_hit() { - return (m_config.gpu_max_cycle_opt && (gpu_tot_sim_cycle + gpu_sim_cycle) >= m_config.gpu_max_cycle_opt) || - (m_config.gpu_max_insn_opt && (gpu_tot_sim_insn + gpu_sim_insn) >= m_config.gpu_max_insn_opt) || - (m_config.gpu_max_cta_opt && (gpu_tot_issued_cta >= m_config.gpu_max_cta_opt) ); - } - void print_stats(); - void update_stats(); - void deadlock_check(); + void init(); + void cycle(); + bool active(); + bool cycle_insn_cta_max_hit() { + return (m_config.gpu_max_cycle_opt && (gpu_tot_sim_cycle + gpu_sim_cycle) >= + m_config.gpu_max_cycle_opt) || + (m_config.gpu_max_insn_opt && + (gpu_tot_sim_insn + gpu_sim_insn) >= m_config.gpu_max_insn_opt) || + (m_config.gpu_max_cta_opt && + (gpu_tot_issued_cta >= m_config.gpu_max_cta_opt)); + } + void print_stats(); + void update_stats(); + void deadlock_check(); - void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ); + void get_pdom_stack_top_info(unsigned sid, unsigned tid, unsigned *pc, + unsigned *rpc); - int shared_mem_size() const; - int shared_mem_per_block() const; - int compute_capability_major() const; - int compute_capability_minor() const; - int num_registers_per_core() const; - int num_registers_per_block() const; - int wrp_size() const; - int shader_clock() const; - int max_cta_per_core() const; - int get_max_cta( const kernel_info_t &k ) const; - const struct cudaDeviceProp *get_prop() const; - enum divergence_support_t simd_model() const; + int shared_mem_size() const; + int shared_mem_per_block() const; + int compute_capability_major() const; + int compute_capability_minor() const; + int num_registers_per_core() const; + int num_registers_per_block() const; + int wrp_size() const; + int shader_clock() const; + int max_cta_per_core() const; + int get_max_cta(const kernel_info_t &k) const; + const struct cudaDeviceProp *get_prop() const; + enum divergence_support_t simd_model() const; - unsigned threads_per_core() const; - bool get_more_cta_left() const; - bool kernel_more_cta_left(kernel_info_t *kernel) const; - bool hit_max_cta_count() const; - kernel_info_t *select_kernel(); + unsigned threads_per_core() const; + bool get_more_cta_left() const; + bool kernel_more_cta_left(kernel_info_t *kernel) const; + bool hit_max_cta_count() const; + kernel_info_t *select_kernel(); - const gpgpu_sim_config &get_config() const { return m_config; } - void gpu_print_stat(); - void dump_pipeline( int mask, int s, int m ) const; + const gpgpu_sim_config &get_config() const { return m_config; } + void gpu_print_stat(); + void dump_pipeline(int mask, int s, int m) const; - void perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ); + void perf_memcpy_to_gpu(size_t dst_start_addr, size_t count); - //The next three functions added to be used by the functional simulation function - - //! Get shader core configuration - /*! - * Returning the configuration of the shader core, used by the functional simulation only so far - */ - const shader_core_config * getShaderCoreConfig(); - - - //! Get shader core Memory Configuration - /*! - * Returning the memory configuration of the shader core, used by the functional simulation only so far - */ - const memory_config * getMemoryConfig(); - - - //! Get shader core SIMT cluster - /*! - * Returning the cluster of of the shader core, used by the functional simulation so far - */ - simt_core_cluster * getSIMTCluster(); + // The next three functions added to be used by the functional simulation + // function - void hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI ); + //! Get shader core configuration + /*! + * Returning the configuration of the shader core, used by the functional + * simulation only so far + */ + const shader_core_config *getShaderCoreConfig(); - // backward pointer - class gpgpu_context* gpgpu_ctx; + //! Get shader core Memory Configuration + /*! + * Returning the memory configuration of the shader core, used by the + * functional simulation only so far + */ + const memory_config *getMemoryConfig(); -private: - // clocks - void reinit_clock_domains(void); - int next_clock_domain(void); - void issue_block2core(); - void print_dram_stats(FILE *fout) const; - void shader_print_runtime_stat( FILE *fout ); - void shader_print_l1_miss_stat( FILE *fout ) const; - void shader_print_cache_stats( FILE *fout ) const; - void shader_print_scheduler_stat( FILE* fout, bool print_dynamic_info ) const; - void visualizer_printstat(); - void print_shader_cycle_distro( FILE *fout ) const; + //! Get shader core SIMT cluster + /*! + * Returning the cluster of of the shader core, used by the functional + * simulation so far + */ + simt_core_cluster *getSIMTCluster(); - void gpgpu_debug(); + void hit_watchpoint(unsigned watchpoint_num, ptx_thread_info *thd, + const ptx_instruction *pI); -///// data ///// + // backward pointer + class gpgpu_context *gpgpu_ctx; - class simt_core_cluster **m_cluster; - class memory_partition_unit **m_memory_partition_unit; - class memory_sub_partition **m_memory_sub_partition; + private: + // clocks + void reinit_clock_domains(void); + int next_clock_domain(void); + void issue_block2core(); + void print_dram_stats(FILE *fout) const; + void shader_print_runtime_stat(FILE *fout); + void shader_print_l1_miss_stat(FILE *fout) const; + void shader_print_cache_stats(FILE *fout) const; + void shader_print_scheduler_stat(FILE *fout, bool print_dynamic_info) const; + void visualizer_printstat(); + void print_shader_cycle_distro(FILE *fout) const; - std::vector<kernel_info_t*> m_running_kernels; - unsigned m_last_issued_kernel; + void gpgpu_debug(); - std::list<unsigned> m_finished_kernel; - // m_total_cta_launched == per-kernel count. gpu_tot_issued_cta == global count. - unsigned long long m_total_cta_launched; - unsigned long long gpu_tot_issued_cta; + ///// data ///// - unsigned m_last_cluster_issue; - float * average_pipeline_duty_cycle; - float * active_sms; - // time of next rising edge - double core_time; - double icnt_time; - double dram_time; - double l2_time; + class simt_core_cluster **m_cluster; + class memory_partition_unit **m_memory_partition_unit; + class memory_sub_partition **m_memory_sub_partition; - // debug - bool gpu_deadlock; + std::vector<kernel_info_t *> m_running_kernels; + unsigned m_last_issued_kernel; - //// configuration parameters //// - const gpgpu_sim_config &m_config; - - const struct cudaDeviceProp *m_cuda_properties; - const shader_core_config *m_shader_config; - const memory_config *m_memory_config; + std::list<unsigned> m_finished_kernel; + // m_total_cta_launched == per-kernel count. gpu_tot_issued_cta == global + // count. + unsigned long long m_total_cta_launched; + unsigned long long gpu_tot_issued_cta; - // stats - class shader_core_stats *m_shader_stats; - class memory_stats_t *m_memory_stats; - class power_stat_t *m_power_stats; - class gpgpu_sim_wrapper *m_gpgpusim_wrapper; - unsigned long long last_gpu_sim_insn; + unsigned m_last_cluster_issue; + float *average_pipeline_duty_cycle; + float *active_sms; + // time of next rising edge + double core_time; + double icnt_time; + double dram_time; + double l2_time; - unsigned long long last_liveness_message_time; + // debug + bool gpu_deadlock; - std::map<std::string, FuncCache> m_special_cache_config; + //// configuration parameters //// + const gpgpu_sim_config &m_config; - std::vector<std::string> m_executed_kernel_names; //< names of kernel for stat printout - std::vector<unsigned> m_executed_kernel_uids; //< uids of kernel launches for stat printout - std::map<unsigned,watchpoint_event> g_watchpoint_hits; + const struct cudaDeviceProp *m_cuda_properties; + const shader_core_config *m_shader_config; + const memory_config *m_memory_config; - std::string executed_kernel_info_string(); //< format the kernel information into a string for stat printout - void clear_executed_kernel_info(); //< clear the kernel information after stat printout + // stats + class shader_core_stats *m_shader_stats; + class memory_stats_t *m_memory_stats; + class power_stat_t *m_power_stats; + class gpgpu_sim_wrapper *m_gpgpusim_wrapper; + unsigned long long last_gpu_sim_insn; + unsigned long long last_liveness_message_time; -public: - unsigned long long gpu_sim_insn; - unsigned long long gpu_tot_sim_insn; - unsigned long long gpu_sim_insn_last_update; - unsigned gpu_sim_insn_last_update_sid; - occupancy_stats gpu_occupancy; - occupancy_stats gpu_tot_occupancy; + std::map<std::string, FuncCache> m_special_cache_config; - // performance counter for stalls due to congestion. - unsigned int gpu_stall_dramfull; - unsigned int gpu_stall_icnt2sh; - unsigned long long partiton_reqs_in_parallel; - unsigned long long partiton_reqs_in_parallel_total; - unsigned long long partiton_reqs_in_parallel_util; - unsigned long long partiton_reqs_in_parallel_util_total; - unsigned long long gpu_sim_cycle_parition_util; - unsigned long long gpu_tot_sim_cycle_parition_util; - unsigned long long partiton_replys_in_parallel; - unsigned long long partiton_replys_in_parallel_total; + std::vector<std::string> + m_executed_kernel_names; //< names of kernel for stat printout + std::vector<unsigned> + m_executed_kernel_uids; //< uids of kernel launches for stat printout + std::map<unsigned, watchpoint_event> g_watchpoint_hits; + std::string executed_kernel_info_string(); //< format the kernel information + //into a string for stat printout + void clear_executed_kernel_info(); //< clear the kernel information after + //stat printout - FuncCache get_cache_config(std::string kernel_name); - void set_cache_config(std::string kernel_name, FuncCache cacheConfig ); - bool has_special_cache_config(std::string kernel_name); - void change_cache_config(FuncCache cache_config); - void set_cache_config(std::string kernel_name); + public: + unsigned long long gpu_sim_insn; + unsigned long long gpu_tot_sim_insn; + unsigned long long gpu_sim_insn_last_update; + unsigned gpu_sim_insn_last_update_sid; + occupancy_stats gpu_occupancy; + occupancy_stats gpu_tot_occupancy; - //Jin: functional simulation for CDP -private: - //set by stream operation every time a functoinal simulation is done - bool m_functional_sim; - kernel_info_t * m_functional_sim_kernel; + // performance counter for stalls due to congestion. + unsigned int gpu_stall_dramfull; + unsigned int gpu_stall_icnt2sh; + unsigned long long partiton_reqs_in_parallel; + unsigned long long partiton_reqs_in_parallel_total; + unsigned long long partiton_reqs_in_parallel_util; + unsigned long long partiton_reqs_in_parallel_util_total; + unsigned long long gpu_sim_cycle_parition_util; + unsigned long long gpu_tot_sim_cycle_parition_util; + unsigned long long partiton_replys_in_parallel; + unsigned long long partiton_replys_in_parallel_total; -public: - bool is_functional_sim() { return m_functional_sim; } - kernel_info_t * get_functional_kernel() { return m_functional_sim_kernel; } - void functional_launch(kernel_info_t * k) { - m_functional_sim = true; - m_functional_sim_kernel = k; - } - void finish_functional_sim(kernel_info_t * k) { - assert(m_functional_sim); - assert(m_functional_sim_kernel == k); - m_functional_sim = false; - m_functional_sim_kernel = NULL; - } -}; + FuncCache get_cache_config(std::string kernel_name); + void set_cache_config(std::string kernel_name, FuncCache cacheConfig); + bool has_special_cache_config(std::string kernel_name); + void change_cache_config(FuncCache cache_config); + void set_cache_config(std::string kernel_name); + // Jin: functional simulation for CDP + private: + // set by stream operation every time a functoinal simulation is done + bool m_functional_sim; + kernel_info_t *m_functional_sim_kernel; + + public: + bool is_functional_sim() { return m_functional_sim; } + kernel_info_t *get_functional_kernel() { return m_functional_sim_kernel; } + void functional_launch(kernel_info_t *k) { + m_functional_sim = true; + m_functional_sim_kernel = k; + } + void finish_functional_sim(kernel_info_t *k) { + assert(m_functional_sim); + assert(m_functional_sim_kernel == k); + m_functional_sim = false; + m_functional_sim_kernel = NULL; + } +}; #endif diff --git a/src/gpgpu-sim/histogram.cc b/src/gpgpu-sim/histogram.cc index 67f5c18..b3dd0b6 100644 --- a/src/gpgpu-sim/histogram.cc +++ b/src/gpgpu-sim/histogram.cc @@ -7,118 +7,131 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "histogram.h" #include <assert.h> -binned_histogram::binned_histogram (std::string name, int nbins, int* bins) - : m_name(name), m_nbins(nbins), m_bins(NULL), m_bin_cnts(new int[m_nbins]), m_maximum(0), m_sum(0) -{ - if (bins) { - m_bins = new int[m_nbins]; - for (int i = 0; i < nbins; i++) { - m_bins[i] = bins[i]; - } - } +binned_histogram::binned_histogram(std::string name, int nbins, int* bins) + : m_name(name), + m_nbins(nbins), + m_bins(NULL), + m_bin_cnts(new int[m_nbins]), + m_maximum(0), + m_sum(0) { + if (bins) { + m_bins = new int[m_nbins]; + for (int i = 0; i < nbins; i++) { + m_bins[i] = bins[i]; + } + } - reset_bins(); + reset_bins(); } -binned_histogram::binned_histogram (const binned_histogram& other) - : m_name(other.m_name), m_nbins(other.m_nbins), m_bins(NULL), - m_bin_cnts(new int[m_nbins]), m_maximum(0), m_sum(0) -{ - for (int i = 0; i < m_nbins; i++) { - m_bin_cnts[i] = other.m_bin_cnts[i]; - } +binned_histogram::binned_histogram(const binned_histogram& other) + : m_name(other.m_name), + m_nbins(other.m_nbins), + m_bins(NULL), + m_bin_cnts(new int[m_nbins]), + m_maximum(0), + m_sum(0) { + for (int i = 0; i < m_nbins; i++) { + m_bin_cnts[i] = other.m_bin_cnts[i]; + } } -void binned_histogram::reset_bins () { - for (int i = 0; i < m_nbins; i++) { - m_bin_cnts[i] = 0; - } +void binned_histogram::reset_bins() { + for (int i = 0; i < m_nbins; i++) { + m_bin_cnts[i] = 0; + } } -void binned_histogram::add2bin (int sample) { - assert(0); - m_maximum = (sample > m_maximum)? sample : m_maximum; +void binned_histogram::add2bin(int sample) { + assert(0); + m_maximum = (sample > m_maximum) ? sample : m_maximum; } -void binned_histogram::fprint (FILE *fout) const -{ - if (m_name.c_str() != NULL) fprintf(fout, "%s = ", m_name.c_str()); - int total_sample = 0; - for (int i = 0; i < m_nbins; i++) { - fprintf(fout, "%d ", m_bin_cnts[i]); - total_sample += m_bin_cnts[i]; - } - fprintf(fout, "max=%d ", m_maximum); - float avg = 0.0f; - if (total_sample > 0) { - avg = (float)m_sum / total_sample; - } - fprintf(fout, "avg=%0.2f ", avg); +void binned_histogram::fprint(FILE* fout) const { + if (m_name.c_str() != NULL) fprintf(fout, "%s = ", m_name.c_str()); + int total_sample = 0; + for (int i = 0; i < m_nbins; i++) { + fprintf(fout, "%d ", m_bin_cnts[i]); + total_sample += m_bin_cnts[i]; + } + fprintf(fout, "max=%d ", m_maximum); + float avg = 0.0f; + if (total_sample > 0) { + avg = (float)m_sum / total_sample; + } + fprintf(fout, "avg=%0.2f ", avg); } -binned_histogram::~binned_histogram () { - if (m_bins) delete[] m_bins; - delete[] m_bin_cnts; +binned_histogram::~binned_histogram() { + if (m_bins) delete[] m_bins; + delete[] m_bin_cnts; } -pow2_histogram::pow2_histogram (std::string name, int nbins, int* bins) - : binned_histogram (name, nbins, bins) {} +pow2_histogram::pow2_histogram(std::string name, int nbins, int* bins) + : binned_histogram(name, nbins, bins) {} -void pow2_histogram::add2bin (int sample) { - assert(sample >= 0); - - int bin; - int v = sample; - register unsigned int shift; +void pow2_histogram::add2bin(int sample) { + assert(sample >= 0); - bin = (v > 0xFFFF) << 4; v >>= bin; - shift = (v > 0xFF ) << 3; v >>= shift; bin |= shift; - shift = (v > 0xF ) << 2; v >>= shift; bin |= shift; - shift = (v > 0x3 ) << 1; v >>= shift; bin |= shift; - bin |= (v >> 1); - bin += (sample > 0)? 1:0; - - m_bin_cnts[bin] += 1; - - m_maximum = (sample > m_maximum)? sample : m_maximum; - m_sum += sample; -} + int bin; + int v = sample; + register unsigned int shift; + + bin = (v > 0xFFFF) << 4; + v >>= bin; + shift = (v > 0xFF) << 3; + v >>= shift; + bin |= shift; + shift = (v > 0xF) << 2; + v >>= shift; + bin |= shift; + shift = (v > 0x3) << 1; + v >>= shift; + bin |= shift; + bin |= (v >> 1); + bin += (sample > 0) ? 1 : 0; + + m_bin_cnts[bin] += 1; -linear_histogram::linear_histogram (int stride, const char *name, int nbins, int* bins) - : binned_histogram (name, nbins, bins), m_stride(stride) -{ + m_maximum = (sample > m_maximum) ? sample : m_maximum; + m_sum += sample; } -void linear_histogram::add2bin (int sample) { - assert(sample >= 0); +linear_histogram::linear_histogram(int stride, const char* name, int nbins, + int* bins) + : binned_histogram(name, nbins, bins), m_stride(stride) {} + +void linear_histogram::add2bin(int sample) { + assert(sample >= 0); + + int bin = sample / m_stride; + if (bin >= m_nbins) bin = m_nbins - 1; + + m_bin_cnts[bin] += 1; - int bin = sample / m_stride; - if (bin >= m_nbins) bin = m_nbins - 1; - - m_bin_cnts[bin] += 1; - - m_maximum = (sample > m_maximum)? sample : m_maximum; - m_sum += sample; + m_maximum = (sample > m_maximum) ? sample : m_maximum; + m_sum += sample; } diff --git a/src/gpgpu-sim/histogram.h b/src/gpgpu-sim/histogram.h index e8fd375..b60388c 100644 --- a/src/gpgpu-sim/histogram.h +++ b/src/gpgpu-sim/histogram.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef HISTOGRAM_H #define HISTOGRAM_H @@ -34,44 +35,46 @@ #include <string> class binned_histogram { -public: - // creators - binned_histogram (std::string name = "", int nbins = 32, int* bins = NULL); - binned_histogram (const binned_histogram& other); - virtual ~binned_histogram (); + public: + // creators + binned_histogram(std::string name = "", int nbins = 32, int* bins = NULL); + binned_histogram(const binned_histogram& other); + virtual ~binned_histogram(); - // modifiers: - void reset_bins (); - void add2bin (int sample); + // modifiers: + void reset_bins(); + void add2bin(int sample); - // accessors: - void fprint (FILE *fout) const; + // accessors: + void fprint(FILE* fout) const; -protected: - std::string m_name; - int m_nbins; - int *m_bins; // bin boundaries - int *m_bin_cnts; // counters - int m_maximum; // the maximum sample - signed long long int m_sum; // for calculating the average + protected: + std::string m_name; + int m_nbins; + int* m_bins; // bin boundaries + int* m_bin_cnts; // counters + int m_maximum; // the maximum sample + signed long long int m_sum; // for calculating the average }; class pow2_histogram : public binned_histogram { -public: - pow2_histogram ( std::string name = "", int nbins = 32, int* bins = NULL); - ~pow2_histogram() {} + public: + pow2_histogram(std::string name = "", int nbins = 32, int* bins = NULL); + ~pow2_histogram() {} - void add2bin (int sample); + void add2bin(int sample); }; class linear_histogram : public binned_histogram { -public: - linear_histogram (int stride = 1, const char *name = NULL, int nbins = 32, int* bins = NULL); - ~linear_histogram() {} + public: + linear_histogram(int stride = 1, const char* name = NULL, int nbins = 32, + int* bins = NULL); + ~linear_histogram() {} - void add2bin (int sample); -private: - int m_stride; + void add2bin(int sample); + + private: + int m_stride; }; #endif diff --git a/src/gpgpu-sim/icnt_wrapper.cc b/src/gpgpu-sim/icnt_wrapper.cc index 67724d0..890638a 100644 --- a/src/gpgpu-sim/icnt_wrapper.cc +++ b/src/gpgpu-sim/icnt_wrapper.cc @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "icnt_wrapper.h" #include <assert.h> @@ -31,196 +32,163 @@ #include "../intersim2/interconnect_interface.hpp" #include "local_interconnect.h" - -icnt_create_p icnt_create; -icnt_init_p icnt_init; -icnt_has_buffer_p icnt_has_buffer; -icnt_push_p icnt_push; -icnt_pop_p icnt_pop; -icnt_transfer_p icnt_transfer; -icnt_busy_p icnt_busy; -icnt_display_stats_p icnt_display_stats; +icnt_create_p icnt_create; +icnt_init_p icnt_init; +icnt_has_buffer_p icnt_has_buffer; +icnt_push_p icnt_push; +icnt_pop_p icnt_pop; +icnt_transfer_p icnt_transfer; +icnt_busy_p icnt_busy; +icnt_display_stats_p icnt_display_stats; icnt_display_overall_stats_p icnt_display_overall_stats; -icnt_display_state_p icnt_display_state; -icnt_get_flit_size_p icnt_get_flit_size; +icnt_display_state_p icnt_display_state; +icnt_get_flit_size_p icnt_get_flit_size; -unsigned g_network_mode; +unsigned g_network_mode; char* g_network_config_filename; - struct inct_config g_inct_config; -LocalInterconnect *g_localicnt_interface; +LocalInterconnect* g_localicnt_interface; #include "../option_parser.h" // Wrapper to intersim2 to accompany old icnt_wrapper // TODO: use delegate/boost/c++11<funtion> instead -static void intersim2_create(unsigned int n_shader, unsigned int n_mem) -{ - g_icnt_interface->CreateInterconnect(n_shader, n_mem); +static void intersim2_create(unsigned int n_shader, unsigned int n_mem) { + g_icnt_interface->CreateInterconnect(n_shader, n_mem); } -static void intersim2_init() -{ - g_icnt_interface->Init(); -} +static void intersim2_init() { g_icnt_interface->Init(); } -static bool intersim2_has_buffer(unsigned input, unsigned int size) -{ - return g_icnt_interface->HasBuffer(input, size); +static bool intersim2_has_buffer(unsigned input, unsigned int size) { + return g_icnt_interface->HasBuffer(input, size); } -static void intersim2_push(unsigned input, unsigned output, void* data, unsigned int size) -{ - g_icnt_interface->Push(input, output, data, size); +static void intersim2_push(unsigned input, unsigned output, void* data, + unsigned int size) { + g_icnt_interface->Push(input, output, data, size); } -static void* intersim2_pop(unsigned output) -{ - return g_icnt_interface->Pop(output); +static void* intersim2_pop(unsigned output) { + return g_icnt_interface->Pop(output); } -static void intersim2_transfer() -{ - g_icnt_interface->Advance(); -} +static void intersim2_transfer() { g_icnt_interface->Advance(); } -static bool intersim2_busy() -{ - return g_icnt_interface->Busy(); -} +static bool intersim2_busy() { return g_icnt_interface->Busy(); } -static void intersim2_display_stats() -{ - g_icnt_interface->DisplayStats(); -} +static void intersim2_display_stats() { g_icnt_interface->DisplayStats(); } -static void intersim2_display_overall_stats() -{ - g_icnt_interface->DisplayOverallStats(); +static void intersim2_display_overall_stats() { + g_icnt_interface->DisplayOverallStats(); } -static void intersim2_display_state(FILE *fp) -{ - g_icnt_interface->DisplayState(fp); +static void intersim2_display_state(FILE* fp) { + g_icnt_interface->DisplayState(fp); } -static unsigned intersim2_get_flit_size() -{ - return g_icnt_interface->GetFlitSize(); +static unsigned intersim2_get_flit_size() { + return g_icnt_interface->GetFlitSize(); } - ////////////////////////////////////////////////////// -static void LocalInterconnect_create(unsigned int n_shader, unsigned int n_mem) -{ - g_localicnt_interface->CreateInterconnect(n_shader, n_mem); +static void LocalInterconnect_create(unsigned int n_shader, + unsigned int n_mem) { + g_localicnt_interface->CreateInterconnect(n_shader, n_mem); } -static void LocalInterconnect_init() -{ - g_localicnt_interface->Init(); -} +static void LocalInterconnect_init() { g_localicnt_interface->Init(); } -static bool LocalInterconnect_has_buffer(unsigned input, unsigned int size) -{ - return g_localicnt_interface->HasBuffer(input, size); +static bool LocalInterconnect_has_buffer(unsigned input, unsigned int size) { + return g_localicnt_interface->HasBuffer(input, size); } -static void LocalInterconnect_push(unsigned input, unsigned output, void* data, unsigned int size) -{ - g_localicnt_interface->Push(input, output, data, size); +static void LocalInterconnect_push(unsigned input, unsigned output, void* data, + unsigned int size) { + g_localicnt_interface->Push(input, output, data, size); } -static void* LocalInterconnect_pop(unsigned output) -{ - return g_localicnt_interface->Pop(output); +static void* LocalInterconnect_pop(unsigned output) { + return g_localicnt_interface->Pop(output); } -static void LocalInterconnect_transfer() -{ - g_localicnt_interface->Advance(); -} +static void LocalInterconnect_transfer() { g_localicnt_interface->Advance(); } -static bool LocalInterconnect_busy() -{ - return g_localicnt_interface->Busy(); -} +static bool LocalInterconnect_busy() { return g_localicnt_interface->Busy(); } -static void LocalInterconnect_display_stats() -{ - g_localicnt_interface->DisplayStats(); +static void LocalInterconnect_display_stats() { + g_localicnt_interface->DisplayStats(); } -static void LocalInterconnect_display_overall_stats() -{ - g_localicnt_interface->DisplayOverallStats(); +static void LocalInterconnect_display_overall_stats() { + g_localicnt_interface->DisplayOverallStats(); } -static void LocalInterconnect_display_state(FILE *fp) -{ - g_localicnt_interface->DisplayState(fp); +static void LocalInterconnect_display_state(FILE* fp) { + g_localicnt_interface->DisplayState(fp); } -static unsigned LocalInterconnect_get_flit_size() -{ - return g_localicnt_interface->GetFlitSize(); +static unsigned LocalInterconnect_get_flit_size() { + return g_localicnt_interface->GetFlitSize(); } - /////////////////////////// -void icnt_reg_options( class OptionParser * opp ) -{ - option_parser_register(opp, "-network_mode", OPT_INT32, &g_network_mode, "Interconnection network mode", "1"); - option_parser_register(opp, "-inter_config_file", OPT_CSTR, &g_network_config_filename, "Interconnection network config file", "mesh"); - - - //parameters for local xbar - option_parser_register(opp, "-inct_in_buffer_limit", OPT_UINT32, &g_inct_config.in_buffer_limit, "in_buffer_limit", "64"); - option_parser_register(opp, "-inct_out_buffer_limit", OPT_UINT32, &g_inct_config.out_buffer_limit, "out_buffer_limit", "64"); - option_parser_register(opp, "-inct_subnets", OPT_UINT32, &g_inct_config.subnets, "subnets", "2"); - option_parser_register(opp, "-arbiter_algo", OPT_UINT32, &g_inct_config.arbiter_algo, "arbiter_algo", "1"); - +void icnt_reg_options(class OptionParser* opp) { + option_parser_register(opp, "-network_mode", OPT_INT32, &g_network_mode, + "Interconnection network mode", "1"); + option_parser_register(opp, "-inter_config_file", OPT_CSTR, + &g_network_config_filename, + "Interconnection network config file", "mesh"); + // parameters for local xbar + option_parser_register(opp, "-inct_in_buffer_limit", OPT_UINT32, + &g_inct_config.in_buffer_limit, "in_buffer_limit", + "64"); + option_parser_register(opp, "-inct_out_buffer_limit", OPT_UINT32, + &g_inct_config.out_buffer_limit, "out_buffer_limit", + "64"); + option_parser_register(opp, "-inct_subnets", OPT_UINT32, + &g_inct_config.subnets, "subnets", "2"); + option_parser_register(opp, "-arbiter_algo", OPT_UINT32, + &g_inct_config.arbiter_algo, "arbiter_algo", "1"); } -void icnt_wrapper_init() -{ - switch (g_network_mode) { - case INTERSIM: - //FIXME: delete the object: may add icnt_done wrapper - g_icnt_interface = InterconnectInterface::New(g_network_config_filename); - icnt_create = intersim2_create; - icnt_init = intersim2_init; - icnt_has_buffer = intersim2_has_buffer; - icnt_push = intersim2_push; - icnt_pop = intersim2_pop; - icnt_transfer = intersim2_transfer; - icnt_busy = intersim2_busy; - icnt_display_stats = intersim2_display_stats; - icnt_display_overall_stats = intersim2_display_overall_stats; - icnt_display_state = intersim2_display_state; - icnt_get_flit_size = intersim2_get_flit_size; - break; - case LOCAL_XBAR: - g_localicnt_interface = LocalInterconnect::New(g_inct_config); - icnt_create = LocalInterconnect_create; - icnt_init = LocalInterconnect_init; - icnt_has_buffer = LocalInterconnect_has_buffer; - icnt_push = LocalInterconnect_push; - icnt_pop = LocalInterconnect_pop; - icnt_transfer = LocalInterconnect_transfer; - icnt_busy = LocalInterconnect_busy; - icnt_display_stats = LocalInterconnect_display_stats; - icnt_display_overall_stats = LocalInterconnect_display_overall_stats; - icnt_display_state = LocalInterconnect_display_state; - icnt_get_flit_size = LocalInterconnect_get_flit_size; - break; - default: - assert(0); - break; - } +void icnt_wrapper_init() { + switch (g_network_mode) { + case INTERSIM: + // FIXME: delete the object: may add icnt_done wrapper + g_icnt_interface = InterconnectInterface::New(g_network_config_filename); + icnt_create = intersim2_create; + icnt_init = intersim2_init; + icnt_has_buffer = intersim2_has_buffer; + icnt_push = intersim2_push; + icnt_pop = intersim2_pop; + icnt_transfer = intersim2_transfer; + icnt_busy = intersim2_busy; + icnt_display_stats = intersim2_display_stats; + icnt_display_overall_stats = intersim2_display_overall_stats; + icnt_display_state = intersim2_display_state; + icnt_get_flit_size = intersim2_get_flit_size; + break; + case LOCAL_XBAR: + g_localicnt_interface = LocalInterconnect::New(g_inct_config); + icnt_create = LocalInterconnect_create; + icnt_init = LocalInterconnect_init; + icnt_has_buffer = LocalInterconnect_has_buffer; + icnt_push = LocalInterconnect_push; + icnt_pop = LocalInterconnect_pop; + icnt_transfer = LocalInterconnect_transfer; + icnt_busy = LocalInterconnect_busy; + icnt_display_stats = LocalInterconnect_display_stats; + icnt_display_overall_stats = LocalInterconnect_display_overall_stats; + icnt_display_state = LocalInterconnect_display_state; + icnt_get_flit_size = LocalInterconnect_get_flit_size; + break; + default: + assert(0); + break; + } } diff --git a/src/gpgpu-sim/icnt_wrapper.h b/src/gpgpu-sim/icnt_wrapper.h index e1086f9..a92f368 100644 --- a/src/gpgpu-sim/icnt_wrapper.h +++ b/src/gpgpu-sim/icnt_wrapper.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ICNT_WRAPPER_H #define ICNT_WRAPPER_H @@ -32,42 +33,37 @@ // functional interface to the interconnect -typedef void (*icnt_create_p)(unsigned n_shader, unsigned n_mem); -typedef void (*icnt_init_p)( ); +typedef void (*icnt_create_p)(unsigned n_shader, unsigned n_mem); +typedef void (*icnt_init_p)(); typedef bool (*icnt_has_buffer_p)(unsigned input, unsigned int size); -typedef void (*icnt_push_p)(unsigned input, unsigned output, void* data, unsigned int size); +typedef void (*icnt_push_p)(unsigned input, unsigned output, void* data, + unsigned int size); typedef void* (*icnt_pop_p)(unsigned output); -typedef void (*icnt_transfer_p)( ); -typedef bool (*icnt_busy_p)( ); -typedef void (*icnt_drain_p)( ); -typedef void (*icnt_display_stats_p)( ); -typedef void (*icnt_display_overall_stats_p)( ); +typedef void (*icnt_transfer_p)(); +typedef bool (*icnt_busy_p)(); +typedef void (*icnt_drain_p)(); +typedef void (*icnt_display_stats_p)(); +typedef void (*icnt_display_overall_stats_p)(); typedef void (*icnt_display_state_p)(FILE* fp); typedef unsigned (*icnt_get_flit_size_p)(); -extern icnt_create_p icnt_create; -extern icnt_init_p icnt_init; +extern icnt_create_p icnt_create; +extern icnt_init_p icnt_init; extern icnt_has_buffer_p icnt_has_buffer; -extern icnt_push_p icnt_push; -extern icnt_pop_p icnt_pop; -extern icnt_transfer_p icnt_transfer; -extern icnt_busy_p icnt_busy; -extern icnt_drain_p icnt_drain; +extern icnt_push_p icnt_push; +extern icnt_pop_p icnt_pop; +extern icnt_transfer_p icnt_transfer; +extern icnt_busy_p icnt_busy; +extern icnt_drain_p icnt_drain; extern icnt_display_stats_p icnt_display_stats; extern icnt_display_overall_stats_p icnt_display_overall_stats; extern icnt_display_state_p icnt_display_state; extern icnt_get_flit_size_p icnt_get_flit_size; extern unsigned g_network_mode; -enum network_mode { - INTERSIM = 1, - LOCAL_XBAR = 2, - N_NETWORK_MODE -}; - - +enum network_mode { INTERSIM = 1, LOCAL_XBAR = 2, N_NETWORK_MODE }; void icnt_wrapper_init(); -void icnt_reg_options( class OptionParser * opp ); +void icnt_reg_options(class OptionParser* opp); #endif diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index fb4ce32..ab6e5c2 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -7,815 +7,854 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include <stdlib.h> #include <stdio.h> +#include <stdlib.h> #include <string.h> #include <list> #include <set> +#include "../abstract_hardware_model.h" #include "../option_parser.h" -#include "mem_fetch.h" +#include "../statwrapper.h" #include "dram.h" #include "gpu-cache.h" +#include "gpu-sim.h" #include "histogram.h" #include "l2cache.h" -#include "../statwrapper.h" -#include "../abstract_hardware_model.h" -#include "gpu-sim.h" -#include "shader.h" -#include "mem_latency_stat.h" #include "l2cache_trace.h" +#include "mem_fetch.h" +#include "mem_latency_stat.h" +#include "shader.h" - -mem_fetch * partition_mf_allocator::alloc(new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const -{ - assert( wr ); - mem_access_t access( type, addr, size, wr, m_memory_config->gpgpu_ctx ); - mem_fetch *mf = new mem_fetch( access, - NULL, - WRITE_PACKET_SIZE, - -1, - -1, - -1, - m_memory_config, - cycle); - return mf; +mem_fetch *partition_mf_allocator::alloc(new_addr_type addr, + mem_access_type type, unsigned size, + bool wr, + unsigned long long cycle) const { + assert(wr); + mem_access_t access(type, addr, size, wr, m_memory_config->gpgpu_ctx); + mem_fetch *mf = new mem_fetch(access, NULL, WRITE_PACKET_SIZE, -1, -1, -1, + m_memory_config, cycle); + return mf; } -memory_partition_unit::memory_partition_unit( unsigned partition_id, - const memory_config *config, - class memory_stats_t *stats, - class gpgpu_sim* gpu) -: m_id(partition_id), m_config(config), m_stats(stats), m_arbitration_metadata(config), m_gpu(gpu) -{ - m_dram = new dram_t(m_id,m_config,m_stats,this,gpu); - - m_sub_partition = new memory_sub_partition*[m_config->m_n_sub_partition_per_memory_channel]; - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - unsigned sub_partition_id = m_id * m_config->m_n_sub_partition_per_memory_channel + p; - m_sub_partition[p] = new memory_sub_partition(sub_partition_id, m_config, stats, gpu); - } +memory_partition_unit::memory_partition_unit(unsigned partition_id, + const memory_config *config, + class memory_stats_t *stats, + class gpgpu_sim *gpu) + : m_id(partition_id), + m_config(config), + m_stats(stats), + m_arbitration_metadata(config), + m_gpu(gpu) { + m_dram = new dram_t(m_id, m_config, m_stats, this, gpu); + m_sub_partition = new memory_sub_partition + *[m_config->m_n_sub_partition_per_memory_channel]; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + unsigned sub_partition_id = + m_id * m_config->m_n_sub_partition_per_memory_channel + p; + m_sub_partition[p] = + new memory_sub_partition(sub_partition_id, m_config, stats, gpu); + } } -void memory_partition_unit::handle_memcpy_to_gpu( size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask ) -{ - unsigned p = global_sub_partition_id_to_local_id(global_subpart_id); - std::string mystring = - mask.to_string<char,std::string::traits_type,std::string::allocator_type>(); - MEMPART_DPRINTF("Copy Engine Request Received For Address=%zx, local_subpart=%u, global_subpart=%u, sector_mask=%s \n", addr, p, global_subpart_id, mystring.c_str()); - m_sub_partition[p]->force_l2_tag_update(addr,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle, mask); +void memory_partition_unit::handle_memcpy_to_gpu( + size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask) { + unsigned p = global_sub_partition_id_to_local_id(global_subpart_id); + std::string mystring = mask.to_string<char, std::string::traits_type, + std::string::allocator_type>(); + MEMPART_DPRINTF( + "Copy Engine Request Received For Address=%zx, local_subpart=%u, " + "global_subpart=%u, sector_mask=%s \n", + addr, p, global_subpart_id, mystring.c_str()); + m_sub_partition[p]->force_l2_tag_update( + addr, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, mask); } -memory_partition_unit::~memory_partition_unit() -{ - delete m_dram; - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - delete m_sub_partition[p]; - } - delete[] m_sub_partition; +memory_partition_unit::~memory_partition_unit() { + delete m_dram; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + delete m_sub_partition[p]; + } + delete[] m_sub_partition; } -memory_partition_unit::arbitration_metadata::arbitration_metadata(const memory_config *config) -: m_last_borrower(config->m_n_sub_partition_per_memory_channel - 1), - m_private_credit(config->m_n_sub_partition_per_memory_channel, 0), - m_shared_credit(0) -{ - // each sub partition get at least 1 credit for forward progress - // the rest is shared among with other partitions - m_private_credit_limit = 1; - m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size - + config->gpgpu_dram_return_queue_size - - (config->m_n_sub_partition_per_memory_channel - 1); - if(config->seperate_write_queue_enabled ) - m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size; - if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 - or config->gpgpu_dram_return_queue_size == 0) - { - m_shared_credit_limit = 0; // no limit if either of the queue has no limit in size - } - assert(m_shared_credit_limit >= 0); +memory_partition_unit::arbitration_metadata::arbitration_metadata( + const memory_config *config) + : m_last_borrower(config->m_n_sub_partition_per_memory_channel - 1), + m_private_credit(config->m_n_sub_partition_per_memory_channel, 0), + m_shared_credit(0) { + // each sub partition get at least 1 credit for forward progress + // the rest is shared among with other partitions + m_private_credit_limit = 1; + m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size + + config->gpgpu_dram_return_queue_size - + (config->m_n_sub_partition_per_memory_channel - 1); + if (config->seperate_write_queue_enabled) + m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size; + if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 or + config->gpgpu_dram_return_queue_size == 0) { + m_shared_credit_limit = + 0; // no limit if either of the queue has no limit in size + } + assert(m_shared_credit_limit >= 0); } -bool memory_partition_unit::arbitration_metadata::has_credits(int inner_sub_partition_id) const -{ - int spid = inner_sub_partition_id; - if (m_private_credit[spid] < m_private_credit_limit) { - return true; - } else if (m_shared_credit_limit == 0 || m_shared_credit < m_shared_credit_limit) { - return true; - } else { - return false; - } +bool memory_partition_unit::arbitration_metadata::has_credits( + int inner_sub_partition_id) const { + int spid = inner_sub_partition_id; + if (m_private_credit[spid] < m_private_credit_limit) { + return true; + } else if (m_shared_credit_limit == 0 || + m_shared_credit < m_shared_credit_limit) { + return true; + } else { + return false; + } } -void memory_partition_unit::arbitration_metadata::borrow_credit(int inner_sub_partition_id) -{ - int spid = inner_sub_partition_id; - if (m_private_credit[spid] < m_private_credit_limit) { - m_private_credit[spid] += 1; - } else if (m_shared_credit_limit == 0 || m_shared_credit < m_shared_credit_limit) { - m_shared_credit += 1; - } else { - assert(0 && "DRAM arbitration error: Borrowing from depleted credit!"); - } - m_last_borrower = spid; +void memory_partition_unit::arbitration_metadata::borrow_credit( + int inner_sub_partition_id) { + int spid = inner_sub_partition_id; + if (m_private_credit[spid] < m_private_credit_limit) { + m_private_credit[spid] += 1; + } else if (m_shared_credit_limit == 0 || + m_shared_credit < m_shared_credit_limit) { + m_shared_credit += 1; + } else { + assert(0 && "DRAM arbitration error: Borrowing from depleted credit!"); + } + m_last_borrower = spid; } -void memory_partition_unit::arbitration_metadata::return_credit(int inner_sub_partition_id) -{ - int spid = inner_sub_partition_id; - if (m_private_credit[spid] > 0) { - m_private_credit[spid] -= 1; - } else { - m_shared_credit -= 1; - } - assert((m_shared_credit >= 0) && "DRAM arbitration error: Returning more than available credits!"); +void memory_partition_unit::arbitration_metadata::return_credit( + int inner_sub_partition_id) { + int spid = inner_sub_partition_id; + if (m_private_credit[spid] > 0) { + m_private_credit[spid] -= 1; + } else { + m_shared_credit -= 1; + } + assert((m_shared_credit >= 0) && + "DRAM arbitration error: Returning more than available credits!"); } -void memory_partition_unit::arbitration_metadata::print( FILE *fp ) const -{ - fprintf(fp, "private_credit = "); - for (unsigned p = 0; p < m_private_credit.size(); p++) { - fprintf(fp, "%d ", m_private_credit[p]); - } - fprintf(fp, "(limit = %d)\n", m_private_credit_limit); - fprintf(fp, "shared_credit = %d (limit = %d)\n", m_shared_credit, m_shared_credit_limit); +void memory_partition_unit::arbitration_metadata::print(FILE *fp) const { + fprintf(fp, "private_credit = "); + for (unsigned p = 0; p < m_private_credit.size(); p++) { + fprintf(fp, "%d ", m_private_credit[p]); + } + fprintf(fp, "(limit = %d)\n", m_private_credit_limit); + fprintf(fp, "shared_credit = %d (limit = %d)\n", m_shared_credit, + m_shared_credit_limit); } -bool memory_partition_unit::busy() const -{ - bool busy = false; - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - if (m_sub_partition[p]->busy()) { - busy = true; - } +bool memory_partition_unit::busy() const { + bool busy = false; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + if (m_sub_partition[p]->busy()) { + busy = true; } - return busy; + } + return busy; } -void memory_partition_unit::cache_cycle(unsigned cycle) -{ - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - m_sub_partition[p]->cache_cycle(cycle); - } +void memory_partition_unit::cache_cycle(unsigned cycle) { + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + m_sub_partition[p]->cache_cycle(cycle); + } } -void memory_partition_unit::visualizer_print( gzFile visualizer_file ) const -{ - m_dram->visualizer_print(visualizer_file); - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - m_sub_partition[p]->visualizer_print(visualizer_file); - } +void memory_partition_unit::visualizer_print(gzFile visualizer_file) const { + m_dram->visualizer_print(visualizer_file); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + m_sub_partition[p]->visualizer_print(visualizer_file); + } } -// determine whether a given subpartition can issue to DRAM -bool memory_partition_unit::can_issue_to_dram(int inner_sub_partition_id) -{ - int spid = inner_sub_partition_id; - bool sub_partition_contention = m_sub_partition[spid]->dram_L2_queue_full(); - bool has_dram_resource = m_arbitration_metadata.has_credits(spid); +// determine whether a given subpartition can issue to DRAM +bool memory_partition_unit::can_issue_to_dram(int inner_sub_partition_id) { + int spid = inner_sub_partition_id; + bool sub_partition_contention = m_sub_partition[spid]->dram_L2_queue_full(); + bool has_dram_resource = m_arbitration_metadata.has_credits(spid); - MEMPART_DPRINTF("sub partition %d sub_partition_contention=%c has_dram_resource=%c\n", - spid, (sub_partition_contention)? 'T':'F', (has_dram_resource)? 'T':'F'); + MEMPART_DPRINTF( + "sub partition %d sub_partition_contention=%c has_dram_resource=%c\n", + spid, (sub_partition_contention) ? 'T' : 'F', + (has_dram_resource) ? 'T' : 'F'); - return (has_dram_resource && !sub_partition_contention); + return (has_dram_resource && !sub_partition_contention); } -int memory_partition_unit::global_sub_partition_id_to_local_id(int global_sub_partition_id) const -{ - return (global_sub_partition_id - m_id * m_config->m_n_sub_partition_per_memory_channel); +int memory_partition_unit::global_sub_partition_id_to_local_id( + int global_sub_partition_id) const { + return (global_sub_partition_id - + m_id * m_config->m_n_sub_partition_per_memory_channel); } -void memory_partition_unit::simple_dram_model_cycle() -{ - - // pop completed memory request from dram and push it to dram-to-L2 queue - // of the original sub partition - if (!m_dram_latency_queue.empty() && ( (m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle )) { - mem_fetch* mf_return = m_dram_latency_queue.front().req; - if( mf_return->get_access_type() != L1_WRBK_ACC && mf_return->get_access_type() != L2_WRBK_ACC ) { - mf_return->set_reply(); +void memory_partition_unit::simple_dram_model_cycle() { + // pop completed memory request from dram and push it to dram-to-L2 queue + // of the original sub partition + if (!m_dram_latency_queue.empty() && + ((m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) >= + m_dram_latency_queue.front().ready_cycle)) { + mem_fetch *mf_return = m_dram_latency_queue.front().req; + if (mf_return->get_access_type() != L1_WRBK_ACC && + mf_return->get_access_type() != L2_WRBK_ACC) { + mf_return->set_reply(); - unsigned dest_global_spid = mf_return->get_sub_partition_id(); - int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid); - assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid); - if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) { - if( mf_return->get_access_type() == L1_WRBK_ACC ) { - m_sub_partition[dest_spid]->set_done(mf_return); - delete mf_return; - } else { - m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return); - mf_return->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_arbitration_metadata.return_credit(dest_spid); - MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf_return, dest_spid); - } - m_dram_latency_queue.pop_front(); - } - - } else { - this->set_done(mf_return); - delete mf_return; - m_dram_latency_queue.pop_front(); - } - } + unsigned dest_global_spid = mf_return->get_sub_partition_id(); + int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid); + assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid); + if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) { + if (mf_return->get_access_type() == L1_WRBK_ACC) { + m_sub_partition[dest_spid]->set_done(mf_return); + delete mf_return; + } else { + m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return); + mf_return->set_status( + IN_PARTITION_DRAM_TO_L2_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_arbitration_metadata.return_credit(dest_spid); + MEMPART_DPRINTF( + "mem_fetch request %p return from dram to sub partition %d\n", + mf_return, dest_spid); + } + m_dram_latency_queue.pop_front(); + } - // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); - //if( !m_dram->full(mf->is_write()) ) { - // L2->DRAM queue to DRAM latency queue - // Arbitrate among multiple L2 subpartitions - int last_issued_partition = m_arbitration_metadata.last_borrower(); - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; - if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { - mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); - if(m_dram->full(mf->is_write()) ) - break; + } else { + this->set_done(mf_return); + delete mf_return; + m_dram_latency_queue.pop_front(); + } + } - m_sub_partition[spid]->L2_dram_queue_pop(); - MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); - dram_delay_t d; - d.req = mf; - d.ready_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle + m_config->dram_latency; - m_dram_latency_queue.push_back(d); - mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_arbitration_metadata.borrow_credit(spid); - break; // the DRAM should only accept one request per cycle - } - } - //} + // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + // if( !m_dram->full(mf->is_write()) ) { + // L2->DRAM queue to DRAM latency queue + // Arbitrate among multiple L2 subpartitions + int last_issued_partition = m_arbitration_metadata.last_borrower(); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + int spid = (p + last_issued_partition + 1) % + m_config->m_n_sub_partition_per_memory_channel; + if (!m_sub_partition[spid]->L2_dram_queue_empty() && + can_issue_to_dram(spid)) { + mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + if (m_dram->full(mf->is_write())) break; + m_sub_partition[spid]->L2_dram_queue_pop(); + MEMPART_DPRINTF( + "Issue mem_fetch request %p from sub partition %d to dram\n", mf, + spid); + dram_delay_t d; + d.req = mf; + d.ready_cycle = m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle + + m_config->dram_latency; + m_dram_latency_queue.push_back(d); + mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_arbitration_metadata.borrow_credit(spid); + break; // the DRAM should only accept one request per cycle + } + } + //} } -void memory_partition_unit::dram_cycle() -{ - // pop completed memory request from dram and push it to dram-to-L2 queue - // of the original sub partition - mem_fetch* mf_return = m_dram->return_queue_top(); - if (mf_return) { - unsigned dest_global_spid = mf_return->get_sub_partition_id(); - int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid); - assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid); - if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) { - if( mf_return->get_access_type() == L1_WRBK_ACC ) { - m_sub_partition[dest_spid]->set_done(mf_return); - delete mf_return; - } else { - m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return); - mf_return->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_arbitration_metadata.return_credit(dest_spid); - MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf_return, dest_spid); - } - m_dram->return_queue_pop(); - } - } else { - m_dram->return_queue_pop(); +void memory_partition_unit::dram_cycle() { + // pop completed memory request from dram and push it to dram-to-L2 queue + // of the original sub partition + mem_fetch *mf_return = m_dram->return_queue_top(); + if (mf_return) { + unsigned dest_global_spid = mf_return->get_sub_partition_id(); + int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid); + assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid); + if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) { + if (mf_return->get_access_type() == L1_WRBK_ACC) { + m_sub_partition[dest_spid]->set_done(mf_return); + delete mf_return; + } else { + m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return); + mf_return->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_arbitration_metadata.return_credit(dest_spid); + MEMPART_DPRINTF( + "mem_fetch request %p return from dram to sub partition %d\n", + mf_return, dest_spid); + } + m_dram->return_queue_pop(); } - - m_dram->cycle(); - m_dram->dram_log(SAMPLELOG); + } else { + m_dram->return_queue_pop(); + } - // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); - //if( !m_dram->full(mf->is_write()) ) { - // L2->DRAM queue to DRAM latency queue - // Arbitrate among multiple L2 subpartitions - int last_issued_partition = m_arbitration_metadata.last_borrower(); - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; - if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { - mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); - if(m_dram->full(mf->is_write()) ) - break; + m_dram->cycle(); + m_dram->dram_log(SAMPLELOG); - m_sub_partition[spid]->L2_dram_queue_pop(); - MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); - dram_delay_t d; - d.req = mf; - d.ready_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle + m_config->dram_latency; - m_dram_latency_queue.push_back(d); - mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_arbitration_metadata.borrow_credit(spid); - break; // the DRAM should only accept one request per cycle - } - } - //} + // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + // if( !m_dram->full(mf->is_write()) ) { + // L2->DRAM queue to DRAM latency queue + // Arbitrate among multiple L2 subpartitions + int last_issued_partition = m_arbitration_metadata.last_borrower(); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + int spid = (p + last_issued_partition + 1) % + m_config->m_n_sub_partition_per_memory_channel; + if (!m_sub_partition[spid]->L2_dram_queue_empty() && + can_issue_to_dram(spid)) { + mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + if (m_dram->full(mf->is_write())) break; - // DRAM latency queue - if( !m_dram_latency_queue.empty() && ( (m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) { - mem_fetch* mf = m_dram_latency_queue.front().req; - m_dram_latency_queue.pop_front(); - m_dram->push(mf); + m_sub_partition[spid]->L2_dram_queue_pop(); + MEMPART_DPRINTF( + "Issue mem_fetch request %p from sub partition %d to dram\n", mf, + spid); + dram_delay_t d; + d.req = mf; + d.ready_cycle = m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle + + m_config->dram_latency; + m_dram_latency_queue.push_back(d); + mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_arbitration_metadata.borrow_credit(spid); + break; // the DRAM should only accept one request per cycle } + } + //} + + // DRAM latency queue + if (!m_dram_latency_queue.empty() && + ((m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) >= + m_dram_latency_queue.front().ready_cycle) && + !m_dram->full(m_dram_latency_queue.front().req->is_write())) { + mem_fetch *mf = m_dram_latency_queue.front().req; + m_dram_latency_queue.pop_front(); + m_dram->push(mf); + } } -void memory_partition_unit::set_done( mem_fetch *mf ) -{ - unsigned global_spid = mf->get_sub_partition_id(); - int spid = global_sub_partition_id_to_local_id(global_spid); - assert(m_sub_partition[spid]->get_id() == global_spid); - if (mf->get_access_type() == L1_WRBK_ACC || mf->get_access_type() == L2_WRBK_ACC) { - m_arbitration_metadata.return_credit(spid); - MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf, spid); - } - m_sub_partition[spid]->set_done(mf); +void memory_partition_unit::set_done(mem_fetch *mf) { + unsigned global_spid = mf->get_sub_partition_id(); + int spid = global_sub_partition_id_to_local_id(global_spid); + assert(m_sub_partition[spid]->get_id() == global_spid); + if (mf->get_access_type() == L1_WRBK_ACC || + mf->get_access_type() == L2_WRBK_ACC) { + m_arbitration_metadata.return_credit(spid); + MEMPART_DPRINTF( + "mem_fetch request %p return from dram to sub partition %d\n", mf, + spid); + } + m_sub_partition[spid]->set_done(mf); } -void memory_partition_unit::set_dram_power_stats(unsigned &n_cmd, - unsigned &n_activity, - unsigned &n_nop, - unsigned &n_act, - unsigned &n_pre, - unsigned &n_rd, - unsigned &n_wr, - unsigned &n_req) const -{ - m_dram->set_dram_power_stats(n_cmd, n_activity, n_nop, n_act, n_pre, n_rd, n_wr, n_req); +void memory_partition_unit::set_dram_power_stats( + unsigned &n_cmd, unsigned &n_activity, unsigned &n_nop, unsigned &n_act, + unsigned &n_pre, unsigned &n_rd, unsigned &n_wr, unsigned &n_req) const { + m_dram->set_dram_power_stats(n_cmd, n_activity, n_nop, n_act, n_pre, n_rd, + n_wr, n_req); } -void memory_partition_unit::print( FILE *fp ) const -{ - fprintf(fp, "Memory Partition %u: \n", m_id); - for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { - m_sub_partition[p]->print(fp); - } - fprintf(fp, "In Dram Latency Queue (total = %zd): \n", m_dram_latency_queue.size()); - for (std::list<dram_delay_t>::const_iterator mf_dlq = m_dram_latency_queue.begin(); - mf_dlq != m_dram_latency_queue.end(); ++mf_dlq) { - mem_fetch *mf = mf_dlq->req; - fprintf(fp, "Ready @ %llu - ", mf_dlq->ready_cycle); - if (mf) - mf->print(fp); - else - fprintf(fp, " <NULL mem_fetch?>\n"); - } - m_dram->print(fp); +void memory_partition_unit::print(FILE *fp) const { + fprintf(fp, "Memory Partition %u: \n", m_id); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; + p++) { + m_sub_partition[p]->print(fp); + } + fprintf(fp, "In Dram Latency Queue (total = %zd): \n", + m_dram_latency_queue.size()); + for (std::list<dram_delay_t>::const_iterator mf_dlq = + m_dram_latency_queue.begin(); + mf_dlq != m_dram_latency_queue.end(); ++mf_dlq) { + mem_fetch *mf = mf_dlq->req; + fprintf(fp, "Ready @ %llu - ", mf_dlq->ready_cycle); + if (mf) + mf->print(fp); + else + fprintf(fp, " <NULL mem_fetch?>\n"); + } + m_dram->print(fp); } -memory_sub_partition::memory_sub_partition( unsigned sub_partition_id, - const memory_config *config, - class memory_stats_t *stats, - class gpgpu_sim* gpu) -{ - m_id = sub_partition_id; - m_config=config; - m_stats=stats; - m_gpu = gpu; - m_memcpy_cycle_offset = 0; +memory_sub_partition::memory_sub_partition(unsigned sub_partition_id, + const memory_config *config, + class memory_stats_t *stats, + class gpgpu_sim *gpu) { + m_id = sub_partition_id; + m_config = config; + m_stats = stats; + m_gpu = gpu; + m_memcpy_cycle_offset = 0; - assert(m_id < m_config->m_n_mem_sub_partition); + assert(m_id < m_config->m_n_mem_sub_partition); - char L2c_name[32]; - snprintf(L2c_name, 32, "L2_bank_%03d", m_id); - m_L2interface = new L2interface(this); - m_mf_allocator = new partition_mf_allocator(config); + char L2c_name[32]; + snprintf(L2c_name, 32, "L2_bank_%03d", m_id); + m_L2interface = new L2interface(this); + m_mf_allocator = new partition_mf_allocator(config); - if(!m_config->m_L2_config.disabled()) - m_L2cache = new l2_cache(L2c_name,m_config->m_L2_config,-1,-1,m_L2interface,m_mf_allocator,IN_PARTITION_L2_MISS_QUEUE, gpu); + if (!m_config->m_L2_config.disabled()) + m_L2cache = + new l2_cache(L2c_name, m_config->m_L2_config, -1, -1, m_L2interface, + m_mf_allocator, IN_PARTITION_L2_MISS_QUEUE, gpu); - unsigned int icnt_L2; - unsigned int L2_dram; - unsigned int dram_L2; - unsigned int L2_icnt; - sscanf(m_config->gpgpu_L2_queue_config,"%u:%u:%u:%u", &icnt_L2,&L2_dram,&dram_L2,&L2_icnt ); - m_icnt_L2_queue = new fifo_pipeline<mem_fetch>("icnt-to-L2",0,icnt_L2); - m_L2_dram_queue = new fifo_pipeline<mem_fetch>("L2-to-dram",0,L2_dram); - m_dram_L2_queue = new fifo_pipeline<mem_fetch>("dram-to-L2",0,dram_L2); - m_L2_icnt_queue = new fifo_pipeline<mem_fetch>("L2-to-icnt",0,L2_icnt); - wb_addr=-1; + unsigned int icnt_L2; + unsigned int L2_dram; + unsigned int dram_L2; + unsigned int L2_icnt; + sscanf(m_config->gpgpu_L2_queue_config, "%u:%u:%u:%u", &icnt_L2, &L2_dram, + &dram_L2, &L2_icnt); + m_icnt_L2_queue = new fifo_pipeline<mem_fetch>("icnt-to-L2", 0, icnt_L2); + m_L2_dram_queue = new fifo_pipeline<mem_fetch>("L2-to-dram", 0, L2_dram); + m_dram_L2_queue = new fifo_pipeline<mem_fetch>("dram-to-L2", 0, dram_L2); + m_L2_icnt_queue = new fifo_pipeline<mem_fetch>("L2-to-icnt", 0, L2_icnt); + wb_addr = -1; } -memory_sub_partition::~memory_sub_partition() -{ - delete m_icnt_L2_queue; - delete m_L2_dram_queue; - delete m_dram_L2_queue; - delete m_L2_icnt_queue; - delete m_L2cache; - delete m_L2interface; +memory_sub_partition::~memory_sub_partition() { + delete m_icnt_L2_queue; + delete m_L2_dram_queue; + delete m_dram_L2_queue; + delete m_L2_icnt_queue; + delete m_L2cache; + delete m_L2interface; } -void memory_sub_partition::cache_cycle( unsigned cycle ) -{ - // L2 fill responses - if( !m_config->m_L2_config.disabled()) { - if ( m_L2cache->access_ready() && !m_L2_icnt_queue->full() ) { - mem_fetch *mf = m_L2cache->next_access(); - if(mf->get_access_type() != L2_WR_ALLOC_R){ // Don't pass write allocate read request back to upper level cache - mf->set_reply(); - mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_icnt_queue->push(mf); - }else{ - if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE) - { - mem_fetch* original_wr_mf = mf->get_original_wr_mf(); - assert(original_wr_mf); - original_wr_mf->set_reply(); - original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_icnt_queue->push(original_wr_mf); - } - m_request_tracker.erase(mf); - delete mf; - } - } +void memory_sub_partition::cache_cycle(unsigned cycle) { + // L2 fill responses + if (!m_config->m_L2_config.disabled()) { + if (m_L2cache->access_ready() && !m_L2_icnt_queue->full()) { + mem_fetch *mf = m_L2cache->next_access(); + if (mf->get_access_type() != + L2_WR_ALLOC_R) { // Don't pass write allocate read request back to + // upper level cache + mf->set_reply(); + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); + } else { + if (m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE) { + mem_fetch *original_wr_mf = mf->get_original_wr_mf(); + assert(original_wr_mf); + original_wr_mf->set_reply(); + original_wr_mf->set_status( + IN_PARTITION_L2_TO_ICNT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_icnt_queue->push(original_wr_mf); + } + m_request_tracker.erase(mf); + delete mf; + } } + } - // DRAM to L2 (texture) and icnt (not texture) - if ( !m_dram_L2_queue->empty() ) { - mem_fetch *mf = m_dram_L2_queue->top(); - if ( !m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf) ) { - if (m_L2cache->fill_port_free()) { - mf->set_status(IN_PARTITION_L2_FILL_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2cache->fill(mf,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle+m_memcpy_cycle_offset); - m_dram_L2_queue->pop(); - } - } else if ( !m_L2_icnt_queue->full() ) { - if(mf->is_write() && mf->get_type() == WRITE_ACK) - mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_icnt_queue->push(mf); - m_dram_L2_queue->pop(); - } + // DRAM to L2 (texture) and icnt (not texture) + if (!m_dram_L2_queue->empty()) { + mem_fetch *mf = m_dram_L2_queue->top(); + if (!m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf)) { + if (m_L2cache->fill_port_free()) { + mf->set_status(IN_PARTITION_L2_FILL_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2cache->fill(mf, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle + + m_memcpy_cycle_offset); + m_dram_L2_queue->pop(); + } + } else if (!m_L2_icnt_queue->full()) { + if (mf->is_write() && mf->get_type() == WRITE_ACK) + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); + m_dram_L2_queue->pop(); } + } - // prior L2 misses inserted into m_L2_dram_queue here - if( !m_config->m_L2_config.disabled() ) - m_L2cache->cycle(); + // prior L2 misses inserted into m_L2_dram_queue here + if (!m_config->m_L2_config.disabled()) m_L2cache->cycle(); - // new L2 texture accesses and/or non-texture accesses - if ( !m_L2_dram_queue->full() && !m_icnt_L2_queue->empty() ) { - mem_fetch *mf = m_icnt_L2_queue->top(); - if ( !m_config->m_L2_config.disabled() && - ( (m_config->m_L2_texure_only && mf->istexture()) || (!m_config->m_L2_texure_only) ) - ) { - // L2 is enabled and access is for L2 - bool output_full = m_L2_icnt_queue->full(); - bool port_free = m_L2cache->data_port_free(); - if ( !output_full && port_free ) { - std::list<cache_event> events; - enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle+m_memcpy_cycle_offset,events); - bool write_sent = was_write_sent(events); - bool read_sent = was_read_sent(events); - MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", mf->get_addr(), status); + // new L2 texture accesses and/or non-texture accesses + if (!m_L2_dram_queue->full() && !m_icnt_L2_queue->empty()) { + mem_fetch *mf = m_icnt_L2_queue->top(); + if (!m_config->m_L2_config.disabled() && + ((m_config->m_L2_texure_only && mf->istexture()) || + (!m_config->m_L2_texure_only))) { + // L2 is enabled and access is for L2 + bool output_full = m_L2_icnt_queue->full(); + bool port_free = m_L2cache->data_port_free(); + if (!output_full && port_free) { + std::list<cache_event> events; + enum cache_request_status status = + m_L2cache->access(mf->get_addr(), mf, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle + + m_memcpy_cycle_offset, + events); + bool write_sent = was_write_sent(events); + bool read_sent = was_read_sent(events); + MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", + mf->get_addr(), status); - if ( status == HIT ) { - if( !write_sent ) { - // L2 cache replies - assert(!read_sent); - if( mf->get_access_type() == L1_WRBK_ACC ) { - m_request_tracker.erase(mf); - delete mf; - } else { - mf->set_reply(); - mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_icnt_queue->push(mf); - } - m_icnt_L2_queue->pop(); - } else { - assert(write_sent); - m_icnt_L2_queue->pop(); - } - } else if ( status != RESERVATION_FAIL ) { - if(mf->is_write() && (m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE || m_config->m_L2_config.m_write_alloc_policy == LAZY_FETCH_ON_READ) && !was_writeallocate_sent(events)) { - mf->set_reply(); - mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_icnt_queue->push(mf); - } - // L2 cache accepted request - m_icnt_L2_queue->pop(); - } else { - assert(!write_sent); - assert(!read_sent); - // L2 cache lock-up: will try again next cycle - } + if (status == HIT) { + if (!write_sent) { + // L2 cache replies + assert(!read_sent); + if (mf->get_access_type() == L1_WRBK_ACC) { + m_request_tracker.erase(mf); + delete mf; + } else { + mf->set_reply(); + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); } - } else { - // L2 is disabled or non-texture access to texture-only L2 - mf->set_status(IN_PARTITION_L2_TO_DRAM_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L2_dram_queue->push(mf); m_icnt_L2_queue->pop(); + } else { + assert(write_sent); + m_icnt_L2_queue->pop(); + } + } else if (status != RESERVATION_FAIL) { + if (mf->is_write() && + (m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE || + m_config->m_L2_config.m_write_alloc_policy == + LAZY_FETCH_ON_READ) && + !was_writeallocate_sent(events)) { + mf->set_reply(); + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); + } + // L2 cache accepted request + m_icnt_L2_queue->pop(); + } else { + assert(!write_sent); + assert(!read_sent); + // L2 cache lock-up: will try again next cycle } + } + } else { + // L2 is disabled or non-texture access to texture-only L2 + mf->set_status(IN_PARTITION_L2_TO_DRAM_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L2_dram_queue->push(mf); + m_icnt_L2_queue->pop(); } + } - // ROP delay queue - if( !m_rop.empty() && (cycle >= m_rop.front().ready_cycle) && !m_icnt_L2_queue->full() ) { - mem_fetch* mf = m_rop.front().req; - m_rop.pop(); - m_icnt_L2_queue->push(mf); - mf->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - } + // ROP delay queue + if (!m_rop.empty() && (cycle >= m_rop.front().ready_cycle) && + !m_icnt_L2_queue->full()) { + mem_fetch *mf = m_rop.front().req; + m_rop.pop(); + m_icnt_L2_queue->push(mf); + mf->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + } } -bool memory_sub_partition::full() const -{ - return m_icnt_L2_queue->full(); -} +bool memory_sub_partition::full() const { return m_icnt_L2_queue->full(); } -bool memory_sub_partition::full(unsigned size) const -{ - return m_icnt_L2_queue->is_avilable_size(size); +bool memory_sub_partition::full(unsigned size) const { + return m_icnt_L2_queue->is_avilable_size(size); } -bool memory_sub_partition::L2_dram_queue_empty() const -{ - return m_L2_dram_queue->empty(); +bool memory_sub_partition::L2_dram_queue_empty() const { + return m_L2_dram_queue->empty(); } -class mem_fetch* memory_sub_partition::L2_dram_queue_top() const -{ - return m_L2_dram_queue->top(); +class mem_fetch *memory_sub_partition::L2_dram_queue_top() const { + return m_L2_dram_queue->top(); } -void memory_sub_partition::L2_dram_queue_pop() -{ - m_L2_dram_queue->pop(); -} +void memory_sub_partition::L2_dram_queue_pop() { m_L2_dram_queue->pop(); } -bool memory_sub_partition::dram_L2_queue_full() const -{ - return m_dram_L2_queue->full(); +bool memory_sub_partition::dram_L2_queue_full() const { + return m_dram_L2_queue->full(); } -void memory_sub_partition::dram_L2_queue_push( class mem_fetch* mf ) -{ - m_dram_L2_queue->push(mf); +void memory_sub_partition::dram_L2_queue_push(class mem_fetch *mf) { + m_dram_L2_queue->push(mf); } -void memory_sub_partition::print_cache_stat(unsigned &accesses, unsigned &misses) const -{ - FILE *fp = stdout; - if( !m_config->m_L2_config.disabled() ) - m_L2cache->print(fp,accesses,misses); +void memory_sub_partition::print_cache_stat(unsigned &accesses, + unsigned &misses) const { + FILE *fp = stdout; + if (!m_config->m_L2_config.disabled()) m_L2cache->print(fp, accesses, misses); } -void memory_sub_partition::print( FILE *fp ) const -{ - if ( !m_request_tracker.empty() ) { - fprintf(fp,"Memory Sub Parition %u: pending memory requests:\n", m_id); - for ( std::set<mem_fetch*>::const_iterator r=m_request_tracker.begin(); r != m_request_tracker.end(); ++r ) { - mem_fetch *mf = *r; - if ( mf ) - mf->print(fp); - else - fprintf(fp," <NULL mem_fetch?>\n"); - } +void memory_sub_partition::print(FILE *fp) const { + if (!m_request_tracker.empty()) { + fprintf(fp, "Memory Sub Parition %u: pending memory requests:\n", m_id); + for (std::set<mem_fetch *>::const_iterator r = m_request_tracker.begin(); + r != m_request_tracker.end(); ++r) { + mem_fetch *mf = *r; + if (mf) + mf->print(fp); + else + fprintf(fp, " <NULL mem_fetch?>\n"); } - if( !m_config->m_L2_config.disabled() ) - m_L2cache->display_state(fp); + } + if (!m_config->m_L2_config.disabled()) m_L2cache->display_state(fp); } -void memory_stats_t::visualizer_print( gzFile visualizer_file ) -{ - gzprintf(visualizer_file, "Ltwowritemiss: %d\n", L2_write_miss); - gzprintf(visualizer_file, "Ltwowritehit: %d\n", L2_write_hit); - gzprintf(visualizer_file, "Ltworeadmiss: %d\n", L2_read_miss); - gzprintf(visualizer_file, "Ltworeadhit: %d\n", L2_read_hit); - clear_L2_stats_pw(); +void memory_stats_t::visualizer_print(gzFile visualizer_file) { + gzprintf(visualizer_file, "Ltwowritemiss: %d\n", L2_write_miss); + gzprintf(visualizer_file, "Ltwowritehit: %d\n", L2_write_hit); + gzprintf(visualizer_file, "Ltworeadmiss: %d\n", L2_read_miss); + gzprintf(visualizer_file, "Ltworeadhit: %d\n", L2_read_hit); + clear_L2_stats_pw(); - if (num_mfs) - gzprintf(visualizer_file, "averagemflatency: %lld\n", mf_total_lat/num_mfs); + if (num_mfs) + gzprintf(visualizer_file, "averagemflatency: %lld\n", + mf_total_lat / num_mfs); } -void memory_stats_t::clear_L2_stats_pw(){ - L2_write_miss = 0; - L2_write_hit = 0; - L2_read_miss = 0; - L2_read_hit = 0; +void memory_stats_t::clear_L2_stats_pw() { + L2_write_miss = 0; + L2_write_hit = 0; + L2_read_miss = 0; + L2_read_hit = 0; } -void gpgpu_sim::print_dram_stats(FILE *fout) const -{ - unsigned cmd=0; - unsigned activity=0; - unsigned nop=0; - unsigned act=0; - unsigned pre=0; - unsigned rd=0; - unsigned wr=0; - unsigned req=0; - unsigned tot_cmd=0; - unsigned tot_nop=0; - unsigned tot_act=0; - unsigned tot_pre=0; - unsigned tot_rd=0; - unsigned tot_wr=0; - unsigned tot_req=0; +void gpgpu_sim::print_dram_stats(FILE *fout) const { + unsigned cmd = 0; + unsigned activity = 0; + unsigned nop = 0; + unsigned act = 0; + unsigned pre = 0; + unsigned rd = 0; + unsigned wr = 0; + unsigned req = 0; + unsigned tot_cmd = 0; + unsigned tot_nop = 0; + unsigned tot_act = 0; + unsigned tot_pre = 0; + unsigned tot_rd = 0; + unsigned tot_wr = 0; + unsigned tot_req = 0; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++){ - m_memory_partition_unit[i]->set_dram_power_stats(cmd,activity,nop,act,pre,rd,wr,req); - tot_cmd+=cmd; - tot_nop+=nop; - tot_act+=act; - tot_pre+=pre; - tot_rd+=rd; - tot_wr+=wr; - tot_req+=req; - } - fprintf(fout,"gpgpu_n_dram_reads = %d\n",tot_rd ); - fprintf(fout,"gpgpu_n_dram_writes = %d\n",tot_wr ); - fprintf(fout,"gpgpu_n_dram_activate = %d\n",tot_act ); - fprintf(fout,"gpgpu_n_dram_commands = %d\n",tot_cmd); - fprintf(fout,"gpgpu_n_dram_noops = %d\n",tot_nop ); - fprintf(fout,"gpgpu_n_dram_precharges = %d\n",tot_pre ); - fprintf(fout,"gpgpu_n_dram_requests = %d\n",tot_req ); -} - -unsigned memory_sub_partition::flushL2() -{ - if (!m_config->m_L2_config.disabled()) { - m_L2cache->flush(); - } - return 0; //TODO: write the flushed data to the main memory + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) { + m_memory_partition_unit[i]->set_dram_power_stats(cmd, activity, nop, act, + pre, rd, wr, req); + tot_cmd += cmd; + tot_nop += nop; + tot_act += act; + tot_pre += pre; + tot_rd += rd; + tot_wr += wr; + tot_req += req; + } + fprintf(fout, "gpgpu_n_dram_reads = %d\n", tot_rd); + fprintf(fout, "gpgpu_n_dram_writes = %d\n", tot_wr); + fprintf(fout, "gpgpu_n_dram_activate = %d\n", tot_act); + fprintf(fout, "gpgpu_n_dram_commands = %d\n", tot_cmd); + fprintf(fout, "gpgpu_n_dram_noops = %d\n", tot_nop); + fprintf(fout, "gpgpu_n_dram_precharges = %d\n", tot_pre); + fprintf(fout, "gpgpu_n_dram_requests = %d\n", tot_req); } -unsigned memory_sub_partition::invalidateL2() -{ - if (!m_config->m_L2_config.disabled()) { - m_L2cache->invalidate(); - } - return 0; +unsigned memory_sub_partition::flushL2() { + if (!m_config->m_L2_config.disabled()) { + m_L2cache->flush(); + } + return 0; // TODO: write the flushed data to the main memory } -bool memory_sub_partition::busy() const -{ - return !m_request_tracker.empty(); +unsigned memory_sub_partition::invalidateL2() { + if (!m_config->m_L2_config.disabled()) { + m_L2cache->invalidate(); + } + return 0; } -std::vector<mem_fetch*> memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch* mf) -{ - std::vector<mem_fetch*> result; +bool memory_sub_partition::busy() const { return !m_request_tracker.empty(); } - if(mf->get_data_size() == SECTOR_SIZE && mf->get_access_sector_mask().count() == 1) { - result.push_back(mf); - } else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) { - //We only accept 32, 64 and 128 bytes reqs - unsigned start=0, end=0; - if(mf->get_data_size() == 128) { - start=0; end=3; - } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "1100") { - start=2; end=3; - } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "0011") { - start=0; end=1; - } else if (mf->get_data_size() == 64 && (mf->get_access_sector_mask().to_string() == "1111" || mf->get_access_sector_mask().to_string() == "0000")) { - if(mf->get_addr() % 128 == 0) { - start=0; end=1; - } else { - start=2; end=3; - } - } else - { - printf("Invalid sector received, address = 0x%06llx, sector mask = %s, data size = %d", - mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size()); - assert(0 && "Undefined sector mask is received"); - } +std::vector<mem_fetch *> +memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch *mf) { + std::vector<mem_fetch *> result; - std::bitset<SECTOR_SIZE*SECTOR_CHUNCK_SIZE> byte_sector_mask; - byte_sector_mask.reset(); - for(unsigned k=start*SECTOR_SIZE; k< SECTOR_SIZE; ++k) - byte_sector_mask.set(k); + if (mf->get_data_size() == SECTOR_SIZE && + mf->get_access_sector_mask().count() == 1) { + result.push_back(mf); + } else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) { + // We only accept 32, 64 and 128 bytes reqs + unsigned start = 0, end = 0; + if (mf->get_data_size() == 128) { + start = 0; + end = 3; + } else if (mf->get_data_size() == 64 && + mf->get_access_sector_mask().to_string() == "1100") { + start = 2; + end = 3; + } else if (mf->get_data_size() == 64 && + mf->get_access_sector_mask().to_string() == "0011") { + start = 0; + end = 1; + } else if (mf->get_data_size() == 64 && + (mf->get_access_sector_mask().to_string() == "1111" || + mf->get_access_sector_mask().to_string() == "0000")) { + if (mf->get_addr() % 128 == 0) { + start = 0; + end = 1; + } else { + start = 2; + end = 3; + } + } else { + printf( + "Invalid sector received, address = 0x%06llx, sector mask = %s, data " + "size = %d", + mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size()); + assert(0 && "Undefined sector mask is received"); + } - for(unsigned j=start, i=0; j<= end ; ++j, ++i){ + std::bitset<SECTOR_SIZE * SECTOR_CHUNCK_SIZE> byte_sector_mask; + byte_sector_mask.reset(); + for (unsigned k = start * SECTOR_SIZE; k < SECTOR_SIZE; ++k) + byte_sector_mask.set(k); - const mem_access_t *ma = new mem_access_t( mf->get_access_type(), - mf->get_addr() + SECTOR_SIZE*i, - SECTOR_SIZE, - mf->is_write(), - mf->get_access_warp_mask(), - mf->get_access_byte_mask() & byte_sector_mask, - std::bitset<SECTOR_CHUNCK_SIZE>().set(j), - m_gpu->gpgpu_ctx); + for (unsigned j = start, i = 0; j <= end; ++j, ++i) { + const mem_access_t *ma = new mem_access_t( + mf->get_access_type(), mf->get_addr() + SECTOR_SIZE * i, SECTOR_SIZE, + mf->is_write(), mf->get_access_warp_mask(), + mf->get_access_byte_mask() & byte_sector_mask, + std::bitset<SECTOR_CHUNCK_SIZE>().set(j), m_gpu->gpgpu_ctx); - mem_fetch *n_mf = new mem_fetch( *ma, - NULL, - mf->get_ctrl_size(), - mf->get_wid(), - mf->get_sid(), - mf->get_tpc(), - mf->get_mem_config(), - m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle, - mf); + mem_fetch *n_mf = + new mem_fetch(*ma, NULL, mf->get_ctrl_size(), mf->get_wid(), + mf->get_sid(), mf->get_tpc(), mf->get_mem_config(), + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, mf); - result.push_back(n_mf); - byte_sector_mask <<= SECTOR_SIZE; - } - } else { - printf("Invalid sector received, address = 0x%06llx, sector mask = %d, byte mask = , data size = %u", - mf->get_addr(), mf->get_access_sector_mask().count(), mf->get_data_size()); - assert(0 && "Undefined data size is received"); - } + result.push_back(n_mf); + byte_sector_mask <<= SECTOR_SIZE; + } + } else { + printf( + "Invalid sector received, address = 0x%06llx, sector mask = %d, byte " + "mask = , data size = %u", + mf->get_addr(), mf->get_access_sector_mask().count(), + mf->get_data_size()); + assert(0 && "Undefined data size is received"); + } - return result; + return result; } -void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle ) -{ - if (m_req) { - m_stats->memlatstat_icnt2mem_pop(m_req); - std::vector<mem_fetch*> reqs; - if(m_config->m_L2_config.m_cache_type == SECTOR) - reqs = breakdown_request_to_sector_requests(m_req); - else - reqs.push_back(m_req); +void memory_sub_partition::push(mem_fetch *m_req, unsigned long long cycle) { + if (m_req) { + m_stats->memlatstat_icnt2mem_pop(m_req); + std::vector<mem_fetch *> reqs; + if (m_config->m_L2_config.m_cache_type == SECTOR) + reqs = breakdown_request_to_sector_requests(m_req); + else + reqs.push_back(m_req); - for(unsigned i=0; i<reqs.size(); ++i) { - mem_fetch* req = reqs[i]; - m_request_tracker.insert(req); - if( req->istexture() ) { - m_icnt_L2_queue->push(req); - req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - } else { - rop_delay_t r; - r.req = req; - r.ready_cycle = cycle + m_config->rop_latency; - m_rop.push(r); - req->set_status(IN_PARTITION_ROP_DELAY,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - } - } + for (unsigned i = 0; i < reqs.size(); ++i) { + mem_fetch *req = reqs[i]; + m_request_tracker.insert(req); + if (req->istexture()) { + m_icnt_L2_queue->push(req); + req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + } else { + rop_delay_t r; + r.req = req; + r.ready_cycle = cycle + m_config->rop_latency; + m_rop.push(r); + req->set_status(IN_PARTITION_ROP_DELAY, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + } } + } } -mem_fetch* memory_sub_partition::pop() -{ - mem_fetch* mf = m_L2_icnt_queue->pop(); - m_request_tracker.erase(mf); - if ( mf && mf->isatomic() ) - mf->do_atomic(); - if( mf && (mf->get_access_type() == L2_WRBK_ACC || mf->get_access_type() == L1_WRBK_ACC) ) { - delete mf; - mf = NULL; - } - return mf; +mem_fetch *memory_sub_partition::pop() { + mem_fetch *mf = m_L2_icnt_queue->pop(); + m_request_tracker.erase(mf); + if (mf && mf->isatomic()) mf->do_atomic(); + if (mf && (mf->get_access_type() == L2_WRBK_ACC || + mf->get_access_type() == L1_WRBK_ACC)) { + delete mf; + mf = NULL; + } + return mf; } -mem_fetch* memory_sub_partition::top() -{ - mem_fetch *mf = m_L2_icnt_queue->top(); - if( mf && (mf->get_access_type() == L2_WRBK_ACC || mf->get_access_type() == L1_WRBK_ACC) ) { - m_L2_icnt_queue->pop(); - m_request_tracker.erase(mf); - delete mf; - mf = NULL; - } - return mf; +mem_fetch *memory_sub_partition::top() { + mem_fetch *mf = m_L2_icnt_queue->top(); + if (mf && (mf->get_access_type() == L2_WRBK_ACC || + mf->get_access_type() == L1_WRBK_ACC)) { + m_L2_icnt_queue->pop(); + m_request_tracker.erase(mf); + delete mf; + mf = NULL; + } + return mf; } -void memory_sub_partition::set_done( mem_fetch *mf ) -{ - m_request_tracker.erase(mf); +void memory_sub_partition::set_done(mem_fetch *mf) { + m_request_tracker.erase(mf); } -void memory_sub_partition::accumulate_L2cache_stats(class cache_stats &l2_stats) const { - if (!m_config->m_L2_config.disabled()) { - l2_stats += m_L2cache->get_stats(); - } +void memory_sub_partition::accumulate_L2cache_stats( + class cache_stats &l2_stats) const { + if (!m_config->m_L2_config.disabled()) { + l2_stats += m_L2cache->get_stats(); + } } -void memory_sub_partition::get_L2cache_sub_stats(struct cache_sub_stats &css) const{ - if (!m_config->m_L2_config.disabled()) { - m_L2cache->get_sub_stats(css); - } +void memory_sub_partition::get_L2cache_sub_stats( + struct cache_sub_stats &css) const { + if (!m_config->m_L2_config.disabled()) { + m_L2cache->get_sub_stats(css); + } } -void memory_sub_partition::get_L2cache_sub_stats_pw(struct cache_sub_stats_pw &css) const{ - if (!m_config->m_L2_config.disabled()) { - m_L2cache->get_sub_stats_pw(css); - } +void memory_sub_partition::get_L2cache_sub_stats_pw( + struct cache_sub_stats_pw &css) const { + if (!m_config->m_L2_config.disabled()) { + m_L2cache->get_sub_stats_pw(css); + } } void memory_sub_partition::clear_L2cache_stats_pw() { - if (!m_config->m_L2_config.disabled()) { - m_L2cache->clear_pw(); - } + if (!m_config->m_L2_config.disabled()) { + m_L2cache->clear_pw(); + } } -void memory_sub_partition::visualizer_print( gzFile visualizer_file ) -{ - // Support for L2 AerialVision stats - // Per-sub-partition stats would be trivial to extend from this - cache_sub_stats_pw temp_sub_stats; - get_L2cache_sub_stats_pw(temp_sub_stats); +void memory_sub_partition::visualizer_print(gzFile visualizer_file) { + // Support for L2 AerialVision stats + // Per-sub-partition stats would be trivial to extend from this + cache_sub_stats_pw temp_sub_stats; + get_L2cache_sub_stats_pw(temp_sub_stats); - m_stats->L2_read_miss += temp_sub_stats.read_misses; - m_stats->L2_write_miss += temp_sub_stats.write_misses; - m_stats->L2_read_hit += temp_sub_stats.read_hits; - m_stats->L2_write_hit += temp_sub_stats.write_hits; + m_stats->L2_read_miss += temp_sub_stats.read_misses; + m_stats->L2_write_miss += temp_sub_stats.write_misses; + m_stats->L2_read_hit += temp_sub_stats.read_hits; + m_stats->L2_write_hit += temp_sub_stats.write_hits; - clear_L2cache_stats_pw(); + clear_L2cache_stats_pw(); } diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h index 0f6fe32..3152db3 100644 --- a/src/gpgpu-sim/l2cache.h +++ b/src/gpgpu-sim/l2cache.h @@ -7,29 +7,30 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef MC_PARTITION_INCLUDED #define MC_PARTITION_INCLUDED -#include "dram.h" #include "../abstract_hardware_model.h" +#include "dram.h" #include <list> #include <queue> @@ -37,222 +38,221 @@ class mem_fetch; class partition_mf_allocator : public mem_fetch_allocator { -public: - partition_mf_allocator( const memory_config *config ) - { - m_memory_config = config; - } - virtual mem_fetch * alloc(const class warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle) const - { - abort(); - return NULL; - } - virtual mem_fetch * alloc(new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle) const; -private: - const memory_config *m_memory_config; + public: + partition_mf_allocator(const memory_config *config) { + m_memory_config = config; + } + virtual mem_fetch *alloc(const class warp_inst_t &inst, + const mem_access_t &access, + unsigned long long cycle) const { + abort(); + return NULL; + } + virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type, + unsigned size, bool wr, + unsigned long long cycle) const; + + private: + const memory_config *m_memory_config; }; -// Memory partition unit contains all the units assolcated with a single DRAM channel. -// - It arbitrates the DRAM channel among multiple sub partitions. -// - It does not connect directly with the interconnection network. -class memory_partition_unit -{ -public: - memory_partition_unit( unsigned partition_id, const memory_config *config, class memory_stats_t *stats, class gpgpu_sim* gpu ); - ~memory_partition_unit(); +// Memory partition unit contains all the units assolcated with a single DRAM +// channel. +// - It arbitrates the DRAM channel among multiple sub partitions. +// - It does not connect directly with the interconnection network. +class memory_partition_unit { + public: + memory_partition_unit(unsigned partition_id, const memory_config *config, + class memory_stats_t *stats, class gpgpu_sim *gpu); + ~memory_partition_unit(); - bool busy() const; + bool busy() const; - void cache_cycle( unsigned cycle ); - void dram_cycle(); - void simple_dram_model_cycle(); + void cache_cycle(unsigned cycle); + void dram_cycle(); + void simple_dram_model_cycle(); - void set_done( mem_fetch *mf ); + void set_done(mem_fetch *mf); - void visualizer_print( gzFile visualizer_file ) const; - void print_stat( FILE *fp ) { m_dram->print_stat(fp); } - void visualize() const { m_dram->visualize(); } - void print( FILE *fp ) const; - void handle_memcpy_to_gpu( size_t dst_start_addr, unsigned subpart_id, mem_access_sector_mask_t mask ); + void visualizer_print(gzFile visualizer_file) const; + void print_stat(FILE *fp) { m_dram->print_stat(fp); } + void visualize() const { m_dram->visualize(); } + void print(FILE *fp) const; + void handle_memcpy_to_gpu(size_t dst_start_addr, unsigned subpart_id, + mem_access_sector_mask_t mask); - class memory_sub_partition * get_sub_partition(int sub_partition_id) - { - return m_sub_partition[sub_partition_id]; - } + class memory_sub_partition *get_sub_partition(int sub_partition_id) { + return m_sub_partition[sub_partition_id]; + } - // Power model - void set_dram_power_stats(unsigned &n_cmd, - unsigned &n_activity, - unsigned &n_nop, - unsigned &n_act, - unsigned &n_pre, - unsigned &n_rd, - unsigned &n_wr, - unsigned &n_req) const; + // Power model + void set_dram_power_stats(unsigned &n_cmd, unsigned &n_activity, + unsigned &n_nop, unsigned &n_act, unsigned &n_pre, + unsigned &n_rd, unsigned &n_wr, + unsigned &n_req) const; - int global_sub_partition_id_to_local_id(int global_sub_partition_id) const; + int global_sub_partition_id_to_local_id(int global_sub_partition_id) const; - unsigned get_mpid() const { return m_id; } + unsigned get_mpid() const { return m_id; } - class gpgpu_sim* get_mgpu() const { return m_gpu; } + class gpgpu_sim *get_mgpu() const { + return m_gpu; + } -private: + private: + unsigned m_id; + const memory_config *m_config; + class memory_stats_t *m_stats; + class memory_sub_partition **m_sub_partition; + class dram_t *m_dram; - unsigned m_id; - const memory_config *m_config; - class memory_stats_t *m_stats; - class memory_sub_partition **m_sub_partition; - class dram_t *m_dram; + class arbitration_metadata { + public: + arbitration_metadata(const memory_config *config); - class arbitration_metadata - { - public: - arbitration_metadata(const memory_config *config); + // check if a subpartition still has credit + bool has_credits(int inner_sub_partition_id) const; + // borrow a credit for a subpartition + void borrow_credit(int inner_sub_partition_id); + // return a credit from a subpartition + void return_credit(int inner_sub_partition_id); - // check if a subpartition still has credit - bool has_credits(int inner_sub_partition_id) const; - // borrow a credit for a subpartition - void borrow_credit(int inner_sub_partition_id); - // return a credit from a subpartition - void return_credit(int inner_sub_partition_id); + // return the last subpartition that borrowed credit + int last_borrower() const { return m_last_borrower; } - // return the last subpartition that borrowed credit - int last_borrower() const { return m_last_borrower; } + void print(FILE *fp) const; - void print( FILE *fp ) const; - private: - // id of the last subpartition that borrowed credit - int m_last_borrower; + private: + // id of the last subpartition that borrowed credit + int m_last_borrower; - int m_shared_credit_limit; - int m_private_credit_limit; + int m_shared_credit_limit; + int m_private_credit_limit; - // credits borrowed by the subpartitions - std::vector<int> m_private_credit; - int m_shared_credit; - }; - arbitration_metadata m_arbitration_metadata; + // credits borrowed by the subpartitions + std::vector<int> m_private_credit; + int m_shared_credit; + }; + arbitration_metadata m_arbitration_metadata; - // determine wheither a given subpartition can issue to DRAM - bool can_issue_to_dram(int inner_sub_partition_id); + // determine wheither a given subpartition can issue to DRAM + bool can_issue_to_dram(int inner_sub_partition_id); - // model DRAM access scheduler latency (fixed latency between L2 and DRAM) - struct dram_delay_t - { - unsigned long long ready_cycle; - class mem_fetch* req; - }; - std::list<dram_delay_t> m_dram_latency_queue; + // model DRAM access scheduler latency (fixed latency between L2 and DRAM) + struct dram_delay_t { + unsigned long long ready_cycle; + class mem_fetch *req; + }; + std::list<dram_delay_t> m_dram_latency_queue; - class gpgpu_sim* m_gpu; + class gpgpu_sim *m_gpu; }; -class memory_sub_partition -{ -public: - memory_sub_partition( unsigned sub_partition_id, const memory_config *config, class memory_stats_t *stats, class gpgpu_sim* gpu ); - ~memory_sub_partition(); +class memory_sub_partition { + public: + memory_sub_partition(unsigned sub_partition_id, const memory_config *config, + class memory_stats_t *stats, class gpgpu_sim *gpu); + ~memory_sub_partition(); - unsigned get_id() const { return m_id; } + unsigned get_id() const { return m_id; } - bool busy() const; + bool busy() const; - void cache_cycle( unsigned cycle ); + void cache_cycle(unsigned cycle); - bool full() const; - bool full(unsigned size) const; - void push( class mem_fetch* mf, unsigned long long clock_cycle ); - class mem_fetch* pop(); - class mem_fetch* top(); - void set_done( mem_fetch *mf ); + bool full() const; + bool full(unsigned size) const; + void push(class mem_fetch *mf, unsigned long long clock_cycle); + class mem_fetch *pop(); + class mem_fetch *top(); + void set_done(mem_fetch *mf); - unsigned flushL2(); - unsigned invalidateL2(); + unsigned flushL2(); + unsigned invalidateL2(); - // interface to L2_dram_queue - bool L2_dram_queue_empty() const; - class mem_fetch* L2_dram_queue_top() const; - void L2_dram_queue_pop(); + // interface to L2_dram_queue + bool L2_dram_queue_empty() const; + class mem_fetch *L2_dram_queue_top() const; + void L2_dram_queue_pop(); - // interface to dram_L2_queue - bool dram_L2_queue_full() const; - void dram_L2_queue_push( class mem_fetch* mf ); + // interface to dram_L2_queue + bool dram_L2_queue_full() const; + void dram_L2_queue_push(class mem_fetch *mf); - void visualizer_print( gzFile visualizer_file ); - void print_cache_stat(unsigned &accesses, unsigned &misses) const; - void print( FILE *fp ) const; + void visualizer_print(gzFile visualizer_file); + void print_cache_stat(unsigned &accesses, unsigned &misses) const; + void print(FILE *fp) const; - void accumulate_L2cache_stats(class cache_stats &l2_stats) const; - void get_L2cache_sub_stats(struct cache_sub_stats &css) const; + void accumulate_L2cache_stats(class cache_stats &l2_stats) const; + void get_L2cache_sub_stats(struct cache_sub_stats &css) const; - // Support for getting per-window L2 stats for AerialVision - void get_L2cache_sub_stats_pw(struct cache_sub_stats_pw &css) const; - void clear_L2cache_stats_pw(); + // Support for getting per-window L2 stats for AerialVision + void get_L2cache_sub_stats_pw(struct cache_sub_stats_pw &css) const; + void clear_L2cache_stats_pw(); - void force_l2_tag_update(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask) - { - m_L2cache->force_tag_access( addr, m_memcpy_cycle_offset + time, mask ); - m_memcpy_cycle_offset += 1; - } + void force_l2_tag_update(new_addr_type addr, unsigned time, + mem_access_sector_mask_t mask) { + m_L2cache->force_tag_access(addr, m_memcpy_cycle_offset + time, mask); + m_memcpy_cycle_offset += 1; + } -private: -// data - unsigned m_id; //< the global sub partition ID - const memory_config *m_config; - class l2_cache *m_L2cache; - class L2interface *m_L2interface; - class gpgpu_sim* m_gpu; - partition_mf_allocator *m_mf_allocator; + private: + // data + unsigned m_id; //< the global sub partition ID + const memory_config *m_config; + class l2_cache *m_L2cache; + class L2interface *m_L2interface; + class gpgpu_sim *m_gpu; + partition_mf_allocator *m_mf_allocator; - // model delay of ROP units with a fixed latency - struct rop_delay_t - { - unsigned long long ready_cycle; - class mem_fetch* req; - }; - std::queue<rop_delay_t> m_rop; + // model delay of ROP units with a fixed latency + struct rop_delay_t { + unsigned long long ready_cycle; + class mem_fetch *req; + }; + std::queue<rop_delay_t> m_rop; - // these are various FIFOs between units within a memory partition - fifo_pipeline<mem_fetch> *m_icnt_L2_queue; - fifo_pipeline<mem_fetch> *m_L2_dram_queue; - fifo_pipeline<mem_fetch> *m_dram_L2_queue; - fifo_pipeline<mem_fetch> *m_L2_icnt_queue; // L2 cache hit response queue + // these are various FIFOs between units within a memory partition + fifo_pipeline<mem_fetch> *m_icnt_L2_queue; + fifo_pipeline<mem_fetch> *m_L2_dram_queue; + fifo_pipeline<mem_fetch> *m_dram_L2_queue; + fifo_pipeline<mem_fetch> *m_L2_icnt_queue; // L2 cache hit response queue - class mem_fetch *L2dramout; - unsigned long long int wb_addr; + class mem_fetch *L2dramout; + unsigned long long int wb_addr; - class memory_stats_t *m_stats; + class memory_stats_t *m_stats; - std::set<mem_fetch*> m_request_tracker; + std::set<mem_fetch *> m_request_tracker; - friend class L2interface; + friend class L2interface; - std::vector<mem_fetch*> breakdown_request_to_sector_requests(mem_fetch* mf); + std::vector<mem_fetch *> breakdown_request_to_sector_requests(mem_fetch *mf); - // This is a cycle offset that has to be applied to the l2 accesses to account for - // the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for kernel execution - // but we want cudamemcpy to go through the L2. Everytime an access is made from cudamemcpy - // this counter is incremented, and when the l2 is accessed (in both cudamemcpyies and otherwise) - // this value is added to the gpgpu-sim cycle counters. - unsigned m_memcpy_cycle_offset; + // This is a cycle offset that has to be applied to the l2 accesses to account + // for the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for + // kernel execution but we want cudamemcpy to go through the L2. Everytime an + // access is made from cudamemcpy this counter is incremented, and when the l2 + // is accessed (in both cudamemcpyies and otherwise) this value is added to + // the gpgpu-sim cycle counters. + unsigned m_memcpy_cycle_offset; }; class L2interface : public mem_fetch_interface { -public: - L2interface( memory_sub_partition *unit ) { m_unit=unit; } - virtual ~L2interface() {} - virtual bool full( unsigned size, bool write) const - { - // assume read and write packets all same size - return m_unit->m_L2_dram_queue->full(); - } - virtual void push(mem_fetch *mf) - { - mf->set_status(IN_PARTITION_L2_TO_DRAM_QUEUE,0/*FIXME*/); - m_unit->m_L2_dram_queue->push(mf); - } -private: - memory_sub_partition *m_unit; + public: + L2interface(memory_sub_partition *unit) { m_unit = unit; } + virtual ~L2interface() {} + virtual bool full(unsigned size, bool write) const { + // assume read and write packets all same size + return m_unit->m_L2_dram_queue->full(); + } + virtual void push(mem_fetch *mf) { + mf->set_status(IN_PARTITION_L2_TO_DRAM_QUEUE, 0 /*FIXME*/); + m_unit->m_L2_dram_queue->push(mf); + } + + private: + memory_sub_partition *m_unit; }; #endif diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h index d2dd948..8dd9241 100644 --- a/src/gpgpu-sim/l2cache_trace.h +++ b/src/gpgpu-sim/l2cache_trace.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Tim Rogers, Wilson W. L. Fung +// Copyright (c) 2009-2011, Tor M. Aamodt, Tim Rogers, Wilson W. L. Fung // The University of British Columbia // All rights reserved. // @@ -7,65 +7,73 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../trace.h" #if TRACING_ON #define MEMPART_PRINT_STR SIM_PRINT_STR " %d - " -#define MEMPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) ) +#define MEMPART_DTRACE(x) \ + (DTRACE(x) && (Trace::sampling_memory_partition == -1 || \ + Trace::sampling_memory_partition == (int)get_mpid())) #define MEM_SUBPART_PRINT_STR SIM_PRINT_STR " %d - " -#define MEM_SUBPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)m_id) ) +#define MEM_SUBPART_DTRACE(x) \ + (DTRACE(x) && (Trace::sampling_memory_partition == -1 || \ + Trace::sampling_memory_partition == (int)m_id)) // Intended to be called from inside components of a memory partition // Depends on a get_mpid() function -#define MEMPART_DPRINTF(...) do {\ - if (MEMPART_DTRACE(MEMORY_PARTITION_UNIT)) {\ - printf( MEMPART_PRINT_STR,\ - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::MEMORY_PARTITION_UNIT],\ - get_mpid() );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define MEMPART_DPRINTF(...) \ + do { \ + if (MEMPART_DTRACE(MEMORY_PARTITION_UNIT)) { \ + printf( \ + MEMPART_PRINT_STR, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::MEMORY_PARTITION_UNIT], get_mpid()); \ + printf(__VA_ARGS__); \ + } \ + } while (0) -#define MEM_SUBPART_DPRINTF(...) do {\ - if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) {\ - printf( MEM_SUBPART_PRINT_STR,\ - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT],\ - m_id );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define MEM_SUBPART_DPRINTF(...) \ + do { \ + if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) { \ + printf(MEM_SUBPART_PRINT_STR, \ + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT], m_id); \ + printf(__VA_ARGS__); \ + } \ + } while (0) #else -#define MEMPART_DTRACE(x) (false) -#define MEMPART_DPRINTF(x, ...) do {} while (0) +#define MEMPART_DTRACE(x) (false) +#define MEMPART_DPRINTF(x, ...) \ + do { \ + } while (0) -#define MEM_SUBPART_DTRACE(x) (false) -#define MEM_SUBPART_DPRINTF(x, ...) do {} while (0) +#define MEM_SUBPART_DTRACE(x) (false) +#define MEM_SUBPART_DPRINTF(x, ...) \ + do { \ + } while (0) #endif - diff --git a/src/gpgpu-sim/local_interconnect.cc b/src/gpgpu-sim/local_interconnect.cc index c70477c..cd32386 100644 --- a/src/gpgpu-sim/local_interconnect.cc +++ b/src/gpgpu-sim/local_interconnect.cc @@ -7,371 +7,362 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include <algorithm> +#include <cmath> #include <fstream> +#include <iomanip> #include <iostream> #include <sstream> -#include <iomanip> -#include <cmath> #include <utility> -#include <algorithm> #include "local_interconnect.h" #include "mem_fetch.h" -xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit, enum Arbiteration_type m_arbit_type) -{ - m_id=router_id; - router_type=m_type; - _n_mem = n_mem; - _n_shader = n_shader; - total_nodes = n_shader+n_mem; - in_buffers.resize(total_nodes); - out_buffers.resize(total_nodes); - next_node.resize(total_nodes,0); - in_buffer_limit = m_in_buffer_limit; - out_buffer_limit = m_out_buffer_limit; - arbit_type = m_arbit_type; - next_node_id=0; - if(m_type == REQ_NET) { - active_in_buffers=n_shader; - active_out_buffers=n_mem; - } - else if(m_type == REPLY_NET) { - active_in_buffers=n_mem; - active_out_buffers=n_shader; - } +xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, + unsigned n_shader, unsigned n_mem, + unsigned m_in_buffer_limit, + unsigned m_out_buffer_limit, + enum Arbiteration_type m_arbit_type) { + m_id = router_id; + router_type = m_type; + _n_mem = n_mem; + _n_shader = n_shader; + total_nodes = n_shader + n_mem; + in_buffers.resize(total_nodes); + out_buffers.resize(total_nodes); + next_node.resize(total_nodes, 0); + in_buffer_limit = m_in_buffer_limit; + out_buffer_limit = m_out_buffer_limit; + arbit_type = m_arbit_type; + next_node_id = 0; + if (m_type == REQ_NET) { + active_in_buffers = n_shader; + active_out_buffers = n_mem; + } else if (m_type == REPLY_NET) { + active_in_buffers = n_mem; + active_out_buffers = n_shader; + } - cycles = 0; - conflicts= 0; - out_buffer_full=0; - in_buffer_full=0; - out_buffer_util=0; - in_buffer_util=0; - packets_num=0; + cycles = 0; + conflicts = 0; + out_buffer_full = 0; + in_buffer_full = 0; + out_buffer_util = 0; + in_buffer_util = 0; + packets_num = 0; } +xbar_router::~xbar_router() {} -xbar_router::~xbar_router() -{ - -} - -void xbar_router::Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size) -{ - assert(input_deviceID < total_nodes); - in_buffers[input_deviceID].push(Packet(data, output_deviceID)); - packets_num++; +void xbar_router::Push(unsigned input_deviceID, unsigned output_deviceID, + void* data, unsigned int size) { + assert(input_deviceID < total_nodes); + in_buffers[input_deviceID].push(Packet(data, output_deviceID)); + packets_num++; } -void* xbar_router::Pop(unsigned ouput_deviceID) -{ - assert(ouput_deviceID < total_nodes); - void* data = NULL; +void* xbar_router::Pop(unsigned ouput_deviceID) { + assert(ouput_deviceID < total_nodes); + void* data = NULL; - if(!out_buffers[ouput_deviceID].empty()) { - data = out_buffers[ouput_deviceID].front().data; - out_buffers[ouput_deviceID].pop(); - } + if (!out_buffers[ouput_deviceID].empty()) { + data = out_buffers[ouput_deviceID].front().data; + out_buffers[ouput_deviceID].pop(); + } - return data; + return data; } +bool xbar_router::Has_Buffer_In(unsigned input_deviceID, unsigned size, + bool update_counter) { + assert(input_deviceID < total_nodes); -bool xbar_router::Has_Buffer_In(unsigned input_deviceID, unsigned size, bool update_counter){ - - assert(input_deviceID < total_nodes); - - bool has_buffer = (in_buffers[input_deviceID].size() + size <= in_buffer_limit); - if(update_counter && !has_buffer) - in_buffer_full++; - - return has_buffer; + bool has_buffer = + (in_buffers[input_deviceID].size() + size <= in_buffer_limit); + if (update_counter && !has_buffer) in_buffer_full++; + return has_buffer; } -bool xbar_router::Has_Buffer_Out(unsigned output_deviceID, unsigned size){ - return (out_buffers[output_deviceID].size() + size <= out_buffer_limit); +bool xbar_router::Has_Buffer_Out(unsigned output_deviceID, unsigned size) { + return (out_buffers[output_deviceID].size() + size <= out_buffer_limit); } void xbar_router::Advance() { - - if(arbit_type == NAIVE_RR) - RR_Advance(); - else if(arbit_type == iSLIP) - iSLIP_Advance(); - else - assert(0); - + if (arbit_type == NAIVE_RR) + RR_Advance(); + else if (arbit_type == iSLIP) + iSLIP_Advance(); + else + assert(0); } void xbar_router::RR_Advance() { - cycles++; + cycles++; - vector<bool> issued(total_nodes, false); + vector<bool> issued(total_nodes, false); - for(unsigned i=0; i<total_nodes; ++i){ - unsigned node_id = (i+next_node_id)%total_nodes; + for (unsigned i = 0; i < total_nodes; ++i) { + unsigned node_id = (i + next_node_id) % total_nodes; - if(!in_buffers[node_id].empty()) { - Packet _packet = in_buffers[node_id].front(); - //ensure that the outbuffer has space and not issued before in this cycle - if(Has_Buffer_Out(_packet.output_deviceID, 1)){ - if(!issued[_packet.output_deviceID]) { - out_buffers[_packet.output_deviceID].push(_packet); - in_buffers[node_id].pop(); - issued[_packet.output_deviceID]=true; - } - else - conflicts++; - } - else { - out_buffer_full++; + if (!in_buffers[node_id].empty()) { + Packet _packet = in_buffers[node_id].front(); + // ensure that the outbuffer has space and not issued before in this cycle + if (Has_Buffer_Out(_packet.output_deviceID, 1)) { + if (!issued[_packet.output_deviceID]) { + out_buffers[_packet.output_deviceID].push(_packet); + in_buffers[node_id].pop(); + issued[_packet.output_deviceID] = true; + } else + conflicts++; + } else { + out_buffer_full++; - if(issued[_packet.output_deviceID]) - conflicts++; - } - } - } + if (issued[_packet.output_deviceID]) conflicts++; + } + } + } - next_node_id = (++next_node_id % total_nodes); + next_node_id = (++next_node_id % total_nodes); - //collect some stats about buffer util - for(unsigned i=0; i<total_nodes; ++i){ - in_buffer_util+=in_buffers[i].size(); - out_buffer_util+=out_buffers[i].size(); - } + // collect some stats about buffer util + for (unsigned i = 0; i < total_nodes; ++i) { + in_buffer_util += in_buffers[i].size(); + out_buffer_util += out_buffers[i].size(); + } } -//iSLIP algorithm -//McKeown, Nick. "The iSLIP scheduling algorithm for input-queued switches." IEEE/ACM transactions on networking 2 (1999): 188-201. -//https://www.cs.rutgers.edu/~sn624/552-F18/papers/islip.pdf +// iSLIP algorithm +// McKeown, Nick. "The iSLIP scheduling algorithm for input-queued switches." +// IEEE/ACM transactions on networking 2 (1999): 188-201. +// https://www.cs.rutgers.edu/~sn624/552-F18/papers/islip.pdf void xbar_router::iSLIP_Advance() { - cycles++; - - vector<unsigned> node_tmp; - + cycles++; - //calcaulte how many conflicts are there for stats - for (unsigned i=0; i<total_nodes; ++i){ - - if(!in_buffers[i].empty()){ - Packet _packet_tmp = in_buffers[i].front(); - if (!node_tmp.empty()){ - if (std::find(node_tmp.begin(), node_tmp.end(), _packet_tmp.output_deviceID)!=node_tmp.end()){ - conflicts++; - } - else - node_tmp.push_back(_packet_tmp.output_deviceID); - } - else{ - node_tmp.push_back(_packet_tmp.output_deviceID); - } - } - } + vector<unsigned> node_tmp; + // calcaulte how many conflicts are there for stats + for (unsigned i = 0; i < total_nodes; ++i) { + if (!in_buffers[i].empty()) { + Packet _packet_tmp = in_buffers[i].front(); + if (!node_tmp.empty()) { + if (std::find(node_tmp.begin(), node_tmp.end(), + _packet_tmp.output_deviceID) != node_tmp.end()) { + conflicts++; + } else + node_tmp.push_back(_packet_tmp.output_deviceID); + } else { + node_tmp.push_back(_packet_tmp.output_deviceID); + } + } + } - //do iSLIP - for(unsigned i=0; i<total_nodes; ++i){ + // do iSLIP + for (unsigned i = 0; i < total_nodes; ++i) { + if (Has_Buffer_Out(i, 1)) { + for (unsigned j = 0; j < total_nodes; ++j) { + unsigned node_id = (j + next_node[i]) % total_nodes; - if(Has_Buffer_Out(i, 1)) { - for(unsigned j=0; j<total_nodes; ++j){ - unsigned node_id = (j+next_node[i])%total_nodes; + if (!in_buffers[node_id].empty()) { + Packet _packet = in_buffers[node_id].front(); + if (_packet.output_deviceID == i) { + out_buffers[_packet.output_deviceID].push(_packet); + in_buffers[node_id].pop(); + next_node[i] = (++node_id % total_nodes); + break; + } + } + } + } else + out_buffer_full++; + } - if(!in_buffers[node_id].empty()) { - Packet _packet = in_buffers[node_id].front(); - if(_packet.output_deviceID==i){ - out_buffers[_packet.output_deviceID].push(_packet); - in_buffers[node_id].pop(); - next_node[i] = (++node_id % total_nodes); - break; - } - } - } - } - else - out_buffer_full++; - } - - //collect some stats about buffer util - for(unsigned i=0; i<total_nodes; ++i){ - in_buffer_util+=in_buffers[i].size(); - out_buffer_util+=out_buffers[i].size(); - } + // collect some stats about buffer util + for (unsigned i = 0; i < total_nodes; ++i) { + in_buffer_util += in_buffers[i].size(); + out_buffer_util += out_buffers[i].size(); + } } - - bool xbar_router::Busy() const { + for (unsigned i = 0; i < total_nodes; ++i) { + if (!in_buffers[i].empty()) return true; - for(unsigned i=0; i<total_nodes; ++i){ - if(!in_buffers[i].empty()) - return true; - - if(!out_buffers[i].empty()) - return true; - } - return false; + if (!out_buffers[i].empty()) return true; + } + return false; } - //////////////////////////////////////////////////// /////////////LocalInterconnect///////////////////// -//assume all the packets are one flit +// assume all the packets are one flit #define LOCAL_INCT_FLIT_SIZE 40 -LocalInterconnect* LocalInterconnect::New(const struct inct_config& m_localinct_config) -{ +LocalInterconnect* LocalInterconnect::New( + const struct inct_config& m_localinct_config) { + LocalInterconnect* icnt_interface = new LocalInterconnect(m_localinct_config); - LocalInterconnect* icnt_interface = new LocalInterconnect(m_localinct_config); - - return icnt_interface; + return icnt_interface; } -LocalInterconnect::LocalInterconnect(const struct inct_config& m_localinct_config): m_inct_config(m_localinct_config){ - n_shader=0; - n_mem=0; - n_subnets = m_localinct_config.subnets; +LocalInterconnect::LocalInterconnect( + const struct inct_config& m_localinct_config) + : m_inct_config(m_localinct_config) { + n_shader = 0; + n_mem = 0; + n_subnets = m_localinct_config.subnets; } -LocalInterconnect::~LocalInterconnect(){ - for (unsigned i = 0; i < m_inct_config.subnets; ++i) { - delete net[i]; - } +LocalInterconnect::~LocalInterconnect() { + for (unsigned i = 0; i < m_inct_config.subnets; ++i) { + delete net[i]; + } } -void LocalInterconnect::CreateInterconnect(unsigned m_n_shader, unsigned m_n_mem){ - n_shader = m_n_shader; - n_mem = m_n_mem; - - net.resize(n_subnets); - for (unsigned i = 0; i < n_subnets; ++i) { - net[i] = new xbar_router( i, static_cast<Interconnect_type>(i), m_n_shader, m_n_mem, m_inct_config.in_buffer_limit, m_inct_config.out_buffer_limit,m_inct_config.arbiter_algo); - } +void LocalInterconnect::CreateInterconnect(unsigned m_n_shader, + unsigned m_n_mem) { + n_shader = m_n_shader; + n_mem = m_n_mem; + net.resize(n_subnets); + for (unsigned i = 0; i < n_subnets; ++i) { + net[i] = new xbar_router(i, static_cast<Interconnect_type>(i), m_n_shader, + m_n_mem, m_inct_config.in_buffer_limit, + m_inct_config.out_buffer_limit, + m_inct_config.arbiter_algo); + } } - void LocalInterconnect::Init() { - //empty - //there is nothing to do - -} - -void LocalInterconnect::Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size){ - - unsigned subnet; - if (n_subnets == 1) { - subnet = 0; - } else { - if (input_deviceID < n_shader ) { - subnet = 0; - } else { - subnet = 1; - } - } - - // it should have free buffer - //assume all the packets have size of one - //no flits are implemented - assert(net[subnet]->Has_Buffer_In(input_deviceID, 1)); - - net[subnet]->Push(input_deviceID, output_deviceID, data, size); - + // empty + // there is nothing to do } -void* LocalInterconnect::Pop(unsigned ouput_deviceID){ +void LocalInterconnect::Push(unsigned input_deviceID, unsigned output_deviceID, + void* data, unsigned int size) { + unsigned subnet; + if (n_subnets == 1) { + subnet = 0; + } else { + if (input_deviceID < n_shader) { + subnet = 0; + } else { + subnet = 1; + } + } - // 0-_n_shader-1 indicates reply(network 1), otherwise request(network 0) - int subnet = 0; - if (ouput_deviceID < n_shader) - subnet = 1; - - return net[subnet]->Pop(ouput_deviceID); + // it should have free buffer + // assume all the packets have size of one + // no flits are implemented + assert(net[subnet]->Has_Buffer_In(input_deviceID, 1)); + net[subnet]->Push(input_deviceID, output_deviceID, data, size); } -void LocalInterconnect::Advance(){ - - for (unsigned i = 0; i < n_subnets; ++i) { - net[i]->Advance(); - } +void* LocalInterconnect::Pop(unsigned ouput_deviceID) { + // 0-_n_shader-1 indicates reply(network 1), otherwise request(network 0) + int subnet = 0; + if (ouput_deviceID < n_shader) subnet = 1; + return net[subnet]->Pop(ouput_deviceID); } -bool LocalInterconnect::Busy() const{ - - for (unsigned i = 0; i < n_subnets; ++i) { - if(net[i]->Busy()) - return true; - } - return false; +void LocalInterconnect::Advance() { + for (unsigned i = 0; i < n_subnets; ++i) { + net[i]->Advance(); + } } -bool LocalInterconnect::HasBuffer(unsigned deviceID, unsigned int size) const{ - - bool has_buffer = false; - - if ((n_subnets>1) && deviceID >= n_shader) // deviceID is memory node - has_buffer = net[REPLY_NET]->Has_Buffer_In(deviceID, 1, true); - else - has_buffer = net[REQ_NET]->Has_Buffer_In(deviceID, 1, true); - - return has_buffer; - +bool LocalInterconnect::Busy() const { + for (unsigned i = 0; i < n_subnets; ++i) { + if (net[i]->Busy()) return true; + } + return false; } -void LocalInterconnect::DisplayStats() const{ - - cout<<"Req_Network_injected_packets_num = "<<net[REQ_NET]->packets_num<<endl; - cout<<"Req_Network_cycles = "<<net[REQ_NET]->cycles<<endl; - cout<<"Req_Network_injected_packets_per_cycle = "<<(float)(net[REQ_NET]->packets_num) / (net[REQ_NET]->cycles)<<endl; - cout<<"Req_Network_conflicts_per_cycle = "<<(float)(net[REQ_NET]->conflicts) / (net[REQ_NET]->cycles)<<endl; - cout<<"Req_Network_in_buffer_full_per_cycle = "<<(float)(net[REQ_NET]->in_buffer_full) / (net[REQ_NET]->cycles)<<endl; - cout<<"Req_Network_in_buffer_avg_util = "<<((float)(net[REQ_NET]->in_buffer_util) / (net[REQ_NET]->cycles) / net[REQ_NET]->active_in_buffers)<<endl; - cout<<"Req_Network_out_buffer_full_per_cycle = "<<(float)(net[REQ_NET]->out_buffer_full) / (net[REQ_NET]->cycles)<<endl; - cout<<"Req_Network_out_buffer_avg_util = "<<((float)(net[REQ_NET]->out_buffer_util) / (net[REQ_NET]->cycles) / net[REQ_NET]->active_out_buffers)<<endl; +bool LocalInterconnect::HasBuffer(unsigned deviceID, unsigned int size) const { + bool has_buffer = false; - cout<<endl; - cout<<"Reply_Network_injected_packets_num = "<<net[REPLY_NET]->packets_num<<endl; - cout<<"Reply_Network_cycles = "<<net[REPLY_NET]->cycles<<endl; - cout<<"Reply_Network_injected_packets_per_cycle = "<<(float)(net[REPLY_NET]->packets_num) / (net[REPLY_NET]->cycles)<<endl; - cout<<"Reply_Network_conflicts_per_cycle = "<<(float)(net[REPLY_NET]->conflicts) / (net[REPLY_NET]->cycles)<<endl; - cout<<"Reply_Network_in_buffer_full_per_cycle = "<<(float)(net[REPLY_NET]->in_buffer_full) / (net[REPLY_NET]->cycles)<<endl; - cout<<"Reply_Network_in_buffer_avg_util = "<<((float)(net[REPLY_NET]->in_buffer_util) / (net[REPLY_NET]->cycles) / net[REPLY_NET]->active_in_buffers)<<endl; - cout<<"Reply_Network_out_buffer_full_per_cycle = "<<(float)(net[REPLY_NET]->out_buffer_full) / (net[REPLY_NET]->cycles)<<endl; - cout<<"Reply_Network_out_buffer_avg_util= "<<((float)(net[REPLY_NET]->out_buffer_util) / (net[REPLY_NET]->cycles) / net[REPLY_NET]->active_out_buffers)<<endl; + if ((n_subnets > 1) && deviceID >= n_shader) // deviceID is memory node + has_buffer = net[REPLY_NET]->Has_Buffer_In(deviceID, 1, true); + else + has_buffer = net[REQ_NET]->Has_Buffer_In(deviceID, 1, true); + return has_buffer; } -void LocalInterconnect::DisplayOverallStats() const{ +void LocalInterconnect::DisplayStats() const { + cout << "Req_Network_injected_packets_num = " << net[REQ_NET]->packets_num + << endl; + cout << "Req_Network_cycles = " << net[REQ_NET]->cycles << endl; + cout << "Req_Network_injected_packets_per_cycle = " + << (float)(net[REQ_NET]->packets_num) / (net[REQ_NET]->cycles) << endl; + cout << "Req_Network_conflicts_per_cycle = " + << (float)(net[REQ_NET]->conflicts) / (net[REQ_NET]->cycles) << endl; + cout << "Req_Network_in_buffer_full_per_cycle = " + << (float)(net[REQ_NET]->in_buffer_full) / (net[REQ_NET]->cycles) + << endl; + cout << "Req_Network_in_buffer_avg_util = " + << ((float)(net[REQ_NET]->in_buffer_util) / (net[REQ_NET]->cycles) / + net[REQ_NET]->active_in_buffers) + << endl; + cout << "Req_Network_out_buffer_full_per_cycle = " + << (float)(net[REQ_NET]->out_buffer_full) / (net[REQ_NET]->cycles) + << endl; + cout << "Req_Network_out_buffer_avg_util = " + << ((float)(net[REQ_NET]->out_buffer_util) / (net[REQ_NET]->cycles) / + net[REQ_NET]->active_out_buffers) + << endl; + cout << endl; + cout << "Reply_Network_injected_packets_num = " << net[REPLY_NET]->packets_num + << endl; + cout << "Reply_Network_cycles = " << net[REPLY_NET]->cycles << endl; + cout << "Reply_Network_injected_packets_per_cycle = " + << (float)(net[REPLY_NET]->packets_num) / (net[REPLY_NET]->cycles) + << endl; + cout << "Reply_Network_conflicts_per_cycle = " + << (float)(net[REPLY_NET]->conflicts) / (net[REPLY_NET]->cycles) << endl; + cout << "Reply_Network_in_buffer_full_per_cycle = " + << (float)(net[REPLY_NET]->in_buffer_full) / (net[REPLY_NET]->cycles) + << endl; + cout << "Reply_Network_in_buffer_avg_util = " + << ((float)(net[REPLY_NET]->in_buffer_util) / (net[REPLY_NET]->cycles) / + net[REPLY_NET]->active_in_buffers) + << endl; + cout << "Reply_Network_out_buffer_full_per_cycle = " + << (float)(net[REPLY_NET]->out_buffer_full) / (net[REPLY_NET]->cycles) + << endl; + cout << "Reply_Network_out_buffer_avg_util= " + << ((float)(net[REPLY_NET]->out_buffer_util) / (net[REPLY_NET]->cycles) / + net[REPLY_NET]->active_out_buffers) + << endl; } -unsigned LocalInterconnect::GetFlitSize() const{ - return LOCAL_INCT_FLIT_SIZE; -} +void LocalInterconnect::DisplayOverallStats() const {} -void LocalInterconnect::DisplayState(FILE* fp) const{ +unsigned LocalInterconnect::GetFlitSize() const { return LOCAL_INCT_FLIT_SIZE; } - fprintf(fp, "GPGPU-Sim uArch: ICNT:Display State: Under implementation\n"); +void LocalInterconnect::DisplayState(FILE* fp) const { + fprintf(fp, "GPGPU-Sim uArch: ICNT:Display State: Under implementation\n"); } - diff --git a/src/gpgpu-sim/local_interconnect.h b/src/gpgpu-sim/local_interconnect.h index a784da8..29cd903 100644 --- a/src/gpgpu-sim/local_interconnect.h +++ b/src/gpgpu-sim/local_interconnect.h @@ -7,134 +7,124 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef _LOCAL_INTERCONNECT_HPP_ #define _LOCAL_INTERCONNECT_HPP_ -#include <vector> -#include <queue> #include <iostream> #include <map> +#include <queue> +#include <vector> using namespace std; +enum Interconnect_type { REQ_NET = 0, REPLY_NET = 1 }; -enum Interconnect_type { - REQ_NET=0, - REPLY_NET=1 -}; - -enum Arbiteration_type { - NAIVE_RR=0, - iSLIP=1 -}; +enum Arbiteration_type { NAIVE_RR = 0, iSLIP = 1 }; -struct inct_config -{ - //config for local interconnect - unsigned in_buffer_limit; - unsigned out_buffer_limit; - unsigned subnets; - Arbiteration_type arbiter_algo; +struct inct_config { + // config for local interconnect + unsigned in_buffer_limit; + unsigned out_buffer_limit; + unsigned subnets; + Arbiteration_type arbiter_algo; }; class xbar_router { + public: + xbar_router(unsigned router_id, enum Interconnect_type m_type, + unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, + unsigned m_out_buffer_limit, enum Arbiteration_type m_arbit_type); + ~xbar_router(); + void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, + unsigned int size); + void* Pop(unsigned ouput_deviceID); + void Advance(); -public: - xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit, enum Arbiteration_type m_arbit_type); - ~xbar_router(); - void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size); - void* Pop(unsigned ouput_deviceID); - void Advance( ); - + bool Busy() const; + bool Has_Buffer_In(unsigned input_deviceID, unsigned size, + bool update_counter = false); + bool Has_Buffer_Out(unsigned output_deviceID, unsigned size); - bool Busy() const; - bool Has_Buffer_In(unsigned input_deviceID, unsigned size, bool update_counter=false); - bool Has_Buffer_Out(unsigned output_deviceID, unsigned size); + // some stats + unsigned long long cycles; + unsigned long long conflicts; + unsigned long long out_buffer_full; + unsigned long long out_buffer_util; + unsigned long long in_buffer_full; + unsigned long long in_buffer_util; + unsigned long long packets_num; - //some stats - unsigned long long cycles; - unsigned long long conflicts; - unsigned long long out_buffer_full; - unsigned long long out_buffer_util; - unsigned long long in_buffer_full; - unsigned long long in_buffer_util; - unsigned long long packets_num; + private: + void iSLIP_Advance(); + void RR_Advance(); -private: - void iSLIP_Advance(); - void RR_Advance(); - - struct Packet{ - Packet(void* m_data, unsigned m_output_deviceID) { - data = m_data; - output_deviceID = m_output_deviceID; - } - void* data; - unsigned output_deviceID; - }; - vector<queue<Packet> > in_buffers; - vector<queue<Packet> > out_buffers; - unsigned _n_shader, _n_mem, total_nodes; - unsigned in_buffer_limit, out_buffer_limit; - vector<unsigned> next_node; //used for iSLIP arbit - unsigned next_node_id; //used for RR arbit - unsigned m_id; - enum Interconnect_type router_type; - unsigned active_in_buffers,active_out_buffers; - Arbiteration_type arbit_type; - - friend class LocalInterconnect; + struct Packet { + Packet(void* m_data, unsigned m_output_deviceID) { + data = m_data; + output_deviceID = m_output_deviceID; + } + void* data; + unsigned output_deviceID; + }; + vector<queue<Packet> > in_buffers; + vector<queue<Packet> > out_buffers; + unsigned _n_shader, _n_mem, total_nodes; + unsigned in_buffer_limit, out_buffer_limit; + vector<unsigned> next_node; // used for iSLIP arbit + unsigned next_node_id; // used for RR arbit + unsigned m_id; + enum Interconnect_type router_type; + unsigned active_in_buffers, active_out_buffers; + Arbiteration_type arbit_type; + friend class LocalInterconnect; }; class LocalInterconnect { -public: - LocalInterconnect(const struct inct_config& m_localinct_config); - ~LocalInterconnect(); - static LocalInterconnect* New(const struct inct_config& m_inct_config); - void CreateInterconnect(unsigned n_shader, unsigned n_mem); - - //node side functions - void Init(); - void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size); - void* Pop(unsigned ouput_deviceID); - void Advance(); - bool Busy() const; - bool HasBuffer(unsigned deviceID, unsigned int size) const; - void DisplayStats() const; - void DisplayOverallStats() const; - unsigned GetFlitSize() const; - - void DisplayState(FILE* fp) const; + public: + LocalInterconnect(const struct inct_config& m_localinct_config); + ~LocalInterconnect(); + static LocalInterconnect* New(const struct inct_config& m_inct_config); + void CreateInterconnect(unsigned n_shader, unsigned n_mem); + // node side functions + void Init(); + void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, + unsigned int size); + void* Pop(unsigned ouput_deviceID); + void Advance(); + bool Busy() const; + bool HasBuffer(unsigned deviceID, unsigned int size) const; + void DisplayStats() const; + void DisplayOverallStats() const; + unsigned GetFlitSize() const; -protected: + void DisplayState(FILE* fp) const; - const inct_config& m_inct_config; - - unsigned n_shader, n_mem; - unsigned n_subnets; - vector<xbar_router *> net; + protected: + const inct_config& m_inct_config; + unsigned n_shader, n_mem; + unsigned n_subnets; + vector<xbar_router*> net; }; #endif - - diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index 6a00889..7cb02cf 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -7,137 +7,131 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "mem_fetch.h" +#include "gpu-sim.h" #include "mem_latency_stat.h" #include "shader.h" #include "visualizer.h" -#include "gpu-sim.h" -unsigned mem_fetch::sm_next_mf_request_uid=1; +unsigned mem_fetch::sm_next_mf_request_uid = 1; -mem_fetch::mem_fetch( const mem_access_t &access, - const warp_inst_t *inst, - unsigned ctrl_size, - unsigned wid, - unsigned sid, - unsigned tpc, - const memory_config *config, - unsigned long long cycle, - mem_fetch *m_original_mf, - mem_fetch *m_original_wr_mf):m_access(access) +mem_fetch::mem_fetch(const mem_access_t &access, const warp_inst_t *inst, + unsigned ctrl_size, unsigned wid, unsigned sid, + unsigned tpc, const memory_config *config, + unsigned long long cycle, mem_fetch *m_original_mf, + mem_fetch *m_original_wr_mf) + : m_access(access) { - m_request_uid = sm_next_mf_request_uid++; - m_access = access; - if( inst ) { - m_inst = *inst; - assert( wid == m_inst.warp_id() ); - } - m_data_size = access.get_size(); - m_ctrl_size = ctrl_size; - m_sid = sid; - m_tpc = tpc; - m_wid = wid; - config->m_address_mapping.addrdec_tlx(access.get_addr(),&m_raw_addr); - m_partition_addr = config->m_address_mapping.partition_address(access.get_addr()); - m_type = m_access.is_write()?WRITE_REQUEST:READ_REQUEST; - m_timestamp = cycle; - m_timestamp2 = 0; - m_status = MEM_FETCH_INITIALIZED; - m_status_change = cycle; - m_mem_config = config; - icnt_flit_size = config->icnt_flit_size; - original_mf = m_original_mf; - original_wr_mf = m_original_wr_mf; + m_request_uid = sm_next_mf_request_uid++; + m_access = access; + if (inst) { + m_inst = *inst; + assert(wid == m_inst.warp_id()); + } + m_data_size = access.get_size(); + m_ctrl_size = ctrl_size; + m_sid = sid; + m_tpc = tpc; + m_wid = wid; + config->m_address_mapping.addrdec_tlx(access.get_addr(), &m_raw_addr); + m_partition_addr = + config->m_address_mapping.partition_address(access.get_addr()); + m_type = m_access.is_write() ? WRITE_REQUEST : READ_REQUEST; + m_timestamp = cycle; + m_timestamp2 = 0; + m_status = MEM_FETCH_INITIALIZED; + m_status_change = cycle; + m_mem_config = config; + icnt_flit_size = config->icnt_flit_size; + original_mf = m_original_mf; + original_wr_mf = m_original_wr_mf; } -mem_fetch::~mem_fetch() -{ - m_status = MEM_FETCH_DELETED; -} +mem_fetch::~mem_fetch() { m_status = MEM_FETCH_DELETED; } -#define MF_TUP_BEGIN(X) static const char* Status_str[] = { +#define MF_TUP_BEGIN(X) static const char *Status_str[] = { #define MF_TUP(X) #X -#define MF_TUP_END(X) }; +#define MF_TUP_END(X) \ + } \ + ; #include "mem_fetch_status.tup" #undef MF_TUP_BEGIN #undef MF_TUP #undef MF_TUP_END -void mem_fetch::print( FILE *fp, bool print_inst ) const -{ - if( this == NULL ) { - fprintf(fp," <NULL mem_fetch pointer>\n"); - return; - } - fprintf(fp," mf: uid=%6u, sid%02u:w%02u, part=%u, ", m_request_uid, m_sid, m_wid, m_raw_addr.chip ); - m_access.print(fp); - if( (unsigned)m_status < NUM_MEM_REQ_STAT ) - fprintf(fp," status = %s (%llu), ", Status_str[m_status], m_status_change ); - else - fprintf(fp," status = %u??? (%llu), ", m_status, m_status_change ); - if( !m_inst.empty() && print_inst ) m_inst.print(fp); - else fprintf(fp,"\n"); +void mem_fetch::print(FILE *fp, bool print_inst) const { + if (this == NULL) { + fprintf(fp, " <NULL mem_fetch pointer>\n"); + return; + } + fprintf(fp, " mf: uid=%6u, sid%02u:w%02u, part=%u, ", m_request_uid, m_sid, + m_wid, m_raw_addr.chip); + m_access.print(fp); + if ((unsigned)m_status < NUM_MEM_REQ_STAT) + fprintf(fp, " status = %s (%llu), ", Status_str[m_status], m_status_change); + else + fprintf(fp, " status = %u??? (%llu), ", m_status, m_status_change); + if (!m_inst.empty() && print_inst) + m_inst.print(fp); + else + fprintf(fp, "\n"); } -void mem_fetch::set_status( enum mem_fetch_status status, unsigned long long cycle ) -{ - m_status = status; - m_status_change = cycle; +void mem_fetch::set_status(enum mem_fetch_status status, + unsigned long long cycle) { + m_status = status; + m_status_change = cycle; } -bool mem_fetch::isatomic() const -{ - if( m_inst.empty() ) return false; - return m_inst.isatomic(); +bool mem_fetch::isatomic() const { + if (m_inst.empty()) return false; + return m_inst.isatomic(); } -void mem_fetch::do_atomic() -{ - m_inst.do_atomic( m_access.get_warp_mask() ); -} +void mem_fetch::do_atomic() { m_inst.do_atomic(m_access.get_warp_mask()); } -bool mem_fetch::istexture() const -{ - if( m_inst.empty() ) return false; - return m_inst.space.get_type() == tex_space; +bool mem_fetch::istexture() const { + if (m_inst.empty()) return false; + return m_inst.space.get_type() == tex_space; } -bool mem_fetch::isconst() const -{ - if( m_inst.empty() ) return false; - return (m_inst.space.get_type() == const_space) || (m_inst.space.get_type() == param_space_kernel); +bool mem_fetch::isconst() const { + if (m_inst.empty()) return false; + return (m_inst.space.get_type() == const_space) || + (m_inst.space.get_type() == param_space_kernel); } -/// Returns number of flits traversing interconnect. simt_to_mem specifies the direction -unsigned mem_fetch::get_num_flits(bool simt_to_mem){ - unsigned sz=0; - // If atomic, write going to memory, or read coming back from memory, size = ctrl + data. Else, only ctrl - if( isatomic() || (simt_to_mem && get_is_write()) || !(simt_to_mem || get_is_write()) ) - sz = size(); - else - sz = get_ctrl_size(); +/// Returns number of flits traversing interconnect. simt_to_mem specifies the +/// direction +unsigned mem_fetch::get_num_flits(bool simt_to_mem) { + unsigned sz = 0; + // If atomic, write going to memory, or read coming back from memory, size = + // ctrl + data. Else, only ctrl + if (isatomic() || (simt_to_mem && get_is_write()) || + !(simt_to_mem || get_is_write())) + sz = size(); + else + sz = get_ctrl_size(); - return (sz/icnt_flit_size) + ( (sz % icnt_flit_size)? 1:0); + return (sz / icnt_flit_size) + ((sz % icnt_flit_size) ? 1 : 0); } - - - diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index 1cab9f2..71d8acd 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -7,41 +7,44 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef MEM_FETCH_H #define MEM_FETCH_H -#include "addrdec.h" -#include "../abstract_hardware_model.h" #include <bitset> +#include "../abstract_hardware_model.h" +#include "addrdec.h" enum mf_type { - READ_REQUEST = 0, - WRITE_REQUEST, - READ_REPLY, // send to shader - WRITE_ACK + READ_REQUEST = 0, + WRITE_REQUEST, + READ_REPLY, // send to shader + WRITE_ACK }; #define MF_TUP_BEGIN(X) enum X { #define MF_TUP(X) X -#define MF_TUP_END(X) }; +#define MF_TUP_END(X) \ + } \ + ; #include "mem_fetch_status.tup" #undef MF_TUP_BEGIN #undef MF_TUP @@ -49,113 +52,124 @@ enum mf_type { class memory_config; class mem_fetch { -public: - mem_fetch( const mem_access_t &access, - const warp_inst_t *inst, - unsigned ctrl_size, - unsigned wid, - unsigned sid, - unsigned tpc, - const memory_config *config, - unsigned long long cycle, - mem_fetch *original_mf = NULL, - mem_fetch *original_wr_mf = NULL); - ~mem_fetch(); - - void set_status( enum mem_fetch_status status, unsigned long long cycle ); - void set_reply() - { - assert( m_access.get_type() != L1_WRBK_ACC && m_access.get_type() != L2_WRBK_ACC ); - if( m_type==READ_REQUEST ) { - assert( !get_is_write() ); - m_type = READ_REPLY; - } else if( m_type == WRITE_REQUEST ) { - assert( get_is_write() ); - m_type = WRITE_ACK; - } - } - void do_atomic(); + public: + mem_fetch(const mem_access_t &access, const warp_inst_t *inst, + unsigned ctrl_size, unsigned wid, unsigned sid, unsigned tpc, + const memory_config *config, unsigned long long cycle, + mem_fetch *original_mf = NULL, mem_fetch *original_wr_mf = NULL); + ~mem_fetch(); - void print( FILE *fp, bool print_inst = true ) const; + void set_status(enum mem_fetch_status status, unsigned long long cycle); + void set_reply() { + assert(m_access.get_type() != L1_WRBK_ACC && + m_access.get_type() != L2_WRBK_ACC); + if (m_type == READ_REQUEST) { + assert(!get_is_write()); + m_type = READ_REPLY; + } else if (m_type == WRITE_REQUEST) { + assert(get_is_write()); + m_type = WRITE_ACK; + } + } + void do_atomic(); - const addrdec_t &get_tlx_addr() const { return m_raw_addr; } - unsigned get_data_size() const { return m_data_size; } - void set_data_size( unsigned size ) { m_data_size=size; } - unsigned get_ctrl_size() const { return m_ctrl_size; } - unsigned size() const { return m_data_size+m_ctrl_size; } - bool is_write() {return m_access.is_write();} - void set_addr(new_addr_type addr) { m_access.set_addr(addr); } - new_addr_type get_addr() const { return m_access.get_addr(); } - unsigned get_access_size() const { return m_access.get_size(); } - new_addr_type get_partition_addr() const { return m_partition_addr; } - unsigned get_sub_partition_id() const { return m_raw_addr.sub_partition; } - bool get_is_write() const { return m_access.is_write(); } - unsigned get_request_uid() const { return m_request_uid; } - unsigned get_sid() const { return m_sid; } - unsigned get_tpc() const { return m_tpc; } - unsigned get_wid() const { return m_wid; } - bool istexture() const; - bool isconst() const; - enum mf_type get_type() const { return m_type; } - bool isatomic() const; + void print(FILE *fp, bool print_inst = true) const; - void set_return_timestamp( unsigned t ) { m_timestamp2=t; } - void set_icnt_receive_time( unsigned t ) { m_icnt_receive_time=t; } - unsigned get_timestamp() const { return m_timestamp; } - unsigned get_return_timestamp() const { return m_timestamp2; } - unsigned get_icnt_receive_time() const { return m_icnt_receive_time; } + const addrdec_t &get_tlx_addr() const { return m_raw_addr; } + unsigned get_data_size() const { return m_data_size; } + void set_data_size(unsigned size) { m_data_size = size; } + unsigned get_ctrl_size() const { return m_ctrl_size; } + unsigned size() const { return m_data_size + m_ctrl_size; } + bool is_write() { return m_access.is_write(); } + void set_addr(new_addr_type addr) { m_access.set_addr(addr); } + new_addr_type get_addr() const { return m_access.get_addr(); } + unsigned get_access_size() const { return m_access.get_size(); } + new_addr_type get_partition_addr() const { return m_partition_addr; } + unsigned get_sub_partition_id() const { return m_raw_addr.sub_partition; } + bool get_is_write() const { return m_access.is_write(); } + unsigned get_request_uid() const { return m_request_uid; } + unsigned get_sid() const { return m_sid; } + unsigned get_tpc() const { return m_tpc; } + unsigned get_wid() const { return m_wid; } + bool istexture() const; + bool isconst() const; + enum mf_type get_type() const { return m_type; } + bool isatomic() const; - enum mem_access_type get_access_type() const { return m_access.get_type(); } - const active_mask_t& get_access_warp_mask() const { return m_access.get_warp_mask(); } - mem_access_byte_mask_t get_access_byte_mask() const { return m_access.get_byte_mask(); } - mem_access_sector_mask_t get_access_sector_mask() const { return m_access.get_sector_mask(); } + void set_return_timestamp(unsigned t) { m_timestamp2 = t; } + void set_icnt_receive_time(unsigned t) { m_icnt_receive_time = t; } + unsigned get_timestamp() const { return m_timestamp; } + unsigned get_return_timestamp() const { return m_timestamp2; } + unsigned get_icnt_receive_time() const { return m_icnt_receive_time; } - address_type get_pc() const { return m_inst.empty()?-1:m_inst.pc; } - const warp_inst_t &get_inst() { return m_inst; } - enum mem_fetch_status get_status() const { return m_status; } + enum mem_access_type get_access_type() const { return m_access.get_type(); } + const active_mask_t &get_access_warp_mask() const { + return m_access.get_warp_mask(); + } + mem_access_byte_mask_t get_access_byte_mask() const { + return m_access.get_byte_mask(); + } + mem_access_sector_mask_t get_access_sector_mask() const { + return m_access.get_sector_mask(); + } - const memory_config *get_mem_config(){return m_mem_config;} + address_type get_pc() const { return m_inst.empty() ? -1 : m_inst.pc; } + const warp_inst_t &get_inst() { return m_inst; } + enum mem_fetch_status get_status() const { return m_status; } - unsigned get_num_flits(bool simt_to_mem); + const memory_config *get_mem_config() { return m_mem_config; } - mem_fetch* get_original_mf() { return original_mf; } - mem_fetch* get_original_wr_mf() { return original_wr_mf; } + unsigned get_num_flits(bool simt_to_mem); -private: - // request source information - unsigned m_request_uid; - unsigned m_sid; - unsigned m_tpc; - unsigned m_wid; + mem_fetch *get_original_mf() { return original_mf; } + mem_fetch *get_original_wr_mf() { return original_wr_mf; } - // where is this request now? - enum mem_fetch_status m_status; - unsigned long long m_status_change; + private: + // request source information + unsigned m_request_uid; + unsigned m_sid; + unsigned m_tpc; + unsigned m_wid; - // request type, address, size, mask - mem_access_t m_access; - unsigned m_data_size; // how much data is being written - unsigned m_ctrl_size; // how big would all this meta data be in hardware (does not necessarily match actual size of mem_fetch) - new_addr_type m_partition_addr; // linear physical address *within* dram partition (partition bank select bits squeezed out) - addrdec_t m_raw_addr; // raw physical address (i.e., decoded DRAM chip-row-bank-column address) - enum mf_type m_type; + // where is this request now? + enum mem_fetch_status m_status; + unsigned long long m_status_change; - // statistics - unsigned m_timestamp; // set to gpu_sim_cycle+gpu_tot_sim_cycle at struct creation - unsigned m_timestamp2; // set to gpu_sim_cycle+gpu_tot_sim_cycle when pushed onto icnt to shader; only used for reads - unsigned m_icnt_receive_time; // set to gpu_sim_cycle + interconnect_latency when fixed icnt latency mode is enabled + // request type, address, size, mask + mem_access_t m_access; + unsigned m_data_size; // how much data is being written + unsigned + m_ctrl_size; // how big would all this meta data be in hardware (does not + // necessarily match actual size of mem_fetch) + new_addr_type + m_partition_addr; // linear physical address *within* dram partition + // (partition bank select bits squeezed out) + addrdec_t m_raw_addr; // raw physical address (i.e., decoded DRAM + // chip-row-bank-column address) + enum mf_type m_type; - // requesting instruction (put last so mem_fetch prints nicer in gdb) - warp_inst_t m_inst; + // statistics + unsigned + m_timestamp; // set to gpu_sim_cycle+gpu_tot_sim_cycle at struct creation + unsigned m_timestamp2; // set to gpu_sim_cycle+gpu_tot_sim_cycle when pushed + // onto icnt to shader; only used for reads + unsigned m_icnt_receive_time; // set to gpu_sim_cycle + interconnect_latency + // when fixed icnt latency mode is enabled - static unsigned sm_next_mf_request_uid; + // requesting instruction (put last so mem_fetch prints nicer in gdb) + warp_inst_t m_inst; - const memory_config *m_mem_config; - unsigned icnt_flit_size; + static unsigned sm_next_mf_request_uid; - mem_fetch* original_mf; //this pointer is set up when a request is divided into sector requests at L2 cache (if the req size > L2 sector size), so the pointer refers to the original request - mem_fetch* original_wr_mf; //this pointer refers to the original write req, when fetch-on-write policy is used + const memory_config *m_mem_config; + unsigned icnt_flit_size; + mem_fetch + *original_mf; // this pointer is set up when a request is divided into + // sector requests at L2 cache (if the req size > L2 sector + // size), so the pointer refers to the original request + mem_fetch *original_wr_mf; // this pointer refers to the original write req, + // when fetch-on-write policy is used }; #endif diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index a1b43a8..e50ad9e 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -8,474 +8,513 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "../abstract_hardware_model.h" #include "mem_latency_stat.h" -#include "gpu-sim.h" -#include "gpu-misc.h" +#include "../abstract_hardware_model.h" +#include "../cuda-sim/ptx-stats.h" +#include "dram.h" #include "gpu-cache.h" -#include "shader.h" +#include "gpu-misc.h" +#include "gpu-sim.h" #include "mem_fetch.h" +#include "shader.h" #include "stat-tool.h" -#include "../cuda-sim/ptx-stats.h" #include "visualizer.h" -#include "dram.h" -#include <string.h> -#include <stdlib.h> #include <stdio.h> +#include <stdlib.h> +#include <string.h> #include "../../libcuda/gpgpu_context.h" -memory_stats_t::memory_stats_t( unsigned n_shader, const shader_core_config *shader_config, const memory_config *mem_config, const class gpgpu_sim* gpu ) -{ - assert( mem_config->m_valid ); - assert( shader_config->m_valid ); +memory_stats_t::memory_stats_t(unsigned n_shader, + const shader_core_config *shader_config, + const memory_config *mem_config, + const class gpgpu_sim *gpu) { + assert(mem_config->m_valid); + assert(shader_config->m_valid); - unsigned i,j; + unsigned i, j; + concurrent_row_access = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + num_activates = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + row_access = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + max_conc_access2samerow = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + max_servicetime2samerow = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); - concurrent_row_access = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - num_activates = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - row_access = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - max_conc_access2samerow = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - max_servicetime2samerow = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); + for (unsigned i = 0; i < mem_config->m_n_mem; i++) { + concurrent_row_access[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + row_access[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + num_activates[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + max_conc_access2samerow[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + max_servicetime2samerow[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + } - for (unsigned i=0;i<mem_config->m_n_mem ;i++ ) { - concurrent_row_access[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - row_access[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - num_activates[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - max_conc_access2samerow[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - max_servicetime2samerow[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - } + m_n_shader = n_shader; + m_memory_config = mem_config; + m_gpu = gpu; + total_n_access = 0; + total_n_reads = 0; + total_n_writes = 0; + max_mrq_latency = 0; + max_dq_latency = 0; + max_mf_latency = 0; + max_icnt2mem_latency = 0; + max_icnt2sh_latency = 0; + tot_icnt2mem_latency = 0; + tot_icnt2sh_latency = 0; + tot_mrq_num = 0; + tot_mrq_latency = 0; + memset(mrq_lat_table, 0, sizeof(unsigned) * 32); + memset(dq_lat_table, 0, sizeof(unsigned) * 32); + memset(mf_lat_table, 0, sizeof(unsigned) * 32); + memset(icnt2mem_lat_table, 0, sizeof(unsigned) * 24); + memset(icnt2sh_lat_table, 0, sizeof(unsigned) * 24); + memset(mf_lat_pw_table, 0, sizeof(unsigned) * 32); + mf_num_lat_pw = 0; + max_warps = + n_shader * + (shader_config->n_thread_per_shader / shader_config->warp_size + 1); + mf_tot_lat_pw = 0; // total latency summed up per window. divide by + // mf_num_lat_pw to obtain average latency Per Window + mf_total_lat = 0; + num_mfs = 0; + printf("*** Initializing Memory Statistics ***\n"); + totalbankreads = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + totalbankwrites = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + totalbankaccesses = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + mf_total_lat_table = (unsigned long long int **)calloc( + mem_config->m_n_mem, sizeof(unsigned long long *)); + mf_max_lat_table = + (unsigned **)calloc(mem_config->m_n_mem, sizeof(unsigned *)); + bankreads = (unsigned int ***)calloc(n_shader, sizeof(unsigned int **)); + bankwrites = (unsigned int ***)calloc(n_shader, sizeof(unsigned int **)); + num_MCBs_accessed = (unsigned int *)calloc( + mem_config->m_n_mem * mem_config->nbk, sizeof(unsigned int)); + if (mem_config->gpgpu_frfcfs_dram_sched_queue_size) { + position_of_mrq_chosen = (unsigned int *)calloc( + mem_config->gpgpu_frfcfs_dram_sched_queue_size, sizeof(unsigned int)); + } else + position_of_mrq_chosen = (unsigned int *)calloc(1024, sizeof(unsigned int)); + for (i = 0; i < n_shader; i++) { + bankreads[i] = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + bankwrites[i] = + (unsigned int **)calloc(mem_config->m_n_mem, sizeof(unsigned int *)); + for (j = 0; j < mem_config->m_n_mem; j++) { + bankreads[i][j] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + bankwrites[i][j] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + } + } + for (i = 0; i < mem_config->m_n_mem; i++) { + totalbankreads[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + totalbankwrites[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + totalbankaccesses[i] = + (unsigned int *)calloc(mem_config->nbk, sizeof(unsigned int)); + mf_total_lat_table[i] = (unsigned long long int *)calloc( + mem_config->nbk, sizeof(unsigned long long int)); + mf_max_lat_table[i] = (unsigned *)calloc(mem_config->nbk, sizeof(unsigned)); + } - m_n_shader=n_shader; - m_memory_config=mem_config; - m_gpu=gpu; - total_n_access=0; - total_n_reads=0; - total_n_writes=0; - max_mrq_latency = 0; - max_dq_latency = 0; - max_mf_latency = 0; - max_icnt2mem_latency = 0; - max_icnt2sh_latency = 0; - tot_icnt2mem_latency = 0; - tot_icnt2sh_latency = 0; - tot_mrq_num = 0; - tot_mrq_latency = 0; - memset(mrq_lat_table, 0, sizeof(unsigned)*32); - memset(dq_lat_table, 0, sizeof(unsigned)*32); - memset(mf_lat_table, 0, sizeof(unsigned)*32); - memset(icnt2mem_lat_table, 0, sizeof(unsigned)*24); - memset(icnt2sh_lat_table, 0, sizeof(unsigned)*24); - memset(mf_lat_pw_table, 0, sizeof(unsigned)*32); - mf_num_lat_pw = 0; - max_warps = n_shader * (shader_config->n_thread_per_shader / shader_config->warp_size+1); - mf_tot_lat_pw = 0; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window - mf_total_lat = 0; - num_mfs = 0; - printf("*** Initializing Memory Statistics ***\n"); - totalbankreads = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - totalbankwrites = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - totalbankaccesses = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - mf_total_lat_table = (unsigned long long int **) calloc(mem_config->m_n_mem, sizeof(unsigned long long *)); - mf_max_lat_table = (unsigned **) calloc(mem_config->m_n_mem, sizeof(unsigned *)); - bankreads = (unsigned int***) calloc(n_shader, sizeof(unsigned int**)); - bankwrites = (unsigned int***) calloc(n_shader, sizeof(unsigned int**)); - num_MCBs_accessed = (unsigned int*) calloc(mem_config->m_n_mem*mem_config->nbk, sizeof(unsigned int)); - if (mem_config->gpgpu_frfcfs_dram_sched_queue_size) { - position_of_mrq_chosen = (unsigned int*) calloc(mem_config->gpgpu_frfcfs_dram_sched_queue_size, sizeof(unsigned int)); - } else - position_of_mrq_chosen = (unsigned int*) calloc(1024, sizeof(unsigned int)); - for (i=0;i<n_shader ;i++ ) { - bankreads[i] = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - bankwrites[i] = (unsigned int**) calloc(mem_config->m_n_mem, sizeof(unsigned int*)); - for (j=0;j<mem_config->m_n_mem ;j++ ) { - bankreads[i][j] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - bankwrites[i][j] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - } - } + mem_access_type_stats = + (unsigned ***)malloc(NUM_MEM_ACCESS_TYPE * sizeof(unsigned **)); + for (i = 0; i < NUM_MEM_ACCESS_TYPE; i++) { + int j; + mem_access_type_stats[i] = + (unsigned **)calloc(mem_config->m_n_mem, sizeof(unsigned *)); + for (j = 0; (unsigned)j < mem_config->m_n_mem; j++) { + mem_access_type_stats[i][j] = + (unsigned *)calloc((mem_config->nbk + 1), sizeof(unsigned *)); + } + } - for (i=0;i<mem_config->m_n_mem ;i++ ) { - totalbankreads[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - totalbankwrites[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - totalbankaccesses[i] = (unsigned int*) calloc(mem_config->nbk, sizeof(unsigned int)); - mf_total_lat_table[i] = (unsigned long long int*) calloc(mem_config->nbk, sizeof(unsigned long long int)); - mf_max_lat_table[i] = (unsigned *) calloc(mem_config->nbk, sizeof(unsigned)); - } + // AerialVision L2 stats + L2_read_miss = 0; + L2_write_miss = 0; + L2_read_hit = 0; + L2_write_hit = 0; - mem_access_type_stats = (unsigned ***) malloc(NUM_MEM_ACCESS_TYPE * sizeof(unsigned **)); - for (i = 0; i < NUM_MEM_ACCESS_TYPE; i++) { - int j; - mem_access_type_stats[i] = (unsigned **) calloc(mem_config->m_n_mem, sizeof(unsigned*)); - for (j=0; (unsigned) j< mem_config->m_n_mem; j++) { - mem_access_type_stats[i][j] = (unsigned *) calloc((mem_config->nbk+1), sizeof(unsigned*)); - } - } - - // AerialVision L2 stats - L2_read_miss = 0; - L2_write_miss = 0; - L2_read_hit = 0; - L2_write_hit = 0; - - L2_cbtoL2length = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); - L2_cbtoL2writelength = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); - L2_L2tocblength = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); - L2_dramtoL2length = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); - L2_dramtoL2writelength = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); - L2_L2todramlength = (unsigned int*) calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_cbtoL2length = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_cbtoL2writelength = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_L2tocblength = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_dramtoL2length = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_dramtoL2writelength = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); + L2_L2todramlength = + (unsigned int *)calloc(mem_config->m_n_mem, sizeof(unsigned int)); } // record the total latency -unsigned memory_stats_t::memlatstat_done(mem_fetch *mf ) -{ - unsigned mf_latency; - mf_latency = (m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle) - mf->get_timestamp(); - mf_num_lat_pw++; - mf_tot_lat_pw += mf_latency; - unsigned idx = LOGB2(mf_latency); - assert(idx<32); - mf_lat_table[idx]++; - shader_mem_lat_log(mf->get_sid(), mf_latency); - mf_total_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] += mf_latency; - if (mf_latency > max_mf_latency) - max_mf_latency = mf_latency; - return mf_latency; +unsigned memory_stats_t::memlatstat_done(mem_fetch *mf) { + unsigned mf_latency; + mf_latency = + (m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle) - mf->get_timestamp(); + mf_num_lat_pw++; + mf_tot_lat_pw += mf_latency; + unsigned idx = LOGB2(mf_latency); + assert(idx < 32); + mf_lat_table[idx]++; + shader_mem_lat_log(mf->get_sid(), mf_latency); + mf_total_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] += + mf_latency; + if (mf_latency > max_mf_latency) max_mf_latency = mf_latency; + return mf_latency; } -void memory_stats_t::memlatstat_read_done(mem_fetch *mf) -{ - if (m_memory_config->gpgpu_memlatency_stat) { - unsigned mf_latency = memlatstat_done(mf); - if (mf_latency > mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk]) - mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency; - unsigned icnt2sh_latency; - icnt2sh_latency = (m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle) - mf->get_return_timestamp(); - tot_icnt2sh_latency += icnt2sh_latency; - icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; - if (icnt2sh_latency > max_icnt2sh_latency) - max_icnt2sh_latency = icnt2sh_latency; - } +void memory_stats_t::memlatstat_read_done(mem_fetch *mf) { + if (m_memory_config->gpgpu_memlatency_stat) { + unsigned mf_latency = memlatstat_done(mf); + if (mf_latency > + mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk]) + mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = + mf_latency; + unsigned icnt2sh_latency; + icnt2sh_latency = (m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle) - + mf->get_return_timestamp(); + tot_icnt2sh_latency += icnt2sh_latency; + icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; + if (icnt2sh_latency > max_icnt2sh_latency) + max_icnt2sh_latency = icnt2sh_latency; + } } -void memory_stats_t::memlatstat_dram_access(mem_fetch *mf) -{ - unsigned dram_id = mf->get_tlx_addr().chip; - unsigned bank = mf->get_tlx_addr().bk; - if (m_memory_config->gpgpu_memlatency_stat) { - if (mf->get_is_write()) { - if ( mf->get_sid() < m_n_shader ) { //do not count L2_writebacks here - bankwrites[mf->get_sid()][dram_id][bank]++; - shader_mem_acc_log( mf->get_sid(), dram_id, bank, 'w'); - } - totalbankwrites[dram_id][bank]++; - } else { - bankreads[mf->get_sid()][dram_id][bank]++; - shader_mem_acc_log( mf->get_sid(), dram_id, bank, 'r'); - totalbankreads[dram_id][bank]++; +void memory_stats_t::memlatstat_dram_access(mem_fetch *mf) { + unsigned dram_id = mf->get_tlx_addr().chip; + unsigned bank = mf->get_tlx_addr().bk; + if (m_memory_config->gpgpu_memlatency_stat) { + if (mf->get_is_write()) { + if (mf->get_sid() < m_n_shader) { // do not count L2_writebacks here + bankwrites[mf->get_sid()][dram_id][bank]++; + shader_mem_acc_log(mf->get_sid(), dram_id, bank, 'w'); } - mem_access_type_stats[mf->get_access_type()][dram_id][bank]++; - } - if (mf->get_pc() != (unsigned)-1) - m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_dram_traffic(mf->get_pc(), mf->get_data_size()); + totalbankwrites[dram_id][bank]++; + } else { + bankreads[mf->get_sid()][dram_id][bank]++; + shader_mem_acc_log(mf->get_sid(), dram_id, bank, 'r'); + totalbankreads[dram_id][bank]++; + } + mem_access_type_stats[mf->get_access_type()][dram_id][bank]++; + } + if (mf->get_pc() != (unsigned)-1) + m_gpu->gpgpu_ctx->stats->ptx_file_line_stats_add_dram_traffic( + mf->get_pc(), mf->get_data_size()); } -void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) -{ - if (m_memory_config->gpgpu_memlatency_stat) { - unsigned icnt2mem_latency; - icnt2mem_latency = (m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle) - mf->get_timestamp(); - tot_icnt2mem_latency += icnt2mem_latency; - icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; - if (icnt2mem_latency > max_icnt2mem_latency) - max_icnt2mem_latency = icnt2mem_latency; - } +void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) { + if (m_memory_config->gpgpu_memlatency_stat) { + unsigned icnt2mem_latency; + icnt2mem_latency = + (m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle) - mf->get_timestamp(); + tot_icnt2mem_latency += icnt2mem_latency; + icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; + if (icnt2mem_latency > max_icnt2mem_latency) + max_icnt2mem_latency = icnt2mem_latency; + } } -void memory_stats_t::memlatstat_lat_pw() -{ - if (mf_num_lat_pw && m_memory_config->gpgpu_memlatency_stat) { - assert(mf_tot_lat_pw); - mf_total_lat += mf_tot_lat_pw; - num_mfs += mf_num_lat_pw; - mf_lat_pw_table[LOGB2(mf_tot_lat_pw/mf_num_lat_pw)]++; - mf_tot_lat_pw = 0; - mf_num_lat_pw = 0; - } +void memory_stats_t::memlatstat_lat_pw() { + if (mf_num_lat_pw && m_memory_config->gpgpu_memlatency_stat) { + assert(mf_tot_lat_pw); + mf_total_lat += mf_tot_lat_pw; + num_mfs += mf_num_lat_pw; + mf_lat_pw_table[LOGB2(mf_tot_lat_pw / mf_num_lat_pw)]++; + mf_tot_lat_pw = 0; + mf_num_lat_pw = 0; + } } +void memory_stats_t::memlatstat_print(unsigned n_mem, unsigned gpu_mem_n_bk) { + unsigned i, j, k, l, m; + unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, + min_chip_accesses; -void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk ) -{ - unsigned i,j,k,l,m; - unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses; + if (m_memory_config->gpgpu_memlatency_stat) { + printf("maxmflatency = %d \n", max_mf_latency); + printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); + printf("maxmrqlatency = %d \n", max_mrq_latency); + // printf("maxdqlatency = %d \n", max_dq_latency); + printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); + if (num_mfs) { + printf("averagemflatency = %lld \n", mf_total_lat / num_mfs); + printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency / num_mfs); + if (tot_mrq_num) + printf("avg_mrq_latency = %lld \n", tot_mrq_latency / tot_mrq_num); - if (m_memory_config->gpgpu_memlatency_stat) { - printf("maxmflatency = %d \n", max_mf_latency); - printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); - printf("maxmrqlatency = %d \n", max_mrq_latency); - //printf("maxdqlatency = %d \n", max_dq_latency); - printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); - if (num_mfs) { - printf("averagemflatency = %lld \n", mf_total_lat/num_mfs); - printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs); - if(tot_mrq_num) - printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num); + printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency / num_mfs); + } + printf("mrq_lat_table:"); + for (i = 0; i < 32; i++) { + printf("%d \t", mrq_lat_table[i]); + } + printf("\n"); + printf("dq_lat_table:"); + for (i = 0; i < 32; i++) { + printf("%d \t", dq_lat_table[i]); + } + printf("\n"); + printf("mf_lat_table:"); + for (i = 0; i < 32; i++) { + printf("%d \t", mf_lat_table[i]); + } + printf("\n"); + printf("icnt2mem_lat_table:"); + for (i = 0; i < 24; i++) { + printf("%d \t", icnt2mem_lat_table[i]); + } + printf("\n"); + printf("icnt2sh_lat_table:"); + for (i = 0; i < 24; i++) { + printf("%d \t", icnt2sh_lat_table[i]); + } + printf("\n"); + printf("mf_lat_pw_table:"); + for (i = 0; i < 32; i++) { + printf("%d \t", mf_lat_pw_table[i]); + } + printf("\n"); - printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs); - } - printf("mrq_lat_table:"); - for (i=0; i< 32; i++) { - printf("%d \t", mrq_lat_table[i]); - } - printf("\n"); - printf("dq_lat_table:"); - for (i=0; i< 32; i++) { - printf("%d \t", dq_lat_table[i]); + /*MAXIMUM CONCURRENT ACCESSES TO SAME ROW*/ + printf("maximum concurrent accesses to same row:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + printf("%9d ", max_conc_access2samerow[i][j]); } printf("\n"); - printf("mf_lat_table:"); - for (i=0; i< 32; i++) { - printf("%d \t", mf_lat_table[i]); - } - printf("\n"); - printf("icnt2mem_lat_table:"); - for (i=0; i< 24; i++) { - printf("%d \t", icnt2mem_lat_table[i]); + } + + /*MAXIMUM SERVICE TIME TO SAME ROW*/ + printf("maximum service time to same row:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + printf("%9d ", max_servicetime2samerow[i][j]); } printf("\n"); - printf("icnt2sh_lat_table:"); - for (i=0; i< 24; i++) { - printf("%d \t", icnt2sh_lat_table[i]); + } + + /*AVERAGE ROW ACCESSES PER ACTIVATE*/ + int total_row_accesses = 0; + int total_num_activates = 0; + printf("average row accesses per activate:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + total_row_accesses += row_access[i][j]; + total_num_activates += num_activates[i][j]; + printf("%9f ", (float)row_access[i][j] / num_activates[i][j]); } printf("\n"); - printf("mf_lat_pw_table:"); - for (i=0; i< 32; i++) { - printf("%d \t", mf_lat_pw_table[i]); + } + printf("average row locality = %d/%d = %f\n", total_row_accesses, + total_num_activates, + (float)total_row_accesses / total_num_activates); + /*MEMORY ACCESSES*/ + k = 0; + l = 0; + m = 0; + max_bank_accesses = 0; + max_chip_accesses = 0; + min_bank_accesses = 0xFFFFFFFF; + min_chip_accesses = 0xFFFFFFFF; + printf("number of total memory accesses made:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + l = totalbankaccesses[i][j]; + if (l < min_bank_accesses) min_bank_accesses = l; + if (l > max_bank_accesses) max_bank_accesses = l; + k += l; + m += l; + printf("%9d ", l); } + if (m < min_chip_accesses) min_chip_accesses = m; + if (m > max_chip_accesses) max_chip_accesses = m; + m = 0; printf("\n"); + } + printf("total accesses: %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, + (float)max_bank_accesses / min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, + (float)max_chip_accesses / min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); - /*MAXIMUM CONCURRENT ACCESSES TO SAME ROW*/ - printf("maximum concurrent accesses to same row:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - printf("%9d ",max_conc_access2samerow[i][j]); - } - printf("\n"); + /*READ ACCESSES*/ + k = 0; + l = 0; + m = 0; + max_bank_accesses = 0; + max_chip_accesses = 0; + min_bank_accesses = 0xFFFFFFFF; + min_chip_accesses = 0xFFFFFFFF; + printf("number of total read accesses:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + l = totalbankreads[i][j]; + if (l < min_bank_accesses) min_bank_accesses = l; + if (l > max_bank_accesses) max_bank_accesses = l; + k += l; + m += l; + printf("%9d ", l); } - - /*MAXIMUM SERVICE TIME TO SAME ROW*/ - printf("maximum service time to same row:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - printf("%9d ",max_servicetime2samerow[i][j]); - } - printf("\n"); - } - - /*AVERAGE ROW ACCESSES PER ACTIVATE*/ - int total_row_accesses = 0; - int total_num_activates = 0; - printf("average row accesses per activate:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - total_row_accesses += row_access[i][j]; - total_num_activates += num_activates[i][j]; - printf("%9f ",(float) row_access[i][j]/num_activates[i][j]); - } - printf("\n"); - } - printf("average row locality = %d/%d = %f\n", total_row_accesses, total_num_activates, (float)total_row_accesses/total_num_activates); - /*MEMORY ACCESSES*/ - k = 0; - l = 0; + if (m < min_chip_accesses) min_chip_accesses = m; + if (m > max_chip_accesses) max_chip_accesses = m; m = 0; - max_bank_accesses = 0; - max_chip_accesses = 0; - min_bank_accesses = 0xFFFFFFFF; - min_chip_accesses = 0xFFFFFFFF; - printf("number of total memory accesses made:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - l = totalbankaccesses[i][j]; - if (l < min_bank_accesses) - min_bank_accesses = l; - if (l > max_bank_accesses) - max_bank_accesses = l; - k += l; - m += l; - printf("%9d ",l); - } - if (m < min_chip_accesses) - min_chip_accesses = m; - if (m > max_chip_accesses) - max_chip_accesses = m; - m = 0; - printf("\n"); - } - printf("total accesses: %d\n", k); - if (min_bank_accesses) - printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); - else - printf("min_bank_accesses = 0!\n"); - if (min_chip_accesses) - printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); - else - printf("min_chip_accesses = 0!\n"); + printf("\n"); + } + printf("total dram reads = %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, + (float)max_bank_accesses / min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, + (float)max_chip_accesses / min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); - /*READ ACCESSES*/ - k = 0; - l = 0; - m = 0; - max_bank_accesses = 0; - max_chip_accesses = 0; - min_bank_accesses = 0xFFFFFFFF; - min_chip_accesses = 0xFFFFFFFF; - printf("number of total read accesses:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - l = totalbankreads[i][j]; - if (l < min_bank_accesses) - min_bank_accesses = l; - if (l > max_bank_accesses) - max_bank_accesses = l; - k += l; - m += l; - printf("%9d ",l); - } - if (m < min_chip_accesses) - min_chip_accesses = m; - if (m > max_chip_accesses) - max_chip_accesses = m; - m = 0; - printf("\n"); + /*WRITE ACCESSES*/ + k = 0; + l = 0; + m = 0; + max_bank_accesses = 0; + max_chip_accesses = 0; + min_bank_accesses = 0xFFFFFFFF; + min_chip_accesses = 0xFFFFFFFF; + printf("number of total write accesses:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + l = totalbankwrites[i][j]; + if (l < min_bank_accesses) min_bank_accesses = l; + if (l > max_bank_accesses) max_bank_accesses = l; + k += l; + m += l; + printf("%9d ", l); } - printf("total dram reads = %d\n", k); - if (min_bank_accesses) - printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); - else - printf("min_bank_accesses = 0!\n"); - if (min_chip_accesses) - printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); - else - printf("min_chip_accesses = 0!\n"); - - /*WRITE ACCESSES*/ - k = 0; - l = 0; + if (m < min_chip_accesses) min_chip_accesses = m; + if (m > max_chip_accesses) max_chip_accesses = m; m = 0; - max_bank_accesses = 0; - max_chip_accesses = 0; - min_bank_accesses = 0xFFFFFFFF; - min_chip_accesses = 0xFFFFFFFF; - printf("number of total write accesses:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - l = totalbankwrites[i][j]; - if (l < min_bank_accesses) - min_bank_accesses = l; - if (l > max_bank_accesses) - max_bank_accesses = l; - k += l; - m += l; - printf("%9d ",l); - } - if (m < min_chip_accesses) - min_chip_accesses = m; - if (m > max_chip_accesses) - max_chip_accesses = m; - m = 0; - printf("\n"); - } - printf("total dram writes = %d\n", k); - if (min_bank_accesses) - printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); - else - printf("min_bank_accesses = 0!\n"); - if (min_chip_accesses) - printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); - else - printf("min_chip_accesses = 0!\n"); - - - /*AVERAGE MF LATENCY PER BANK*/ - printf("average mf latency per bank:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - k = totalbankwrites[i][j] + totalbankreads[i][j]; - if (k) - printf("%10lld", mf_total_lat_table[i][j] / k); - else - printf(" none "); - } - printf("\n"); - } + printf("\n"); + } + printf("total dram writes = %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, + (float)max_bank_accesses / min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, + (float)max_chip_accesses / min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); - /*MAXIMUM MF LATENCY PER BANK*/ - printf("maximum mf latency per bank:\n"); - for (i=0;i<n_mem ;i++ ) { - printf("dram[%d]: ", i); - for (j=0;j<gpu_mem_n_bk;j++ ) { - printf("%10d", mf_max_lat_table[i][j]); - } - printf("\n"); + /*AVERAGE MF LATENCY PER BANK*/ + printf("average mf latency per bank:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + k = totalbankwrites[i][j] + totalbankreads[i][j]; + if (k) + printf("%10lld", mf_total_lat_table[i][j] / k); + else + printf(" none "); } - } + printf("\n"); + } - if (m_memory_config->gpgpu_memlatency_stat & GPU_MEMLATSTAT_MC) { - printf("\nNumber of Memory Banks Accessed per Memory Operation per Warp (from 0):\n"); - unsigned long long accum_MCBs_accessed = 0; - unsigned long long tot_mem_ops_per_warp = 0; - for (i=0;i< n_mem*gpu_mem_n_bk ; i++ ) { - accum_MCBs_accessed += i*num_MCBs_accessed[i]; - tot_mem_ops_per_warp += num_MCBs_accessed[i]; - printf("%d\t", num_MCBs_accessed[i]); + /*MAXIMUM MF LATENCY PER BANK*/ + printf("maximum mf latency per bank:\n"); + for (i = 0; i < n_mem; i++) { + printf("dram[%d]: ", i); + for (j = 0; j < gpu_mem_n_bk; j++) { + printf("%10d", mf_max_lat_table[i][j]); } + printf("\n"); + } + } - printf("\nAverage # of Memory Banks Accessed per Memory Operation per Warp=%f\n", (float)accum_MCBs_accessed/tot_mem_ops_per_warp); + if (m_memory_config->gpgpu_memlatency_stat & GPU_MEMLATSTAT_MC) { + printf( + "\nNumber of Memory Banks Accessed per Memory Operation per Warp (from " + "0):\n"); + unsigned long long accum_MCBs_accessed = 0; + unsigned long long tot_mem_ops_per_warp = 0; + for (i = 0; i < n_mem * gpu_mem_n_bk; i++) { + accum_MCBs_accessed += i * num_MCBs_accessed[i]; + tot_mem_ops_per_warp += num_MCBs_accessed[i]; + printf("%d\t", num_MCBs_accessed[i]); + } - //printf("\nAverage Difference Between First and Last Response from Memory System per warp = "); + printf( + "\nAverage # of Memory Banks Accessed per Memory Operation per " + "Warp=%f\n", + (float)accum_MCBs_accessed / tot_mem_ops_per_warp); + // printf("\nAverage Difference Between First and Last Response from Memory + // System per warp = "); - printf("\nposition of mrq chosen\n"); + printf("\nposition of mrq chosen\n"); - if (!m_memory_config->gpgpu_frfcfs_dram_sched_queue_size) - j = 1024; - else - j = m_memory_config->gpgpu_frfcfs_dram_sched_queue_size; - k=0;l=0; - for (i=0;i< j; i++ ) { - printf("%d\t", position_of_mrq_chosen[i]); - k += position_of_mrq_chosen[i]; - l += i*position_of_mrq_chosen[i]; - } - printf("\n"); - printf("\naverage position of mrq chosen = %f\n", (float)l/k); - } + if (!m_memory_config->gpgpu_frfcfs_dram_sched_queue_size) + j = 1024; + else + j = m_memory_config->gpgpu_frfcfs_dram_sched_queue_size; + k = 0; + l = 0; + for (i = 0; i < j; i++) { + printf("%d\t", position_of_mrq_chosen[i]); + k += position_of_mrq_chosen[i]; + l += i * position_of_mrq_chosen[i]; + } + printf("\n"); + printf("\naverage position of mrq chosen = %f\n", (float)l / k); + } } diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h index 0c84972..34385b2 100644 --- a/src/gpgpu-sim/mem_latency_stat.h +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef MEM_LATENCY_STAT_H #define MEM_LATENCY_STAT_H @@ -34,87 +35,94 @@ class memory_config; class memory_stats_t { -public: - memory_stats_t( unsigned n_shader, - const class shader_core_config *shader_config, - const memory_config *mem_config, - const class gpgpu_sim* gpu); + public: + memory_stats_t(unsigned n_shader, + const class shader_core_config *shader_config, + const memory_config *mem_config, const class gpgpu_sim *gpu); - unsigned memlatstat_done( class mem_fetch *mf ); - void memlatstat_read_done( class mem_fetch *mf ); - void memlatstat_dram_access( class mem_fetch *mf ); - void memlatstat_icnt2mem_pop( class mem_fetch *mf); - void memlatstat_lat_pw(); - void memlatstat_print(unsigned n_mem, unsigned gpu_mem_n_bk); + unsigned memlatstat_done(class mem_fetch *mf); + void memlatstat_read_done(class mem_fetch *mf); + void memlatstat_dram_access(class mem_fetch *mf); + void memlatstat_icnt2mem_pop(class mem_fetch *mf); + void memlatstat_lat_pw(); + void memlatstat_print(unsigned n_mem, unsigned gpu_mem_n_bk); - void visualizer_print( gzFile visualizer_file ); + void visualizer_print(gzFile visualizer_file); - // Reset local L2 stats that are aggregated each sampling window - void clear_L2_stats_pw(); + // Reset local L2 stats that are aggregated each sampling window + void clear_L2_stats_pw(); - unsigned m_n_shader; + unsigned m_n_shader; - const shader_core_config *m_shader_config; - const memory_config *m_memory_config; - const class gpgpu_sim* m_gpu; + const shader_core_config *m_shader_config; + const memory_config *m_memory_config; + const class gpgpu_sim *m_gpu; - unsigned max_mrq_latency; - unsigned max_dq_latency; - unsigned max_mf_latency; - unsigned max_icnt2mem_latency; - unsigned long long int tot_icnt2mem_latency; - unsigned long long int tot_icnt2sh_latency; - unsigned long long int tot_mrq_latency; - unsigned long long int tot_mrq_num; - unsigned max_icnt2sh_latency; - unsigned mrq_lat_table[32]; - unsigned dq_lat_table[32]; - unsigned mf_lat_table[32]; - unsigned icnt2mem_lat_table[24]; - unsigned icnt2sh_lat_table[24]; - unsigned mf_lat_pw_table[32]; //table storing values of mf latency Per Window - unsigned mf_num_lat_pw; - unsigned max_warps; - unsigned mf_tot_lat_pw; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window - unsigned long long int mf_total_lat; - unsigned long long int ** mf_total_lat_table; //mf latency sums[dram chip id][bank id] - unsigned ** mf_max_lat_table; //mf latency sums[dram chip id][bank id] - unsigned num_mfs; - unsigned int ***bankwrites; //bankwrites[shader id][dram chip id][bank id] - unsigned int ***bankreads; //bankreads[shader id][dram chip id][bank id] - unsigned int **totalbankwrites; //bankwrites[dram chip id][bank id] - unsigned int **totalbankreads; //bankreads[dram chip id][bank id] - unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id] - unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache - unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen - - unsigned ***mem_access_type_stats; // dram access type classification + unsigned max_mrq_latency; + unsigned max_dq_latency; + unsigned max_mf_latency; + unsigned max_icnt2mem_latency; + unsigned long long int tot_icnt2mem_latency; + unsigned long long int tot_icnt2sh_latency; + unsigned long long int tot_mrq_latency; + unsigned long long int tot_mrq_num; + unsigned max_icnt2sh_latency; + unsigned mrq_lat_table[32]; + unsigned dq_lat_table[32]; + unsigned mf_lat_table[32]; + unsigned icnt2mem_lat_table[24]; + unsigned icnt2sh_lat_table[24]; + unsigned mf_lat_pw_table[32]; // table storing values of mf latency Per + // Window + unsigned mf_num_lat_pw; + unsigned max_warps; + unsigned mf_tot_lat_pw; // total latency summed up per window. divide by + // mf_num_lat_pw to obtain average latency Per Window + unsigned long long int mf_total_lat; + unsigned long long int * + *mf_total_lat_table; // mf latency sums[dram chip id][bank id] + unsigned **mf_max_lat_table; // mf latency sums[dram chip id][bank id] + unsigned num_mfs; + unsigned int ***bankwrites; // bankwrites[shader id][dram chip id][bank id] + unsigned int ***bankreads; // bankreads[shader id][dram chip id][bank id] + unsigned int **totalbankwrites; // bankwrites[dram chip id][bank id] + unsigned int **totalbankreads; // bankreads[dram chip id][bank id] + unsigned int **totalbankaccesses; // bankaccesses[dram chip id][bank id] + unsigned int + *num_MCBs_accessed; // tracks how many memory controllers are accessed + // whenever any thread in a warp misses in cache + unsigned int *position_of_mrq_chosen; // position of mrq in m_queue chosen - // AerialVision L2 stats - unsigned L2_read_miss; - unsigned L2_write_miss; - unsigned L2_read_hit; - unsigned L2_write_hit; + unsigned ***mem_access_type_stats; // dram access type classification - // L2 cache stats - unsigned int *L2_cbtoL2length; - unsigned int *L2_cbtoL2writelength; - unsigned int *L2_L2tocblength; - unsigned int *L2_dramtoL2length; - unsigned int *L2_dramtoL2writelength; - unsigned int *L2_L2todramlength; + // AerialVision L2 stats + unsigned L2_read_miss; + unsigned L2_write_miss; + unsigned L2_read_hit; + unsigned L2_write_hit; - // DRAM access row locality stats - unsigned int **concurrent_row_access; //concurrent_row_access[dram chip id][bank id] - unsigned int **num_activates; //num_activates[dram chip id][bank id] - unsigned int **row_access; //row_access[dram chip id][bank id] - unsigned int **max_conc_access2samerow; //max_conc_access2samerow[dram chip id][bank id] - unsigned int **max_servicetime2samerow; //max_servicetime2samerow[dram chip id][bank id] + // L2 cache stats + unsigned int *L2_cbtoL2length; + unsigned int *L2_cbtoL2writelength; + unsigned int *L2_L2tocblength; + unsigned int *L2_dramtoL2length; + unsigned int *L2_dramtoL2writelength; + unsigned int *L2_L2todramlength; - // Power stats - unsigned total_n_access; - unsigned total_n_reads; - unsigned total_n_writes; + // DRAM access row locality stats + unsigned int * + *concurrent_row_access; // concurrent_row_access[dram chip id][bank id] + unsigned int **num_activates; // num_activates[dram chip id][bank id] + unsigned int **row_access; // row_access[dram chip id][bank id] + unsigned int **max_conc_access2samerow; // max_conc_access2samerow[dram chip + // id][bank id] + unsigned int **max_servicetime2samerow; // max_servicetime2samerow[dram chip + // id][bank id] + + // Power stats + unsigned total_n_access; + unsigned total_n_reads; + unsigned total_n_writes; }; #endif /*MEM_LATENCY_STAT_H*/ diff --git a/src/gpgpu-sim/power_interface.cc b/src/gpgpu-sim/power_interface.cc index 0272aa6..c637d84 100644 --- a/src/gpgpu-sim/power_interface.cc +++ b/src/gpgpu-sim/power_interface.cc @@ -7,119 +7,148 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "power_interface.h" -void init_mcpat(const gpgpu_sim_config &config, class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq, unsigned tot_inst, unsigned inst){ - - wrapper->init_mcpat(config.g_power_config_name, config.g_power_filename, config.g_power_trace_filename, - config.g_metric_trace_filename,config.g_steady_state_tracking_filename,config.g_power_simulation_enabled, - config.g_power_trace_enabled,config.g_steady_power_levels_enabled,config.g_power_per_cycle_dump, - config.gpu_steady_power_deviation,config.gpu_steady_min_period,config.g_power_trace_zlevel, - tot_inst+inst,stat_sample_freq - ); - +void init_mcpat(const gpgpu_sim_config &config, + class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq, + unsigned tot_inst, unsigned inst) { + wrapper->init_mcpat( + config.g_power_config_name, config.g_power_filename, + config.g_power_trace_filename, config.g_metric_trace_filename, + config.g_steady_state_tracking_filename, + config.g_power_simulation_enabled, config.g_power_trace_enabled, + config.g_steady_power_levels_enabled, config.g_power_per_cycle_dump, + config.gpu_steady_power_deviation, config.gpu_steady_min_period, + config.g_power_trace_zlevel, tot_inst + inst, stat_sample_freq); } -void mcpat_cycle(const gpgpu_sim_config &config, const shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, unsigned stat_sample_freq, unsigned tot_cycle, unsigned cycle, unsigned tot_inst, unsigned inst){ - - static bool mcpat_init=true; - - if(mcpat_init){ // If first cycle, don't have any power numbers yet - mcpat_init=false; - return; - } - - if ((tot_cycle+cycle) % stat_sample_freq == 0) { - - wrapper->set_inst_power(shdr_config->gpgpu_clock_gated_lanes, - stat_sample_freq, stat_sample_freq, - power_stats->get_total_inst(), power_stats->get_total_int_inst(), - power_stats->get_total_fp_inst(), power_stats->get_l1d_read_accesses(), - power_stats->get_l1d_write_accesses(), power_stats->get_committed_inst()); - - // Single RF for both int and fp ops - wrapper->set_regfile_power(power_stats->get_regfile_reads(), power_stats->get_regfile_writes(), power_stats->get_non_regfile_operands()); - - //Instruction cache stats - wrapper->set_icache_power(power_stats->get_inst_c_hits(), power_stats->get_inst_c_misses()); - - //Constant Cache, shared memory, texture cache - wrapper->set_ccache_power(power_stats->get_constant_c_hits(), power_stats->get_constant_c_misses()); - wrapper->set_tcache_power(power_stats->get_texture_c_hits(), power_stats->get_texture_c_misses()); - wrapper->set_shrd_mem_power(power_stats->get_shmem_read_access()); +void mcpat_cycle(const gpgpu_sim_config &config, + const shader_core_config *shdr_config, + class gpgpu_sim_wrapper *wrapper, + class power_stat_t *power_stats, unsigned stat_sample_freq, + unsigned tot_cycle, unsigned cycle, unsigned tot_inst, + unsigned inst) { + static bool mcpat_init = true; - wrapper->set_l1cache_power(power_stats->get_l1d_read_hits(), power_stats->get_l1d_read_misses(), - power_stats->get_l1d_write_hits(), power_stats->get_l1d_write_misses()); + if (mcpat_init) { // If first cycle, don't have any power numbers yet + mcpat_init = false; + return; + } + if ((tot_cycle + cycle) % stat_sample_freq == 0) { + wrapper->set_inst_power( + shdr_config->gpgpu_clock_gated_lanes, stat_sample_freq, + stat_sample_freq, power_stats->get_total_inst(), + power_stats->get_total_int_inst(), power_stats->get_total_fp_inst(), + power_stats->get_l1d_read_accesses(), + power_stats->get_l1d_write_accesses(), + power_stats->get_committed_inst()); - wrapper->set_l2cache_power(power_stats->get_l2_read_hits(), power_stats->get_l2_read_misses(), - power_stats->get_l2_write_hits(), power_stats->get_l2_write_misses()); + // Single RF for both int and fp ops + wrapper->set_regfile_power(power_stats->get_regfile_reads(), + power_stats->get_regfile_writes(), + power_stats->get_non_regfile_operands()); + // Instruction cache stats + wrapper->set_icache_power(power_stats->get_inst_c_hits(), + power_stats->get_inst_c_misses()); - float active_sms=(*power_stats->m_active_sms)/stat_sample_freq; - float num_cores = shdr_config->num_shader(); - float num_idle_core = num_cores - active_sms; - wrapper->set_idle_core_power(num_idle_core); + // Constant Cache, shared memory, texture cache + wrapper->set_ccache_power(power_stats->get_constant_c_hits(), + power_stats->get_constant_c_misses()); + wrapper->set_tcache_power(power_stats->get_texture_c_hits(), + power_stats->get_texture_c_misses()); + wrapper->set_shrd_mem_power(power_stats->get_shmem_read_access()); - //pipeline power - pipeline_duty_cycle *= percent_active_sms; - float pipeline_duty_cycle=((*power_stats->m_average_pipeline_duty_cycle/( stat_sample_freq)) < 0.8)?((*power_stats->m_average_pipeline_duty_cycle)/stat_sample_freq):0.8; - wrapper->set_duty_cycle_power(pipeline_duty_cycle); + wrapper->set_l1cache_power( + power_stats->get_l1d_read_hits(), power_stats->get_l1d_read_misses(), + power_stats->get_l1d_write_hits(), power_stats->get_l1d_write_misses()); - //Memory Controller - wrapper->set_mem_ctrl_power(power_stats->get_dram_rd(), power_stats->get_dram_wr(), power_stats->get_dram_pre()); + wrapper->set_l2cache_power( + power_stats->get_l2_read_hits(), power_stats->get_l2_read_misses(), + power_stats->get_l2_write_hits(), power_stats->get_l2_write_misses()); - //Execution pipeline accesses - //FPU (SP) accesses, Integer ALU (not present in Tesla), Sfu accesses - wrapper->set_exec_unit_power(power_stats->get_tot_fpu_accessess(), power_stats->get_ialu_accessess(), power_stats->get_tot_sfu_accessess()); + float active_sms = (*power_stats->m_active_sms) / stat_sample_freq; + float num_cores = shdr_config->num_shader(); + float num_idle_core = num_cores - active_sms; + wrapper->set_idle_core_power(num_idle_core); - //Average active lanes for sp and sfu pipelines - float avg_sp_active_lanes=(power_stats->get_sp_active_lanes())/stat_sample_freq; - float avg_sfu_active_lanes=(power_stats->get_sfu_active_lanes())/stat_sample_freq; - assert(avg_sp_active_lanes<=32); - assert(avg_sfu_active_lanes<=32); - wrapper->set_active_lanes_power((power_stats->get_sp_active_lanes())/stat_sample_freq, - (power_stats->get_sfu_active_lanes())/stat_sample_freq); + // pipeline power - pipeline_duty_cycle *= percent_active_sms; + float pipeline_duty_cycle = + ((*power_stats->m_average_pipeline_duty_cycle / (stat_sample_freq)) < + 0.8) + ? ((*power_stats->m_average_pipeline_duty_cycle) / stat_sample_freq) + : 0.8; + wrapper->set_duty_cycle_power(pipeline_duty_cycle); + // Memory Controller + wrapper->set_mem_ctrl_power(power_stats->get_dram_rd(), + power_stats->get_dram_wr(), + power_stats->get_dram_pre()); - double n_icnt_simt_to_mem = (double)power_stats->get_icnt_simt_to_mem(); // # flits from SIMT clusters to memory partitions - double n_icnt_mem_to_simt = (double)power_stats->get_icnt_mem_to_simt(); // # flits from memory partitions to SIMT clusters - wrapper->set_NoC_power(n_icnt_mem_to_simt, n_icnt_simt_to_mem); // Number of flits traversing the interconnect + // Execution pipeline accesses + // FPU (SP) accesses, Integer ALU (not present in Tesla), Sfu accesses + wrapper->set_exec_unit_power(power_stats->get_tot_fpu_accessess(), + power_stats->get_ialu_accessess(), + power_stats->get_tot_sfu_accessess()); - wrapper->compute(); + // Average active lanes for sp and sfu pipelines + float avg_sp_active_lanes = + (power_stats->get_sp_active_lanes()) / stat_sample_freq; + float avg_sfu_active_lanes = + (power_stats->get_sfu_active_lanes()) / stat_sample_freq; + assert(avg_sp_active_lanes <= 32); + assert(avg_sfu_active_lanes <= 32); + wrapper->set_active_lanes_power( + (power_stats->get_sp_active_lanes()) / stat_sample_freq, + (power_stats->get_sfu_active_lanes()) / stat_sample_freq); + double n_icnt_simt_to_mem = + (double) + power_stats->get_icnt_simt_to_mem(); // # flits from SIMT clusters + // to memory partitions + double n_icnt_mem_to_simt = + (double) + power_stats->get_icnt_mem_to_simt(); // # flits from memory + // partitions to SIMT clusters + wrapper->set_NoC_power( + n_icnt_mem_to_simt, + n_icnt_simt_to_mem); // Number of flits traversing the interconnect - wrapper->update_components_power(); - wrapper->print_trace_files(); - power_stats->save_stats(); + wrapper->compute(); - wrapper->detect_print_steady_state(0,tot_inst+inst); + wrapper->update_components_power(); + wrapper->print_trace_files(); + power_stats->save_stats(); - wrapper->power_metrics_calculations(); + wrapper->detect_print_steady_state(0, tot_inst + inst); + wrapper->power_metrics_calculations(); - wrapper->dump(); - } - //wrapper->close_files(); + wrapper->dump(); + } + // wrapper->close_files(); } -void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper){ - wrapper->reset_counters(); +void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper) { + wrapper->reset_counters(); } diff --git a/src/gpgpu-sim/power_interface.h b/src/gpgpu-sim/power_interface.h index a388c23..2bfd4d5 100644 --- a/src/gpgpu-sim/power_interface.h +++ b/src/gpgpu-sim/power_interface.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef POWER_INTERFACE_H_ #define POWER_INTERFACE_H_ @@ -32,12 +33,17 @@ #include "power_stat.h" #include "shader.h" - #include "gpgpu_sim_wrapper.h" -void init_mcpat(const gpgpu_sim_config &config, class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq, unsigned tot_inst, unsigned inst); -void mcpat_cycle(const gpgpu_sim_config &config, const shader_core_config *shdr_config, class gpgpu_sim_wrapper *wrapper, class power_stat_t *power_stats, - unsigned stat_sample_freq, unsigned tot_cycle, unsigned cycle, unsigned tot_inst, unsigned inst); +void init_mcpat(const gpgpu_sim_config &config, + class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq, + unsigned tot_inst, unsigned inst); +void mcpat_cycle(const gpgpu_sim_config &config, + const shader_core_config *shdr_config, + class gpgpu_sim_wrapper *wrapper, + class power_stat_t *power_stats, unsigned stat_sample_freq, + unsigned tot_cycle, unsigned cycle, unsigned tot_inst, + unsigned inst); void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper); #endif /* POWER_INTERFACE_H_ */ diff --git a/src/gpgpu-sim/power_stat.cc b/src/gpgpu-sim/power_stat.cc index 2c02082..7b60ddf 100644 --- a/src/gpgpu-sim/power_stat.cc +++ b/src/gpgpu-sim/power_stat.cc @@ -7,287 +7,365 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "../abstract_hardware_model.h" #include "power_stat.h" -#include "gpu-sim.h" +#include "../abstract_hardware_model.h" +#include "../cuda-sim/ptx-stats.h" +#include "dram.h" #include "gpu-misc.h" -#include "shader.h" +#include "gpu-sim.h" #include "mem_fetch.h" +#include "shader.h" #include "stat-tool.h" -#include "../cuda-sim/ptx-stats.h" #include "visualizer.h" -#include "dram.h" -#include <string.h> -#include <stdlib.h> #include <stdio.h> +#include <stdlib.h> +#include <string.h> +power_mem_stat_t::power_mem_stat_t(const memory_config *mem_config, + const shader_core_config *shdr_config, + memory_stats_t *mem_stats, + shader_core_stats *shdr_stats) { + assert(mem_config->m_valid); + m_mem_stats = mem_stats; + m_config = mem_config; + m_core_stats = shdr_stats; + m_core_config = shdr_config; - -power_mem_stat_t::power_mem_stat_t(const memory_config *mem_config, const shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats){ - assert( mem_config->m_valid ); - m_mem_stats = mem_stats; - m_config = mem_config; - m_core_stats = shdr_stats; - m_core_config = shdr_config; - - init(); + init(); } -void power_mem_stat_t::init(){ - - shmem_read_access[CURRENT_STAT_IDX] = m_core_stats->gpgpu_n_shmem_bank_access; // Shared memory access - shmem_read_access[PREV_STAT_IDX] = (unsigned *)calloc(m_core_config->num_shader(),sizeof(unsigned)); +void power_mem_stat_t::init() { + shmem_read_access[CURRENT_STAT_IDX] = + m_core_stats->gpgpu_n_shmem_bank_access; // Shared memory access + shmem_read_access[PREV_STAT_IDX] = + (unsigned *)calloc(m_core_config->num_shader(), sizeof(unsigned)); - for(unsigned i=0; i<NUM_STAT_IDX; ++i){ - core_cache_stats[i].clear(); - l2_cache_stats[i].clear(); + for (unsigned i = 0; i < NUM_STAT_IDX; ++i) { + core_cache_stats[i].clear(); + l2_cache_stats[i].clear(); - n_cmd[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_activity[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_nop[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_act[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_pre[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_rd[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_wr[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); - n_req[i] = (unsigned *)calloc(m_config->m_n_mem,sizeof(unsigned)); + n_cmd[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_activity[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_nop[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_act[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_pre[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_rd[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_wr[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); + n_req[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned)); - // Interconnect stats - n_mem_to_simt[i] = (long *)calloc(m_core_config->n_simt_clusters,sizeof(long)); // Counted at SM - n_simt_to_mem[i] = (long *)calloc(m_core_config->n_simt_clusters,sizeof(long)); // Counted at SM - } + // Interconnect stats + n_mem_to_simt[i] = (long *)calloc(m_core_config->n_simt_clusters, + sizeof(long)); // Counted at SM + n_simt_to_mem[i] = (long *)calloc(m_core_config->n_simt_clusters, + sizeof(long)); // Counted at SM + } } -void power_mem_stat_t::save_stats(){ +void power_mem_stat_t::save_stats() { + core_cache_stats[PREV_STAT_IDX] = core_cache_stats[CURRENT_STAT_IDX]; + l2_cache_stats[PREV_STAT_IDX] = l2_cache_stats[CURRENT_STAT_IDX]; - core_cache_stats[PREV_STAT_IDX] = core_cache_stats[CURRENT_STAT_IDX]; - l2_cache_stats[PREV_STAT_IDX] = l2_cache_stats[CURRENT_STAT_IDX]; + for (unsigned i = 0; i < m_core_config->num_shader(); ++i) { + shmem_read_access[PREV_STAT_IDX][i] = + shmem_read_access[CURRENT_STAT_IDX][i]; // Shared memory access + } - for(unsigned i=0; i<m_core_config->num_shader(); ++i){ - shmem_read_access[PREV_STAT_IDX][i] = shmem_read_access[CURRENT_STAT_IDX][i] ; // Shared memory access - } + for (unsigned i = 0; i < m_config->m_n_mem; ++i) { + n_cmd[PREV_STAT_IDX][i] = n_cmd[CURRENT_STAT_IDX][i]; + n_activity[PREV_STAT_IDX][i] = n_activity[CURRENT_STAT_IDX][i]; + n_nop[PREV_STAT_IDX][i] = n_nop[CURRENT_STAT_IDX][i]; + n_act[PREV_STAT_IDX][i] = n_act[CURRENT_STAT_IDX][i]; + n_pre[PREV_STAT_IDX][i] = n_pre[CURRENT_STAT_IDX][i]; + n_rd[PREV_STAT_IDX][i] = n_rd[CURRENT_STAT_IDX][i]; + n_wr[PREV_STAT_IDX][i] = n_wr[CURRENT_STAT_IDX][i]; + n_req[PREV_STAT_IDX][i] = n_req[CURRENT_STAT_IDX][i]; + } - for(unsigned i=0; i<m_config->m_n_mem; ++i){ - n_cmd[PREV_STAT_IDX][i] = n_cmd[CURRENT_STAT_IDX][i]; - n_activity[PREV_STAT_IDX][i] = n_activity[CURRENT_STAT_IDX][i]; - n_nop[PREV_STAT_IDX][i] = n_nop[CURRENT_STAT_IDX][i]; - n_act[PREV_STAT_IDX][i] = n_act[CURRENT_STAT_IDX][i]; - n_pre[PREV_STAT_IDX][i] = n_pre[CURRENT_STAT_IDX][i]; - n_rd[PREV_STAT_IDX][i] = n_rd[CURRENT_STAT_IDX][i]; - n_wr[PREV_STAT_IDX][i] = n_wr[CURRENT_STAT_IDX][i]; - n_req[PREV_STAT_IDX][i] = n_req[CURRENT_STAT_IDX][i]; - } - - for(unsigned i=0; i<m_core_config->n_simt_clusters;i++){ - n_simt_to_mem[PREV_STAT_IDX][i] = n_simt_to_mem[CURRENT_STAT_IDX][i]; // Interconnect - n_mem_to_simt[PREV_STAT_IDX][i] = n_mem_to_simt[CURRENT_STAT_IDX][i]; // Interconnect - } + for (unsigned i = 0; i < m_core_config->n_simt_clusters; i++) { + n_simt_to_mem[PREV_STAT_IDX][i] = + n_simt_to_mem[CURRENT_STAT_IDX][i]; // Interconnect + n_mem_to_simt[PREV_STAT_IDX][i] = + n_mem_to_simt[CURRENT_STAT_IDX][i]; // Interconnect + } } -void power_mem_stat_t::visualizer_print( gzFile power_visualizer_file ){ - -} +void power_mem_stat_t::visualizer_print(gzFile power_visualizer_file) {} -void power_mem_stat_t::print (FILE *fout) const { - fprintf(fout, "\n\n==========Power Metrics -- Memory==========\n"); - unsigned total_mem_reads=0; - unsigned total_mem_writes=0; - for(unsigned i=0; i<m_config->m_n_mem; ++i){ - total_mem_reads += n_rd[CURRENT_STAT_IDX][i]; - total_mem_writes += n_wr[CURRENT_STAT_IDX][i]; - } - fprintf(fout, "Total memory controller accesses: %u\n", total_mem_reads+total_mem_writes); - fprintf(fout, "Total memory controller reads: %u\n", total_mem_reads); - fprintf(fout, "Total memory controller writes: %u\n", total_mem_writes); +void power_mem_stat_t::print(FILE *fout) const { + fprintf(fout, "\n\n==========Power Metrics -- Memory==========\n"); + unsigned total_mem_reads = 0; + unsigned total_mem_writes = 0; + for (unsigned i = 0; i < m_config->m_n_mem; ++i) { + total_mem_reads += n_rd[CURRENT_STAT_IDX][i]; + total_mem_writes += n_wr[CURRENT_STAT_IDX][i]; + } + fprintf(fout, "Total memory controller accesses: %u\n", + total_mem_reads + total_mem_writes); + fprintf(fout, "Total memory controller reads: %u\n", total_mem_reads); + fprintf(fout, "Total memory controller writes: %u\n", total_mem_writes); - fprintf(fout, "Core cache stats:\n"); - core_cache_stats->print_stats(fout); - fprintf(fout, "L2 cache stats:\n"); - l2_cache_stats->print_stats(fout); + fprintf(fout, "Core cache stats:\n"); + core_cache_stats->print_stats(fout); + fprintf(fout, "L2 cache stats:\n"); + l2_cache_stats->print_stats(fout); } +power_core_stat_t::power_core_stat_t(const shader_core_config *shader_config, + shader_core_stats *core_stats) { + assert(shader_config->m_valid); + m_config = shader_config; + shader_core_power_stats_pod *pod = this; + memset(pod, 0, sizeof(shader_core_power_stats_pod)); + m_core_stats = core_stats; -power_core_stat_t::power_core_stat_t( const shader_core_config *shader_config, shader_core_stats *core_stats ) -{ - assert( shader_config->m_valid ); - m_config = shader_config; - shader_core_power_stats_pod *pod = this; - memset(pod,0,sizeof(shader_core_power_stats_pod)); - m_core_stats=core_stats; - - init(); - + init(); } -void power_core_stat_t::visualizer_print( gzFile visualizer_file ) -{ - -} +void power_core_stat_t::visualizer_print(gzFile visualizer_file) {} -void power_core_stat_t::print (FILE *fout) -{ - // per core statistics - fprintf(fout,"Power Metrics: \n"); - for(unsigned i=0; i<m_config->num_shader();i++){ - fprintf(fout,"core %u:\n",i); - fprintf(fout,"\tpipeline duty cycle =%f\n",m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal Deocded Instructions=%u\n",m_num_decoded_insn[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal FP Deocded Instructions=%u\n",m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal INT Deocded Instructions=%u\n",m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal LOAD Queued Instructions=%u\n",m_num_loadqueued_insn[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal STORE Queued Instructions=%u\n",m_num_storequeued_insn[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal IALU Acesses=%u\n",m_num_ialu_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal FP Acesses=%u\n",m_num_fp_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal IMUL Acesses=%u\n",m_num_imul_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal IMUL24 Acesses=%u\n",m_num_imul24_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal IMUL32 Acesses=%u\n",m_num_imul32_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal IDIV Acesses=%u\n",m_num_idiv_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal FPMUL Acesses=%u\n",m_num_fpmul_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal SFU Acesses=%u\n",m_num_trans_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal FPDIV Acesses=%u\n",m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal SFU Acesses=%u\n",m_num_sfu_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal SP Acesses=%u\n",m_num_sp_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal MEM Acesses=%u\n",m_num_mem_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal SFU Commissions=%u\n",m_num_sfu_committed[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal SP Commissions=%u\n",m_num_sp_committed[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal MEM Commissions=%u\n",m_num_mem_committed[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal REG Reads=%u\n",m_read_regfile_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal REG Writes=%u\n",m_write_regfile_acesses[CURRENT_STAT_IDX][i]); - fprintf(fout,"\tTotal NON REG=%u\n",m_non_rf_operands[CURRENT_STAT_IDX][i]); - } +void power_core_stat_t::print(FILE *fout) { + // per core statistics + fprintf(fout, "Power Metrics: \n"); + for (unsigned i = 0; i < m_config->num_shader(); i++) { + fprintf(fout, "core %u:\n", i); + fprintf(fout, "\tpipeline duty cycle =%f\n", + m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal Deocded Instructions=%u\n", + m_num_decoded_insn[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal FP Deocded Instructions=%u\n", + m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal INT Deocded Instructions=%u\n", + m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal LOAD Queued Instructions=%u\n", + m_num_loadqueued_insn[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal STORE Queued Instructions=%u\n", + m_num_storequeued_insn[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal IALU Acesses=%u\n", + m_num_ialu_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal FP Acesses=%u\n", + m_num_fp_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal IMUL Acesses=%u\n", + m_num_imul_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal IMUL24 Acesses=%u\n", + m_num_imul24_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal IMUL32 Acesses=%u\n", + m_num_imul32_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal IDIV Acesses=%u\n", + m_num_idiv_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal FPMUL Acesses=%u\n", + m_num_fpmul_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal SFU Acesses=%u\n", + m_num_trans_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal FPDIV Acesses=%u\n", + m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal SFU Acesses=%u\n", + m_num_sfu_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal SP Acesses=%u\n", + m_num_sp_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal MEM Acesses=%u\n", + m_num_mem_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal SFU Commissions=%u\n", + m_num_sfu_committed[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal SP Commissions=%u\n", + m_num_sp_committed[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal MEM Commissions=%u\n", + m_num_mem_committed[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal REG Reads=%u\n", + m_read_regfile_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal REG Writes=%u\n", + m_write_regfile_acesses[CURRENT_STAT_IDX][i]); + fprintf(fout, "\tTotal NON REG=%u\n", + m_non_rf_operands[CURRENT_STAT_IDX][i]); + } } -void power_core_stat_t::init() -{ - m_pipeline_duty_cycle[CURRENT_STAT_IDX]=m_core_stats->m_pipeline_duty_cycle; - m_num_decoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_decoded_insn; - m_num_FPdecoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_FPdecoded_insn; - m_num_INTdecoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_INTdecoded_insn; - m_num_storequeued_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_storequeued_insn; - m_num_loadqueued_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_loadqueued_insn; - m_num_ialu_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_ialu_acesses; - m_num_fp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fp_acesses; - m_num_imul_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul_acesses; - m_num_imul24_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul24_acesses; - m_num_imul32_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul32_acesses; - m_num_fpmul_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fpmul_acesses; - m_num_idiv_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_idiv_acesses; - m_num_fpdiv_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fpdiv_acesses; - m_num_sp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sp_acesses; - m_num_sfu_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sfu_acesses; - m_num_trans_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_trans_acesses; - m_num_mem_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_mem_acesses; - m_num_sp_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_sp_committed; - m_num_sfu_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_sfu_committed; - m_num_mem_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_mem_committed; - m_read_regfile_acesses[CURRENT_STAT_IDX]=m_core_stats->m_read_regfile_acesses; - m_write_regfile_acesses[CURRENT_STAT_IDX]=m_core_stats->m_write_regfile_acesses; - m_non_rf_operands[CURRENT_STAT_IDX]=m_core_stats->m_non_rf_operands; - m_active_sp_lanes[CURRENT_STAT_IDX]=m_core_stats->m_active_sp_lanes; - m_active_sfu_lanes[CURRENT_STAT_IDX]=m_core_stats->m_active_sfu_lanes; - m_num_tex_inst[CURRENT_STAT_IDX]=m_core_stats->m_num_tex_inst; - +void power_core_stat_t::init() { + m_pipeline_duty_cycle[CURRENT_STAT_IDX] = m_core_stats->m_pipeline_duty_cycle; + m_num_decoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_decoded_insn; + m_num_FPdecoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_FPdecoded_insn; + m_num_INTdecoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_INTdecoded_insn; + m_num_storequeued_insn[CURRENT_STAT_IDX] = + m_core_stats->m_num_storequeued_insn; + m_num_loadqueued_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_loadqueued_insn; + m_num_ialu_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_ialu_acesses; + m_num_fp_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fp_acesses; + m_num_imul_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul_acesses; + m_num_imul24_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul24_acesses; + m_num_imul32_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul32_acesses; + m_num_fpmul_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fpmul_acesses; + m_num_idiv_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_idiv_acesses; + m_num_fpdiv_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fpdiv_acesses; + m_num_sp_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_sp_acesses; + m_num_sfu_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_sfu_acesses; + m_num_trans_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_trans_acesses; + m_num_mem_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_mem_acesses; + m_num_sp_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_sp_committed; + m_num_sfu_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_sfu_committed; + m_num_mem_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_mem_committed; + m_read_regfile_acesses[CURRENT_STAT_IDX] = + m_core_stats->m_read_regfile_acesses; + m_write_regfile_acesses[CURRENT_STAT_IDX] = + m_core_stats->m_write_regfile_acesses; + m_non_rf_operands[CURRENT_STAT_IDX] = m_core_stats->m_non_rf_operands; + m_active_sp_lanes[CURRENT_STAT_IDX] = m_core_stats->m_active_sp_lanes; + m_active_sfu_lanes[CURRENT_STAT_IDX] = m_core_stats->m_active_sfu_lanes; + m_num_tex_inst[CURRENT_STAT_IDX] = m_core_stats->m_num_tex_inst; - m_pipeline_duty_cycle[PREV_STAT_IDX]=(float*)calloc(m_config->num_shader(),sizeof(float)); - m_num_decoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_FPdecoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_INTdecoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_storequeued_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_loadqueued_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_ialu_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_fp_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_tex_inst[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_imul_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_imul24_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_imul32_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_fpmul_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_idiv_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_fpdiv_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_sp_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_sfu_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_trans_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_mem_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_sp_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_sfu_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_num_mem_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_read_regfile_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_write_regfile_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_non_rf_operands[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_active_sp_lanes[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); - m_active_sfu_lanes[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned)); + m_pipeline_duty_cycle[PREV_STAT_IDX] = + (float *)calloc(m_config->num_shader(), sizeof(float)); + m_num_decoded_insn[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_FPdecoded_insn[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_INTdecoded_insn[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_storequeued_insn[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_loadqueued_insn[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_ialu_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_fp_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_tex_inst[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_imul_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_imul24_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_imul32_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_fpmul_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_idiv_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_fpdiv_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_sp_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_sfu_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_trans_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_mem_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_sp_committed[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_sfu_committed[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_num_mem_committed[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_read_regfile_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_write_regfile_acesses[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_non_rf_operands[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_active_sp_lanes[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); + m_active_sfu_lanes[PREV_STAT_IDX] = + (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned)); } -void power_core_stat_t::save_stats(){ -for(unsigned i=0; i<m_config->num_shader(); ++i){ - m_pipeline_duty_cycle[PREV_STAT_IDX][i]=m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]; - m_num_decoded_insn[PREV_STAT_IDX][i]= m_num_decoded_insn[CURRENT_STAT_IDX][i]; - m_num_FPdecoded_insn[PREV_STAT_IDX][i]=m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]; - m_num_INTdecoded_insn[PREV_STAT_IDX][i]=m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]; - m_num_storequeued_insn[PREV_STAT_IDX][i]=m_num_storequeued_insn[CURRENT_STAT_IDX][i]; - m_num_loadqueued_insn[PREV_STAT_IDX][i]=m_num_loadqueued_insn[CURRENT_STAT_IDX][i]; - m_num_ialu_acesses[PREV_STAT_IDX][i]=m_num_ialu_acesses[CURRENT_STAT_IDX][i]; - m_num_fp_acesses[PREV_STAT_IDX][i]=m_num_fp_acesses[CURRENT_STAT_IDX][i]; - m_num_tex_inst[PREV_STAT_IDX][i]=m_num_tex_inst[CURRENT_STAT_IDX][i]; - m_num_imul_acesses[PREV_STAT_IDX][i]=m_num_imul_acesses[CURRENT_STAT_IDX][i]; - m_num_imul24_acesses[PREV_STAT_IDX][i]=m_num_imul24_acesses[CURRENT_STAT_IDX][i]; - m_num_imul32_acesses[PREV_STAT_IDX][i]=m_num_imul32_acesses[CURRENT_STAT_IDX][i]; - m_num_fpmul_acesses[PREV_STAT_IDX][i]=m_num_fpmul_acesses[CURRENT_STAT_IDX][i]; - m_num_idiv_acesses[PREV_STAT_IDX][i]=m_num_idiv_acesses[CURRENT_STAT_IDX][i]; - m_num_fpdiv_acesses[PREV_STAT_IDX][i]=m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]; - m_num_sp_acesses[PREV_STAT_IDX][i]=m_num_sp_acesses[CURRENT_STAT_IDX][i]; - m_num_sfu_acesses[PREV_STAT_IDX][i]=m_num_sfu_acesses[CURRENT_STAT_IDX][i]; - m_num_trans_acesses[PREV_STAT_IDX][i]=m_num_trans_acesses[CURRENT_STAT_IDX][i]; - m_num_mem_acesses[PREV_STAT_IDX][i]=m_num_mem_acesses[CURRENT_STAT_IDX][i]; - m_num_sp_committed[PREV_STAT_IDX][i]=m_num_sp_committed[CURRENT_STAT_IDX][i]; - m_num_sfu_committed[PREV_STAT_IDX][i]=m_num_sfu_committed[CURRENT_STAT_IDX][i]; - m_num_mem_committed[PREV_STAT_IDX][i]=m_num_mem_committed[CURRENT_STAT_IDX][i]; - m_read_regfile_acesses[PREV_STAT_IDX][i]=m_read_regfile_acesses[CURRENT_STAT_IDX][i]; - m_write_regfile_acesses[PREV_STAT_IDX][i]=m_write_regfile_acesses[CURRENT_STAT_IDX][i]; - m_non_rf_operands[PREV_STAT_IDX][i]=m_non_rf_operands[CURRENT_STAT_IDX][i]; - m_active_sp_lanes[PREV_STAT_IDX][i]=m_active_sp_lanes[CURRENT_STAT_IDX][i]; - m_active_sfu_lanes[PREV_STAT_IDX][i]=m_active_sfu_lanes[CURRENT_STAT_IDX][i]; - } +void power_core_stat_t::save_stats() { + for (unsigned i = 0; i < m_config->num_shader(); ++i) { + m_pipeline_duty_cycle[PREV_STAT_IDX][i] = + m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]; + m_num_decoded_insn[PREV_STAT_IDX][i] = + m_num_decoded_insn[CURRENT_STAT_IDX][i]; + m_num_FPdecoded_insn[PREV_STAT_IDX][i] = + m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]; + m_num_INTdecoded_insn[PREV_STAT_IDX][i] = + m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]; + m_num_storequeued_insn[PREV_STAT_IDX][i] = + m_num_storequeued_insn[CURRENT_STAT_IDX][i]; + m_num_loadqueued_insn[PREV_STAT_IDX][i] = + m_num_loadqueued_insn[CURRENT_STAT_IDX][i]; + m_num_ialu_acesses[PREV_STAT_IDX][i] = + m_num_ialu_acesses[CURRENT_STAT_IDX][i]; + m_num_fp_acesses[PREV_STAT_IDX][i] = m_num_fp_acesses[CURRENT_STAT_IDX][i]; + m_num_tex_inst[PREV_STAT_IDX][i] = m_num_tex_inst[CURRENT_STAT_IDX][i]; + m_num_imul_acesses[PREV_STAT_IDX][i] = + m_num_imul_acesses[CURRENT_STAT_IDX][i]; + m_num_imul24_acesses[PREV_STAT_IDX][i] = + m_num_imul24_acesses[CURRENT_STAT_IDX][i]; + m_num_imul32_acesses[PREV_STAT_IDX][i] = + m_num_imul32_acesses[CURRENT_STAT_IDX][i]; + m_num_fpmul_acesses[PREV_STAT_IDX][i] = + m_num_fpmul_acesses[CURRENT_STAT_IDX][i]; + m_num_idiv_acesses[PREV_STAT_IDX][i] = + m_num_idiv_acesses[CURRENT_STAT_IDX][i]; + m_num_fpdiv_acesses[PREV_STAT_IDX][i] = + m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]; + m_num_sp_acesses[PREV_STAT_IDX][i] = m_num_sp_acesses[CURRENT_STAT_IDX][i]; + m_num_sfu_acesses[PREV_STAT_IDX][i] = + m_num_sfu_acesses[CURRENT_STAT_IDX][i]; + m_num_trans_acesses[PREV_STAT_IDX][i] = + m_num_trans_acesses[CURRENT_STAT_IDX][i]; + m_num_mem_acesses[PREV_STAT_IDX][i] = + m_num_mem_acesses[CURRENT_STAT_IDX][i]; + m_num_sp_committed[PREV_STAT_IDX][i] = + m_num_sp_committed[CURRENT_STAT_IDX][i]; + m_num_sfu_committed[PREV_STAT_IDX][i] = + m_num_sfu_committed[CURRENT_STAT_IDX][i]; + m_num_mem_committed[PREV_STAT_IDX][i] = + m_num_mem_committed[CURRENT_STAT_IDX][i]; + m_read_regfile_acesses[PREV_STAT_IDX][i] = + m_read_regfile_acesses[CURRENT_STAT_IDX][i]; + m_write_regfile_acesses[PREV_STAT_IDX][i] = + m_write_regfile_acesses[CURRENT_STAT_IDX][i]; + m_non_rf_operands[PREV_STAT_IDX][i] = + m_non_rf_operands[CURRENT_STAT_IDX][i]; + m_active_sp_lanes[PREV_STAT_IDX][i] = + m_active_sp_lanes[CURRENT_STAT_IDX][i]; + m_active_sfu_lanes[PREV_STAT_IDX][i] = + m_active_sfu_lanes[CURRENT_STAT_IDX][i]; + } } -power_stat_t::power_stat_t( const shader_core_config *shader_config,float * average_pipeline_duty_cycle,float *active_sms,shader_core_stats * shader_stats, const memory_config *mem_config,memory_stats_t * memory_stats) -{ - assert( shader_config->m_valid ); - assert( mem_config->m_valid ); - pwr_core_stat= new power_core_stat_t(shader_config,shader_stats); - pwr_mem_stat= new power_mem_stat_t(mem_config,shader_config, memory_stats, shader_stats); - m_average_pipeline_duty_cycle=average_pipeline_duty_cycle; - m_active_sms=active_sms; - m_config = shader_config; - m_mem_config = mem_config; +power_stat_t::power_stat_t(const shader_core_config *shader_config, + float *average_pipeline_duty_cycle, + float *active_sms, shader_core_stats *shader_stats, + const memory_config *mem_config, + memory_stats_t *memory_stats) { + assert(shader_config->m_valid); + assert(mem_config->m_valid); + pwr_core_stat = new power_core_stat_t(shader_config, shader_stats); + pwr_mem_stat = new power_mem_stat_t(mem_config, shader_config, memory_stats, + shader_stats); + m_average_pipeline_duty_cycle = average_pipeline_duty_cycle; + m_active_sms = active_sms; + m_config = shader_config; + m_mem_config = mem_config; } -void power_stat_t::visualizer_print( gzFile visualizer_file ) -{ - pwr_core_stat->visualizer_print(visualizer_file); - pwr_mem_stat->visualizer_print(visualizer_file); +void power_stat_t::visualizer_print(gzFile visualizer_file) { + pwr_core_stat->visualizer_print(visualizer_file); + pwr_mem_stat->visualizer_print(visualizer_file); } -void power_stat_t::print (FILE *fout) const -{ - fprintf(fout,"average_pipeline_duty_cycle=%f\n",*m_average_pipeline_duty_cycle); - pwr_core_stat->print(fout); - pwr_mem_stat->print(fout); +void power_stat_t::print(FILE *fout) const { + fprintf(fout, "average_pipeline_duty_cycle=%f\n", + *m_average_pipeline_duty_cycle); + pwr_core_stat->print(fout); + pwr_mem_stat->print(fout); } - diff --git a/src/gpgpu-sim/power_stat.h b/src/gpgpu-sim/power_stat.h index 24ade99..c469db3 100644 --- a/src/gpgpu-sim/power_stat.h +++ b/src/gpgpu-sim/power_stat.h @@ -7,622 +7,774 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef POWER_STAT_H #define POWER_STAT_H #include <stdio.h> #include <zlib.h> -#include "mem_latency_stat.h" #include "gpu-sim.h" +#include "mem_latency_stat.h" -typedef enum _stat_idx{ - CURRENT_STAT_IDX = 0, // Current activity count - PREV_STAT_IDX, // Previous sample activity count - NUM_STAT_IDX // Total number of samples -}stat_idx; - +typedef enum _stat_idx { + CURRENT_STAT_IDX = 0, // Current activity count + PREV_STAT_IDX, // Previous sample activity count + NUM_STAT_IDX // Total number of samples +} stat_idx; struct shader_core_power_stats_pod { - // [CURRENT_STAT_IDX] = CURRENT_STAT_IDX stat, [PREV_STAT_IDX] = last reading - float *m_pipeline_duty_cycle[NUM_STAT_IDX]; - unsigned *m_num_decoded_insn[NUM_STAT_IDX]; // number of instructions committed by this shader core - unsigned *m_num_FPdecoded_insn[NUM_STAT_IDX]; // number of instructions committed by this shader core - unsigned *m_num_INTdecoded_insn[NUM_STAT_IDX]; // number of instructions committed by this shader core - unsigned *m_num_storequeued_insn[NUM_STAT_IDX]; - unsigned *m_num_loadqueued_insn[NUM_STAT_IDX]; - unsigned *m_num_ialu_acesses[NUM_STAT_IDX]; - unsigned *m_num_fp_acesses[NUM_STAT_IDX]; - unsigned *m_num_tex_inst[NUM_STAT_IDX]; - unsigned *m_num_imul_acesses[NUM_STAT_IDX]; - unsigned *m_num_imul32_acesses[NUM_STAT_IDX]; - unsigned *m_num_imul24_acesses[NUM_STAT_IDX]; - unsigned *m_num_fpmul_acesses[NUM_STAT_IDX]; - unsigned *m_num_idiv_acesses[NUM_STAT_IDX]; - unsigned *m_num_fpdiv_acesses[NUM_STAT_IDX]; - unsigned *m_num_sp_acesses[NUM_STAT_IDX]; - unsigned *m_num_sfu_acesses[NUM_STAT_IDX]; - unsigned *m_num_trans_acesses[NUM_STAT_IDX]; - unsigned *m_num_mem_acesses[NUM_STAT_IDX]; - unsigned *m_num_sp_committed[NUM_STAT_IDX]; - unsigned *m_num_sfu_committed[NUM_STAT_IDX]; - unsigned *m_num_mem_committed[NUM_STAT_IDX]; - unsigned *m_active_sp_lanes[NUM_STAT_IDX]; - unsigned *m_active_sfu_lanes[NUM_STAT_IDX]; - unsigned *m_read_regfile_acesses[NUM_STAT_IDX]; - unsigned *m_write_regfile_acesses[NUM_STAT_IDX]; - unsigned *m_non_rf_operands[NUM_STAT_IDX]; + // [CURRENT_STAT_IDX] = CURRENT_STAT_IDX stat, [PREV_STAT_IDX] = last reading + float *m_pipeline_duty_cycle[NUM_STAT_IDX]; + unsigned *m_num_decoded_insn[NUM_STAT_IDX]; // number of instructions + // committed by this shader core + unsigned + *m_num_FPdecoded_insn[NUM_STAT_IDX]; // number of instructions committed + // by this shader core + unsigned + *m_num_INTdecoded_insn[NUM_STAT_IDX]; // number of instructions committed + // by this shader core + unsigned *m_num_storequeued_insn[NUM_STAT_IDX]; + unsigned *m_num_loadqueued_insn[NUM_STAT_IDX]; + unsigned *m_num_ialu_acesses[NUM_STAT_IDX]; + unsigned *m_num_fp_acesses[NUM_STAT_IDX]; + unsigned *m_num_tex_inst[NUM_STAT_IDX]; + unsigned *m_num_imul_acesses[NUM_STAT_IDX]; + unsigned *m_num_imul32_acesses[NUM_STAT_IDX]; + unsigned *m_num_imul24_acesses[NUM_STAT_IDX]; + unsigned *m_num_fpmul_acesses[NUM_STAT_IDX]; + unsigned *m_num_idiv_acesses[NUM_STAT_IDX]; + unsigned *m_num_fpdiv_acesses[NUM_STAT_IDX]; + unsigned *m_num_sp_acesses[NUM_STAT_IDX]; + unsigned *m_num_sfu_acesses[NUM_STAT_IDX]; + unsigned *m_num_trans_acesses[NUM_STAT_IDX]; + unsigned *m_num_mem_acesses[NUM_STAT_IDX]; + unsigned *m_num_sp_committed[NUM_STAT_IDX]; + unsigned *m_num_sfu_committed[NUM_STAT_IDX]; + unsigned *m_num_mem_committed[NUM_STAT_IDX]; + unsigned *m_active_sp_lanes[NUM_STAT_IDX]; + unsigned *m_active_sfu_lanes[NUM_STAT_IDX]; + unsigned *m_read_regfile_acesses[NUM_STAT_IDX]; + unsigned *m_write_regfile_acesses[NUM_STAT_IDX]; + unsigned *m_non_rf_operands[NUM_STAT_IDX]; }; class power_core_stat_t : public shader_core_power_stats_pod { -public: - power_core_stat_t(const shader_core_config *shader_config, shader_core_stats *core_stats); - void visualizer_print( gzFile visualizer_file ); - void print (FILE *fout); - void init(); - void save_stats(); - -private: - shader_core_stats * m_core_stats; - const shader_core_config *m_config; - float average_duty_cycle; - + public: + power_core_stat_t(const shader_core_config *shader_config, + shader_core_stats *core_stats); + void visualizer_print(gzFile visualizer_file); + void print(FILE *fout); + void init(); + void save_stats(); + private: + shader_core_stats *m_core_stats; + const shader_core_config *m_config; + float average_duty_cycle; }; -struct mem_power_stats_pod{ - // [CURRENT_STAT_IDX] = CURRENT_STAT_IDX stat, [PREV_STAT_IDX] = last reading - class cache_stats core_cache_stats[NUM_STAT_IDX]; // Total core stats - class cache_stats l2_cache_stats[NUM_STAT_IDX]; // Total L2 partition stats +struct mem_power_stats_pod { + // [CURRENT_STAT_IDX] = CURRENT_STAT_IDX stat, [PREV_STAT_IDX] = last reading + class cache_stats core_cache_stats[NUM_STAT_IDX]; // Total core stats + class cache_stats l2_cache_stats[NUM_STAT_IDX]; // Total L2 partition stats - unsigned *shmem_read_access[NUM_STAT_IDX]; // Shared memory access + unsigned *shmem_read_access[NUM_STAT_IDX]; // Shared memory access - // Low level DRAM stats - unsigned *n_cmd[NUM_STAT_IDX]; - unsigned *n_activity[NUM_STAT_IDX]; - unsigned *n_nop[NUM_STAT_IDX]; - unsigned *n_act[NUM_STAT_IDX]; - unsigned *n_pre[NUM_STAT_IDX]; - unsigned *n_rd[NUM_STAT_IDX]; - unsigned *n_wr[NUM_STAT_IDX]; - unsigned *n_req[NUM_STAT_IDX]; + // Low level DRAM stats + unsigned *n_cmd[NUM_STAT_IDX]; + unsigned *n_activity[NUM_STAT_IDX]; + unsigned *n_nop[NUM_STAT_IDX]; + unsigned *n_act[NUM_STAT_IDX]; + unsigned *n_pre[NUM_STAT_IDX]; + unsigned *n_rd[NUM_STAT_IDX]; + unsigned *n_wr[NUM_STAT_IDX]; + unsigned *n_req[NUM_STAT_IDX]; - // Interconnect stats - long *n_simt_to_mem[NUM_STAT_IDX]; - long *n_mem_to_simt[NUM_STAT_IDX]; + // Interconnect stats + long *n_simt_to_mem[NUM_STAT_IDX]; + long *n_mem_to_simt[NUM_STAT_IDX]; }; +class power_mem_stat_t : public mem_power_stats_pod { + public: + power_mem_stat_t(const memory_config *mem_config, + const shader_core_config *shdr_config, + memory_stats_t *mem_stats, shader_core_stats *shdr_stats); + void visualizer_print(gzFile visualizer_file); + void print(FILE *fout) const; + void init(); + void save_stats(); - -class power_mem_stat_t : public mem_power_stats_pod{ -public: - power_mem_stat_t(const memory_config *mem_config, const shader_core_config *shdr_config, memory_stats_t *mem_stats, shader_core_stats *shdr_stats); - void visualizer_print( gzFile visualizer_file ); - void print (FILE *fout) const; - void init(); - void save_stats(); -private: - memory_stats_t *m_mem_stats; - shader_core_stats * m_core_stats; - const memory_config *m_config; - const shader_core_config *m_core_config; + private: + memory_stats_t *m_mem_stats; + shader_core_stats *m_core_stats; + const memory_config *m_config; + const shader_core_config *m_core_config; }; - class power_stat_t { -public: - power_stat_t( const shader_core_config *shader_config,float * average_pipeline_duty_cycle,float * active_sms,shader_core_stats * shader_stats, const memory_config *mem_config,memory_stats_t * memory_stats); - void visualizer_print( gzFile visualizer_file ); - void print (FILE *fout) const; - void save_stats(){ - pwr_core_stat->save_stats(); - pwr_mem_stat->save_stats(); - *m_average_pipeline_duty_cycle=0; - *m_active_sms=0; - } + public: + power_stat_t(const shader_core_config *shader_config, + float *average_pipeline_duty_cycle, float *active_sms, + shader_core_stats *shader_stats, const memory_config *mem_config, + memory_stats_t *memory_stats); + void visualizer_print(gzFile visualizer_file); + void print(FILE *fout) const; + void save_stats() { + pwr_core_stat->save_stats(); + pwr_mem_stat->save_stats(); + *m_average_pipeline_duty_cycle = 0; + *m_active_sms = 0; + } - unsigned get_total_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_decoded_insn[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_decoded_insn[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_total_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_decoded_insn[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_decoded_insn[PREV_STAT_IDX][i]); } - unsigned get_total_int_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_INTdecoded_insn[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_total_int_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_INTdecoded_insn[PREV_STAT_IDX][i]); } - unsigned get_total_fp_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_FPdecoded_insn[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_total_fp_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_FPdecoded_insn[PREV_STAT_IDX][i]); } - unsigned get_total_load_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_loadqueued_insn[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_loadqueued_insn[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_total_load_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_num_loadqueued_insn[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_loadqueued_insn[PREV_STAT_IDX][i]); } - unsigned get_total_store_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_storequeued_insn[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_storequeued_insn[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_total_store_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_num_storequeued_insn[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_storequeued_insn[PREV_STAT_IDX][i]); } - unsigned get_sp_committed_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sp_committed[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_sp_committed_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sp_committed[PREV_STAT_IDX][i]); } - unsigned get_sfu_committed_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_sfu_committed_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]); } - unsigned get_mem_committed_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_mem_committed_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]); } - unsigned get_committed_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]) - +(pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]) - +(pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sp_committed[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_committed_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sp_committed[PREV_STAT_IDX][i]); } - unsigned get_regfile_reads(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_read_regfile_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_read_regfile_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_regfile_reads() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_read_regfile_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_read_regfile_acesses[PREV_STAT_IDX][i]); } - unsigned get_regfile_writes(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_write_regfile_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_write_regfile_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_regfile_writes() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_write_regfile_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_write_regfile_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - float get_pipeline_duty(){ - float total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_pipeline_duty_cycle[PREV_STAT_IDX][i]); - } - return total_inst; + float get_pipeline_duty() { + float total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += + (pwr_core_stat->m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_pipeline_duty_cycle[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_non_regfile_operands(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_non_rf_operands[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_non_rf_operands[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_non_regfile_operands() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_non_rf_operands[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_non_rf_operands[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_sp_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_sp_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sp_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_sp_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_sp_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sp_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_sfu_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_sfu_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sfu_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_sfu_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_sfu_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_sfu_acesses[PREV_STAT_IDX][i]); } - unsigned get_trans_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + return total_inst; + } + unsigned get_trans_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_mem_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_mem_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_mem_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_mem_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_mem_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_mem_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_intdiv_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_intdiv_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_fpdiv_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_fpdiv_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_intmul32_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_intmul32_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_intmul24_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_intmul24_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_intmul_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_intmul_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_fpmul_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_fpmul_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - float get_sp_active_lanes(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_active_sp_lanes[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_active_sp_lanes[PREV_STAT_IDX][i]); - } - return (total_inst/m_config->num_shader())/m_config->gpgpu_num_sp_units; + float get_sp_active_lanes() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_active_sp_lanes[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_active_sp_lanes[PREV_STAT_IDX][i]); } + return (total_inst / m_config->num_shader()) / m_config->gpgpu_num_sp_units; + } - float get_sfu_active_lanes(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_active_sfu_lanes[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_active_sfu_lanes[PREV_STAT_IDX][i]); - } - - return (total_inst/m_config->num_shader())/m_config->gpgpu_num_sfu_units; + float get_sfu_active_lanes() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_active_sfu_lanes[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_active_sfu_lanes[PREV_STAT_IDX][i]); } - unsigned get_tot_fpu_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_fpmul_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]); - } - total_inst += get_total_load_inst()+get_total_store_inst()+get_tex_inst(); - return total_inst; + return (total_inst / m_config->num_shader()) / + m_config->gpgpu_num_sfu_units; + } + + unsigned get_tot_fpu_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_fpmul_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]); } + total_inst += + get_total_load_inst() + get_total_store_inst() + get_tex_inst(); + return total_inst; + } - unsigned get_tot_sfu_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+= (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i])+ - (pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_tot_sfu_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]) + + (pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_ialu_accessess(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_ialu_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_ialu_acesses[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_ialu_accessess() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_ialu_acesses[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_ialu_acesses[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_tex_inst(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_core_stat->m_num_tex_inst[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_tex_inst[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_tex_inst() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_core_stat->m_num_tex_inst[CURRENT_STAT_IDX][i]) - + (pwr_core_stat->m_num_tex_inst[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_constant_c_accesses(){ - enum mem_access_type access_type[] = {CONST_ACC_R}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_constant_c_accesses() { + enum mem_access_type access_type[] = {CONST_ACC_R}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_constant_c_misses(){ - enum mem_access_type access_type[] = {CONST_ACC_R}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_constant_c_misses() { + enum mem_access_type access_type[] = {CONST_ACC_R}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_constant_c_hits(){ - return (get_constant_c_accesses()-get_constant_c_misses()); - } - unsigned get_texture_c_accesses(){ - enum mem_access_type access_type[] = {TEXTURE_ACC_R}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_constant_c_hits() { + return (get_constant_c_accesses() - get_constant_c_misses()); + } + unsigned get_texture_c_accesses() { + enum mem_access_type access_type[] = {TEXTURE_ACC_R}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_texture_c_misses(){ - enum mem_access_type access_type[] = {TEXTURE_ACC_R}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_texture_c_misses() { + enum mem_access_type access_type[] = {TEXTURE_ACC_R}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_texture_c_hits(){ - return ( get_texture_c_accesses()- get_texture_c_misses()); - } - unsigned get_inst_c_accesses(){ - enum mem_access_type access_type[] = {INST_ACC_R}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_texture_c_hits() { + return (get_texture_c_accesses() - get_texture_c_misses()); + } + unsigned get_inst_c_accesses() { + enum mem_access_type access_type[] = {INST_ACC_R}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_inst_c_misses(){ - enum mem_access_type access_type[] = {INST_ACC_R}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_inst_c_misses() { + enum mem_access_type access_type[] = {INST_ACC_R}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_inst_c_hits(){ - return (get_inst_c_accesses()-get_inst_c_misses()); - } + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_inst_c_hits() { + return (get_inst_c_accesses() - get_inst_c_misses()); + } - unsigned get_l1d_read_accesses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_l1d_read_accesses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_l1d_read_misses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_l1d_read_misses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_l1d_read_hits(){ - return (get_l1d_read_accesses()-get_l1d_read_misses()); - } - unsigned get_l1d_write_accesses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_l1d_read_hits() { + return (get_l1d_read_accesses() - get_l1d_read_misses()); + } + unsigned get_l1d_write_accesses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_l1d_write_misses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_l1d_write_misses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_l1d_write_hits(){ - return (get_l1d_write_accesses()-get_l1d_write_misses()); - } - unsigned get_cache_misses(){ - return get_l1d_read_misses()+get_constant_c_misses()+get_l1d_write_misses()+get_texture_c_misses(); - } - - unsigned get_cache_read_misses(){ - return get_l1d_read_misses()+get_constant_c_misses()+get_texture_c_misses(); - } + return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_l1d_write_hits() { + return (get_l1d_write_accesses() - get_l1d_write_misses()); + } + unsigned get_cache_misses() { + return get_l1d_read_misses() + get_constant_c_misses() + + get_l1d_write_misses() + get_texture_c_misses(); + } - unsigned get_cache_write_misses(){ - return get_l1d_write_misses(); - } + unsigned get_cache_read_misses() { + return get_l1d_read_misses() + get_constant_c_misses() + + get_texture_c_misses(); + } - unsigned get_shmem_read_access(){ - unsigned total_inst=0; - for(unsigned i=0; i<m_config->num_shader();i++){ - total_inst+=(pwr_mem_stat->shmem_read_access[CURRENT_STAT_IDX][i]) - (pwr_mem_stat->shmem_read_access[PREV_STAT_IDX][i]); - } - return total_inst; + unsigned get_cache_write_misses() { return get_l1d_write_misses(); } + + unsigned get_shmem_read_access() { + unsigned total_inst = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + total_inst += (pwr_mem_stat->shmem_read_access[CURRENT_STAT_IDX][i]) - + (pwr_mem_stat->shmem_read_access[PREV_STAT_IDX][i]); } + return total_inst; + } - unsigned get_l2_read_accesses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_l2_read_accesses() { + enum mem_access_type access_type[] = { + GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } + return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } - unsigned get_l2_read_misses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_l2_read_misses() { + enum mem_access_type access_type[] = { + GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } + return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } - unsigned get_l2_read_hits(){ - return (get_l2_read_accesses()-get_l2_read_misses()); - } + unsigned get_l2_read_hits() { + return (get_l2_read_accesses() - get_l2_read_misses()); + } - unsigned get_l2_write_accesses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W, L1_WRBK_ACC}; - enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_l2_write_accesses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W, + L1_WRBK_ACC}; + enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } + return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } - unsigned get_l2_write_misses(){ - enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W, L1_WRBK_ACC}; - enum cache_request_status request_status[] = {MISS}; - unsigned num_access_type = sizeof(access_type)/sizeof(enum mem_access_type); - unsigned num_request_status = sizeof(request_status)/sizeof(enum cache_request_status); + unsigned get_l2_write_misses() { + enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W, + L1_WRBK_ACC}; + enum cache_request_status request_status[] = {MISS}; + unsigned num_access_type = + sizeof(access_type) / sizeof(enum mem_access_type); + unsigned num_request_status = + sizeof(request_status) / sizeof(enum cache_request_status); - return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)) - - (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(access_type, num_access_type, request_status, num_request_status)); - } - unsigned get_l2_write_hits(){ - return (get_l2_write_accesses()-get_l2_write_misses()); + return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)) - + (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats( + access_type, num_access_type, request_status, + num_request_status)); + } + unsigned get_l2_write_hits() { + return (get_l2_write_accesses() - get_l2_write_misses()); + } + unsigned get_dram_cmd() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_cmd[PREV_STAT_IDX][i]); } - unsigned get_dram_cmd(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_cmd[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_activity() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_activity[PREV_STAT_IDX][i]); } - unsigned get_dram_activity(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_activity[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_nop() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_nop[PREV_STAT_IDX][i]); } - unsigned get_dram_nop(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_nop[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_act() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_act[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_act[PREV_STAT_IDX][i]); } - unsigned get_dram_act(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_act[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_act[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_pre() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_pre[PREV_STAT_IDX][i]); } - unsigned get_dram_pre(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_pre[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_rd() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_rd[PREV_STAT_IDX][i]); } - unsigned get_dram_rd(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_rd[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_wr() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_wr[PREV_STAT_IDX][i]); } - unsigned get_dram_wr(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_wr[PREV_STAT_IDX][i]); - } - return total; - } - unsigned get_dram_req(){ - unsigned total=0; - for(unsigned i=0; i<m_mem_config->m_n_mem; ++i){ - total += (pwr_mem_stat->n_req[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_req[PREV_STAT_IDX][i]); - } - return total; + return total; + } + unsigned get_dram_req() { + unsigned total = 0; + for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) { + total += (pwr_mem_stat->n_req[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_req[PREV_STAT_IDX][i]); } + return total; + } - long get_icnt_simt_to_mem(){ - long total=0; - for(unsigned i=0; i<m_config->n_simt_clusters; ++i){ - total += (pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_simt_to_mem[PREV_STAT_IDX][i]); - } - return total; + long get_icnt_simt_to_mem() { + long total = 0; + for (unsigned i = 0; i < m_config->n_simt_clusters; ++i) { + total += (pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_simt_to_mem[PREV_STAT_IDX][i]); } + return total; + } - long get_icnt_mem_to_simt(){ - long total=0; - for(unsigned i=0; i<m_config->n_simt_clusters; ++i){ - total += (pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i] - pwr_mem_stat->n_mem_to_simt[PREV_STAT_IDX][i]); - } - return total; + long get_icnt_mem_to_simt() { + long total = 0; + for (unsigned i = 0; i < m_config->n_simt_clusters; ++i) { + total += (pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i] - + pwr_mem_stat->n_mem_to_simt[PREV_STAT_IDX][i]); } + return total; + } - power_core_stat_t * pwr_core_stat; - power_mem_stat_t * pwr_mem_stat; - float * m_average_pipeline_duty_cycle; - float * m_active_sms; - const shader_core_config *m_config; - const memory_config *m_mem_config; + power_core_stat_t *pwr_core_stat; + power_mem_stat_t *pwr_mem_stat; + float *m_average_pipeline_duty_cycle; + float *m_active_sms; + const shader_core_config *m_config; + const memory_config *m_mem_config; }; - #endif /*POWER_LATENCY_STAT_H*/ diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc index 1017e75..44a9980 100644 --- a/src/gpgpu-sim/scoreboard.cc +++ b/src/gpgpu-sim/scoreboard.cc @@ -7,159 +7,148 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "scoreboard.h" -#include "shader.h" #include "../cuda-sim/ptx_sim.h" +#include "shader.h" #include "shader_trace.h" +// Constructor +Scoreboard::Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t* gpu) + : longopregs() { + m_sid = sid; + // Initialize size of table + reg_table.resize(n_warps); + longopregs.resize(n_warps); -//Constructor -Scoreboard::Scoreboard( unsigned sid, unsigned n_warps, class gpgpu_t* gpu ) -: longopregs() -{ - m_sid = sid; - //Initialize size of table - reg_table.resize(n_warps); - longopregs.resize(n_warps); - - m_gpu = gpu; + m_gpu = gpu; } // Print scoreboard contents -void Scoreboard::printContents() const -{ - printf("scoreboard contents (sid=%d): \n", m_sid); - for(unsigned i=0; i<reg_table.size(); i++) { - if(reg_table[i].size() == 0 ) continue; - printf(" wid = %2d: ", i); - std::set<unsigned>::const_iterator it; - for( it=reg_table[i].begin() ; it != reg_table[i].end(); it++ ) - printf("%u ", *it); - printf("\n"); - } +void Scoreboard::printContents() const { + printf("scoreboard contents (sid=%d): \n", m_sid); + for (unsigned i = 0; i < reg_table.size(); i++) { + if (reg_table[i].size() == 0) continue; + printf(" wid = %2d: ", i); + std::set<unsigned>::const_iterator it; + for (it = reg_table[i].begin(); it != reg_table[i].end(); it++) + printf("%u ", *it); + printf("\n"); + } } -void Scoreboard::reserveRegister(unsigned wid, unsigned regnum) -{ - if( !(reg_table[wid].find(regnum) == reg_table[wid].end()) ){ - printf("Error: trying to reserve an already reserved register (sid=%d, wid=%d, regnum=%d).", m_sid, wid, regnum); - abort(); - } - SHADER_DPRINTF( SCOREBOARD, - "Reserved Register - warp:%d, reg: %d\n", wid, regnum ); - reg_table[wid].insert(regnum); +void Scoreboard::reserveRegister(unsigned wid, unsigned regnum) { + if (!(reg_table[wid].find(regnum) == reg_table[wid].end())) { + printf( + "Error: trying to reserve an already reserved register (sid=%d, " + "wid=%d, regnum=%d).", + m_sid, wid, regnum); + abort(); + } + SHADER_DPRINTF(SCOREBOARD, "Reserved Register - warp:%d, reg: %d\n", wid, + regnum); + reg_table[wid].insert(regnum); } // Unmark register as write-pending -void Scoreboard::releaseRegister(unsigned wid, unsigned regnum) -{ - if( !(reg_table[wid].find(regnum) != reg_table[wid].end()) ) - return; - SHADER_DPRINTF( SCOREBOARD, - "Release register - warp:%d, reg: %d\n", wid, regnum ); - reg_table[wid].erase(regnum); +void Scoreboard::releaseRegister(unsigned wid, unsigned regnum) { + if (!(reg_table[wid].find(regnum) != reg_table[wid].end())) return; + SHADER_DPRINTF(SCOREBOARD, "Release register - warp:%d, reg: %d\n", wid, + regnum); + reg_table[wid].erase(regnum); } -const bool Scoreboard::islongop (unsigned warp_id,unsigned regnum) { - return longopregs[warp_id].find(regnum) != longopregs[warp_id].end(); +const bool Scoreboard::islongop(unsigned warp_id, unsigned regnum) { + return longopregs[warp_id].find(regnum) != longopregs[warp_id].end(); } -void Scoreboard::reserveRegisters(const class warp_inst_t* inst) -{ - for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) { - if(inst->out[r] > 0) { - reserveRegister(inst->warp_id(), inst->out[r]); - SHADER_DPRINTF( SCOREBOARD, - "Reserved register - warp:%d, reg: %d\n", - inst->warp_id(), - inst->out[r] ); - } +void Scoreboard::reserveRegisters(const class warp_inst_t* inst) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + reserveRegister(inst->warp_id(), inst->out[r]); + SHADER_DPRINTF(SCOREBOARD, "Reserved register - warp:%d, reg: %d\n", + inst->warp_id(), inst->out[r]); } + } - //Keep track of long operations - if (inst->is_load() && - ( inst->space.get_type() == global_space || - inst->space.get_type() == local_space || - inst->space.get_type() == param_space_kernel || - inst->space.get_type() == param_space_local || - inst->space.get_type() == param_space_unclassified || - inst->space.get_type() == tex_space)){ - for ( unsigned r=0; r<MAX_OUTPUT_VALUES; r++) { - if(inst->out[r] > 0) { - SHADER_DPRINTF( SCOREBOARD, - "New longopreg marked - warp:%d, reg: %d\n", - inst->warp_id(), - inst->out[r] ); - longopregs[inst->warp_id()].insert(inst->out[r]); - } - } + // Keep track of long operations + if (inst->is_load() && (inst->space.get_type() == global_space || + inst->space.get_type() == local_space || + inst->space.get_type() == param_space_kernel || + inst->space.get_type() == param_space_local || + inst->space.get_type() == param_space_unclassified || + inst->space.get_type() == tex_space)) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + SHADER_DPRINTF(SCOREBOARD, "New longopreg marked - warp:%d, reg: %d\n", + inst->warp_id(), inst->out[r]); + longopregs[inst->warp_id()].insert(inst->out[r]); + } } + } } // Release registers for an instruction -void Scoreboard::releaseRegisters(const class warp_inst_t *inst) -{ - for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) { - if(inst->out[r] > 0) { - SHADER_DPRINTF( SCOREBOARD, - "Register Released - warp:%d, reg: %d\n", - inst->warp_id(), - inst->out[r] ); - releaseRegister(inst->warp_id(), inst->out[r]); - longopregs[inst->warp_id()].erase(inst->out[r]); - } +void Scoreboard::releaseRegisters(const class warp_inst_t* inst) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + SHADER_DPRINTF(SCOREBOARD, "Register Released - warp:%d, reg: %d\n", + inst->warp_id(), inst->out[r]); + releaseRegister(inst->warp_id(), inst->out[r]); + longopregs[inst->warp_id()].erase(inst->out[r]); } + } } -/** - * Checks to see if registers used by an instruction are reserved in the scoreboard - * - * @return +/** + * Checks to see if registers used by an instruction are reserved in the + *scoreboard + * + * @return * true if WAW or RAW hazard (no WAR since in-order issue) - **/ -bool Scoreboard::checkCollision( unsigned wid, const class inst_t *inst ) const -{ - // Get list of all input and output registers - std::set<int> inst_regs; + **/ +bool Scoreboard::checkCollision(unsigned wid, const class inst_t* inst) const { + // Get list of all input and output registers + std::set<int> inst_regs; - for(unsigned iii=0; iii < inst->outcount; iii++) - inst_regs.insert(inst->out[iii]); + for (unsigned iii = 0; iii < inst->outcount; iii++) + inst_regs.insert(inst->out[iii]); - for(unsigned jjj=0;jjj<inst->incount;jjj++) - inst_regs.insert(inst->in[jjj]); + for (unsigned jjj = 0; jjj < inst->incount; jjj++) + inst_regs.insert(inst->in[jjj]); - if(inst->pred > 0) inst_regs.insert(inst->pred); - if(inst->ar1 > 0) inst_regs.insert(inst->ar1); - if(inst->ar2 > 0) inst_regs.insert(inst->ar2); + if (inst->pred > 0) inst_regs.insert(inst->pred); + if (inst->ar1 > 0) inst_regs.insert(inst->ar1); + if (inst->ar2 > 0) inst_regs.insert(inst->ar2); - // Check for collision, get the intersection of reserved registers and instruction registers - std::set<int>::const_iterator it2; - for ( it2=inst_regs.begin() ; it2 != inst_regs.end(); it2++ ) - if(reg_table[wid].find(*it2) != reg_table[wid].end()) { - return true; - } - return false; + // Check for collision, get the intersection of reserved registers and + // instruction registers + std::set<int>::const_iterator it2; + for (it2 = inst_regs.begin(); it2 != inst_regs.end(); it2++) + if (reg_table[wid].find(*it2) != reg_table[wid].end()) { + return true; + } + return false; } -bool Scoreboard::pendingWrites(unsigned wid) const -{ - return !reg_table[wid].empty(); +bool Scoreboard::pendingWrites(unsigned wid) const { + return !reg_table[wid].empty(); } diff --git a/src/gpgpu-sim/scoreboard.h b/src/gpgpu-sim/scoreboard.h index a4baa19..1fe0923 100644 --- a/src/gpgpu-sim/scoreboard.h +++ b/src/gpgpu-sim/scoreboard.h @@ -7,28 +7,29 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include <stdio.h> #include <stdlib.h> -#include <vector> #include <set> +#include <vector> #include "assert.h" #ifndef SCOREBOARD_H_ @@ -37,31 +38,31 @@ #include "../abstract_hardware_model.h" class Scoreboard { -public: - Scoreboard( unsigned sid, unsigned n_warps, class gpgpu_t* gpu ); + public: + Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t *gpu); - void reserveRegisters(const warp_inst_t *inst); - void releaseRegisters(const warp_inst_t *inst); - void releaseRegister(unsigned wid, unsigned regnum); + void reserveRegisters(const warp_inst_t *inst); + void releaseRegisters(const warp_inst_t *inst); + void releaseRegister(unsigned wid, unsigned regnum); - bool checkCollision(unsigned wid, const inst_t *inst) const; - bool pendingWrites(unsigned wid) const; - void printContents() const; - const bool islongop(unsigned warp_id, unsigned regnum); -private: - void reserveRegister(unsigned wid, unsigned regnum); - int get_sid() const { return m_sid; } + bool checkCollision(unsigned wid, const inst_t *inst) const; + bool pendingWrites(unsigned wid) const; + void printContents() const; + const bool islongop(unsigned warp_id, unsigned regnum); - unsigned m_sid; + private: + void reserveRegister(unsigned wid, unsigned regnum); + int get_sid() const { return m_sid; } - // keeps track of pending writes to registers - // indexed by warp id, reg_id => pending write count - std::vector< std::set<unsigned> > reg_table; - //Register that depend on a long operation (global, local or tex memory) - std::vector< std::set<unsigned> > longopregs; + unsigned m_sid; - class gpgpu_t* m_gpu; -}; + // keeps track of pending writes to registers + // indexed by warp id, reg_id => pending write count + std::vector<std::set<unsigned> > reg_table; + // Register that depend on a long operation (global, local or tex memory) + std::vector<std::set<unsigned> > longopregs; + class gpgpu_t *m_gpu; +}; #endif /* SCOREBOARD_H_ */ diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 6a0e3d6..45e6790 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda, -// George L. Yuan, Andrew Turner, Inderpreet Singh +// George L. Yuan, Andrew Turner, Inderpreet Singh // The University of British Columbia // All rights reserved. // @@ -8,2060 +8,2132 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include <float.h> #include "shader.h" +#include <float.h> +#include <limits.h> +#include <string.h> +#include "../../libcuda/gpgpu_context.h" +#include "../cuda-sim/cuda-sim.h" +#include "../cuda-sim/ptx-stats.h" +#include "../cuda-sim/ptx_sim.h" +#include "../statwrapper.h" #include "addrdec.h" #include "dram.h" -#include "stat-tool.h" #include "gpu-misc.h" -#include "../cuda-sim/ptx_sim.h" -#include "../cuda-sim/ptx-stats.h" -#include "../cuda-sim/cuda-sim.h" #include "gpu-sim.h" +#include "icnt_wrapper.h" #include "mem_fetch.h" #include "mem_latency_stat.h" -#include "visualizer.h" -#include "../statwrapper.h" -#include "icnt_wrapper.h" -#include <string.h> -#include <limits.h> -#include "traffic_breakdown.h" #include "shader_trace.h" -#include "../../libcuda/gpgpu_context.h" +#include "stat-tool.h" +#include "traffic_breakdown.h" +#include "visualizer.h" #define PRIORITIZE_MSHR_OVER_WB 1 -#define MAX(a,b) (((a)>(b))?(a):(b)) -#define MIN(a,b) (((a)<(b))?(a):(b)) - +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -mem_fetch *shader_core_mem_fetch_allocator::alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const -{ - mem_access_t access( type, addr, size, wr, m_memory_config->gpgpu_ctx); - mem_fetch *mf = new mem_fetch( access, - NULL, - wr?WRITE_PACKET_SIZE:READ_PACKET_SIZE, - -1, - m_core_id, - m_cluster_id, - m_memory_config, - cycle); - return mf; +mem_fetch *shader_core_mem_fetch_allocator::alloc( + new_addr_type addr, mem_access_type type, unsigned size, bool wr, + unsigned long long cycle) const { + mem_access_t access(type, addr, size, wr, m_memory_config->gpgpu_ctx); + mem_fetch *mf = + new mem_fetch(access, NULL, wr ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, -1, + m_core_id, m_cluster_id, m_memory_config, cycle); + return mf; } ///////////////////////////////////////////////////////////////////////////// -std::list<unsigned> shader_core_ctx::get_regs_written( const inst_t &fvt ) const -{ - std::list<unsigned> result; - for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) { - int reg_num = fvt.arch_reg.dst[op]; // this math needs to match that used in function_info::ptx_decode_inst - if( reg_num >= 0 ) // valid register - result.push_back(reg_num); - } - return result; +std::list<unsigned> shader_core_ctx::get_regs_written(const inst_t &fvt) const { + std::list<unsigned> result; + for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) { + int reg_num = fvt.arch_reg.dst[op]; // this math needs to match that used + // in function_info::ptx_decode_inst + if (reg_num >= 0) // valid register + result.push_back(reg_num); + } + return result; } -shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, - class simt_core_cluster *cluster, - unsigned shader_id, - unsigned tpc_id, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats ) - : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ), - m_barriers( this, config->max_warps_per_shader, config->max_cta_per_core, config->max_barriers_per_cta, config->warp_size ), - m_active_warps(0), m_dynamic_warp_id(0) -{ - m_cluster = cluster; - m_config = config; - m_memory_config = mem_config; - m_stats = stats; - unsigned warp_size=config->warp_size; - Issue_Prio = 0; - - m_sid = shader_id; - m_tpc = tpc_id; - - m_pipeline_reg.reserve(N_PIPELINE_STAGES); - for (int j = 0; j<N_PIPELINE_STAGES; j++) { - m_pipeline_reg.push_back(register_set(m_config->pipe_widths[j],pipeline_stage_name_decode[j])); +shader_core_ctx::shader_core_ctx(class gpgpu_sim *gpu, + class simt_core_cluster *cluster, + unsigned shader_id, unsigned tpc_id, + const shader_core_config *config, + const memory_config *mem_config, + shader_core_stats *stats) + : core_t(gpu, NULL, config->warp_size, config->n_thread_per_shader), + m_barriers(this, config->max_warps_per_shader, config->max_cta_per_core, + config->max_barriers_per_cta, config->warp_size), + m_active_warps(0), + m_dynamic_warp_id(0) { + m_cluster = cluster; + m_config = config; + m_memory_config = mem_config; + m_stats = stats; + unsigned warp_size = config->warp_size; + Issue_Prio = 0; + + m_sid = shader_id; + m_tpc = tpc_id; + + m_pipeline_reg.reserve(N_PIPELINE_STAGES); + for (int j = 0; j < N_PIPELINE_STAGES; j++) { + m_pipeline_reg.push_back( + register_set(m_config->pipe_widths[j], pipeline_stage_name_decode[j])); + } + if (m_config->sub_core_model) { + // in subcore model, each scheduler should has its own issue register, so + // num scheduler = reg width + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_SP].get_size()); + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_SFU].get_size()); + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_MEM].get_size()); + if (m_config->gpgpu_tensor_core_avail) + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_TENSOR_CORE].get_size()); + if (m_config->gpgpu_num_dp_units > 0) + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_DP].get_size()); + if (m_config->gpgpu_num_int_units > 0) + assert(m_config->gpgpu_num_sched_per_core == + m_pipeline_reg[ID_OC_INT].get_size()); + } + + m_threadState = + (thread_ctx_t *)calloc(sizeof(thread_ctx_t), config->n_thread_per_shader); + + m_not_completed = 0; + m_active_threads.reset(); + m_n_active_cta = 0; + for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) m_cta_status[i] = 0; + for (unsigned i = 0; i < config->n_thread_per_shader; i++) { + m_thread[i] = NULL; + m_threadState[i].m_cta_id = -1; + m_threadState[i].m_active = false; + } + + // m_icnt = new shader_memory_interface(this,cluster); + if (m_config->gpgpu_perfect_mem) { + m_icnt = new perfect_memory_interface(this, cluster); + } else { + m_icnt = new shader_memory_interface(this, cluster); + } + m_mem_fetch_allocator = + new shader_core_mem_fetch_allocator(shader_id, tpc_id, mem_config); + + // fetch + m_last_warp_fetched = 0; + +#define STRSIZE 1024 + char name[STRSIZE]; + snprintf(name, STRSIZE, "L1I_%03d", m_sid); + m_L1I = new read_only_cache(name, m_config->m_L1I_config, m_sid, + get_shader_instruction_cache_id(), m_icnt, + IN_L1I_MISS_QUEUE); + + m_warp.resize(m_config->max_warps_per_shader, shd_warp_t(this, warp_size)); + m_scoreboard = new Scoreboard(m_sid, m_config->max_warps_per_shader, gpu); + + // scedulers + // must currently occur after all inputs have been initialized. + std::string sched_config = m_config->gpgpu_scheduler_string; + const concrete_scheduler scheduler = + sched_config.find("lrr") != std::string::npos + ? CONCRETE_SCHEDULER_LRR + : sched_config.find("two_level_active") != std::string::npos + ? CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE + : sched_config.find("gto") != std::string::npos + ? CONCRETE_SCHEDULER_GTO + : sched_config.find("old") != std::string::npos + ? CONCRETE_SCHEDULER_OLDEST_FIRST + : sched_config.find("warp_limiting") != + std::string::npos + ? CONCRETE_SCHEDULER_WARP_LIMITING + : NUM_CONCRETE_SCHEDULERS; + assert(scheduler != NUM_CONCRETE_SCHEDULERS); + + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) { + switch (scheduler) { + case CONCRETE_SCHEDULER_LRR: + schedulers.push_back(new lrr_scheduler( + m_stats, this, m_scoreboard, m_simt_stack, &m_warp, + &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i)); + break; + case CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE: + schedulers.push_back(new two_level_active_scheduler( + m_stats, this, m_scoreboard, m_simt_stack, &m_warp, + &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i, + config->gpgpu_scheduler_string)); + break; + case CONCRETE_SCHEDULER_GTO: + schedulers.push_back(new gto_scheduler( + m_stats, this, m_scoreboard, m_simt_stack, &m_warp, + &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i)); + break; + case CONCRETE_SCHEDULER_OLDEST_FIRST: + schedulers.push_back(new oldest_scheduler( + m_stats, this, m_scoreboard, m_simt_stack, &m_warp, + &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i)); + break; + case CONCRETE_SCHEDULER_WARP_LIMITING: + schedulers.push_back(new swl_scheduler( + m_stats, this, m_scoreboard, m_simt_stack, &m_warp, + &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i, + config->gpgpu_scheduler_string)); + break; + default: + abort(); + }; + } + + for (unsigned i = 0; i < m_warp.size(); i++) { + // distribute i's evenly though schedulers; + schedulers[i % m_config->gpgpu_num_sched_per_core]->add_supervised_warp_id( + i); + } + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; ++i) { + schedulers[i]->done_adding_supervised_warps(); + } + + // op collector configuration + + enum { SP_CUS, DP_CUS, SFU_CUS, TENSOR_CORE_CUS, INT_CUS, MEM_CUS, GEN_CUS }; + + opndcoll_rfu_t::port_vector_t in_ports; + opndcoll_rfu_t::port_vector_t out_ports; + opndcoll_rfu_t::uint_vector_t cu_sets; + + // configure generic collectors + m_operand_collector.add_cu_set( + GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, + m_config->gpgpu_operand_collector_num_out_ports_gen); + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_gen; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); + in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); + in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); + out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); + if (m_config->gpgpu_tensor_core_avail) { + in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); } - if(m_config->sub_core_model) { - //in subcore model, each scheduler should has its own issue register, so num scheduler = reg width - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SFU].get_size() ); - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_MEM].get_size() ); - if(m_config->gpgpu_tensor_core_avail) - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_TENSOR_CORE].get_size() ); - if(m_config->gpgpu_num_dp_units > 0) - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_DP].get_size() ); - if(m_config->gpgpu_num_int_units > 0) - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_INT].get_size() ); + if (m_config->gpgpu_num_dp_units > 0) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); } - - m_threadState = (thread_ctx_t*) calloc(sizeof(thread_ctx_t), config->n_thread_per_shader); - - m_not_completed = 0; - m_active_threads.reset(); - m_n_active_cta = 0; - for ( unsigned i = 0; i<MAX_CTA_PER_SHADER; i++ ) - m_cta_status[i]=0; - for (unsigned i = 0; i<config->n_thread_per_shader; i++) { - m_thread[i]= NULL; - m_threadState[i].m_cta_id = -1; - m_threadState[i].m_active = false; + if (m_config->gpgpu_num_int_units > 0) { + in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); + out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); } - - // m_icnt = new shader_memory_interface(this,cluster); - if ( m_config->gpgpu_perfect_mem ) { - m_icnt = new perfect_memory_interface(this,cluster); - } else { - m_icnt = new shader_memory_interface(this,cluster); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); + } + + if (m_config->enable_specialized_operand_collector) { + m_operand_collector.add_cu_set( + SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, + m_config->gpgpu_operand_collector_num_out_ports_sp); + m_operand_collector.add_cu_set( + DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, + m_config->gpgpu_operand_collector_num_out_ports_dp); + m_operand_collector.add_cu_set( + TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, + config->gpgpu_operand_collector_num_out_ports_tensor_core); + m_operand_collector.add_cu_set( + SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, + m_config->gpgpu_operand_collector_num_out_ports_sfu); + m_operand_collector.add_cu_set( + MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, + m_config->gpgpu_operand_collector_num_out_ports_mem); + m_operand_collector.add_cu_set( + INT_CUS, m_config->gpgpu_operand_collector_num_units_int, + m_config->gpgpu_operand_collector_num_out_ports_int); + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); + cu_sets.push_back((unsigned)SP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); } - m_mem_fetch_allocator = new shader_core_mem_fetch_allocator(shader_id,tpc_id,mem_config); - - // fetch - m_last_warp_fetched = 0; - - #define STRSIZE 1024 - char name[STRSIZE]; - snprintf(name, STRSIZE, "L1I_%03d", m_sid); - m_L1I = new read_only_cache( name,m_config->m_L1I_config,m_sid,get_shader_instruction_cache_id(),m_icnt,IN_L1I_MISS_QUEUE); - - m_warp.resize(m_config->max_warps_per_shader, shd_warp_t(this, warp_size)); - m_scoreboard = new Scoreboard(m_sid, m_config->max_warps_per_shader, gpu); - - //scedulers - //must currently occur after all inputs have been initialized. - std::string sched_config = m_config->gpgpu_scheduler_string; - const concrete_scheduler scheduler = sched_config.find("lrr") != std::string::npos ? - CONCRETE_SCHEDULER_LRR : - sched_config.find("two_level_active") != std::string::npos ? - CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE : - sched_config.find("gto") != std::string::npos ? - CONCRETE_SCHEDULER_GTO : - sched_config.find("old") != std::string::npos ? - CONCRETE_SCHEDULER_OLDEST_FIRST : - sched_config.find("warp_limiting") != std::string::npos ? - CONCRETE_SCHEDULER_WARP_LIMITING: - NUM_CONCRETE_SCHEDULERS; - assert ( scheduler != NUM_CONCRETE_SCHEDULERS ); - - for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) { - switch( scheduler ) - { - case CONCRETE_SCHEDULER_LRR: - schedulers.push_back( - new lrr_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_DP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_INT], - &m_pipeline_reg[ID_OC_TENSOR_CORE], - &m_pipeline_reg[ID_OC_MEM], - i - ) - ); - break; - case CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE: - schedulers.push_back( - new two_level_active_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_DP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_INT], - &m_pipeline_reg[ID_OC_TENSOR_CORE], - &m_pipeline_reg[ID_OC_MEM], - i, - config->gpgpu_scheduler_string - ) - ); - break; - case CONCRETE_SCHEDULER_GTO: - schedulers.push_back( - new gto_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_DP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_INT], - &m_pipeline_reg[ID_OC_TENSOR_CORE], - &m_pipeline_reg[ID_OC_MEM], - i - ) - ); - break; - case CONCRETE_SCHEDULER_OLDEST_FIRST: - schedulers.push_back( - new oldest_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_DP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_INT], - &m_pipeline_reg[ID_OC_TENSOR_CORE], - &m_pipeline_reg[ID_OC_MEM], - i - ) - ); - break; - case CONCRETE_SCHEDULER_WARP_LIMITING: - schedulers.push_back( - new swl_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_DP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_INT], - &m_pipeline_reg[ID_OC_TENSOR_CORE], - &m_pipeline_reg[ID_OC_MEM], - i, - config->gpgpu_scheduler_string - ) - ); - break; - default: - abort(); - }; + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); + cu_sets.push_back((unsigned)DP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); } - - for (unsigned i = 0; i < m_warp.size(); i++) { - //distribute i's evenly though schedulers; - schedulers[i%m_config->gpgpu_num_sched_per_core]->add_supervised_warp_id(i); + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); + cu_sets.push_back((unsigned)SFU_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); } - for ( unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; ++i ) { - schedulers[i]->done_adding_supervised_warps(); + + for (unsigned i = 0; + i < config->gpgpu_operand_collector_num_in_ports_tensor_core; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); + cu_sets.push_back((unsigned)TENSOR_CORE_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); } - - //op collector configuration - enum { SP_CUS, DP_CUS, SFU_CUS, TENSOR_CORE_CUS, INT_CUS, MEM_CUS, GEN_CUS }; + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); + out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); + cu_sets.push_back((unsigned)MEM_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); + } - opndcoll_rfu_t::port_vector_t in_ports; - opndcoll_rfu_t::port_vector_t out_ports; - opndcoll_rfu_t::uint_vector_t cu_sets; + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_int; + i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); + out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); + cu_sets.push_back((unsigned)INT_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports, out_ports, cu_sets); + in_ports.clear(), out_ports.clear(), cu_sets.clear(); + } + } - //configure generic collectors - m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen); + m_operand_collector.init(m_config->gpgpu_num_reg_banks, this); - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_gen; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); - in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); - in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); - out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); - if(m_config->gpgpu_tensor_core_avail) { - in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); - out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); - } - if(m_config->gpgpu_num_dp_units > 0) { - in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); - } - if(m_config->gpgpu_num_int_units > 0) { - in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); - out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); - } - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + m_num_function_units = + m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + + m_config->gpgpu_num_int_units + + 1; // sp_unit, sfu, dp, tensor, int, ldst_unit + // m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; + // m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; + + // m_fu = new simd_function_unit*[m_num_function_units]; + + for (int k = 0; k < m_config->gpgpu_num_sp_units; k++) { + m_fu.push_back(new sp_unit(&m_pipeline_reg[EX_WB], m_config, this)); + m_dispatch_port.push_back(ID_OC_SP); + m_issue_port.push_back(OC_EX_SP); + } - if(m_config->enable_specialized_operand_collector) { - m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); - m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp); - m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core); - m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); - m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); - m_operand_collector.add_cu_set(INT_CUS, m_config->gpgpu_operand_collector_num_units_int, m_config->gpgpu_operand_collector_num_out_ports_int); + for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) { + m_fu.push_back(new dp_unit(&m_pipeline_reg[EX_WB], m_config, this)); + m_dispatch_port.push_back(ID_OC_DP); + m_issue_port.push_back(OC_EX_DP); + } + for (int k = 0; k < m_config->gpgpu_num_int_units; k++) { + m_fu.push_back(new int_unit(&m_pipeline_reg[EX_WB], m_config, this)); + m_dispatch_port.push_back(ID_OC_INT); + m_issue_port.push_back(OC_EX_INT); + } - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); - cu_sets.push_back((unsigned)SP_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { + m_fu.push_back(new sfu(&m_pipeline_reg[EX_WB], m_config, this)); + m_dispatch_port.push_back(ID_OC_SFU); + m_issue_port.push_back(OC_EX_SFU); + } - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); - cu_sets.push_back((unsigned)DP_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + for (int k = 0; k < config->gpgpu_num_tensor_core_units; k++) { + m_fu.push_back(new tensor_core(&m_pipeline_reg[EX_WB], m_config, this)); + m_dispatch_port.push_back(ID_OC_TENSOR_CORE); + m_issue_port.push_back(OC_EX_TENSOR_CORE); + } - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); - cu_sets.push_back((unsigned)SFU_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + m_ldst_unit = + new ldst_unit(m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, + m_scoreboard, config, mem_config, stats, shader_id, tpc_id); + m_fu.push_back(m_ldst_unit); + m_dispatch_port.push_back(ID_OC_MEM); + m_issue_port.push_back(OC_EX_MEM); - for (unsigned i = 0; i < config->gpgpu_operand_collector_num_in_ports_tensor_core; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); - out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); - cu_sets.push_back((unsigned)TENSOR_CORE_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + assert(m_num_function_units == m_fu.size() and + m_fu.size() == m_dispatch_port.size() and + m_fu.size() == m_issue_port.size()); - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); - out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); - cu_sets.push_back((unsigned)MEM_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } + // there are as many result buses as the width of the EX_WB stage + num_result_bus = config->pipe_widths[EX_WB]; + for (unsigned i = 0; i < num_result_bus; i++) { + this->m_result_bus.push_back(new std::bitset<MAX_ALU_LATENCY>()); + } - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_int; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); - out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); - cu_sets.push_back((unsigned)INT_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } - } - - m_operand_collector.init( m_config->gpgpu_num_reg_banks, this ); - - m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + m_config->gpgpu_num_int_units + 1; // sp_unit, sfu, dp, tensor, int, ldst_unit - //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; - //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; - - //m_fu = new simd_function_unit*[m_num_function_units]; - - for (int k = 0; k < m_config->gpgpu_num_sp_units; k++) { - m_fu.push_back(new sp_unit( &m_pipeline_reg[EX_WB], m_config, this )); - m_dispatch_port.push_back(ID_OC_SP); - m_issue_port.push_back(OC_EX_SP); - } - - for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) { - m_fu.push_back(new dp_unit( &m_pipeline_reg[EX_WB], m_config, this )); - m_dispatch_port.push_back(ID_OC_DP); - m_issue_port.push_back(OC_EX_DP); - } - for (int k = 0; k < m_config->gpgpu_num_int_units; k++) { - m_fu.push_back(new int_unit( &m_pipeline_reg[EX_WB], m_config, this )); - m_dispatch_port.push_back(ID_OC_INT); - m_issue_port.push_back(OC_EX_INT); - } + m_last_inst_gpu_sim_cycle = 0; + m_last_inst_gpu_tot_sim_cycle = 0; - for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { - m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this )); - m_dispatch_port.push_back(ID_OC_SFU); - m_issue_port.push_back(OC_EX_SFU); - } - - for (int k = 0; k < config->gpgpu_num_tensor_core_units; k++) { - m_fu.push_back(new tensor_core( &m_pipeline_reg[EX_WB], m_config, this )); - m_dispatch_port.push_back(ID_OC_TENSOR_CORE); - m_issue_port.push_back(OC_EX_TENSOR_CORE); - } + // Jin: for concurrent kernels on a SM + m_occupied_n_threads = 0; + m_occupied_shmem = 0; + m_occupied_regs = 0; + m_occupied_ctas = 0; + m_occupied_hwtid.reset(); + m_occupied_cta_to_hwtid.clear(); +} - m_ldst_unit = new ldst_unit( m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, m_scoreboard, config, mem_config, stats, shader_id, tpc_id ); - m_fu.push_back(m_ldst_unit); - m_dispatch_port.push_back(ID_OC_MEM); - m_issue_port.push_back(OC_EX_MEM); - - assert(m_num_function_units == m_fu.size() and m_fu.size() == m_dispatch_port.size() and m_fu.size() == m_issue_port.size()); - - //there are as many result buses as the width of the EX_WB stage - num_result_bus = config->pipe_widths[EX_WB]; - for(unsigned i=0; i<num_result_bus; i++){ - this->m_result_bus.push_back(new std::bitset<MAX_ALU_LATENCY>()); - } - - m_last_inst_gpu_sim_cycle = 0; - m_last_inst_gpu_tot_sim_cycle = 0; +void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, + bool reset_not_completed) { + if (reset_not_completed) { + m_not_completed = 0; + m_active_threads.reset(); - //Jin: for concurrent kernels on a SM + // Jin: for concurrent kernels on a SM m_occupied_n_threads = 0; m_occupied_shmem = 0; m_occupied_regs = 0; m_occupied_ctas = 0; m_occupied_hwtid.reset(); m_occupied_cta_to_hwtid.clear(); + m_active_warps = 0; + } + for (unsigned i = start_thread; i < end_thread; i++) { + m_threadState[i].n_insn = 0; + m_threadState[i].m_cta_id = -1; + } + for (unsigned i = start_thread / m_config->warp_size; + i < end_thread / m_config->warp_size; ++i) { + m_warp[i].reset(); + m_simt_stack[i]->reset(); + } } -void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool reset_not_completed ) -{ - if( reset_not_completed ) { - m_not_completed = 0; - m_active_threads.reset(); - - //Jin: for concurrent kernels on a SM - m_occupied_n_threads = 0; - m_occupied_shmem = 0; - m_occupied_regs = 0; - m_occupied_ctas = 0; - m_occupied_hwtid.reset(); - m_occupied_cta_to_hwtid.clear(); - m_active_warps = 0; - - } - for (unsigned i = start_thread; i<end_thread; i++) { - m_threadState[i].n_insn = 0; - m_threadState[i].m_cta_id = -1; - } - for (unsigned i = start_thread / m_config->warp_size; i < end_thread / m_config->warp_size; ++i) { - m_warp[i].reset(); - m_simt_stack[i]->reset(); - } -} - -void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id ) -{ - address_type start_pc = next_pc(start_thread); - if (m_config->model == POST_DOMINATOR) { - unsigned start_warp = start_thread / m_config->warp_size; - unsigned warp_per_cta = cta_size / m_config->warp_size; - unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0); - for (unsigned i = start_warp; i < end_warp; ++i) { - unsigned n_active=0; - simt_mask_t active_threads; - for (unsigned t = 0; t < m_config->warp_size; t++) { - unsigned hwtid = i * m_config->warp_size + t; - if ( hwtid < end_thread ) { - n_active++; - assert( !m_active_threads.test(hwtid) ); - m_active_threads.set( hwtid ); - active_threads.set(t); - } - } - m_simt_stack[i]->launch(start_pc,active_threads); +void shader_core_ctx::init_warps(unsigned cta_id, unsigned start_thread, + unsigned end_thread, unsigned ctaid, + int cta_size, unsigned kernel_id) { + address_type start_pc = next_pc(start_thread); + if (m_config->model == POST_DOMINATOR) { + unsigned start_warp = start_thread / m_config->warp_size; + unsigned warp_per_cta = cta_size / m_config->warp_size; + unsigned end_warp = end_thread / m_config->warp_size + + ((end_thread % m_config->warp_size) ? 1 : 0); + for (unsigned i = start_warp; i < end_warp; ++i) { + unsigned n_active = 0; + simt_mask_t active_threads; + for (unsigned t = 0; t < m_config->warp_size; t++) { + unsigned hwtid = i * m_config->warp_size + t; + if (hwtid < end_thread) { + n_active++; + assert(!m_active_threads.test(hwtid)); + m_active_threads.set(hwtid); + active_threads.set(t); + } + } + m_simt_stack[i]->launch(start_pc, active_threads); - if(m_gpu->resume_option == 1 && kernel_id == m_gpu->resume_kernel && ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t ) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid ); - unsigned pc,rpc; - m_simt_stack[i]->resume(fname); - m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc); - for (unsigned t = 0; t < m_config->warp_size; t++) { - m_thread[i * m_config->warp_size + t]->set_npc(pc); - m_thread[i * m_config->warp_size + t]->update_pc(); - } - start_pc=pc; - } - - m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id); - ++m_dynamic_warp_id; - m_not_completed += n_active; - ++m_active_warps; + if (m_gpu->resume_option == 1 && kernel_id == m_gpu->resume_kernel && + ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", + i % warp_per_cta, ctaid); + unsigned pc, rpc; + m_simt_stack[i]->resume(fname); + m_simt_stack[i]->get_pdom_stack_top_info(&pc, &rpc); + for (unsigned t = 0; t < m_config->warp_size; t++) { + m_thread[i * m_config->warp_size + t]->set_npc(pc); + m_thread[i * m_config->warp_size + t]->update_pc(); + } + start_pc = pc; } - } + + m_warp[i].init(start_pc, cta_id, i, active_threads, m_dynamic_warp_id); + ++m_dynamic_warp_id; + m_not_completed += n_active; + ++m_active_warps; + } + } } -// return the next pc of a thread -address_type shader_core_ctx::next_pc( int tid ) const -{ - if( tid == -1 ) - return -1; - ptx_thread_info *the_thread = m_thread[tid]; - if ( the_thread == NULL ) - return -1; - return the_thread->get_pc(); // PC should already be updatd to next PC at this point (was set in shader_decode() last time thread ran) +// return the next pc of a thread +address_type shader_core_ctx::next_pc(int tid) const { + if (tid == -1) return -1; + ptx_thread_info *the_thread = m_thread[tid]; + if (the_thread == NULL) return -1; + return the_thread + ->get_pc(); // PC should already be updatd to next PC at this point (was + // set in shader_decode() last time thread ran) } -void gpgpu_sim::get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ) -{ - unsigned cluster_id = m_shader_config->sid_to_cluster(sid); - m_cluster[cluster_id]->get_pdom_stack_top_info(sid,tid,pc,rpc); +void gpgpu_sim::get_pdom_stack_top_info(unsigned sid, unsigned tid, + unsigned *pc, unsigned *rpc) { + unsigned cluster_id = m_shader_config->sid_to_cluster(sid); + m_cluster[cluster_id]->get_pdom_stack_top_info(sid, tid, pc, rpc); } -void shader_core_ctx::get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsigned *rpc ) const -{ - unsigned warp_id = tid/m_config->warp_size; - m_simt_stack[warp_id]->get_pdom_stack_top_info(pc,rpc); +void shader_core_ctx::get_pdom_stack_top_info(unsigned tid, unsigned *pc, + unsigned *rpc) const { + unsigned warp_id = tid / m_config->warp_size; + m_simt_stack[warp_id]->get_pdom_stack_top_info(pc, rpc); } -float shader_core_ctx::get_current_occupancy( unsigned long long & active, unsigned long long & total ) const -{ - // To match the achieved_occupancy in nvprof, only SMs that are active are counted toward the occupancy. - if ( m_active_warps > 0 ) { - total += m_warp.size(); - active += m_active_warps; - return float(active) / float(total); - } else { - return 0; - } +float shader_core_ctx::get_current_occupancy(unsigned long long &active, + unsigned long long &total) const { + // To match the achieved_occupancy in nvprof, only SMs that are active are + // counted toward the occupancy. + if (m_active_warps > 0) { + total += m_warp.size(); + active += m_active_warps; + return float(active) / float(total); + } else { + return 0; + } } -void shader_core_stats::print( FILE* fout ) const -{ - unsigned long long thread_icount_uarch=0; - unsigned long long warp_icount_uarch=0; +void shader_core_stats::print(FILE *fout) const { + unsigned long long thread_icount_uarch = 0; + unsigned long long warp_icount_uarch = 0; - for(unsigned i=0; i < m_config->num_shader(); i++) { - thread_icount_uarch += m_num_sim_insn[i]; - warp_icount_uarch += m_num_sim_winsn[i]; - } - fprintf(fout,"gpgpu_n_tot_thrd_icount = %lld\n", thread_icount_uarch); - fprintf(fout,"gpgpu_n_tot_w_icount = %lld\n", warp_icount_uarch); + for (unsigned i = 0; i < m_config->num_shader(); i++) { + thread_icount_uarch += m_num_sim_insn[i]; + warp_icount_uarch += m_num_sim_winsn[i]; + } + fprintf(fout, "gpgpu_n_tot_thrd_icount = %lld\n", thread_icount_uarch); + fprintf(fout, "gpgpu_n_tot_w_icount = %lld\n", warp_icount_uarch); - fprintf(fout,"gpgpu_n_stall_shd_mem = %d\n", gpgpu_n_stall_shd_mem ); - fprintf(fout,"gpgpu_n_mem_read_local = %d\n", gpgpu_n_mem_read_local); - fprintf(fout,"gpgpu_n_mem_write_local = %d\n", gpgpu_n_mem_write_local); - fprintf(fout,"gpgpu_n_mem_read_global = %d\n", gpgpu_n_mem_read_global); - fprintf(fout,"gpgpu_n_mem_write_global = %d\n", gpgpu_n_mem_write_global); - fprintf(fout,"gpgpu_n_mem_texture = %d\n", gpgpu_n_mem_texture); - fprintf(fout,"gpgpu_n_mem_const = %d\n", gpgpu_n_mem_const); + fprintf(fout, "gpgpu_n_stall_shd_mem = %d\n", gpgpu_n_stall_shd_mem); + fprintf(fout, "gpgpu_n_mem_read_local = %d\n", gpgpu_n_mem_read_local); + fprintf(fout, "gpgpu_n_mem_write_local = %d\n", gpgpu_n_mem_write_local); + fprintf(fout, "gpgpu_n_mem_read_global = %d\n", gpgpu_n_mem_read_global); + fprintf(fout, "gpgpu_n_mem_write_global = %d\n", gpgpu_n_mem_write_global); + fprintf(fout, "gpgpu_n_mem_texture = %d\n", gpgpu_n_mem_texture); + fprintf(fout, "gpgpu_n_mem_const = %d\n", gpgpu_n_mem_const); - fprintf(fout, "gpgpu_n_load_insn = %d\n", gpgpu_n_load_insn); - fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn); - fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn); - fprintf(fout, "gpgpu_n_sstarr_insn = %d\n", gpgpu_n_sstarr_insn); - fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn); - fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn); - fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn); + fprintf(fout, "gpgpu_n_load_insn = %d\n", gpgpu_n_load_insn); + fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn); + fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn); + fprintf(fout, "gpgpu_n_sstarr_insn = %d\n", gpgpu_n_sstarr_insn); + fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn); + fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn); + fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn); - fprintf(fout, "gpgpu_n_shmem_bkconflict = %d\n", gpgpu_n_shmem_bkconflict); - fprintf(fout, "gpgpu_n_cache_bkconflict = %d\n", gpgpu_n_cache_bkconflict); + fprintf(fout, "gpgpu_n_shmem_bkconflict = %d\n", gpgpu_n_shmem_bkconflict); + fprintf(fout, "gpgpu_n_cache_bkconflict = %d\n", gpgpu_n_cache_bkconflict); - fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge); - fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict); + fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", + gpgpu_n_intrawarp_mshr_merge); + fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); - //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); - //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); - fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]); - fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n", - gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] + - gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] + - gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] + - gpu_stall_shd_mem_breakdown[L_MEM_ST][BK_CONF] - ); // coalescing stall at data cache - fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][coal_stall] = %d\n", - gpu_stall_shd_mem_breakdown[G_MEM_LD][COAL_STALL] + - gpu_stall_shd_mem_breakdown[G_MEM_ST][COAL_STALL] + - gpu_stall_shd_mem_breakdown[L_MEM_LD][COAL_STALL] + - gpu_stall_shd_mem_breakdown[L_MEM_ST][COAL_STALL] - ); // coalescing stall + bank conflict at data cache - fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][data_port_stall] = %d\n", - gpu_stall_shd_mem_breakdown[G_MEM_LD][DATA_PORT_STALL] + - gpu_stall_shd_mem_breakdown[G_MEM_ST][DATA_PORT_STALL] + - gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] + - gpu_stall_shd_mem_breakdown[L_MEM_ST][DATA_PORT_STALL] - ); // data port stall at data cache - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); - //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n", + gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); + // fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", + // gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); fprintf(fout, + // "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", + // gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); + fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", + gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]); + fprintf( + fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n", + gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] + + gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] + + gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] + + gpu_stall_shd_mem_breakdown[L_MEM_ST][BK_CONF]); // coalescing stall + // at data cache + fprintf( + fout, "gpgpu_stall_shd_mem[gl_mem][coal_stall] = %d\n", + gpu_stall_shd_mem_breakdown[G_MEM_LD][COAL_STALL] + + gpu_stall_shd_mem_breakdown[G_MEM_ST][COAL_STALL] + + gpu_stall_shd_mem_breakdown[L_MEM_LD][COAL_STALL] + + gpu_stall_shd_mem_breakdown[L_MEM_ST] + [COAL_STALL]); // coalescing stall + bank + // conflict at data cache + fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][data_port_stall] = %d\n", + gpu_stall_shd_mem_breakdown[G_MEM_LD][DATA_PORT_STALL] + + gpu_stall_shd_mem_breakdown[G_MEM_ST][DATA_PORT_STALL] + + gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] + + gpu_stall_shd_mem_breakdown[L_MEM_ST] + [DATA_PORT_STALL]); // data port stall + // at data cache + // fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", + // gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); fprintf(fout, + // "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", + // gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls); + fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", + gpu_reg_bank_conflict_stalls); - fprintf(fout, "Warp Occupancy Distribution:\n"); - fprintf(fout, "Stall:%d\t", shader_cycle_distro[2]); - fprintf(fout, "W0_Idle:%d\t", shader_cycle_distro[0]); - fprintf(fout, "W0_Scoreboard:%d", shader_cycle_distro[1]); - for (unsigned i = 3; i < m_config->warp_size + 3; i++) - fprintf(fout, "\tW%d:%d", i-2, shader_cycle_distro[i]); - fprintf(fout, "\n"); - fprintf(fout, "single_issue_nums: "); - for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) - fprintf(fout, "WS%d:%d\t", i, single_issue_nums[i]); - fprintf(fout, "\n"); - fprintf(fout, "dual_issue_nums: "); - for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) - fprintf(fout, "WS%d:%d\t", i, dual_issue_nums[i]); - fprintf(fout, "\n"); + fprintf(fout, "Warp Occupancy Distribution:\n"); + fprintf(fout, "Stall:%d\t", shader_cycle_distro[2]); + fprintf(fout, "W0_Idle:%d\t", shader_cycle_distro[0]); + fprintf(fout, "W0_Scoreboard:%d", shader_cycle_distro[1]); + for (unsigned i = 3; i < m_config->warp_size + 3; i++) + fprintf(fout, "\tW%d:%d", i - 2, shader_cycle_distro[i]); + fprintf(fout, "\n"); + fprintf(fout, "single_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, single_issue_nums[i]); + fprintf(fout, "\n"); + fprintf(fout, "dual_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, dual_issue_nums[i]); + fprintf(fout, "\n"); - m_outgoing_traffic_stats->print(fout); - m_incoming_traffic_stats->print(fout); + m_outgoing_traffic_stats->print(fout); + m_incoming_traffic_stats->print(fout); } -void shader_core_stats::event_warp_issued( unsigned s_id, unsigned warp_id, unsigned num_issued, unsigned dynamic_warp_id ) { - assert( warp_id <= m_config->max_warps_per_shader ); - for ( unsigned i = 0; i < num_issued; ++i ) { - if ( m_shader_dynamic_warp_issue_distro[ s_id ].size() <= dynamic_warp_id ) { - m_shader_dynamic_warp_issue_distro[ s_id ].resize(dynamic_warp_id + 1); - } - ++m_shader_dynamic_warp_issue_distro[ s_id ][ dynamic_warp_id ]; - if ( m_shader_warp_slot_issue_distro[ s_id ].size() <= warp_id ) { - m_shader_warp_slot_issue_distro[ s_id ].resize(warp_id + 1); - } - ++m_shader_warp_slot_issue_distro[ s_id ][ warp_id ]; +void shader_core_stats::event_warp_issued(unsigned s_id, unsigned warp_id, + unsigned num_issued, + unsigned dynamic_warp_id) { + assert(warp_id <= m_config->max_warps_per_shader); + for (unsigned i = 0; i < num_issued; ++i) { + if (m_shader_dynamic_warp_issue_distro[s_id].size() <= dynamic_warp_id) { + m_shader_dynamic_warp_issue_distro[s_id].resize(dynamic_warp_id + 1); + } + ++m_shader_dynamic_warp_issue_distro[s_id][dynamic_warp_id]; + if (m_shader_warp_slot_issue_distro[s_id].size() <= warp_id) { + m_shader_warp_slot_issue_distro[s_id].resize(warp_id + 1); } + ++m_shader_warp_slot_issue_distro[s_id][warp_id]; + } } -void shader_core_stats::visualizer_print( gzFile visualizer_file ) -{ - // warp divergence breakdown - gzprintf(visualizer_file, "WarpDivergenceBreakdown:"); - unsigned int total=0; - unsigned int cf = (m_config->gpgpu_warpdistro_shader==-1)?m_config->num_shader():1; - gzprintf(visualizer_file, " %d", (shader_cycle_distro[0] - last_shader_cycle_distro[0]) / cf ); - gzprintf(visualizer_file, " %d", (shader_cycle_distro[1] - last_shader_cycle_distro[1]) / cf ); - gzprintf(visualizer_file, " %d", (shader_cycle_distro[2] - last_shader_cycle_distro[2]) / cf ); - for (unsigned i=0; i<m_config->warp_size+3; i++) { - if ( i>=3 ) { - total += (shader_cycle_distro[i] - last_shader_cycle_distro[i]); - if ( ((i-3) % (m_config->warp_size/8)) == ((m_config->warp_size/8)-1) ) { - gzprintf(visualizer_file, " %d", total / cf ); - total=0; - } - } - last_shader_cycle_distro[i] = shader_cycle_distro[i]; +void shader_core_stats::visualizer_print(gzFile visualizer_file) { + // warp divergence breakdown + gzprintf(visualizer_file, "WarpDivergenceBreakdown:"); + unsigned int total = 0; + unsigned int cf = + (m_config->gpgpu_warpdistro_shader == -1) ? m_config->num_shader() : 1; + gzprintf(visualizer_file, " %d", + (shader_cycle_distro[0] - last_shader_cycle_distro[0]) / cf); + gzprintf(visualizer_file, " %d", + (shader_cycle_distro[1] - last_shader_cycle_distro[1]) / cf); + gzprintf(visualizer_file, " %d", + (shader_cycle_distro[2] - last_shader_cycle_distro[2]) / cf); + for (unsigned i = 0; i < m_config->warp_size + 3; i++) { + if (i >= 3) { + total += (shader_cycle_distro[i] - last_shader_cycle_distro[i]); + if (((i - 3) % (m_config->warp_size / 8)) == + ((m_config->warp_size / 8) - 1)) { + gzprintf(visualizer_file, " %d", total / cf); + total = 0; + } } - gzprintf(visualizer_file,"\n"); + last_shader_cycle_distro[i] = shader_cycle_distro[i]; + } + gzprintf(visualizer_file, "\n"); - // warp issue breakdown - unsigned sid = m_config->gpgpu_warp_issue_shader; - unsigned count = 0; - unsigned warp_id_issued_sum = 0; - gzprintf(visualizer_file, "WarpIssueSlotBreakdown:"); - if(m_shader_warp_slot_issue_distro[sid].size() > 0){ - for ( std::vector<unsigned>::const_iterator iter = m_shader_warp_slot_issue_distro[ sid ].begin(); - iter != m_shader_warp_slot_issue_distro[ sid ].end(); iter++, count++ ) { - unsigned diff = count < m_last_shader_warp_slot_issue_distro.size() ? - *iter - m_last_shader_warp_slot_issue_distro[ count ] : - *iter; - gzprintf( visualizer_file, " %d", diff ); - warp_id_issued_sum += diff; - } - m_last_shader_warp_slot_issue_distro = m_shader_warp_slot_issue_distro[ sid ]; - }else{ - gzprintf( visualizer_file, " 0"); + // warp issue breakdown + unsigned sid = m_config->gpgpu_warp_issue_shader; + unsigned count = 0; + unsigned warp_id_issued_sum = 0; + gzprintf(visualizer_file, "WarpIssueSlotBreakdown:"); + if (m_shader_warp_slot_issue_distro[sid].size() > 0) { + for (std::vector<unsigned>::const_iterator iter = + m_shader_warp_slot_issue_distro[sid].begin(); + iter != m_shader_warp_slot_issue_distro[sid].end(); iter++, count++) { + unsigned diff = count < m_last_shader_warp_slot_issue_distro.size() + ? *iter - m_last_shader_warp_slot_issue_distro[count] + : *iter; + gzprintf(visualizer_file, " %d", diff); + warp_id_issued_sum += diff; } - gzprintf(visualizer_file,"\n"); + m_last_shader_warp_slot_issue_distro = m_shader_warp_slot_issue_distro[sid]; + } else { + gzprintf(visualizer_file, " 0"); + } + gzprintf(visualizer_file, "\n"); - #define DYNAMIC_WARP_PRINT_RESOLUTION 32 - unsigned total_issued_this_resolution = 0; - unsigned dynamic_id_issued_sum = 0; - count = 0; - gzprintf(visualizer_file, "WarpIssueDynamicIdBreakdown:"); - if(m_shader_dynamic_warp_issue_distro[sid].size() > 0){ - for ( std::vector<unsigned>::const_iterator iter = m_shader_dynamic_warp_issue_distro[ sid ].begin(); - iter != m_shader_dynamic_warp_issue_distro[ sid ].end(); iter++, count++ ) { - unsigned diff = count < m_last_shader_dynamic_warp_issue_distro.size() ? - *iter - m_last_shader_dynamic_warp_issue_distro[ count ] : - *iter; - total_issued_this_resolution += diff; - if ( ( count + 1 ) % DYNAMIC_WARP_PRINT_RESOLUTION == 0 ) { - gzprintf( visualizer_file, " %d", total_issued_this_resolution ); - dynamic_id_issued_sum += total_issued_this_resolution; - total_issued_this_resolution = 0; - } - } - if ( count % DYNAMIC_WARP_PRINT_RESOLUTION != 0 ) { - gzprintf( visualizer_file, " %d", total_issued_this_resolution ); - dynamic_id_issued_sum += total_issued_this_resolution; - } - m_last_shader_dynamic_warp_issue_distro = m_shader_dynamic_warp_issue_distro[ sid ]; - assert( warp_id_issued_sum == dynamic_id_issued_sum ); - }else{ - gzprintf( visualizer_file, " 0"); +#define DYNAMIC_WARP_PRINT_RESOLUTION 32 + unsigned total_issued_this_resolution = 0; + unsigned dynamic_id_issued_sum = 0; + count = 0; + gzprintf(visualizer_file, "WarpIssueDynamicIdBreakdown:"); + if (m_shader_dynamic_warp_issue_distro[sid].size() > 0) { + for (std::vector<unsigned>::const_iterator iter = + m_shader_dynamic_warp_issue_distro[sid].begin(); + iter != m_shader_dynamic_warp_issue_distro[sid].end(); + iter++, count++) { + unsigned diff = + count < m_last_shader_dynamic_warp_issue_distro.size() + ? *iter - m_last_shader_dynamic_warp_issue_distro[count] + : *iter; + total_issued_this_resolution += diff; + if ((count + 1) % DYNAMIC_WARP_PRINT_RESOLUTION == 0) { + gzprintf(visualizer_file, " %d", total_issued_this_resolution); + dynamic_id_issued_sum += total_issued_this_resolution; + total_issued_this_resolution = 0; + } } - gzprintf(visualizer_file,"\n"); - - // overall cache miss rates - gzprintf(visualizer_file, "gpgpu_n_cache_bkconflict: %d\n", gpgpu_n_cache_bkconflict); - gzprintf(visualizer_file, "gpgpu_n_shmem_bkconflict: %d\n", gpgpu_n_shmem_bkconflict); + if (count % DYNAMIC_WARP_PRINT_RESOLUTION != 0) { + gzprintf(visualizer_file, " %d", total_issued_this_resolution); + dynamic_id_issued_sum += total_issued_this_resolution; + } + m_last_shader_dynamic_warp_issue_distro = + m_shader_dynamic_warp_issue_distro[sid]; + assert(warp_id_issued_sum == dynamic_id_issued_sum); + } else { + gzprintf(visualizer_file, " 0"); + } + gzprintf(visualizer_file, "\n"); + // overall cache miss rates + gzprintf(visualizer_file, "gpgpu_n_cache_bkconflict: %d\n", + gpgpu_n_cache_bkconflict); + gzprintf(visualizer_file, "gpgpu_n_shmem_bkconflict: %d\n", + gpgpu_n_shmem_bkconflict); - // instruction count per shader core - gzprintf(visualizer_file, "shaderinsncount: "); - for (unsigned i=0;i<m_config->num_shader();i++) - gzprintf(visualizer_file, "%u ", m_num_sim_insn[i] ); - gzprintf(visualizer_file, "\n"); - // warp instruction count per shader core - gzprintf(visualizer_file, "shaderwarpinsncount: "); - for (unsigned i=0;i<m_config->num_shader();i++) - gzprintf(visualizer_file, "%u ", m_num_sim_winsn[i] ); - gzprintf(visualizer_file, "\n"); - // warp divergence per shader core - gzprintf(visualizer_file, "shaderwarpdiv: "); - for (unsigned i=0;i<m_config->num_shader();i++) - gzprintf(visualizer_file, "%u ", m_n_diverge[i] ); - gzprintf(visualizer_file, "\n"); + // instruction count per shader core + gzprintf(visualizer_file, "shaderinsncount: "); + for (unsigned i = 0; i < m_config->num_shader(); i++) + gzprintf(visualizer_file, "%u ", m_num_sim_insn[i]); + gzprintf(visualizer_file, "\n"); + // warp instruction count per shader core + gzprintf(visualizer_file, "shaderwarpinsncount: "); + for (unsigned i = 0; i < m_config->num_shader(); i++) + gzprintf(visualizer_file, "%u ", m_num_sim_winsn[i]); + gzprintf(visualizer_file, "\n"); + // warp divergence per shader core + gzprintf(visualizer_file, "shaderwarpdiv: "); + for (unsigned i = 0; i < m_config->num_shader(); i++) + gzprintf(visualizer_file, "%u ", m_n_diverge[i]); + gzprintf(visualizer_file, "\n"); } -#define PROGRAM_MEM_START 0xF0000000 /* should be distinct from other memory spaces... - check ptx_ir.h to verify this does not overlap - other memory spaces */ -void shader_core_ctx::decode() -{ - if( m_inst_fetch_buffer.m_valid ) { - // decode 1 or 2 instructions and place them into ibuffer - address_type pc = m_inst_fetch_buffer.m_pc; - const warp_inst_t* pI1 = m_gpu->gpgpu_ctx->ptx_fetch_inst(pc); - m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(0,pI1); +#define PROGRAM_MEM_START \ + 0xF0000000 /* should be distinct from other memory spaces... \ + check ptx_ir.h to verify this does not overlap \ + other memory spaces */ +void shader_core_ctx::decode() { + if (m_inst_fetch_buffer.m_valid) { + // decode 1 or 2 instructions and place them into ibuffer + address_type pc = m_inst_fetch_buffer.m_pc; + const warp_inst_t *pI1 = m_gpu->gpgpu_ctx->ptx_fetch_inst(pc); + m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(0, pI1); + m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline(); + if (pI1) { + m_stats->m_num_decoded_insn[m_sid]++; + if (pI1->oprnd_type == INT_OP) { + m_stats->m_num_INTdecoded_insn[m_sid]++; + } else if (pI1->oprnd_type == FP_OP) { + m_stats->m_num_FPdecoded_insn[m_sid]++; + } + const warp_inst_t *pI2 = + m_gpu->gpgpu_ctx->ptx_fetch_inst(pc + pI1->isize); + if (pI2) { + m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(1, pI2); m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline(); - if( pI1 ) { - m_stats->m_num_decoded_insn[m_sid]++; - if(pI1->oprnd_type==INT_OP){ - m_stats->m_num_INTdecoded_insn[m_sid]++; - }else if(pI1->oprnd_type==FP_OP) { - m_stats->m_num_FPdecoded_insn[m_sid]++; - } - const warp_inst_t* pI2 = m_gpu->gpgpu_ctx->ptx_fetch_inst(pc+pI1->isize); - if( pI2 ) { - m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(1,pI2); - m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline(); - m_stats->m_num_decoded_insn[m_sid]++; - if(pI2->oprnd_type==INT_OP){ - m_stats->m_num_INTdecoded_insn[m_sid]++; - }else if(pI2->oprnd_type==FP_OP) { - m_stats->m_num_FPdecoded_insn[m_sid]++; - } - } + m_stats->m_num_decoded_insn[m_sid]++; + if (pI2->oprnd_type == INT_OP) { + m_stats->m_num_INTdecoded_insn[m_sid]++; + } else if (pI2->oprnd_type == FP_OP) { + m_stats->m_num_FPdecoded_insn[m_sid]++; } - m_inst_fetch_buffer.m_valid = false; + } } + m_inst_fetch_buffer.m_valid = false; + } } -void shader_core_ctx::fetch() -{ +void shader_core_ctx::fetch() { + if (!m_inst_fetch_buffer.m_valid) { + if (m_L1I->access_ready()) { + mem_fetch *mf = m_L1I->next_access(); + m_warp[mf->get_wid()].clear_imiss_pending(); + m_inst_fetch_buffer = ifetch_buffer_t( + m_warp[mf->get_wid()].get_pc(), mf->get_access_size(), mf->get_wid()); + assert(m_warp[mf->get_wid()].get_pc() == + (mf->get_addr() - + PROGRAM_MEM_START)); // Verify that we got the instruction we + // were expecting. + m_inst_fetch_buffer.m_valid = true; + m_warp[mf->get_wid()].set_last_fetch(m_gpu->gpu_sim_cycle); + delete mf; + } else { + // find an active warp with space in instruction buffer that is not + // already waiting on a cache miss and get next 1-2 instructions from + // i-cache... + for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { + unsigned warp_id = + (m_last_warp_fetched + 1 + i) % m_config->max_warps_per_shader; - if( !m_inst_fetch_buffer.m_valid ) { - if( m_L1I->access_ready() ) { - mem_fetch *mf = m_L1I->next_access(); - m_warp[mf->get_wid()].clear_imiss_pending(); - m_inst_fetch_buffer = ifetch_buffer_t(m_warp[mf->get_wid()].get_pc(), mf->get_access_size(), mf->get_wid()); - assert( m_warp[mf->get_wid()].get_pc() == (mf->get_addr()-PROGRAM_MEM_START)); // Verify that we got the instruction we were expecting. - m_inst_fetch_buffer.m_valid = true; - m_warp[mf->get_wid()].set_last_fetch(m_gpu->gpu_sim_cycle); - delete mf; + // this code checks if this warp has finished executing and can be + // reclaimed + if (m_warp[warp_id].hardware_done() && + !m_scoreboard->pendingWrites(warp_id) && + !m_warp[warp_id].done_exit()) { + bool did_exit = false; + for (unsigned t = 0; t < m_config->warp_size; t++) { + unsigned tid = warp_id * m_config->warp_size + t; + if (m_threadState[tid].m_active == true) { + m_threadState[tid].m_active = false; + unsigned cta_id = m_warp[warp_id].get_cta_id(); + register_cta_thread_exit(cta_id, &(m_thread[tid]->get_kernel())); + m_not_completed -= 1; + m_active_threads.reset(tid); + assert(m_thread[tid] != NULL); + did_exit = true; + } + } + if (did_exit) m_warp[warp_id].set_done_exit(); + --m_active_warps; + assert(m_active_warps >= 0); } - else { - // find an active warp with space in instruction buffer that is not already waiting on a cache miss - // and get next 1-2 instructions from i-cache... - for( unsigned i=0; i < m_config->max_warps_per_shader; i++ ) { - unsigned warp_id = (m_last_warp_fetched+1+i) % m_config->max_warps_per_shader; - - // this code checks if this warp has finished executing and can be reclaimed - if( m_warp[warp_id].hardware_done() && !m_scoreboard->pendingWrites(warp_id) && !m_warp[warp_id].done_exit() ) { - bool did_exit=false; - for( unsigned t=0; t<m_config->warp_size;t++) { - unsigned tid=warp_id*m_config->warp_size+t; - if( m_threadState[tid].m_active == true ) { - m_threadState[tid].m_active = false; - unsigned cta_id = m_warp[warp_id].get_cta_id(); - register_cta_thread_exit(cta_id, &(m_thread[tid]->get_kernel())); - m_not_completed -= 1; - m_active_threads.reset(tid); - assert( m_thread[tid]!= NULL ); - did_exit=true; - } - } - if( did_exit ) - m_warp[warp_id].set_done_exit(); - --m_active_warps; - assert(m_active_warps >= 0); - } - // this code fetches instructions from the i-cache or generates memory requests - if( !m_warp[warp_id].functional_done() && !m_warp[warp_id].imiss_pending() && m_warp[warp_id].ibuffer_empty() ) { - address_type pc = m_warp[warp_id].get_pc(); - address_type ppc = pc + PROGRAM_MEM_START; - unsigned nbytes=16; - unsigned offset_in_block = pc & (m_config->m_L1I_config.get_line_sz()-1); - if( (offset_in_block+nbytes) > m_config->m_L1I_config.get_line_sz() ) - nbytes = (m_config->m_L1I_config.get_line_sz()-offset_in_block); + // this code fetches instructions from the i-cache or generates memory + // requests + if (!m_warp[warp_id].functional_done() && + !m_warp[warp_id].imiss_pending() && + m_warp[warp_id].ibuffer_empty()) { + address_type pc = m_warp[warp_id].get_pc(); + address_type ppc = pc + PROGRAM_MEM_START; + unsigned nbytes = 16; + unsigned offset_in_block = + pc & (m_config->m_L1I_config.get_line_sz() - 1); + if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz()) + nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block); - // TODO: replace with use of allocator - // mem_fetch *mf = m_mem_fetch_allocator->alloc() - mem_access_t acc(INST_ACC_R,ppc,nbytes,false, m_gpu->gpgpu_ctx); - mem_fetch *mf = new mem_fetch(acc, - NULL/*we don't have an instruction yet*/, - READ_PACKET_SIZE, - warp_id, - m_sid, - m_tpc, - m_memory_config, - m_gpu->gpu_tot_sim_cycle+m_gpu->gpu_sim_cycle - ); - std::list<cache_event> events; - enum cache_request_status status = m_L1I->access( (new_addr_type)ppc, mf, m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle,events); - if( status == MISS ) { - m_last_warp_fetched=warp_id; - m_warp[warp_id].set_imiss_pending(); - m_warp[warp_id].set_last_fetch(m_gpu->gpu_sim_cycle); - } else if( status == HIT ) { - m_last_warp_fetched=warp_id; - m_inst_fetch_buffer = ifetch_buffer_t(pc,nbytes,warp_id); - m_warp[warp_id].set_last_fetch(m_gpu->gpu_sim_cycle); - delete mf; - } else { - m_last_warp_fetched=warp_id; - assert( status == RESERVATION_FAIL ); - delete mf; - } - break; - } - } + // TODO: replace with use of allocator + // mem_fetch *mf = m_mem_fetch_allocator->alloc() + mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx); + mem_fetch *mf = new mem_fetch( + acc, NULL /*we don't have an instruction yet*/, READ_PACKET_SIZE, + warp_id, m_sid, m_tpc, m_memory_config, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + std::list<cache_event> events; + enum cache_request_status status = m_L1I->access( + (new_addr_type)ppc, mf, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, events); + if (status == MISS) { + m_last_warp_fetched = warp_id; + m_warp[warp_id].set_imiss_pending(); + m_warp[warp_id].set_last_fetch(m_gpu->gpu_sim_cycle); + } else if (status == HIT) { + m_last_warp_fetched = warp_id; + m_inst_fetch_buffer = ifetch_buffer_t(pc, nbytes, warp_id); + m_warp[warp_id].set_last_fetch(m_gpu->gpu_sim_cycle); + delete mf; + } else { + m_last_warp_fetched = warp_id; + assert(status == RESERVATION_FAIL); + delete mf; + } + break; } + } } + } - m_L1I->cycle(); + m_L1I->cycle(); } -void shader_core_ctx::func_exec_inst( warp_inst_t &inst ) -{ - execute_warp_inst_t(inst); - if( inst.is_load() || inst.is_store() ) - { - inst.generate_mem_accesses(); - //inst.print_m_accessq(); - } +void shader_core_ctx::func_exec_inst(warp_inst_t &inst) { + execute_warp_inst_t(inst); + if (inst.is_load() || inst.is_store()) { + inst.generate_mem_accesses(); + // inst.print_m_accessq(); + } } -void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* next_inst, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id ) -{ - warp_inst_t** pipe_reg = pipe_reg_set.get_free(m_config->sub_core_model, sch_id); - assert(pipe_reg); +void shader_core_ctx::issue_warp(register_set &pipe_reg_set, + const warp_inst_t *next_inst, + const active_mask_t &active_mask, + unsigned warp_id, unsigned sch_id) { + warp_inst_t **pipe_reg = + pipe_reg_set.get_free(m_config->sub_core_model, sch_id); + assert(pipe_reg); - m_warp[warp_id].ibuffer_free(); - assert(next_inst->valid()); - **pipe_reg = *next_inst; // static instruction information - (*pipe_reg)->issue( active_mask, warp_id, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, m_warp[warp_id].get_dynamic_warp_id(), sch_id ); // dynamic instruction information - m_stats->shader_cycle_distro[2+(*pipe_reg)->active_count()]++; - func_exec_inst( **pipe_reg ); - if( next_inst->op == BARRIER_OP ){ - m_warp[warp_id].store_info_of_last_inst_at_barrier(*pipe_reg); - m_barriers.warp_reaches_barrier(m_warp[warp_id].get_cta_id(),warp_id,const_cast<warp_inst_t*> (next_inst)); + m_warp[warp_id].ibuffer_free(); + assert(next_inst->valid()); + **pipe_reg = *next_inst; // static instruction information + (*pipe_reg)->issue(active_mask, warp_id, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, + m_warp[warp_id].get_dynamic_warp_id(), + sch_id); // dynamic instruction information + m_stats->shader_cycle_distro[2 + (*pipe_reg)->active_count()]++; + func_exec_inst(**pipe_reg); + if (next_inst->op == BARRIER_OP) { + m_warp[warp_id].store_info_of_last_inst_at_barrier(*pipe_reg); + m_barriers.warp_reaches_barrier(m_warp[warp_id].get_cta_id(), warp_id, + const_cast<warp_inst_t *>(next_inst)); - }else if( next_inst->op == MEMORY_BARRIER_OP ){ - m_warp[warp_id].set_membar(); - } + } else if (next_inst->op == MEMORY_BARRIER_OP) { + m_warp[warp_id].set_membar(); + } - updateSIMTStack(warp_id,*pipe_reg); - m_scoreboard->reserveRegisters(*pipe_reg); - m_warp[warp_id].set_next_pc(next_inst->pc + next_inst->isize); + updateSIMTStack(warp_id, *pipe_reg); + m_scoreboard->reserveRegisters(*pipe_reg); + m_warp[warp_id].set_next_pc(next_inst->pc + next_inst->isize); } -void shader_core_ctx::issue(){ - - //Ensure fair round robin issu between schedulers - unsigned j; - for (unsigned i = 0; i < schedulers.size(); i++) { - j = (Issue_Prio + i) % schedulers.size(); - schedulers[j]->cycle(); - } - Issue_Prio = (Issue_Prio+1)% schedulers.size(); - - //really is issue; - //for (unsigned i = 0; i < schedulers.size(); i++) { - // schedulers[i]->cycle(); - //} -} +void shader_core_ctx::issue() { + // Ensure fair round robin issu between schedulers + unsigned j; + for (unsigned i = 0; i < schedulers.size(); i++) { + j = (Issue_Prio + i) % schedulers.size(); + schedulers[j]->cycle(); + } + Issue_Prio = (Issue_Prio + 1) % schedulers.size(); -shd_warp_t& scheduler_unit::warp(int i){ - return (*m_warp)[i]; + // really is issue; + // for (unsigned i = 0; i < schedulers.size(); i++) { + // schedulers[i]->cycle(); + //} } +shd_warp_t &scheduler_unit::warp(int i) { return (*m_warp)[i]; } /** - * A general function to order things in a Loose Round Robin way. The simplist use of this - * function would be to implement a loose RR scheduler between all the warps assigned to this core. - * A more sophisticated usage would be to order a set of "fetch groups" in a RR fashion. - * In the first case, the templated class variable would be a simple unsigned int representing the - * warp_id. In the 2lvl case, T could be a struct or a list representing a set of warp_ids. - * @param result_list: The resultant list the caller wants returned. This list is cleared and then populated - * in a loose round robin way - * @param input_list: The list of things that should be put into the result_list. For a simple scheduler - * this can simply be the m_supervised_warps list. - * @param last_issued_from_input: An iterator pointing the last member in the input_list that issued. - * Since this function orders in a RR fashion, the object pointed - * to by this iterator will be last in the prioritization list - * @param num_warps_to_add: The number of warps you want the scheudler to pick between this cycle. - * Normally, this will be all the warps availible on the core, i.e. - * m_supervised_warps.size(). However, a more sophisticated scheduler may wish to - * limit this number. If the number if < m_supervised_warps.size(), then only - * the warps with highest RR priority will be placed in the result_list. + * A general function to order things in a Loose Round Robin way. The simplist + * use of this function would be to implement a loose RR scheduler between all + * the warps assigned to this core. A more sophisticated usage would be to order + * a set of "fetch groups" in a RR fashion. In the first case, the templated + * class variable would be a simple unsigned int representing the warp_id. In + * the 2lvl case, T could be a struct or a list representing a set of warp_ids. + * @param result_list: The resultant list the caller wants returned. This list + * is cleared and then populated in a loose round robin way + * @param input_list: The list of things that should be put into the + * result_list. For a simple scheduler this can simply be the m_supervised_warps + * list. + * @param last_issued_from_input: An iterator pointing the last member in the + * input_list that issued. Since this function orders in a RR fashion, the + * object pointed to by this iterator will be last in the prioritization list + * @param num_warps_to_add: The number of warps you want the scheudler to pick + * between this cycle. Normally, this will be all the warps availible on the + * core, i.e. m_supervised_warps.size(). However, a more sophisticated scheduler + * may wish to limit this number. If the number if < m_supervised_warps.size(), + * then only the warps with highest RR priority will be placed in the + * result_list. */ - template < class T > -void scheduler_unit::order_lrr( std::vector< T >& result_list, - const typename std::vector< T >& input_list, - const typename std::vector< T >::const_iterator& last_issued_from_input, - unsigned num_warps_to_add ) -{ - assert( num_warps_to_add <= input_list.size() ); - result_list.clear(); - typename std::vector< T >::const_iterator iter - = ( last_issued_from_input == input_list.end() ) ? input_list.begin() - : last_issued_from_input + 1; +template <class T> +void scheduler_unit::order_lrr( + std::vector<T> &result_list, const typename std::vector<T> &input_list, + const typename std::vector<T>::const_iterator &last_issued_from_input, + unsigned num_warps_to_add) { + assert(num_warps_to_add <= input_list.size()); + result_list.clear(); + typename std::vector<T>::const_iterator iter = + (last_issued_from_input == input_list.end()) ? input_list.begin() + : last_issued_from_input + 1; - for ( unsigned count = 0; - count < num_warps_to_add; - ++iter, ++count) { - if ( iter == input_list.end() ) { - iter = input_list.begin(); - } - result_list.push_back( *iter ); + for (unsigned count = 0; count < num_warps_to_add; ++iter, ++count) { + if (iter == input_list.end()) { + iter = input_list.begin(); } + result_list.push_back(*iter); + } } /** * A general function to order things in an priority-based way. * The core usage of the function is similar to order_lrr. - * The explanation of the additional parameters (beyond order_lrr) explains the further extensions. - * @param ordering: An enum that determines how the age function will be treated in prioritization - * see the definition of OrderingType. - * @param priority_function: This function is used to sort the input_list. It is passed to stl::sort as - * the sorting fucntion. So, if you wanted to sort a list of integer warp_ids - * with the oldest warps having the most priority, then the priority_function - * would compare the age of the two warps. + * The explanation of the additional parameters (beyond order_lrr) explains the + * further extensions. + * @param ordering: An enum that determines how the age function will be treated + * in prioritization see the definition of OrderingType. + * @param priority_function: This function is used to sort the input_list. It + * is passed to stl::sort as the sorting fucntion. So, if you wanted to sort a + * list of integer warp_ids with the oldest warps having the most priority, then + * the priority_function would compare the age of the two warps. */ - template < class T > -void scheduler_unit::order_by_priority( std::vector< T >& result_list, - const typename std::vector< T >& input_list, - const typename std::vector< T >::const_iterator& last_issued_from_input, - unsigned num_warps_to_add, - OrderingType ordering, - bool (*priority_func)(T lhs, T rhs) ) -{ - assert( num_warps_to_add <= input_list.size() ); - result_list.clear(); - typename std::vector< T > temp = input_list; +template <class T> +void scheduler_unit::order_by_priority( + std::vector<T> &result_list, const typename std::vector<T> &input_list, + const typename std::vector<T>::const_iterator &last_issued_from_input, + unsigned num_warps_to_add, OrderingType ordering, + bool (*priority_func)(T lhs, T rhs)) { + assert(num_warps_to_add <= input_list.size()); + result_list.clear(); + typename std::vector<T> temp = input_list; - if ( ORDERING_GREEDY_THEN_PRIORITY_FUNC == ordering ) { - T greedy_value = *last_issued_from_input; - result_list.push_back( greedy_value ); + if (ORDERING_GREEDY_THEN_PRIORITY_FUNC == ordering) { + T greedy_value = *last_issued_from_input; + result_list.push_back(greedy_value); - std::sort( temp.begin(), temp.end(), priority_func ); - typename std::vector< T >::iterator iter = temp.begin(); - for ( unsigned count = 0; count < num_warps_to_add; ++count, ++iter ) { - if ( *iter != greedy_value ) { - result_list.push_back( *iter ); - } - } - } else if ( ORDERED_PRIORITY_FUNC_ONLY == ordering ) { - std::sort( temp.begin(), temp.end(), priority_func ); - typename std::vector< T >::iterator iter = temp.begin(); - for ( unsigned count = 0; count < num_warps_to_add; ++count, ++iter ) { - result_list.push_back( *iter ); - } - } else { - fprintf( stderr, "Unknown ordering - %d\n", ordering ); - abort(); + std::sort(temp.begin(), temp.end(), priority_func); + typename std::vector<T>::iterator iter = temp.begin(); + for (unsigned count = 0; count < num_warps_to_add; ++count, ++iter) { + if (*iter != greedy_value) { + result_list.push_back(*iter); + } + } + } else if (ORDERED_PRIORITY_FUNC_ONLY == ordering) { + std::sort(temp.begin(), temp.end(), priority_func); + typename std::vector<T>::iterator iter = temp.begin(); + for (unsigned count = 0; count < num_warps_to_add; ++count, ++iter) { + result_list.push_back(*iter); } + } else { + fprintf(stderr, "Unknown ordering - %d\n", ordering); + abort(); + } } -void scheduler_unit::cycle() -{ - SCHED_DPRINTF( "scheduler_unit::cycle()\n" ); - bool valid_inst = false; // there was one warp with a valid instruction to issue (didn't require flush due to control hazard) - bool ready_inst = false; // of the valid instructions, there was one not waiting for pending register writes - bool issued_inst = false; // of these we issued one - - order_warps(); - for ( std::vector< shd_warp_t* >::const_iterator iter = m_next_cycle_prioritized_warps.begin(); - iter != m_next_cycle_prioritized_warps.end(); - iter++ ) { - // Don't consider warps that are not yet valid - if ( (*iter) == NULL || (*iter)->done_exit() ) { - continue; - } - SCHED_DPRINTF( "Testing (warp_id %u, dynamic_warp_id %u)\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - unsigned warp_id = (*iter)->get_warp_id(); - unsigned checked=0; - unsigned issued=0; - exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; - unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; - bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; //In tis mode, we only allow dual issue to diff execution units (as in Maxwell and Pascal) - - while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { - const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); - //Jin: handle cdp latency; - if(pI && pI->m_is_cdp && warp(warp_id).m_cdp_latency > 0) { - assert(warp(warp_id).m_cdp_dummy); - warp(warp_id).m_cdp_latency--; - break; - } +void scheduler_unit::cycle() { + SCHED_DPRINTF("scheduler_unit::cycle()\n"); + bool valid_inst = + false; // there was one warp with a valid instruction to issue (didn't + // require flush due to control hazard) + bool ready_inst = false; // of the valid instructions, there was one not + // waiting for pending register writes + bool issued_inst = false; // of these we issued one - bool valid = warp(warp_id).ibuffer_next_valid(); - bool warp_inst_issued = false; - unsigned pc,rpc; - m_simt_stack[warp_id]->get_pdom_stack_top_info(&pc,&rpc); - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) has valid instruction (%s)\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(), - m_shader->m_config->gpgpu_ctx->func_sim->ptx_get_insn_str( pc).c_str() ); - if( pI ) { - assert(valid); - if( pc != pI->pc ) { - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) control hazard instruction flush\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - // control hazard - warp(warp_id).set_next_pc(pc); - warp(warp_id).ibuffer_flush(); - } else { - valid_inst = true; - if ( !m_scoreboard->checkCollision(warp_id, pI) ) { - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) passes scoreboard\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - ready_inst = true; - const active_mask_t &active_mask = m_simt_stack[warp_id]->get_active_mask(); - assert( warp(warp_id).inst_in_pipeline() ); - - if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP)||(pI->op==TENSOR_CORE_LOAD_OP)||(pI->op==TENSOR_CORE_STORE_OP) ) { - if( m_mem_out->has_free(m_shader->m_config->sub_core_model, m_id) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { - m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::MEM; - } - } else { + order_warps(); + for (std::vector<shd_warp_t *>::const_iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != m_next_cycle_prioritized_warps.end(); iter++) { + // Don't consider warps that are not yet valid + if ((*iter) == NULL || (*iter)->done_exit()) { + continue; + } + SCHED_DPRINTF("Testing (warp_id %u, dynamic_warp_id %u)\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + unsigned warp_id = (*iter)->get_warp_id(); + unsigned checked = 0; + unsigned issued = 0; + exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; + unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + bool diff_exec_units = + m_shader->m_config + ->gpgpu_dual_issue_diff_exec_units; // In tis mode, we only allow + // dual issue to diff execution + // units (as in Maxwell and + // Pascal) - bool sp_pipe_avail = m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool sfu_pipe_avail = m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool tensor_core_pipe_avail = m_tensor_core_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool dp_pipe_avail = m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool int_pipe_avail = m_int_out->has_free(m_shader->m_config->sub_core_model, m_id); + while (!warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && + (checked < max_issue) && (checked <= issued) && + (issued < max_issue)) { + const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); + // Jin: handle cdp latency; + if (pI && pI->m_is_cdp && warp(warp_id).m_cdp_latency > 0) { + assert(warp(warp_id).m_cdp_dummy); + warp(warp_id).m_cdp_latency--; + break; + } - //This code need to be refactored - if(pI->op != TENSOR_CORE_OP && pI->op != SFU_OP && pI->op != DP_OP) { - - bool execute_on_SP = false; - bool execute_on_INT = false; + bool valid = warp(warp_id).ibuffer_next_valid(); + bool warp_inst_issued = false; + unsigned pc, rpc; + m_simt_stack[warp_id]->get_pdom_stack_top_info(&pc, &rpc); + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) has valid instruction (%s)\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(), + m_shader->m_config->gpgpu_ctx->func_sim->ptx_get_insn_str(pc) + .c_str()); + if (pI) { + assert(valid); + if (pc != pI->pc) { + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) control hazard " + "instruction flush\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + // control hazard + warp(warp_id).set_next_pc(pc); + warp(warp_id).ibuffer_flush(); + } else { + valid_inst = true; + if (!m_scoreboard->checkCollision(warp_id, pI)) { + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) passes scoreboard\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + ready_inst = true; + const active_mask_t &active_mask = + m_simt_stack[warp_id]->get_active_mask(); + assert(warp(warp_id).inst_in_pipeline()); - //if INT unit pipline exist, then execute ALU and INT operations on INT unit and SP-FPU on SP unit (like in Volta) - //if INT unit pipline does not exist, then execute all ALU, INT and SP operations on SP unit (as in Fermi, Pascal GPUs) - if(m_shader->m_config->gpgpu_num_int_units > 0 && - int_pipe_avail && - pI->op != SP_OP && - !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::INT)) - execute_on_INT = true; - else if (sp_pipe_avail && - (m_shader->m_config->gpgpu_num_int_units == 0 || - (m_shader->m_config->gpgpu_num_int_units > 0 && pI->op == SP_OP)) && - !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) - execute_on_SP = true; + if ((pI->op == LOAD_OP) || (pI->op == STORE_OP) || + (pI->op == MEMORY_BARRIER_OP) || + (pI->op == TENSOR_CORE_LOAD_OP) || + (pI->op == TENSOR_CORE_STORE_OP)) { + if (m_mem_out->has_free(m_shader->m_config->sub_core_model, + m_id) && + (!diff_exec_units || + previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { + m_shader->issue_warp(*m_mem_out, pI, active_mask, warp_id, + m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::MEM; + } + } else { + bool sp_pipe_avail = + m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool sfu_pipe_avail = + m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool tensor_core_pipe_avail = m_tensor_core_out->has_free( + m_shader->m_config->sub_core_model, m_id); + bool dp_pipe_avail = + m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool int_pipe_avail = + m_int_out->has_free(m_shader->m_config->sub_core_model, m_id); + // This code need to be refactored + if (pI->op != TENSOR_CORE_OP && pI->op != SFU_OP && + pI->op != DP_OP) { + bool execute_on_SP = false; + bool execute_on_INT = false; - if(execute_on_INT || execute_on_SP) { - //Jin: special for CDP api - if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { - assert(warp(warp_id).m_cdp_latency == 0); + // if INT unit pipline exist, then execute ALU and INT + // operations on INT unit and SP-FPU on SP unit (like in Volta) + // if INT unit pipline does not exist, then execute all ALU, INT + // and SP operations on SP unit (as in Fermi, Pascal GPUs) + if (m_shader->m_config->gpgpu_num_int_units > 0 && + int_pipe_avail && pI->op != SP_OP && + !(diff_exec_units && + previous_issued_inst_exec_type == exec_unit_type_t::INT)) + execute_on_INT = true; + else if (sp_pipe_avail && + (m_shader->m_config->gpgpu_num_int_units == 0 || + (m_shader->m_config->gpgpu_num_int_units > 0 && + pI->op == SP_OP)) && + !(diff_exec_units && previous_issued_inst_exec_type == + exec_unit_type_t::SP)) + execute_on_SP = true; - if(pI->m_is_cdp == 1) - warp(warp_id).m_cdp_latency = m_shader->m_config->gpgpu_ctx->func_sim->cdp_latency[pI->m_is_cdp - 1]; - else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2 - warp(warp_id).m_cdp_latency = m_shader->m_config->gpgpu_ctx->func_sim->cdp_latency[pI->m_is_cdp - 1] - + m_shader->m_config->gpgpu_ctx->func_sim->cdp_latency[pI->m_is_cdp] * active_mask.count(); - warp(warp_id).m_cdp_dummy = true; - break; - } - else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) { - assert(warp(warp_id).m_cdp_latency == 0); - warp(warp_id).m_cdp_dummy = false; - } - } + if (execute_on_INT || execute_on_SP) { + // Jin: special for CDP api + if (pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { + assert(warp(warp_id).m_cdp_latency == 0); - if(execute_on_SP) { - m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::SP; - } else if (execute_on_INT) { - m_shader->issue_warp(*m_int_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::INT; - } - } else if ( (m_shader->m_config->gpgpu_num_dp_units > 0) && (pI->op == DP_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::DP)) { - if( dp_pipe_avail ) { - m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::DP; - } - } //If the DP units = 0 (like in Fermi archi), then execute DP inst on SFU unit - else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SFU)) { - if( sfu_pipe_avail ) { - m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::SFU; - } - } - else if ( (pI->op == TENSOR_CORE_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) { - if( tensor_core_pipe_avail ) { - m_shader->issue_warp(*m_tensor_core_out,pI,active_mask,warp_id,m_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::TENSOR; - } - } - }//end of else - } else { + if (pI->m_is_cdp == 1) + warp(warp_id).m_cdp_latency = + m_shader->m_config->gpgpu_ctx->func_sim + ->cdp_latency[pI->m_is_cdp - 1]; + else // cudaLaunchDeviceV2 and cudaGetParameterBufferV2 + warp(warp_id).m_cdp_latency = + m_shader->m_config->gpgpu_ctx->func_sim + ->cdp_latency[pI->m_is_cdp - 1] + + m_shader->m_config->gpgpu_ctx->func_sim + ->cdp_latency[pI->m_is_cdp] * + active_mask.count(); + warp(warp_id).m_cdp_dummy = true; + break; + } else if (pI->m_is_cdp && warp(warp_id).m_cdp_dummy) { + assert(warp(warp_id).m_cdp_latency == 0); + warp(warp_id).m_cdp_dummy = false; + } + } - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - } + if (execute_on_SP) { + m_shader->issue_warp(*m_sp_out, pI, active_mask, warp_id, + m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::SP; + } else if (execute_on_INT) { + m_shader->issue_warp(*m_int_out, pI, active_mask, warp_id, + m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::INT; } - } else if( valid ) { - // this case can happen after a return instruction in diverged warp - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) return from diverged warp flush\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - warp(warp_id).set_next_pc(pc); - warp(warp_id).ibuffer_flush(); - } - if(warp_inst_issued) { - SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) issued %u instructions\n", - (*iter)->get_warp_id(), - (*iter)->get_dynamic_warp_id(), - issued ); - do_on_warp_issued( warp_id, issued, iter ); - } - checked++; - } - if ( issued ) { - // This might be a bit inefficient, but we need to maintain - // two ordered list for proper scheduler execution. - // We could remove the need for this loop by associating a - // supervised_is index with each entry in the m_next_cycle_prioritized_warps - // vector. For now, just run through until you find the right warp_id - for ( std::vector< shd_warp_t* >::const_iterator supervised_iter = m_supervised_warps.begin(); - supervised_iter != m_supervised_warps.end(); - ++supervised_iter ) { - if ( *iter == *supervised_iter ) { - m_last_supervised_issued = supervised_iter; + } else if ((m_shader->m_config->gpgpu_num_dp_units > 0) && + (pI->op == DP_OP) && + !(diff_exec_units && previous_issued_inst_exec_type == + exec_unit_type_t::DP)) { + if (dp_pipe_avail) { + m_shader->issue_warp(*m_dp_out, pI, active_mask, warp_id, + m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::DP; } - } + } // If the DP units = 0 (like in Fermi archi), then execute DP + // inst on SFU unit + else if (((m_shader->m_config->gpgpu_num_dp_units == 0 && + pI->op == DP_OP) || + (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && + !(diff_exec_units && previous_issued_inst_exec_type == + exec_unit_type_t::SFU)) { + if (sfu_pipe_avail) { + m_shader->issue_warp(*m_sfu_out, pI, active_mask, warp_id, + m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::SFU; + } + } else if ((pI->op == TENSOR_CORE_OP) && + !(diff_exec_units && previous_issued_inst_exec_type == + exec_unit_type_t::SP)) { + if (tensor_core_pipe_avail) { + m_shader->issue_warp(*m_tensor_core_out, pI, active_mask, + warp_id, m_id); + issued++; + issued_inst = true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::TENSOR; + } + } + } // end of else + } else { + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + } + } + } else if (valid) { + // this case can happen after a return instruction in diverged warp + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) return from diverged warp " + "flush\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + warp(warp_id).set_next_pc(pc); + warp(warp_id).ibuffer_flush(); + } + if (warp_inst_issued) { + SCHED_DPRINTF( + "Warp (warp_id %u, dynamic_warp_id %u) issued %u instructions\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(), issued); + do_on_warp_issued(warp_id, issued, iter); + } + checked++; + } + if (issued) { + // This might be a bit inefficient, but we need to maintain + // two ordered list for proper scheduler execution. + // We could remove the need for this loop by associating a + // supervised_is index with each entry in the + // m_next_cycle_prioritized_warps vector. For now, just run through until + // you find the right warp_id + for (std::vector<shd_warp_t *>::const_iterator supervised_iter = + m_supervised_warps.begin(); + supervised_iter != m_supervised_warps.end(); ++supervised_iter) { + if (*iter == *supervised_iter) { + m_last_supervised_issued = supervised_iter; + } + } - if(issued == 1) - m_stats->single_issue_nums[m_id]++; - else if(issued > 1) - m_stats->dual_issue_nums[m_id]++; - else - abort(); //issued should be > 0 + if (issued == 1) + m_stats->single_issue_nums[m_id]++; + else if (issued > 1) + m_stats->dual_issue_nums[m_id]++; + else + abort(); // issued should be > 0 - break; - } + break; } + } - // issue stall statistics: - if( !valid_inst ) - m_stats->shader_cycle_distro[0]++; // idle or control hazard - else if( !ready_inst ) - m_stats->shader_cycle_distro[1]++; // waiting for RAW hazards (possibly due to memory) - else if( !issued_inst ) - m_stats->shader_cycle_distro[2]++; // pipeline stalled + // issue stall statistics: + if (!valid_inst) + m_stats->shader_cycle_distro[0]++; // idle or control hazard + else if (!ready_inst) + m_stats->shader_cycle_distro[1]++; // waiting for RAW hazards (possibly due + // to memory) + else if (!issued_inst) + m_stats->shader_cycle_distro[2]++; // pipeline stalled } -void scheduler_unit::do_on_warp_issued( unsigned warp_id, - unsigned num_issued, - const std::vector< shd_warp_t* >::const_iterator& prioritized_iter ) -{ - m_stats->event_warp_issued( m_shader->get_sid(), - warp_id, - num_issued, - warp(warp_id).get_dynamic_warp_id() ); - warp(warp_id).ibuffer_step(); +void scheduler_unit::do_on_warp_issued( + unsigned warp_id, unsigned num_issued, + const std::vector<shd_warp_t *>::const_iterator &prioritized_iter) { + m_stats->event_warp_issued(m_shader->get_sid(), warp_id, num_issued, + warp(warp_id).get_dynamic_warp_id()); + warp(warp_id).ibuffer_step(); } -bool scheduler_unit::sort_warps_by_oldest_dynamic_id(shd_warp_t* lhs, shd_warp_t* rhs) -{ - if (rhs && lhs) { - if ( lhs->done_exit() || lhs->waiting() ) { - return false; - } else if ( rhs->done_exit() || rhs->waiting() ) { - return true; - } else { - return lhs->get_dynamic_warp_id() < rhs->get_dynamic_warp_id(); - } +bool scheduler_unit::sort_warps_by_oldest_dynamic_id(shd_warp_t *lhs, + shd_warp_t *rhs) { + if (rhs && lhs) { + if (lhs->done_exit() || lhs->waiting()) { + return false; + } else if (rhs->done_exit() || rhs->waiting()) { + return true; } else { - return lhs < rhs; + return lhs->get_dynamic_warp_id() < rhs->get_dynamic_warp_id(); } + } else { + return lhs < rhs; + } } -void lrr_scheduler::order_warps() -{ - order_lrr( m_next_cycle_prioritized_warps, - m_supervised_warps, - m_last_supervised_issued, - m_supervised_warps.size() ); +void lrr_scheduler::order_warps() { + order_lrr(m_next_cycle_prioritized_warps, m_supervised_warps, + m_last_supervised_issued, m_supervised_warps.size()); } -void gto_scheduler::order_warps() -{ - order_by_priority( m_next_cycle_prioritized_warps, - m_supervised_warps, - m_last_supervised_issued, - m_supervised_warps.size(), - ORDERING_GREEDY_THEN_PRIORITY_FUNC, - scheduler_unit::sort_warps_by_oldest_dynamic_id ); +void gto_scheduler::order_warps() { + order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps, + m_last_supervised_issued, m_supervised_warps.size(), + ORDERING_GREEDY_THEN_PRIORITY_FUNC, + scheduler_unit::sort_warps_by_oldest_dynamic_id); } -void oldest_scheduler::order_warps() -{ - order_by_priority( m_next_cycle_prioritized_warps, - m_supervised_warps, - m_last_supervised_issued, - m_supervised_warps.size(), - ORDERED_PRIORITY_FUNC_ONLY, - scheduler_unit::sort_warps_by_oldest_dynamic_id ); +void oldest_scheduler::order_warps() { + order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps, + m_last_supervised_issued, m_supervised_warps.size(), + ORDERED_PRIORITY_FUNC_ONLY, + scheduler_unit::sort_warps_by_oldest_dynamic_id); } -void -two_level_active_scheduler::do_on_warp_issued( unsigned warp_id, - unsigned num_issued, - const std::vector< shd_warp_t* >::const_iterator& prioritized_iter ) -{ - scheduler_unit::do_on_warp_issued( warp_id, num_issued, prioritized_iter ); - if ( SCHEDULER_PRIORITIZATION_LRR == m_inner_level_prioritization ) { - std::vector< shd_warp_t* > new_active; - order_lrr( new_active, - m_next_cycle_prioritized_warps, - prioritized_iter, - m_next_cycle_prioritized_warps.size() ); - m_next_cycle_prioritized_warps = new_active; - } else { - fprintf( stderr, - "Unimplemented m_inner_level_prioritization: %d\n", - m_inner_level_prioritization ); - abort(); - } +void two_level_active_scheduler::do_on_warp_issued( + unsigned warp_id, unsigned num_issued, + const std::vector<shd_warp_t *>::const_iterator &prioritized_iter) { + scheduler_unit::do_on_warp_issued(warp_id, num_issued, prioritized_iter); + if (SCHEDULER_PRIORITIZATION_LRR == m_inner_level_prioritization) { + std::vector<shd_warp_t *> new_active; + order_lrr(new_active, m_next_cycle_prioritized_warps, prioritized_iter, + m_next_cycle_prioritized_warps.size()); + m_next_cycle_prioritized_warps = new_active; + } else { + fprintf(stderr, "Unimplemented m_inner_level_prioritization: %d\n", + m_inner_level_prioritization); + abort(); + } } -void two_level_active_scheduler::order_warps() -{ - //Move waiting warps to m_pending_warps - unsigned num_demoted = 0; - for ( std::vector< shd_warp_t* >::iterator iter = m_next_cycle_prioritized_warps.begin(); - iter != m_next_cycle_prioritized_warps.end(); ) { - bool waiting = (*iter)->waiting(); - for (int i=0; i<MAX_INPUT_VALUES; i++){ - const warp_inst_t* inst = (*iter)->ibuffer_next_inst(); - //Is the instruction waiting on a long operation? - if ( inst && inst->in[i] > 0 && this->m_scoreboard->islongop((*iter)->get_warp_id(), inst->in[i])){ - waiting = true; - } - } - - if( waiting ) { - m_pending_warps.push_back(*iter); - iter = m_next_cycle_prioritized_warps.erase(iter); - SCHED_DPRINTF( "DEMOTED warp_id=%d, dynamic_warp_id=%d\n", - (*iter)->get_warp_id(), - (*iter)->get_dynamic_warp_id() ); - ++num_demoted; - } else { - ++iter; - } +void two_level_active_scheduler::order_warps() { + // Move waiting warps to m_pending_warps + unsigned num_demoted = 0; + for (std::vector<shd_warp_t *>::iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != m_next_cycle_prioritized_warps.end();) { + bool waiting = (*iter)->waiting(); + for (int i = 0; i < MAX_INPUT_VALUES; i++) { + const warp_inst_t *inst = (*iter)->ibuffer_next_inst(); + // Is the instruction waiting on a long operation? + if (inst && inst->in[i] > 0 && + this->m_scoreboard->islongop((*iter)->get_warp_id(), inst->in[i])) { + waiting = true; + } } - //If there is space in m_next_cycle_prioritized_warps, promote the next m_pending_warps - unsigned num_promoted = 0; - if ( SCHEDULER_PRIORITIZATION_SRR == m_outer_level_prioritization ) { - while ( m_next_cycle_prioritized_warps.size() < m_max_active_warps ) { - m_next_cycle_prioritized_warps.push_back(m_pending_warps.front()); - m_pending_warps.pop_front(); - SCHED_DPRINTF( "PROMOTED warp_id=%d, dynamic_warp_id=%d\n", - (m_next_cycle_prioritized_warps.back())->get_warp_id(), - (m_next_cycle_prioritized_warps.back())->get_dynamic_warp_id() ); - ++num_promoted; - } + if (waiting) { + m_pending_warps.push_back(*iter); + iter = m_next_cycle_prioritized_warps.erase(iter); + SCHED_DPRINTF("DEMOTED warp_id=%d, dynamic_warp_id=%d\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); + ++num_demoted; } else { - fprintf( stderr, - "Unimplemented m_outer_level_prioritization: %d\n", - m_outer_level_prioritization ); - abort(); + ++iter; } - assert( num_promoted == num_demoted ); -} + } -swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id, - char* config_string ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ) -{ - unsigned m_prioritization_readin; - int ret = sscanf( config_string, - "warp_limiting:%d:%d", - &m_prioritization_readin, - &m_num_warps_to_limit - ); - assert( 2 == ret ); - m_prioritization = (scheduler_prioritization_type)m_prioritization_readin; - // Currently only GTO is implemented - assert( m_prioritization == SCHEDULER_PRIORITIZATION_GTO ); - assert( m_num_warps_to_limit <= shader->get_config()->max_warps_per_shader ); + // If there is space in m_next_cycle_prioritized_warps, promote the next + // m_pending_warps + unsigned num_promoted = 0; + if (SCHEDULER_PRIORITIZATION_SRR == m_outer_level_prioritization) { + while (m_next_cycle_prioritized_warps.size() < m_max_active_warps) { + m_next_cycle_prioritized_warps.push_back(m_pending_warps.front()); + m_pending_warps.pop_front(); + SCHED_DPRINTF( + "PROMOTED warp_id=%d, dynamic_warp_id=%d\n", + (m_next_cycle_prioritized_warps.back())->get_warp_id(), + (m_next_cycle_prioritized_warps.back())->get_dynamic_warp_id()); + ++num_promoted; + } + } else { + fprintf(stderr, "Unimplemented m_outer_level_prioritization: %d\n", + m_outer_level_prioritization); + abort(); + } + assert(num_promoted == num_demoted); } -void swl_scheduler::order_warps() -{ - if ( SCHEDULER_PRIORITIZATION_GTO == m_prioritization ) { - order_by_priority( m_next_cycle_prioritized_warps, - m_supervised_warps, - m_last_supervised_issued, - MIN( m_num_warps_to_limit, m_supervised_warps.size() ), - ORDERING_GREEDY_THEN_PRIORITY_FUNC, - scheduler_unit::sort_warps_by_oldest_dynamic_id ); - } else { - fprintf(stderr, "swl_scheduler m_prioritization = %d\n", m_prioritization); - abort(); - } +swl_scheduler::swl_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, + register_set *sp_out, register_set *dp_out, + register_set *sfu_out, register_set *int_out, + register_set *tensor_core_out, + register_set *mem_out, int id, char *config_string) + : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out, + sfu_out, int_out, tensor_core_out, mem_out, id) { + unsigned m_prioritization_readin; + int ret = sscanf(config_string, "warp_limiting:%d:%d", + &m_prioritization_readin, &m_num_warps_to_limit); + assert(2 == ret); + m_prioritization = (scheduler_prioritization_type)m_prioritization_readin; + // Currently only GTO is implemented + assert(m_prioritization == SCHEDULER_PRIORITIZATION_GTO); + assert(m_num_warps_to_limit <= shader->get_config()->max_warps_per_shader); } -void shader_core_ctx::read_operands() -{ +void swl_scheduler::order_warps() { + if (SCHEDULER_PRIORITIZATION_GTO == m_prioritization) { + order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps, + m_last_supervised_issued, + MIN(m_num_warps_to_limit, m_supervised_warps.size()), + ORDERING_GREEDY_THEN_PRIORITY_FUNC, + scheduler_unit::sort_warps_by_oldest_dynamic_id); + } else { + fprintf(stderr, "swl_scheduler m_prioritization = %d\n", m_prioritization); + abort(); + } } -address_type coalesced_segment(address_type addr, unsigned segment_size_lg2bytes) -{ - return (addr >> segment_size_lg2bytes); +void shader_core_ctx::read_operands() {} + +address_type coalesced_segment(address_type addr, + unsigned segment_size_lg2bytes) { + return (addr >> segment_size_lg2bytes); } -// Returns numbers of addresses in translated_addrs, each addr points to a 4B (32-bit) word -unsigned shader_core_ctx::translate_local_memaddr( address_type localaddr, unsigned tid, unsigned num_shader, unsigned datasize, new_addr_type* translated_addrs ) -{ - // During functional execution, each thread sees its own memory space for local memory, but these - // need to be mapped to a shared address space for timing simulation. We do that mapping here. +// Returns numbers of addresses in translated_addrs, each addr points to a 4B +// (32-bit) word +unsigned shader_core_ctx::translate_local_memaddr( + address_type localaddr, unsigned tid, unsigned num_shader, + unsigned datasize, new_addr_type *translated_addrs) { + // During functional execution, each thread sees its own memory space for + // local memory, but these need to be mapped to a shared address space for + // timing simulation. We do that mapping here. - address_type thread_base = 0; - unsigned max_concurrent_threads=0; - if (m_config->gpgpu_local_mem_map) { - // Dnew = D*N + T%nTpC + nTpC*C - // N = nTpC*nCpS*nS (max concurent threads) - // C = nS*K + S (hw cta number per gpu) - // K = T/nTpC (hw cta number per core) - // D = data index - // T = thread - // nTpC = number of threads per CTA - // nCpS = number of CTA per shader - // - // for a given local memory address threads in a CTA map to contiguous addresses, - // then distribute across memory space by CTAs from successive shader cores first, - // then by successive CTA in same shader core - thread_base = 4*(kernel_padded_threads_per_cta * (m_sid + num_shader * (tid / kernel_padded_threads_per_cta)) - + tid % kernel_padded_threads_per_cta); - max_concurrent_threads = kernel_padded_threads_per_cta * kernel_max_cta_per_shader * num_shader; - } else { - // legacy mapping that maps the same address in the local memory space of all threads - // to a single contiguous address region - thread_base = 4*(m_config->n_thread_per_shader * m_sid + tid); - max_concurrent_threads = num_shader * m_config->n_thread_per_shader; - } - assert( thread_base < 4/*word size*/*max_concurrent_threads ); + address_type thread_base = 0; + unsigned max_concurrent_threads = 0; + if (m_config->gpgpu_local_mem_map) { + // Dnew = D*N + T%nTpC + nTpC*C + // N = nTpC*nCpS*nS (max concurent threads) + // C = nS*K + S (hw cta number per gpu) + // K = T/nTpC (hw cta number per core) + // D = data index + // T = thread + // nTpC = number of threads per CTA + // nCpS = number of CTA per shader + // + // for a given local memory address threads in a CTA map to contiguous + // addresses, then distribute across memory space by CTAs from successive + // shader cores first, then by successive CTA in same shader core + thread_base = + 4 * (kernel_padded_threads_per_cta * + (m_sid + num_shader * (tid / kernel_padded_threads_per_cta)) + + tid % kernel_padded_threads_per_cta); + max_concurrent_threads = + kernel_padded_threads_per_cta * kernel_max_cta_per_shader * num_shader; + } else { + // legacy mapping that maps the same address in the local memory space of + // all threads to a single contiguous address region + thread_base = 4 * (m_config->n_thread_per_shader * m_sid + tid); + max_concurrent_threads = num_shader * m_config->n_thread_per_shader; + } + assert(thread_base < 4 /*word size*/ * max_concurrent_threads); - // If requested datasize > 4B, split into multiple 4B accesses - // otherwise do one sub-4 byte memory access - unsigned num_accesses = 0; + // If requested datasize > 4B, split into multiple 4B accesses + // otherwise do one sub-4 byte memory access + unsigned num_accesses = 0; - if(datasize >= 4) { - // >4B access, split into 4B chunks - assert(datasize%4 == 0); // Must be a multiple of 4B - num_accesses = datasize/4; - assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); // max 32B - assert(localaddr%4 == 0); // Address must be 4B aligned - required if accessing 4B per request, otherwise access will overflow into next thread's space - for(unsigned i=0; i<num_accesses; i++) { - address_type local_word = localaddr/4 + i; - address_type linear_address = local_word*max_concurrent_threads*4 + thread_base + LOCAL_GENERIC_START; - translated_addrs[i] = linear_address; - } - } else { - // Sub-4B access, do only one access - assert(datasize > 0); - num_accesses = 1; - address_type local_word = localaddr/4; - address_type local_word_offset = localaddr%4; - assert( (localaddr+datasize-1)/4 == local_word ); // Make sure access doesn't overflow into next 4B chunk - address_type linear_address = local_word*max_concurrent_threads*4 + local_word_offset + thread_base + LOCAL_GENERIC_START; - translated_addrs[0] = linear_address; - } - return num_accesses; + if (datasize >= 4) { + // >4B access, split into 4B chunks + assert(datasize % 4 == 0); // Must be a multiple of 4B + num_accesses = datasize / 4; + assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); // max 32B + assert( + localaddr % 4 == + 0); // Address must be 4B aligned - required if accessing 4B per + // request, otherwise access will overflow into next thread's space + for (unsigned i = 0; i < num_accesses; i++) { + address_type local_word = localaddr / 4 + i; + address_type linear_address = local_word * max_concurrent_threads * 4 + + thread_base + LOCAL_GENERIC_START; + translated_addrs[i] = linear_address; + } + } else { + // Sub-4B access, do only one access + assert(datasize > 0); + num_accesses = 1; + address_type local_word = localaddr / 4; + address_type local_word_offset = localaddr % 4; + assert((localaddr + datasize - 1) / 4 == + local_word); // Make sure access doesn't overflow into next 4B chunk + address_type linear_address = local_word * max_concurrent_threads * 4 + + local_word_offset + thread_base + + LOCAL_GENERIC_START; + translated_addrs[0] = linear_address; + } + return num_accesses; } ///////////////////////////////////////////////////////////////////////////////////////// -int shader_core_ctx::test_res_bus(int latency){ - for(unsigned i=0; i<num_result_bus; i++){ - if(!m_result_bus[i]->test(latency)){return i;} - } - return -1; +int shader_core_ctx::test_res_bus(int latency) { + for (unsigned i = 0; i < num_result_bus; i++) { + if (!m_result_bus[i]->test(latency)) { + return i; + } + } + return -1; } -void shader_core_ctx::execute() -{ - for(unsigned i=0; i<num_result_bus; i++){ - *(m_result_bus[i]) >>=1; - } - for( unsigned n=0; n < m_num_function_units; n++ ) { - unsigned multiplier = m_fu[n]->clock_multiplier(); - for( unsigned c=0; c < multiplier; c++ ) - m_fu[n]->cycle(); - m_fu[n]->active_lanes_in_pipeline(); - enum pipeline_stage_name_t issue_port = m_issue_port[n]; - register_set& issue_inst = m_pipeline_reg[ issue_port ]; - warp_inst_t** ready_reg = issue_inst.get_ready(); - if( issue_inst.has_ready() && m_fu[n]->can_issue( **ready_reg ) ) { - bool schedule_wb_now = !m_fu[n]->stallable(); - int resbus = -1; - if( schedule_wb_now && (resbus=test_res_bus( (*ready_reg)->latency ))!=-1 ) { - assert( (*ready_reg)->latency < MAX_ALU_LATENCY ); - m_result_bus[resbus]->set( (*ready_reg)->latency ); - m_fu[n]->issue( issue_inst ); - } else if( !schedule_wb_now ) { - m_fu[n]->issue( issue_inst ); - } else { - // stall issue (cannot reserve result bus) - } - } +void shader_core_ctx::execute() { + for (unsigned i = 0; i < num_result_bus; i++) { + *(m_result_bus[i]) >>= 1; + } + for (unsigned n = 0; n < m_num_function_units; n++) { + unsigned multiplier = m_fu[n]->clock_multiplier(); + for (unsigned c = 0; c < multiplier; c++) m_fu[n]->cycle(); + m_fu[n]->active_lanes_in_pipeline(); + enum pipeline_stage_name_t issue_port = m_issue_port[n]; + register_set &issue_inst = m_pipeline_reg[issue_port]; + warp_inst_t **ready_reg = issue_inst.get_ready(); + if (issue_inst.has_ready() && m_fu[n]->can_issue(**ready_reg)) { + bool schedule_wb_now = !m_fu[n]->stallable(); + int resbus = -1; + if (schedule_wb_now && + (resbus = test_res_bus((*ready_reg)->latency)) != -1) { + assert((*ready_reg)->latency < MAX_ALU_LATENCY); + m_result_bus[resbus]->set((*ready_reg)->latency); + m_fu[n]->issue(issue_inst); + } else if (!schedule_wb_now) { + m_fu[n]->issue(issue_inst); + } else { + // stall issue (cannot reserve result bus) + } } + } } -void ldst_unit::print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ) { - if( m_L1D ) { - m_L1D->print( fp, dl1_accesses, dl1_misses ); - } +void ldst_unit::print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses) { + if (m_L1D) { + m_L1D->print(fp, dl1_accesses, dl1_misses); + } } void ldst_unit::get_cache_stats(cache_stats &cs) { - // Adds stats to 'cs' from each cache - if(m_L1D) - cs += m_L1D->get_stats(); - if(m_L1C) - cs += m_L1C->get_stats(); - if(m_L1T) - cs += m_L1T->get_stats(); - + // Adds stats to 'cs' from each cache + if (m_L1D) cs += m_L1D->get_stats(); + if (m_L1C) cs += m_L1C->get_stats(); + if (m_L1T) cs += m_L1T->get_stats(); } -void ldst_unit::get_L1D_sub_stats(struct cache_sub_stats &css) const{ - if(m_L1D) - m_L1D->get_sub_stats(css); +void ldst_unit::get_L1D_sub_stats(struct cache_sub_stats &css) const { + if (m_L1D) m_L1D->get_sub_stats(css); } -void ldst_unit::get_L1C_sub_stats(struct cache_sub_stats &css) const{ - if(m_L1C) - m_L1C->get_sub_stats(css); +void ldst_unit::get_L1C_sub_stats(struct cache_sub_stats &css) const { + if (m_L1C) m_L1C->get_sub_stats(css); } -void ldst_unit::get_L1T_sub_stats(struct cache_sub_stats &css) const{ - if(m_L1T) - m_L1T->get_sub_stats(css); +void ldst_unit::get_L1T_sub_stats(struct cache_sub_stats &css) const { + if (m_L1T) m_L1T->get_sub_stats(css); } -void shader_core_ctx::warp_inst_complete(const warp_inst_t &inst) -{ - - #if 0 +void shader_core_ctx::warp_inst_complete(const warp_inst_t &inst) { +#if 0 printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu issued@%llu\n", inst.get_uid(), m_sid, inst.warp_id(), inst.pc, gpu_tot_sim_cycle + gpu_sim_cycle, inst.get_issue_cycle()); - #endif +#endif - if(inst.op_pipe==SP__OP) - m_stats->m_num_sp_committed[m_sid]++; - else if(inst.op_pipe==SFU__OP) - m_stats->m_num_sfu_committed[m_sid]++; - else if(inst.op_pipe==MEM__OP) - m_stats->m_num_mem_committed[m_sid]++; + if (inst.op_pipe == SP__OP) + m_stats->m_num_sp_committed[m_sid]++; + else if (inst.op_pipe == SFU__OP) + m_stats->m_num_sfu_committed[m_sid]++; + else if (inst.op_pipe == MEM__OP) + m_stats->m_num_mem_committed[m_sid]++; - if(m_config->gpgpu_clock_gated_lanes==false) - m_stats->m_num_sim_insn[m_sid] += m_config->warp_size; + if (m_config->gpgpu_clock_gated_lanes == false) + m_stats->m_num_sim_insn[m_sid] += m_config->warp_size; else - m_stats->m_num_sim_insn[m_sid] += inst.active_count(); + m_stats->m_num_sim_insn[m_sid] += inst.active_count(); m_stats->m_num_sim_winsn[m_sid]++; m_gpu->gpu_sim_insn += inst.active_count(); inst.completed(m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); } -void shader_core_ctx::writeback() -{ - - unsigned max_committed_thread_instructions=m_config->warp_size * (m_config->pipe_widths[EX_WB]); //from the functional units - m_stats->m_pipeline_duty_cycle[m_sid]=((float)(m_stats->m_num_sim_insn[m_sid]-m_stats->m_last_num_sim_insn[m_sid]))/max_committed_thread_instructions; +void shader_core_ctx::writeback() { + unsigned max_committed_thread_instructions = + m_config->warp_size * + (m_config->pipe_widths[EX_WB]); // from the functional units + m_stats->m_pipeline_duty_cycle[m_sid] = + ((float)(m_stats->m_num_sim_insn[m_sid] - + m_stats->m_last_num_sim_insn[m_sid])) / + max_committed_thread_instructions; - m_stats->m_last_num_sim_insn[m_sid]=m_stats->m_num_sim_insn[m_sid]; - m_stats->m_last_num_sim_winsn[m_sid]=m_stats->m_num_sim_winsn[m_sid]; + m_stats->m_last_num_sim_insn[m_sid] = m_stats->m_num_sim_insn[m_sid]; + m_stats->m_last_num_sim_winsn[m_sid] = m_stats->m_num_sim_winsn[m_sid]; - warp_inst_t** preg = m_pipeline_reg[EX_WB].get_ready(); - warp_inst_t* pipe_reg = (preg==NULL)? NULL:*preg; - while( preg and !pipe_reg->empty()) { - /* - * Right now, the writeback stage drains all waiting instructions - * assuming there are enough ports in the register file or the - * conflicts are resolved at issue. - */ - /* - * The operand collector writeback can generally generate a stall - * However, here, the pipelines should be un-stallable. This is - * guaranteed because this is the first time the writeback function - * is called after the operand collector's step function, which - * resets the allocations. There is one case which could result in - * the writeback function returning false (stall), which is when - * an instruction tries to modify two registers (GPR and predicate) - * To handle this case, we ignore the return value (thus allowing - * no stalling). - */ + warp_inst_t **preg = m_pipeline_reg[EX_WB].get_ready(); + warp_inst_t *pipe_reg = (preg == NULL) ? NULL : *preg; + while (preg and !pipe_reg->empty()) { + /* + * Right now, the writeback stage drains all waiting instructions + * assuming there are enough ports in the register file or the + * conflicts are resolved at issue. + */ + /* + * The operand collector writeback can generally generate a stall + * However, here, the pipelines should be un-stallable. This is + * guaranteed because this is the first time the writeback function + * is called after the operand collector's step function, which + * resets the allocations. There is one case which could result in + * the writeback function returning false (stall), which is when + * an instruction tries to modify two registers (GPR and predicate) + * To handle this case, we ignore the return value (thus allowing + * no stalling). + */ - m_operand_collector.writeback(*pipe_reg); - unsigned warp_id = pipe_reg->warp_id(); - m_scoreboard->releaseRegisters( pipe_reg ); - m_warp[warp_id].dec_inst_in_pipeline(); - warp_inst_complete(*pipe_reg); - m_gpu->gpu_sim_insn_last_update_sid = m_sid; - m_gpu->gpu_sim_insn_last_update = m_gpu->gpu_sim_cycle; - m_last_inst_gpu_sim_cycle = m_gpu->gpu_sim_cycle; - m_last_inst_gpu_tot_sim_cycle = m_gpu->gpu_tot_sim_cycle; - pipe_reg->clear(); - preg = m_pipeline_reg[EX_WB].get_ready(); - pipe_reg = (preg==NULL)? NULL:*preg; - } + m_operand_collector.writeback(*pipe_reg); + unsigned warp_id = pipe_reg->warp_id(); + m_scoreboard->releaseRegisters(pipe_reg); + m_warp[warp_id].dec_inst_in_pipeline(); + warp_inst_complete(*pipe_reg); + m_gpu->gpu_sim_insn_last_update_sid = m_sid; + m_gpu->gpu_sim_insn_last_update = m_gpu->gpu_sim_cycle; + m_last_inst_gpu_sim_cycle = m_gpu->gpu_sim_cycle; + m_last_inst_gpu_tot_sim_cycle = m_gpu->gpu_tot_sim_cycle; + pipe_reg->clear(); + preg = m_pipeline_reg[EX_WB].get_ready(); + pipe_reg = (preg == NULL) ? NULL : *preg; + } } -bool ldst_unit::shared_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) -{ - if( inst.space.get_type() != shared_space ) - return true; +bool ldst_unit::shared_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type) { + if (inst.space.get_type() != shared_space) return true; - if(inst.has_dispatch_delay()){ - m_stats->gpgpu_n_shmem_bank_access[m_sid]++; - } + if (inst.has_dispatch_delay()) { + m_stats->gpgpu_n_shmem_bank_access[m_sid]++; + } - bool stall = inst.dispatch_delay(); - if( stall ) { - fail_type = S_MEM; - rc_fail = BK_CONF; - } else - rc_fail = NO_RC_FAIL; - return !stall; + bool stall = inst.dispatch_delay(); + if (stall) { + fail_type = S_MEM; + rc_fail = BK_CONF; + } else + rc_fail = NO_RC_FAIL; + return !stall; } -mem_stage_stall_type -ldst_unit::process_cache_access( cache_t* cache, - new_addr_type address, - warp_inst_t &inst, - std::list<cache_event>& events, - mem_fetch *mf, - enum cache_request_status status ) -{ - mem_stage_stall_type result = NO_RC_FAIL; - bool write_sent = was_write_sent(events); - bool read_sent = was_read_sent(events); - if( write_sent ) { - unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? - (mf->get_data_size()/SECTOR_SIZE) : 1; - - for(unsigned i=0; i< inc_ack; ++i) - m_core->inc_store_req( inst.warp_id() ); +mem_stage_stall_type ldst_unit::process_cache_access( + cache_t *cache, new_addr_type address, warp_inst_t &inst, + std::list<cache_event> &events, mem_fetch *mf, + enum cache_request_status status) { + mem_stage_stall_type result = NO_RC_FAIL; + bool write_sent = was_write_sent(events); + bool read_sent = was_read_sent(events); + if (write_sent) { + unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) + ? (mf->get_data_size() / SECTOR_SIZE) + : 1; + for (unsigned i = 0; i < inc_ack; ++i) + m_core->inc_store_req(inst.warp_id()); + } + if (status == HIT) { + assert(!read_sent); + inst.accessq_pop_back(); + if (inst.is_load()) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) + if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--; } - if ( status == HIT ) { - assert( !read_sent ); - inst.accessq_pop_back(); - if ( inst.is_load() ) { - for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) - if (inst.out[r] > 0) - m_pending_writes[inst.warp_id()][inst.out[r]]--; - } - if( !write_sent ) - delete mf; - } else if ( status == RESERVATION_FAIL ) { - result = BK_CONF; - assert( !read_sent ); - assert( !write_sent ); - delete mf; - } else { - assert( status == MISS || status == HIT_RESERVED ); - //inst.clear_active( access.get_warp_mask() ); // threads in mf writeback when mf returns - inst.accessq_pop_back(); - } - if( !inst.accessq_empty() && result == NO_RC_FAIL) - result = COAL_STALL; - return result; + if (!write_sent) delete mf; + } else if (status == RESERVATION_FAIL) { + result = BK_CONF; + assert(!read_sent); + assert(!write_sent); + delete mf; + } else { + assert(status == MISS || status == HIT_RESERVED); + // inst.clear_active( access.get_warp_mask() ); // threads in mf writeback + // when mf returns + inst.accessq_pop_back(); + } + if (!inst.accessq_empty() && result == NO_RC_FAIL) result = COAL_STALL; + return result; } -mem_stage_stall_type ldst_unit::process_memory_access_queue( cache_t *cache, warp_inst_t &inst ) -{ - mem_stage_stall_type result = NO_RC_FAIL; - if( inst.accessq_empty() ) - return result; +mem_stage_stall_type ldst_unit::process_memory_access_queue(cache_t *cache, + warp_inst_t &inst) { + mem_stage_stall_type result = NO_RC_FAIL; + if (inst.accessq_empty()) return result; - if( !cache->data_port_free() ) - return DATA_PORT_STALL; + if (!cache->data_port_free()) return DATA_PORT_STALL; - //const mem_access_t &access = inst.accessq_back(); - mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - std::list<cache_event> events; - enum cache_request_status status = cache->access(mf->get_addr(),mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events); - return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); + // const mem_access_t &access = inst.accessq_back(); + mem_fetch *mf = m_mf_allocator->alloc( + inst, inst.accessq_back(), + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle); + std::list<cache_event> events; + enum cache_request_status status = cache->access( + mf->get_addr(), mf, + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle, + events); + return process_cache_access(cache, mf->get_addr(), inst, events, mf, status); } -mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ) -{ - mem_stage_stall_type result = NO_RC_FAIL; - if( inst.accessq_empty() ) - return result; - - if(m_config->m_L1D_config.l1_latency > 0) - { - for(int j=0; j<m_config->m_L1D_config.l1_banks; j++) { //We can handle at max l1_banks reqs per cycle +mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( + l1_cache *cache, warp_inst_t &inst) { + mem_stage_stall_type result = NO_RC_FAIL; + if (inst.accessq_empty()) return result; - if( inst.accessq_empty() ) - return result; + if (m_config->m_L1D_config.l1_latency > 0) { + for (int j = 0; j < m_config->m_L1D_config.l1_banks; + j++) { // We can handle at max l1_banks reqs per cycle - mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - unsigned bank_id = m_config->m_L1D_config.set_bank(mf->get_addr()); - assert(bank_id < m_config->m_L1D_config.l1_banks); + if (inst.accessq_empty()) return result; - if((l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency-1]) == NULL) - { - l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency-1] = mf; + mem_fetch *mf = + m_mf_allocator->alloc(inst, inst.accessq_back(), + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + unsigned bank_id = m_config->m_L1D_config.set_bank(mf->get_addr()); + assert(bank_id < m_config->m_L1D_config.l1_banks); - if( mf->get_inst().is_store() ) { - unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? - (mf->get_data_size()/SECTOR_SIZE) : 1; + if ((l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1]) == + NULL) { + l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1] = mf; - for(unsigned i=0; i< inc_ack; ++i) - m_core->inc_store_req( inst.warp_id() ); - } + if (mf->get_inst().is_store()) { + unsigned inc_ack = + (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) + ? (mf->get_data_size() / SECTOR_SIZE) + : 1; - inst.accessq_pop_back(); - } - else - { - result = BK_CONF; - delete mf; - break; //do not try again, just break from the loop and try the next cycle - } - } - if( !inst.accessq_empty() && result !=BK_CONF) - result = COAL_STALL; + for (unsigned i = 0; i < inc_ack; ++i) + m_core->inc_store_req(inst.warp_id()); + } - return result; - } - else - { - mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - std::list<cache_event> events; - enum cache_request_status status = cache->access(mf->get_addr(),mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events); - return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); + inst.accessq_pop_back(); + } else { + result = BK_CONF; + delete mf; + break; // do not try again, just break from the loop and try the next + // cycle + } } -} - -void ldst_unit::L1_latency_queue_cycle() -{ - for(int j=0; j<m_config->m_L1D_config.l1_banks; j++) { - if((l1_latency_queue[j][0]) != NULL) - { - mem_fetch* mf_next = l1_latency_queue[j][0]; - std::list<cache_event> events; - enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events); - - bool write_sent = was_write_sent(events); - bool read_sent = was_read_sent(events); + if (!inst.accessq_empty() && result != BK_CONF) result = COAL_STALL; - if ( status == HIT ) { - assert( !read_sent ); - l1_latency_queue[j][0] = NULL; - if ( mf_next->get_inst().is_load() ) { - for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) - if (mf_next->get_inst().out[r] > 0) - { - assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0); - unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]; - if(!still_pending) - { - m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]); - m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]); - m_core->warp_inst_complete(mf_next->get_inst()); - } - } - } - - //For write hit in WB policy - if(mf_next->get_inst().is_store() && !write_sent) - { - unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? - (mf_next->get_data_size()/SECTOR_SIZE) : 1; + return result; + } else { + mem_fetch *mf = + m_mf_allocator->alloc(inst, inst.accessq_back(), + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + std::list<cache_event> events; + enum cache_request_status status = cache->access( + mf->get_addr(), mf, + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle, + events); + return process_cache_access(cache, mf->get_addr(), inst, events, mf, + status); + } +} - mf_next->set_reply(); +void ldst_unit::L1_latency_queue_cycle() { + for (int j = 0; j < m_config->m_L1D_config.l1_banks; j++) { + if ((l1_latency_queue[j][0]) != NULL) { + mem_fetch *mf_next = l1_latency_queue[j][0]; + std::list<cache_event> events; + enum cache_request_status status = + m_L1D->access(mf_next->get_addr(), mf_next, + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle, + events); - for(unsigned i=0; i< dec_ack; ++i) - m_core->store_ack(mf_next); - } + bool write_sent = was_write_sent(events); + bool read_sent = was_read_sent(events); - if( !write_sent ) - delete mf_next; + if (status == HIT) { + assert(!read_sent); + l1_latency_queue[j][0] = NULL; + if (mf_next->get_inst().is_load()) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) + if (mf_next->get_inst().out[r] > 0) { + assert(m_pending_writes[mf_next->get_inst().warp_id()] + [mf_next->get_inst().out[r]] > 0); + unsigned still_pending = + --m_pending_writes[mf_next->get_inst().warp_id()] + [mf_next->get_inst().out[r]]; + if (!still_pending) { + m_pending_writes[mf_next->get_inst().warp_id()].erase( + mf_next->get_inst().out[r]); + m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(), + mf_next->get_inst().out[r]); + m_core->warp_inst_complete(mf_next->get_inst()); + } + } + } - } else if ( status == RESERVATION_FAIL ) { - assert( !read_sent ); - assert( !write_sent ); - } else { - assert( status == MISS || status == HIT_RESERVED ); - l1_latency_queue[j][0] = NULL; - } - } + // For write hit in WB policy + if (mf_next->get_inst().is_store() && !write_sent) { + unsigned dec_ack = + (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) + ? (mf_next->get_data_size() / SECTOR_SIZE) + : 1; - for( unsigned stage = 0; stage<m_config->m_L1D_config.l1_latency-1; ++stage) - if( l1_latency_queue[j][stage] == NULL) { - l1_latency_queue[j][stage] = l1_latency_queue[j][stage+1] ; - l1_latency_queue[j][stage+1] = NULL; - } - } + mf_next->set_reply(); -} + for (unsigned i = 0; i < dec_ack; ++i) m_core->store_ack(mf_next); + } + if (!write_sent) delete mf_next; + } else if (status == RESERVATION_FAIL) { + assert(!read_sent); + assert(!write_sent); + } else { + assert(status == MISS || status == HIT_RESERVED); + l1_latency_queue[j][0] = NULL; + } + } -bool ldst_unit::constant_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) -{ - if( inst.empty() || ((inst.space.get_type() != const_space) && (inst.space.get_type() != param_space_kernel)) ) - return true; - if( inst.active_count() == 0 ) - return true; - mem_stage_stall_type fail = process_memory_access_queue(m_L1C,inst); - if (fail != NO_RC_FAIL){ - rc_fail = fail; //keep other fails if this didn't fail. - fail_type = C_MEM; - if (rc_fail == BK_CONF or rc_fail == COAL_STALL) { - m_stats->gpgpu_n_cmem_portconflict++; //coal stalls aren't really a bank conflict, but this maintains previous behavior. + for (unsigned stage = 0; stage < m_config->m_L1D_config.l1_latency - 1; + ++stage) + if (l1_latency_queue[j][stage] == NULL) { + l1_latency_queue[j][stage] = l1_latency_queue[j][stage + 1]; + l1_latency_queue[j][stage + 1] = NULL; } - } - return inst.accessq_empty(); //done if empty. + } } -bool ldst_unit::texture_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) -{ - if( inst.empty() || inst.space.get_type() != tex_space ) - return true; - if( inst.active_count() == 0 ) - return true; - mem_stage_stall_type fail = process_memory_access_queue(m_L1T,inst); - if (fail != NO_RC_FAIL){ - rc_fail = fail; //keep other fails if this didn't fail. - fail_type = T_MEM; - } - return inst.accessq_empty(); //done if empty. +bool ldst_unit::constant_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type) { + if (inst.empty() || ((inst.space.get_type() != const_space) && + (inst.space.get_type() != param_space_kernel))) + return true; + if (inst.active_count() == 0) return true; + mem_stage_stall_type fail = process_memory_access_queue(m_L1C, inst); + if (fail != NO_RC_FAIL) { + rc_fail = fail; // keep other fails if this didn't fail. + fail_type = C_MEM; + if (rc_fail == BK_CONF or rc_fail == COAL_STALL) { + m_stats->gpgpu_n_cmem_portconflict++; // coal stalls aren't really a bank + // conflict, but this maintains + // previous behavior. + } + } + return inst.accessq_empty(); // done if empty. } -bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_reason, mem_stage_access_type &access_type ) -{ - if( inst.empty() || - ((inst.space.get_type() != global_space) && - (inst.space.get_type() != local_space) && - (inst.space.get_type() != param_space_local)) ) - return true; - if( inst.active_count() == 0 ) - return true; - assert( !inst.accessq_empty() ); - mem_stage_stall_type stall_cond = NO_RC_FAIL; - const mem_access_t &access = inst.accessq_back(); - - bool bypassL1D = false; - if ( CACHE_GLOBAL == inst.cache_op || (m_L1D == NULL) ) { - bypassL1D = true; - } else if (inst.space.is_global()) { // global memory access - // skip L1 cache if the option is enabled - if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op)) - bypassL1D = true; - } - if( bypassL1D ) { - // bypass L1 cache - unsigned control_size = inst.is_store() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE; - unsigned size = access.get_size() + control_size; - //printf("Interconnect:Addr: %x, size=%d\n",access.get_addr(),size); - if( m_icnt->full(size, inst.is_store() || inst.isatomic()) ) { - stall_cond = ICNT_RC_FAIL; - } else { - mem_fetch *mf = m_mf_allocator->alloc(inst,access,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_icnt->push(mf); - inst.accessq_pop_back(); - //inst.clear_active( access.get_warp_mask() ); - if( inst.is_load() ) { - for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) - if(inst.out[r] > 0) - assert( m_pending_writes[inst.warp_id()][inst.out[r]] > 0 ); - } else if( inst.is_store() ) - m_core->inc_store_req( inst.warp_id() ); - } - } else { - assert( CACHE_UNDEFINED != inst.cache_op ); - stall_cond = process_memory_access_queue_l1cache(m_L1D,inst); - } - if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL) - stall_cond = COAL_STALL; - if (stall_cond != NO_RC_FAIL) { - stall_reason = stall_cond; - bool iswrite = inst.is_store(); - if (inst.space.is_local()) - access_type = (iswrite)?L_MEM_ST:L_MEM_LD; - else - access_type = (iswrite)?G_MEM_ST:G_MEM_LD; - } - return inst.accessq_empty(); +bool ldst_unit::texture_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type) { + if (inst.empty() || inst.space.get_type() != tex_space) return true; + if (inst.active_count() == 0) return true; + mem_stage_stall_type fail = process_memory_access_queue(m_L1T, inst); + if (fail != NO_RC_FAIL) { + rc_fail = fail; // keep other fails if this didn't fail. + fail_type = T_MEM; + } + return inst.accessq_empty(); // done if empty. } +bool ldst_unit::memory_cycle(warp_inst_t &inst, + mem_stage_stall_type &stall_reason, + mem_stage_access_type &access_type) { + if (inst.empty() || ((inst.space.get_type() != global_space) && + (inst.space.get_type() != local_space) && + (inst.space.get_type() != param_space_local))) + return true; + if (inst.active_count() == 0) return true; + assert(!inst.accessq_empty()); + mem_stage_stall_type stall_cond = NO_RC_FAIL; + const mem_access_t &access = inst.accessq_back(); -bool ldst_unit::response_buffer_full() const -{ - return m_response_fifo.size() >= m_config->ldst_unit_response_queue_size; + bool bypassL1D = false; + if (CACHE_GLOBAL == inst.cache_op || (m_L1D == NULL)) { + bypassL1D = true; + } else if (inst.space.is_global()) { // global memory access + // skip L1 cache if the option is enabled + if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op)) + bypassL1D = true; + } + if (bypassL1D) { + // bypass L1 cache + unsigned control_size = + inst.is_store() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE; + unsigned size = access.get_size() + control_size; + // printf("Interconnect:Addr: %x, size=%d\n",access.get_addr(),size); + if (m_icnt->full(size, inst.is_store() || inst.isatomic())) { + stall_cond = ICNT_RC_FAIL; + } else { + mem_fetch *mf = + m_mf_allocator->alloc(inst, access, + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_icnt->push(mf); + inst.accessq_pop_back(); + // inst.clear_active( access.get_warp_mask() ); + if (inst.is_load()) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) + if (inst.out[r] > 0) + assert(m_pending_writes[inst.warp_id()][inst.out[r]] > 0); + } else if (inst.is_store()) + m_core->inc_store_req(inst.warp_id()); + } + } else { + assert(CACHE_UNDEFINED != inst.cache_op); + stall_cond = process_memory_access_queue_l1cache(m_L1D, inst); + } + if (!inst.accessq_empty() && stall_cond == NO_RC_FAIL) + stall_cond = COAL_STALL; + if (stall_cond != NO_RC_FAIL) { + stall_reason = stall_cond; + bool iswrite = inst.is_store(); + if (inst.space.is_local()) + access_type = (iswrite) ? L_MEM_ST : L_MEM_LD; + else + access_type = (iswrite) ? G_MEM_ST : G_MEM_LD; + } + return inst.accessq_empty(); } -void ldst_unit::fill( mem_fetch *mf ) -{ - mf->set_status(IN_SHADER_LDST_RESPONSE_FIFO,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_response_fifo.push_back(mf); +bool ldst_unit::response_buffer_full() const { + return m_response_fifo.size() >= m_config->ldst_unit_response_queue_size; } -void ldst_unit::flush(){ - // Flush L1D cache - m_L1D->flush(); +void ldst_unit::fill(mem_fetch *mf) { + mf->set_status( + IN_SHADER_LDST_RESPONSE_FIFO, + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle); + m_response_fifo.push_back(mf); } -void ldst_unit::invalidate(){ - // Flush L1D cache - m_L1D->invalidate(); +void ldst_unit::flush() { + // Flush L1D cache + m_L1D->flush(); } -simd_function_unit::simd_function_unit( const shader_core_config *config ) -{ - m_config=config; - m_dispatch_reg = new warp_inst_t(config); +void ldst_unit::invalidate() { + // Flush L1D cache + m_L1D->invalidate(); } +simd_function_unit::simd_function_unit(const shader_core_config *config) { + m_config = config; + m_dispatch_reg = new warp_inst_t(config); +} -sfu:: sfu( register_set* result_port, const shader_core_config *config,shader_core_ctx *core ) - : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core) -{ - m_name = "SFU"; +sfu::sfu(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core) + : pipelined_simd_unit(result_port, config, config->max_sfu_latency, core) { + m_name = "SFU"; } -tensor_core:: tensor_core( register_set* result_port, const shader_core_config *config,shader_core_ctx *core ) - : pipelined_simd_unit(result_port,config,config->max_tensor_core_latency,core) -{ - m_name = "TENSOR_CORE"; +tensor_core::tensor_core(register_set *result_port, + const shader_core_config *config, + shader_core_ctx *core) + : pipelined_simd_unit(result_port, config, config->max_tensor_core_latency, + core) { + m_name = "TENSOR_CORE"; } -void sfu::issue( register_set& source_reg ) -{ - warp_inst_t** ready_reg = source_reg.get_ready(); - //m_core->incexecstat((*ready_reg)); +void sfu::issue(register_set &source_reg) { + warp_inst_t **ready_reg = source_reg.get_ready(); + // m_core->incexecstat((*ready_reg)); - (*ready_reg)->op_pipe=SFU__OP; - m_core->incsfu_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); - pipelined_simd_unit::issue(source_reg); + (*ready_reg)->op_pipe = SFU__OP; + m_core->incsfu_stat(m_core->get_config()->warp_size, (*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); } -void tensor_core::issue( register_set& source_reg ) -{ - warp_inst_t** ready_reg = source_reg.get_ready(); - //m_core->incexecstat((*ready_reg)); +void tensor_core::issue(register_set &source_reg) { + warp_inst_t **ready_reg = source_reg.get_ready(); + // m_core->incexecstat((*ready_reg)); - (*ready_reg)->op_pipe= TENSOR_CORE__OP; - m_core->incsfu_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); - pipelined_simd_unit::issue(source_reg); + (*ready_reg)->op_pipe = TENSOR_CORE__OP; + m_core->incsfu_stat(m_core->get_config()->warp_size, (*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); } -unsigned pipelined_simd_unit::get_active_lanes_in_pipeline(){ - active_mask_t active_lanes; - active_lanes.reset(); - if(m_core->get_gpu()->get_config().g_power_simulation_enabled){ - for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ){ - if( !m_pipeline_reg[stage]->empty() ) - active_lanes|=m_pipeline_reg[stage]->get_active_mask(); - } - } - return active_lanes.count(); +unsigned pipelined_simd_unit::get_active_lanes_in_pipeline() { + active_mask_t active_lanes; + active_lanes.reset(); + if (m_core->get_gpu()->get_config().g_power_simulation_enabled) { + for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++) { + if (!m_pipeline_reg[stage]->empty()) + active_lanes |= m_pipeline_reg[stage]->get_active_mask(); + } + } + return active_lanes.count(); } -void ldst_unit::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incfumemactivelanes_stat(active_count); +void ldst_unit::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incfumemactivelanes_stat(active_count); } -void sp_unit::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incspactivelanes_stat(active_count); - m_core->incfuactivelanes_stat(active_count); - m_core->incfumemactivelanes_stat(active_count); +void sp_unit::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); } -void dp_unit::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incspactivelanes_stat(active_count); - m_core->incfuactivelanes_stat(active_count); - m_core->incfumemactivelanes_stat(active_count); +void dp_unit::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); } -void int_unit::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incspactivelanes_stat(active_count); - m_core->incfuactivelanes_stat(active_count); - m_core->incfumemactivelanes_stat(active_count); +void int_unit::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); } -void sfu::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incsfuactivelanes_stat(active_count); - m_core->incfuactivelanes_stat(active_count); - m_core->incfumemactivelanes_stat(active_count); +void sfu::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incsfuactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); } -void tensor_core::active_lanes_in_pipeline(){ - unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); - assert(active_count<=m_core->get_config()->warp_size); - m_core->incsfuactivelanes_stat(active_count); - m_core->incfuactivelanes_stat(active_count); - m_core->incfumemactivelanes_stat(active_count); +void tensor_core::active_lanes_in_pipeline() { + unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count <= m_core->get_config()->warp_size); + m_core->incsfuactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); } - -sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) - : pipelined_simd_unit(result_port,config,config->max_sp_latency,core) -{ - m_name = "SP "; +sp_unit::sp_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core) + : pipelined_simd_unit(result_port, config, config->max_sp_latency, core) { + m_name = "SP "; } -dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) - : pipelined_simd_unit(result_port,config,config->max_dp_latency,core) -{ - m_name = "DP "; +dp_unit::dp_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core) + : pipelined_simd_unit(result_port, config, config->max_dp_latency, core) { + m_name = "DP "; } -int_unit::int_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) - : pipelined_simd_unit(result_port,config,config->max_int_latency,core) -{ - m_name = "INT "; +int_unit::int_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core) + : pipelined_simd_unit(result_port, config, config->max_int_latency, core) { + m_name = "INT "; } -void sp_unit :: issue(register_set& source_reg) -{ - warp_inst_t** ready_reg = source_reg.get_ready(); - //m_core->incexecstat((*ready_reg)); - (*ready_reg)->op_pipe=SP__OP; - m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); - pipelined_simd_unit::issue(source_reg); +void sp_unit ::issue(register_set &source_reg) { + warp_inst_t **ready_reg = source_reg.get_ready(); + // m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe = SP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); } -void dp_unit :: issue(register_set& source_reg) -{ - warp_inst_t** ready_reg = source_reg.get_ready(); - //m_core->incexecstat((*ready_reg)); - (*ready_reg)->op_pipe=DP__OP; - m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); - pipelined_simd_unit::issue(source_reg); +void dp_unit ::issue(register_set &source_reg) { + warp_inst_t **ready_reg = source_reg.get_ready(); + // m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe = DP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); } -void int_unit :: issue(register_set& source_reg) -{ - warp_inst_t** ready_reg = source_reg.get_ready(); - //m_core->incexecstat((*ready_reg)); - (*ready_reg)->op_pipe=INTP__OP; - m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); - pipelined_simd_unit::issue(source_reg); +void int_unit ::issue(register_set &source_reg) { + warp_inst_t **ready_reg = source_reg.get_ready(); + // m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe = INTP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); } -pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core ) - : simd_function_unit(config) -{ - m_result_port = result_port; - m_pipeline_depth = max_latency; - m_pipeline_reg = new warp_inst_t*[m_pipeline_depth]; - for( unsigned i=0; i < m_pipeline_depth; i++ ) - m_pipeline_reg[i] = new warp_inst_t( config ); - m_core=core; - active_insts_in_pipeline=0; +pipelined_simd_unit::pipelined_simd_unit(register_set *result_port, + const shader_core_config *config, + unsigned max_latency, + shader_core_ctx *core) + : simd_function_unit(config) { + m_result_port = result_port; + m_pipeline_depth = max_latency; + m_pipeline_reg = new warp_inst_t *[m_pipeline_depth]; + for (unsigned i = 0; i < m_pipeline_depth; i++) + m_pipeline_reg[i] = new warp_inst_t(config); + m_core = core; + active_insts_in_pipeline = 0; } -void pipelined_simd_unit::cycle() -{ - if( !m_pipeline_reg[0]->empty() ){ - m_result_port->move_in(m_pipeline_reg[0]); - assert(active_insts_in_pipeline > 0); - active_insts_in_pipeline--; - } - if(active_insts_in_pipeline){ - for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ) - move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]); - } - if( !m_dispatch_reg->empty() ) { - if( !m_dispatch_reg->dispatch_delay()){ - int start_stage = m_dispatch_reg->latency - m_dispatch_reg->initiation_interval; - move_warp(m_pipeline_reg[start_stage],m_dispatch_reg); - active_insts_in_pipeline++; - } +void pipelined_simd_unit::cycle() { + if (!m_pipeline_reg[0]->empty()) { + m_result_port->move_in(m_pipeline_reg[0]); + assert(active_insts_in_pipeline > 0); + active_insts_in_pipeline--; + } + if (active_insts_in_pipeline) { + for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++) + move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]); + } + if (!m_dispatch_reg->empty()) { + if (!m_dispatch_reg->dispatch_delay()) { + int start_stage = + m_dispatch_reg->latency - m_dispatch_reg->initiation_interval; + move_warp(m_pipeline_reg[start_stage], m_dispatch_reg); + active_insts_in_pipeline++; } - occupied >>=1; + } + occupied >>= 1; } - -void pipelined_simd_unit::issue( register_set& source_reg ) -{ - //move_warp(m_dispatch_reg,source_reg); - warp_inst_t** ready_reg = source_reg.get_ready(); - m_core->incexecstat((*ready_reg)); - //source_reg.move_out_to(m_dispatch_reg); - simd_function_unit::issue(source_reg); +void pipelined_simd_unit::issue(register_set &source_reg) { + // move_warp(m_dispatch_reg,source_reg); + warp_inst_t **ready_reg = source_reg.get_ready(); + m_core->incexecstat((*ready_reg)); + // source_reg.move_out_to(m_dispatch_reg); + simd_function_unit::issue(source_reg); } /* @@ -2073,253 +2145,228 @@ void pipelined_simd_unit::issue( register_set& source_reg ) } */ -void ldst_unit::init( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - unsigned sid, - unsigned tpc ) -{ - m_memory_config = mem_config; - m_icnt = icnt; - m_mf_allocator=mf_allocator; - m_core = core; - m_operand_collector = operand_collector; - m_scoreboard = scoreboard; - m_stats = stats; - m_sid = sid; - m_tpc = tpc; - #define STRSIZE 1024 - char L1T_name[STRSIZE]; - char L1C_name[STRSIZE]; - snprintf(L1T_name, STRSIZE, "L1T_%03d", m_sid); - snprintf(L1C_name, STRSIZE, "L1C_%03d", m_sid); - m_L1T = new tex_cache(L1T_name,m_config->m_L1T_config,m_sid,get_shader_texture_cache_id(),icnt,IN_L1T_MISS_QUEUE,IN_SHADER_L1T_ROB); - m_L1C = new read_only_cache(L1C_name,m_config->m_L1C_config,m_sid,get_shader_constant_cache_id(),icnt,IN_L1C_MISS_QUEUE); - m_L1D = NULL; - m_mem_rc = NO_RC_FAIL; - m_num_writeback_clients=5; // = shared memory, global/local (uncached), L1D, L1T, L1C - m_writeback_arb = 0; - m_next_global=NULL; - m_last_inst_gpu_sim_cycle=0; - m_last_inst_gpu_tot_sim_cycle=0; +void ldst_unit::init(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + unsigned sid, unsigned tpc) { + m_memory_config = mem_config; + m_icnt = icnt; + m_mf_allocator = mf_allocator; + m_core = core; + m_operand_collector = operand_collector; + m_scoreboard = scoreboard; + m_stats = stats; + m_sid = sid; + m_tpc = tpc; +#define STRSIZE 1024 + char L1T_name[STRSIZE]; + char L1C_name[STRSIZE]; + snprintf(L1T_name, STRSIZE, "L1T_%03d", m_sid); + snprintf(L1C_name, STRSIZE, "L1C_%03d", m_sid); + m_L1T = new tex_cache(L1T_name, m_config->m_L1T_config, m_sid, + get_shader_texture_cache_id(), icnt, IN_L1T_MISS_QUEUE, + IN_SHADER_L1T_ROB); + m_L1C = new read_only_cache(L1C_name, m_config->m_L1C_config, m_sid, + get_shader_constant_cache_id(), icnt, + IN_L1C_MISS_QUEUE); + m_L1D = NULL; + m_mem_rc = NO_RC_FAIL; + m_num_writeback_clients = + 5; // = shared memory, global/local (uncached), L1D, L1T, L1C + m_writeback_arb = 0; + m_next_global = NULL; + m_last_inst_gpu_sim_cycle = 0; + m_last_inst_gpu_tot_sim_cycle = 0; } +ldst_unit::ldst_unit(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + unsigned sid, unsigned tpc) + : pipelined_simd_unit(NULL, config, config->smem_latency, core), + m_next_wb(config) { + assert(config->smem_latency > 1); + init(icnt, mf_allocator, core, operand_collector, scoreboard, config, + mem_config, stats, sid, tpc); + if (!m_config->m_L1D_config.disabled()) { + char L1D_name[STRSIZE]; + snprintf(L1D_name, STRSIZE, "L1D_%03d", m_sid); + m_L1D = new l1_cache(L1D_name, m_config->m_L1D_config, m_sid, + get_shader_normal_cache_id(), m_icnt, m_mf_allocator, + IN_L1D_MISS_QUEUE, core->get_gpu()); -ldst_unit::ldst_unit( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - unsigned sid, - unsigned tpc ) : pipelined_simd_unit(NULL,config,config->smem_latency,core), m_next_wb(config) -{ - assert(config->smem_latency > 1); - init( icnt, - mf_allocator, - core, - operand_collector, - scoreboard, - config, - mem_config, - stats, - sid, - tpc ); - if( !m_config->m_L1D_config.disabled() ) { - char L1D_name[STRSIZE]; - snprintf(L1D_name, STRSIZE, "L1D_%03d", m_sid); - m_L1D = new l1_cache( L1D_name, - m_config->m_L1D_config, - m_sid, - get_shader_normal_cache_id(), - m_icnt, - m_mf_allocator, - IN_L1D_MISS_QUEUE, - core->get_gpu()); - - l1_latency_queue.resize(m_config->m_L1D_config.l1_banks); - assert(m_config->m_L1D_config.l1_latency > 0); - - for(unsigned j = 0; j < m_config->m_L1D_config.l1_banks; j++ ) - l1_latency_queue[j].resize(m_config->m_L1D_config.l1_latency,(mem_fetch*)NULL); + l1_latency_queue.resize(m_config->m_L1D_config.l1_banks); + assert(m_config->m_L1D_config.l1_latency > 0); - } - m_name = "MEM "; + for (unsigned j = 0; j < m_config->m_L1D_config.l1_banks; j++) + l1_latency_queue[j].resize(m_config->m_L1D_config.l1_latency, + (mem_fetch *)NULL); + } + m_name = "MEM "; } -ldst_unit::ldst_unit( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - unsigned sid, - unsigned tpc, - l1_cache* new_l1d_cache ) - : pipelined_simd_unit(NULL,config,3,core), m_L1D(new_l1d_cache), m_next_wb(config) -{ - init( icnt, - mf_allocator, - core, - operand_collector, - scoreboard, - config, - mem_config, - stats, - sid, - tpc ); +ldst_unit::ldst_unit(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + unsigned sid, unsigned tpc, l1_cache *new_l1d_cache) + : pipelined_simd_unit(NULL, config, 3, core), + m_L1D(new_l1d_cache), + m_next_wb(config) { + init(icnt, mf_allocator, core, operand_collector, scoreboard, config, + mem_config, stats, sid, tpc); } -void ldst_unit:: issue( register_set ®_set ) -{ - warp_inst_t* inst = *(reg_set.get_ready()); +void ldst_unit::issue(register_set ®_set) { + warp_inst_t *inst = *(reg_set.get_ready()); - // record how many pending register writes/memory accesses there are for this instruction - assert(inst->empty() == false); - if (inst->is_load() and inst->space.get_type() != shared_space) { - unsigned warp_id = inst->warp_id(); - unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = inst->out[r]; - if (reg_id > 0) { - m_pending_writes[warp_id][reg_id] += n_accesses; - } + // record how many pending register writes/memory accesses there are for this + // instruction + assert(inst->empty() == false); + if (inst->is_load() and inst->space.get_type() != shared_space) { + unsigned warp_id = inst->warp_id(); + unsigned n_accesses = inst->accessq_count(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) { + m_pending_writes[warp_id][reg_id] += n_accesses; } - } - + } + } - inst->op_pipe=MEM__OP; - // stat collection - m_core->mem_instruction_stats(*inst); - m_core->incmem_stat(m_core->get_config()->warp_size,1); - pipelined_simd_unit::issue(reg_set); + inst->op_pipe = MEM__OP; + // stat collection + m_core->mem_instruction_stats(*inst); + m_core->incmem_stat(m_core->get_config()->warp_size, 1); + pipelined_simd_unit::issue(reg_set); } -void ldst_unit::writeback() -{ - // process next instruction that is going to writeback - if( !m_next_wb.empty() ) { - if( m_operand_collector->writeback(m_next_wb) ) { - bool insn_completed = false; - for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++ ) { - if( m_next_wb.out[r] > 0 ) { - if( m_next_wb.space.get_type() != shared_space ) { - assert( m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0 ); - unsigned still_pending = --m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]]; - if( !still_pending ) { - m_pending_writes[m_next_wb.warp_id()].erase(m_next_wb.out[r]); - m_scoreboard->releaseRegister( m_next_wb.warp_id(), m_next_wb.out[r] ); - insn_completed = true; - } - } else { // shared - m_scoreboard->releaseRegister( m_next_wb.warp_id(), m_next_wb.out[r] ); - insn_completed = true; - } - } - } - if( insn_completed ) { - m_core->warp_inst_complete(m_next_wb); +void ldst_unit::writeback() { + // process next instruction that is going to writeback + if (!m_next_wb.empty()) { + if (m_operand_collector->writeback(m_next_wb)) { + bool insn_completed = false; + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (m_next_wb.out[r] > 0) { + if (m_next_wb.space.get_type() != shared_space) { + assert(m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0); + unsigned still_pending = + --m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]]; + if (!still_pending) { + m_pending_writes[m_next_wb.warp_id()].erase(m_next_wb.out[r]); + m_scoreboard->releaseRegister(m_next_wb.warp_id(), + m_next_wb.out[r]); + insn_completed = true; } - m_next_wb.clear(); - m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle; - m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle; + } else { // shared + m_scoreboard->releaseRegister(m_next_wb.warp_id(), + m_next_wb.out[r]); + insn_completed = true; + } } + } + if (insn_completed) { + m_core->warp_inst_complete(m_next_wb); + } + m_next_wb.clear(); + m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle; + m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle; } + } - unsigned serviced_client = -1; - for( unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients); c++ ) { - unsigned next_client = (c+m_writeback_arb)%m_num_writeback_clients; - switch( next_client ) { - case 0: // shared memory - if( !m_pipeline_reg[0]->empty() ) { - m_next_wb = *m_pipeline_reg[0]; - if(m_next_wb.isatomic()) { - m_next_wb.do_atomic(); - m_core->decrement_atomic_count(m_next_wb.warp_id(), m_next_wb.active_count()); - } - m_core->dec_inst_in_pipeline(m_pipeline_reg[0]->warp_id()); - m_pipeline_reg[0]->clear(); - serviced_client = next_client; - } - break; - case 1: // texture response - if( m_L1T->access_ready() ) { - mem_fetch *mf = m_L1T->next_access(); - m_next_wb = mf->get_inst(); - delete mf; - serviced_client = next_client; - } - break; - case 2: // const cache response - if( m_L1C->access_ready() ) { - mem_fetch *mf = m_L1C->next_access(); - m_next_wb = mf->get_inst(); - delete mf; - serviced_client = next_client; - } - break; - case 3: // global/local - if( m_next_global ) { - m_next_wb = m_next_global->get_inst(); - if( m_next_global->isatomic() ) - m_core->decrement_atomic_count(m_next_global->get_wid(),m_next_global->get_access_warp_mask().count()); - delete m_next_global; - m_next_global = NULL; - serviced_client = next_client; - } - break; - case 4: - if( m_L1D && m_L1D->access_ready() ) { - mem_fetch *mf = m_L1D->next_access(); - m_next_wb = mf->get_inst(); - delete mf; - serviced_client = next_client; - } - break; - default: abort(); + unsigned serviced_client = -1; + for (unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients); + c++) { + unsigned next_client = (c + m_writeback_arb) % m_num_writeback_clients; + switch (next_client) { + case 0: // shared memory + if (!m_pipeline_reg[0]->empty()) { + m_next_wb = *m_pipeline_reg[0]; + if (m_next_wb.isatomic()) { + m_next_wb.do_atomic(); + m_core->decrement_atomic_count(m_next_wb.warp_id(), + m_next_wb.active_count()); + } + m_core->dec_inst_in_pipeline(m_pipeline_reg[0]->warp_id()); + m_pipeline_reg[0]->clear(); + serviced_client = next_client; } + break; + case 1: // texture response + if (m_L1T->access_ready()) { + mem_fetch *mf = m_L1T->next_access(); + m_next_wb = mf->get_inst(); + delete mf; + serviced_client = next_client; + } + break; + case 2: // const cache response + if (m_L1C->access_ready()) { + mem_fetch *mf = m_L1C->next_access(); + m_next_wb = mf->get_inst(); + delete mf; + serviced_client = next_client; + } + break; + case 3: // global/local + if (m_next_global) { + m_next_wb = m_next_global->get_inst(); + if (m_next_global->isatomic()) + m_core->decrement_atomic_count( + m_next_global->get_wid(), + m_next_global->get_access_warp_mask().count()); + delete m_next_global; + m_next_global = NULL; + serviced_client = next_client; + } + break; + case 4: + if (m_L1D && m_L1D->access_ready()) { + mem_fetch *mf = m_L1D->next_access(); + m_next_wb = mf->get_inst(); + delete mf; + serviced_client = next_client; + } + break; + default: + abort(); } - // update arbitration priority only if: - // 1. the writeback buffer was available - // 2. a client was serviced - if (serviced_client != (unsigned)-1) { - m_writeback_arb = (serviced_client + 1) % m_num_writeback_clients; - } + } + // update arbitration priority only if: + // 1. the writeback buffer was available + // 2. a client was serviced + if (serviced_client != (unsigned)-1) { + m_writeback_arb = (serviced_client + 1) % m_num_writeback_clients; + } } -unsigned ldst_unit::clock_multiplier() const -{ - //to model multiple read port, we give multiple cycles for the memory units - if(m_config->mem_unit_ports) - return m_config->mem_unit_ports; - else - return m_config->mem_warp_parts; +unsigned ldst_unit::clock_multiplier() const { + // to model multiple read port, we give multiple cycles for the memory units + if (m_config->mem_unit_ports) + return m_config->mem_unit_ports; + else + return m_config->mem_warp_parts; } /* void ldst_unit::issue( register_set ®_set ) { - warp_inst_t* inst = *(reg_set.get_ready()); + warp_inst_t* inst = *(reg_set.get_ready()); // stat collection - m_core->mem_instruction_stats(*inst); + m_core->mem_instruction_stats(*inst); - // record how many pending register writes/memory accesses there are for this instruction - assert(inst->empty() == false); - if (inst->is_load() and inst->space.get_type() != shared_space) { - unsigned warp_id = inst->warp_id(); - unsigned n_accesses = inst->accessq_count(); + // record how many pending register writes/memory accesses there are for this +instruction assert(inst->empty() == false); if (inst->is_load() and +inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id(); + unsigned n_accesses = inst->accessq_count(); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = inst->out[r]; + unsigned reg_id = inst->out[r]; if (reg_id > 0) { - m_pending_writes[warp_id][reg_id] += n_accesses; + m_pending_writes[warp_id][reg_id] += n_accesses; } } } @@ -2327,1820 +2374,1874 @@ void ldst_unit::issue( register_set ®_set ) pipelined_simd_unit::issue(reg_set); } */ -void ldst_unit::cycle() -{ - writeback(); - m_operand_collector->step(); - for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ) - if( m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage+1]->empty() ) - move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]); +void ldst_unit::cycle() { + writeback(); + m_operand_collector->step(); + for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++) + if (m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage + 1]->empty()) + move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]); - if( !m_response_fifo.empty() ) { - mem_fetch *mf = m_response_fifo.front(); - if (mf->get_access_type() == TEXTURE_ACC_R) { - if (m_L1T->fill_port_free()) { - m_L1T->fill(mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_response_fifo.pop_front(); - } - } else if (mf->get_access_type() == CONST_ACC_R) { - if (m_L1C->fill_port_free()) { - mf->set_status(IN_SHADER_FETCHED,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_L1C->fill(mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_response_fifo.pop_front(); - } - } else { - if( mf->get_type() == WRITE_ACK || ( m_config->gpgpu_perfect_mem && mf->get_is_write() )) { - m_core->store_ack(mf); - m_response_fifo.pop_front(); - delete mf; - } else { - assert( !mf->get_is_write() ); // L1 cache is write evict, allocate line on load miss only + if (!m_response_fifo.empty()) { + mem_fetch *mf = m_response_fifo.front(); + if (mf->get_access_type() == TEXTURE_ACC_R) { + if (m_L1T->fill_port_free()) { + m_L1T->fill(mf, m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_response_fifo.pop_front(); + } + } else if (mf->get_access_type() == CONST_ACC_R) { + if (m_L1C->fill_port_free()) { + mf->set_status(IN_SHADER_FETCHED, + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_L1C->fill(mf, m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_response_fifo.pop_front(); + } + } else { + if (mf->get_type() == WRITE_ACK || + (m_config->gpgpu_perfect_mem && mf->get_is_write())) { + m_core->store_ack(mf); + m_response_fifo.pop_front(); + delete mf; + } else { + assert(!mf->get_is_write()); // L1 cache is write evict, allocate line + // on load miss only - bool bypassL1D = false; - if ( CACHE_GLOBAL == mf->get_inst().cache_op || (m_L1D == NULL) ) { - bypassL1D = true; - } else if (mf->get_access_type() == GLOBAL_ACC_R || mf->get_access_type() == GLOBAL_ACC_W) { // global memory access - if (m_core->get_config()->gmem_skip_L1D) - bypassL1D = true; - } - if( bypassL1D ) { - if ( m_next_global == NULL ) { - mf->set_status(IN_SHADER_FETCHED,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_response_fifo.pop_front(); - m_next_global = mf; - } - } else { - if (m_L1D->fill_port_free()) { - m_L1D->fill(mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle); - m_response_fifo.pop_front(); - } - } - } - } - } + bool bypassL1D = false; + if (CACHE_GLOBAL == mf->get_inst().cache_op || (m_L1D == NULL)) { + bypassL1D = true; + } else if (mf->get_access_type() == GLOBAL_ACC_R || + mf->get_access_type() == + GLOBAL_ACC_W) { // global memory access + if (m_core->get_config()->gmem_skip_L1D) bypassL1D = true; + } + if (bypassL1D) { + if (m_next_global == NULL) { + mf->set_status(IN_SHADER_FETCHED, + m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_response_fifo.pop_front(); + m_next_global = mf; + } + } else { + if (m_L1D->fill_port_free()) { + m_L1D->fill(mf, m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle); + m_response_fifo.pop_front(); + } + } + } + } + } - m_L1T->cycle(); - m_L1C->cycle(); - if( m_L1D ) { - m_L1D->cycle(); - if(m_config->m_L1D_config.l1_latency > 0) - L1_latency_queue_cycle(); - } + m_L1T->cycle(); + m_L1C->cycle(); + if (m_L1D) { + m_L1D->cycle(); + if (m_config->m_L1D_config.l1_latency > 0) L1_latency_queue_cycle(); + } - warp_inst_t &pipe_reg = *m_dispatch_reg; - enum mem_stage_stall_type rc_fail = NO_RC_FAIL; - mem_stage_access_type type; - bool done = true; - done &= shared_cycle(pipe_reg, rc_fail, type); - done &= constant_cycle(pipe_reg, rc_fail, type); - done &= texture_cycle(pipe_reg, rc_fail, type); - done &= memory_cycle(pipe_reg, rc_fail, type); - m_mem_rc = rc_fail; + warp_inst_t &pipe_reg = *m_dispatch_reg; + enum mem_stage_stall_type rc_fail = NO_RC_FAIL; + mem_stage_access_type type; + bool done = true; + done &= shared_cycle(pipe_reg, rc_fail, type); + done &= constant_cycle(pipe_reg, rc_fail, type); + done &= texture_cycle(pipe_reg, rc_fail, type); + done &= memory_cycle(pipe_reg, rc_fail, type); + m_mem_rc = rc_fail; - if (!done) { // log stall types and return - assert(rc_fail != NO_RC_FAIL); - m_stats->gpgpu_n_stall_shd_mem++; - m_stats->gpu_stall_shd_mem_breakdown[type][rc_fail]++; - return; - } + if (!done) { // log stall types and return + assert(rc_fail != NO_RC_FAIL); + m_stats->gpgpu_n_stall_shd_mem++; + m_stats->gpu_stall_shd_mem_breakdown[type][rc_fail]++; + return; + } - if( !pipe_reg.empty() ) { - unsigned warp_id = pipe_reg.warp_id(); - if( pipe_reg.is_load() ) { - if( pipe_reg.space.get_type() == shared_space ) { - if( m_pipeline_reg[m_config->smem_latency-1]->empty() ) { - // new shared memory request - move_warp(m_pipeline_reg[m_config->smem_latency-1],m_dispatch_reg); - m_dispatch_reg->clear(); - } - } else { - //if( pipe_reg.active_count() > 0 ) { - // if( !m_operand_collector->writeback(pipe_reg) ) - // return; - //} + if (!pipe_reg.empty()) { + unsigned warp_id = pipe_reg.warp_id(); + if (pipe_reg.is_load()) { + if (pipe_reg.space.get_type() == shared_space) { + if (m_pipeline_reg[m_config->smem_latency - 1]->empty()) { + // new shared memory request + move_warp(m_pipeline_reg[m_config->smem_latency - 1], m_dispatch_reg); + m_dispatch_reg->clear(); + } + } else { + // if( pipe_reg.active_count() > 0 ) { + // if( !m_operand_collector->writeback(pipe_reg) ) + // return; + //} - bool pending_requests=false; - for( unsigned r=0; r<MAX_OUTPUT_VALUES; r++ ) { - unsigned reg_id = pipe_reg.out[r]; - if( reg_id > 0 ) { - if( m_pending_writes[warp_id].find(reg_id) != m_pending_writes[warp_id].end() ) { - if ( m_pending_writes[warp_id][reg_id] > 0 ) { - pending_requests=true; - break; - } else { - // this instruction is done already - m_pending_writes[warp_id].erase(reg_id); - } - } - } - } - if( !pending_requests ) { - m_core->warp_inst_complete(*m_dispatch_reg); - m_scoreboard->releaseRegisters(m_dispatch_reg); - } - m_core->dec_inst_in_pipeline(warp_id); - m_dispatch_reg->clear(); - } - } else { - // stores exit pipeline here - m_core->dec_inst_in_pipeline(warp_id); - m_core->warp_inst_complete(*m_dispatch_reg); - m_dispatch_reg->clear(); - } - } + bool pending_requests = false; + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = pipe_reg.out[r]; + if (reg_id > 0) { + if (m_pending_writes[warp_id].find(reg_id) != + m_pending_writes[warp_id].end()) { + if (m_pending_writes[warp_id][reg_id] > 0) { + pending_requests = true; + break; + } else { + // this instruction is done already + m_pending_writes[warp_id].erase(reg_id); + } + } + } + } + if (!pending_requests) { + m_core->warp_inst_complete(*m_dispatch_reg); + m_scoreboard->releaseRegisters(m_dispatch_reg); + } + m_core->dec_inst_in_pipeline(warp_id); + m_dispatch_reg->clear(); + } + } else { + // stores exit pipeline here + m_core->dec_inst_in_pipeline(warp_id); + m_core->warp_inst_complete(*m_dispatch_reg); + m_dispatch_reg->clear(); + } + } } -void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t * kernel) -{ - assert( m_cta_status[cta_num] > 0 ); - m_cta_status[cta_num]--; - if (!m_cta_status[cta_num]) { - m_n_active_cta--; - m_barriers.deallocate_barrier(cta_num); - shader_CTA_count_unlog(m_sid, 1); +void shader_core_ctx::register_cta_thread_exit(unsigned cta_num, + kernel_info_t *kernel) { + assert(m_cta_status[cta_num] > 0); + m_cta_status[cta_num]--; + if (!m_cta_status[cta_num]) { + m_n_active_cta--; + m_barriers.deallocate_barrier(cta_num); + shader_CTA_count_unlog(m_sid, 1); - SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Finished CTA #%u (%lld,%lld), %u CTAs running\n", - cta_num, m_gpu->gpu_sim_cycle, m_gpu->gpu_tot_sim_cycle, m_n_active_cta); + SHADER_DPRINTF( + LIVENESS, + "GPGPU-Sim uArch: Finished CTA #%u (%lld,%lld), %u CTAs running\n", + cta_num, m_gpu->gpu_sim_cycle, m_gpu->gpu_tot_sim_cycle, + m_n_active_cta); - if( m_n_active_cta == 0 ) { - SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Empty (last released kernel %u \'%s\').\n", - kernel->get_uid(), kernel->name().c_str()); - fflush(stdout); + if (m_n_active_cta == 0) { + SHADER_DPRINTF( + LIVENESS, + "GPGPU-Sim uArch: Empty (last released kernel %u \'%s\').\n", + kernel->get_uid(), kernel->name().c_str()); + fflush(stdout); - //Shader can only be empty when no more cta are dispatched - if(kernel != m_kernel) { - assert(m_kernel == NULL || !m_gpu->kernel_more_cta_left(m_kernel)); - } - m_kernel = NULL; + // Shader can only be empty when no more cta are dispatched + if (kernel != m_kernel) { + assert(m_kernel == NULL || !m_gpu->kernel_more_cta_left(m_kernel)); } + m_kernel = NULL; + } - //Jin: for concurrent kernels on sm - release_shader_resource_1block(cta_num, *kernel); - kernel->dec_running(); - if( !m_gpu->kernel_more_cta_left(kernel) ) { - if( !kernel->running() ) { - SHADER_DPRINTF(LIVENESS, - "GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(), - kernel->name().c_str(), m_sid); + // Jin: for concurrent kernels on sm + release_shader_resource_1block(cta_num, *kernel); + kernel->dec_running(); + if (!m_gpu->kernel_more_cta_left(kernel)) { + if (!kernel->running()) { + SHADER_DPRINTF(LIVENESS, + "GPGPU-Sim uArch: GPU detected kernel %u \'%s\' " + "finished on shader %u.\n", + kernel->get_uid(), kernel->name().c_str(), m_sid); - if(m_kernel == kernel) - m_kernel = NULL; - m_gpu->set_kernel_done( kernel ); - } + if (m_kernel == kernel) m_kernel = NULL; + m_gpu->set_kernel_done(kernel); } - - } + } + } } -void gpgpu_sim::shader_print_runtime_stat( FILE *fout ) -{ - /* - fprintf(fout, "SHD_INSN: "); - for (unsigned i=0;i<m_n_shader;i++) - fprintf(fout, "%u ",m_sc[i]->get_num_sim_insn()); - fprintf(fout, "\n"); - fprintf(fout, "SHD_THDS: "); - for (unsigned i=0;i<m_n_shader;i++) - fprintf(fout, "%u ",m_sc[i]->get_not_completed()); - fprintf(fout, "\n"); - fprintf(fout, "SHD_DIVG: "); - for (unsigned i=0;i<m_n_shader;i++) - fprintf(fout, "%u ",m_sc[i]->get_n_diverge()); - fprintf(fout, "\n"); +void gpgpu_sim::shader_print_runtime_stat(FILE *fout) { + /* + fprintf(fout, "SHD_INSN: "); + for (unsigned i=0;i<m_n_shader;i++) + fprintf(fout, "%u ",m_sc[i]->get_num_sim_insn()); + fprintf(fout, "\n"); + fprintf(fout, "SHD_THDS: "); + for (unsigned i=0;i<m_n_shader;i++) + fprintf(fout, "%u ",m_sc[i]->get_not_completed()); + fprintf(fout, "\n"); + fprintf(fout, "SHD_DIVG: "); + for (unsigned i=0;i<m_n_shader;i++) + fprintf(fout, "%u ",m_sc[i]->get_n_diverge()); + fprintf(fout, "\n"); - fprintf(fout, "THD_INSN: "); - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) - fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn(i) ); - fprintf(fout, "\n"); - */ + fprintf(fout, "THD_INSN: "); + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) + fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn(i) ); + fprintf(fout, "\n"); + */ } +void gpgpu_sim::shader_print_scheduler_stat(FILE *fout, + bool print_dynamic_info) const { + // Print out the stats from the sampling shader core + const unsigned scheduler_sampling_core = + m_shader_config->gpgpu_warp_issue_shader; +#define STR_SIZE 55 + char name_buff[STR_SIZE]; + name_buff[STR_SIZE - 1] = '\0'; + const std::vector<unsigned> &distro = + print_dynamic_info + ? m_shader_stats->get_dynamic_warp_issue()[scheduler_sampling_core] + : m_shader_stats->get_warp_slot_issue()[scheduler_sampling_core]; + if (print_dynamic_info) { + snprintf(name_buff, STR_SIZE - 1, "dynamic_warp_id"); + } else { + snprintf(name_buff, STR_SIZE - 1, "warp_id"); + } + fprintf(fout, "Shader %d %s issue ditsribution:\n", scheduler_sampling_core, + name_buff); + const unsigned num_warp_ids = distro.size(); + // First print out the warp ids + fprintf(fout, "%s:\n", name_buff); + for (unsigned warp_id = 0; warp_id < num_warp_ids; ++warp_id) { + fprintf(fout, "%d, ", warp_id); + } -void gpgpu_sim::shader_print_scheduler_stat( FILE* fout, bool print_dynamic_info ) const -{ - // Print out the stats from the sampling shader core - const unsigned scheduler_sampling_core = m_shader_config->gpgpu_warp_issue_shader; - #define STR_SIZE 55 - char name_buff[ STR_SIZE ]; - name_buff[ STR_SIZE - 1 ] = '\0'; - const std::vector< unsigned >& distro - = print_dynamic_info ? - m_shader_stats->get_dynamic_warp_issue()[ scheduler_sampling_core ] : - m_shader_stats->get_warp_slot_issue()[ scheduler_sampling_core ]; - if ( print_dynamic_info ) { - snprintf( name_buff, STR_SIZE - 1, "dynamic_warp_id" ); - } else { - snprintf( name_buff, STR_SIZE - 1, "warp_id" ); - } - fprintf( fout, - "Shader %d %s issue ditsribution:\n", - scheduler_sampling_core, - name_buff ); - const unsigned num_warp_ids = distro.size(); - // First print out the warp ids - fprintf( fout, "%s:\n", name_buff ); - for ( unsigned warp_id = 0; - warp_id < num_warp_ids; - ++warp_id ) { - fprintf( fout, "%d, ", warp_id ); - } - - fprintf( fout, "\ndistro:\n" ); - // Then print out the distribution of instuctions issued - for ( std::vector< unsigned >::const_iterator iter = distro.begin(); - iter != distro.end(); - iter++ ) { - fprintf( fout, "%d, ", *iter ); - } - fprintf( fout, "\n" ); + fprintf(fout, "\ndistro:\n"); + // Then print out the distribution of instuctions issued + for (std::vector<unsigned>::const_iterator iter = distro.begin(); + iter != distro.end(); iter++) { + fprintf(fout, "%d, ", *iter); + } + fprintf(fout, "\n"); } -void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{ - - // L1I - struct cache_sub_stats total_css; - struct cache_sub_stats css; +void gpgpu_sim::shader_print_cache_stats(FILE *fout) const { + // L1I + struct cache_sub_stats total_css; + struct cache_sub_stats css; - if(!m_shader_config->m_L1I_config.disabled()){ - total_css.clear(); - css.clear(); - fprintf(fout, "\n========= Core cache stats =========\n"); - fprintf(fout, "L1I_cache:\n"); - for ( unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i ) { - m_cluster[i]->get_L1I_sub_stats(css); - total_css += css; - } - fprintf(fout, "\tL1I_total_cache_accesses = %llu\n", total_css.accesses); - fprintf(fout, "\tL1I_total_cache_misses = %llu\n", total_css.misses); - if(total_css.accesses > 0){ - fprintf(fout, "\tL1I_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses); - } - fprintf(fout, "\tL1I_total_cache_pending_hits = %llu\n", total_css.pending_hits); - fprintf(fout, "\tL1I_total_cache_reservation_fails = %llu\n", total_css.res_fails); + if (!m_shader_config->m_L1I_config.disabled()) { + total_css.clear(); + css.clear(); + fprintf(fout, "\n========= Core cache stats =========\n"); + fprintf(fout, "L1I_cache:\n"); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) { + m_cluster[i]->get_L1I_sub_stats(css); + total_css += css; } + fprintf(fout, "\tL1I_total_cache_accesses = %llu\n", total_css.accesses); + fprintf(fout, "\tL1I_total_cache_misses = %llu\n", total_css.misses); + if (total_css.accesses > 0) { + fprintf(fout, "\tL1I_total_cache_miss_rate = %.4lf\n", + (double)total_css.misses / (double)total_css.accesses); + } + fprintf(fout, "\tL1I_total_cache_pending_hits = %llu\n", + total_css.pending_hits); + fprintf(fout, "\tL1I_total_cache_reservation_fails = %llu\n", + total_css.res_fails); + } - // L1D - if(!m_shader_config->m_L1D_config.disabled()){ - total_css.clear(); - css.clear(); - fprintf(fout, "L1D_cache:\n"); - for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++){ - m_cluster[i]->get_L1D_sub_stats(css); + // L1D + if (!m_shader_config->m_L1D_config.disabled()) { + total_css.clear(); + css.clear(); + fprintf(fout, "L1D_cache:\n"); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + m_cluster[i]->get_L1D_sub_stats(css); - fprintf( stdout, "\tL1D_cache_core[%d]: Access = %llu, Miss = %llu, Miss_rate = %.3lf, Pending_hits = %llu, Reservation_fails = %llu\n", - i, css.accesses, css.misses, (double)css.misses / (double)css.accesses, css.pending_hits, css.res_fails); + fprintf(stdout, + "\tL1D_cache_core[%d]: Access = %llu, Miss = %llu, Miss_rate = " + "%.3lf, Pending_hits = %llu, Reservation_fails = %llu\n", + i, css.accesses, css.misses, + (double)css.misses / (double)css.accesses, css.pending_hits, + css.res_fails); - total_css += css; - } - fprintf(fout, "\tL1D_total_cache_accesses = %llu\n", total_css.accesses); - fprintf(fout, "\tL1D_total_cache_misses = %llu\n", total_css.misses); - if(total_css.accesses > 0){ - fprintf(fout, "\tL1D_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses); - } - fprintf(fout, "\tL1D_total_cache_pending_hits = %llu\n", total_css.pending_hits); - fprintf(fout, "\tL1D_total_cache_reservation_fails = %llu\n", total_css.res_fails); - total_css.print_port_stats(fout, "\tL1D_cache"); + total_css += css; + } + fprintf(fout, "\tL1D_total_cache_accesses = %llu\n", total_css.accesses); + fprintf(fout, "\tL1D_total_cache_misses = %llu\n", total_css.misses); + if (total_css.accesses > 0) { + fprintf(fout, "\tL1D_total_cache_miss_rate = %.4lf\n", + (double)total_css.misses / (double)total_css.accesses); } + fprintf(fout, "\tL1D_total_cache_pending_hits = %llu\n", + total_css.pending_hits); + fprintf(fout, "\tL1D_total_cache_reservation_fails = %llu\n", + total_css.res_fails); + total_css.print_port_stats(fout, "\tL1D_cache"); + } - // L1C - if(!m_shader_config->m_L1C_config.disabled()){ - total_css.clear(); - css.clear(); - fprintf(fout, "L1C_cache:\n"); - for ( unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i ) { - m_cluster[i]->get_L1C_sub_stats(css); - total_css += css; - } - fprintf(fout, "\tL1C_total_cache_accesses = %llu\n", total_css.accesses); - fprintf(fout, "\tL1C_total_cache_misses = %llu\n", total_css.misses); - if(total_css.accesses > 0){ - fprintf(fout, "\tL1C_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses); - } - fprintf(fout, "\tL1C_total_cache_pending_hits = %llu\n", total_css.pending_hits); - fprintf(fout, "\tL1C_total_cache_reservation_fails = %llu\n", total_css.res_fails); + // L1C + if (!m_shader_config->m_L1C_config.disabled()) { + total_css.clear(); + css.clear(); + fprintf(fout, "L1C_cache:\n"); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) { + m_cluster[i]->get_L1C_sub_stats(css); + total_css += css; + } + fprintf(fout, "\tL1C_total_cache_accesses = %llu\n", total_css.accesses); + fprintf(fout, "\tL1C_total_cache_misses = %llu\n", total_css.misses); + if (total_css.accesses > 0) { + fprintf(fout, "\tL1C_total_cache_miss_rate = %.4lf\n", + (double)total_css.misses / (double)total_css.accesses); } + fprintf(fout, "\tL1C_total_cache_pending_hits = %llu\n", + total_css.pending_hits); + fprintf(fout, "\tL1C_total_cache_reservation_fails = %llu\n", + total_css.res_fails); + } - // L1T - if(!m_shader_config->m_L1T_config.disabled()){ - total_css.clear(); - css.clear(); - fprintf(fout, "L1T_cache:\n"); - for ( unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i ) { - m_cluster[i]->get_L1T_sub_stats(css); - total_css += css; - } - fprintf(fout, "\tL1T_total_cache_accesses = %llu\n", total_css.accesses); - fprintf(fout, "\tL1T_total_cache_misses = %llu\n", total_css.misses); - if(total_css.accesses > 0){ - fprintf(fout, "\tL1T_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses); - } - fprintf(fout, "\tL1T_total_cache_pending_hits = %llu\n", total_css.pending_hits); - fprintf(fout, "\tL1T_total_cache_reservation_fails = %llu\n", total_css.res_fails); + // L1T + if (!m_shader_config->m_L1T_config.disabled()) { + total_css.clear(); + css.clear(); + fprintf(fout, "L1T_cache:\n"); + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) { + m_cluster[i]->get_L1T_sub_stats(css); + total_css += css; + } + fprintf(fout, "\tL1T_total_cache_accesses = %llu\n", total_css.accesses); + fprintf(fout, "\tL1T_total_cache_misses = %llu\n", total_css.misses); + if (total_css.accesses > 0) { + fprintf(fout, "\tL1T_total_cache_miss_rate = %.4lf\n", + (double)total_css.misses / (double)total_css.accesses); } + fprintf(fout, "\tL1T_total_cache_pending_hits = %llu\n", + total_css.pending_hits); + fprintf(fout, "\tL1T_total_cache_reservation_fails = %llu\n", + total_css.res_fails); + } } -void gpgpu_sim::shader_print_l1_miss_stat( FILE *fout ) const -{ - unsigned total_d1_misses = 0, total_d1_accesses = 0; - for ( unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i ) { - unsigned custer_d1_misses = 0, cluster_d1_accesses = 0; - m_cluster[ i ]->print_cache_stats( fout, cluster_d1_accesses, custer_d1_misses ); - total_d1_misses += custer_d1_misses; - total_d1_accesses += cluster_d1_accesses; - } - fprintf( fout, "total_dl1_misses=%d\n", total_d1_misses ); - fprintf( fout, "total_dl1_accesses=%d\n", total_d1_accesses ); - fprintf( fout, "total_dl1_miss_rate= %f\n", (float)total_d1_misses / (float)total_d1_accesses ); - /* - fprintf(fout, "THD_INSN_AC: "); - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) - fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn_ac(i)); - fprintf(fout, "\n"); - fprintf(fout, "T_L1_Mss: "); //l1 miss rate per thread - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) - fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i)); - fprintf(fout, "\n"); - fprintf(fout, "T_L1_Mgs: "); //l1 merged miss rate per thread - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) - fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i) - m_sc[0]->get_thread_n_l1_mrghit_ac(i)); - fprintf(fout, "\n"); - fprintf(fout, "T_L1_Acc: "); //l1 access per thread - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) - fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_access_ac(i)); - fprintf(fout, "\n"); +void gpgpu_sim::shader_print_l1_miss_stat(FILE *fout) const { + unsigned total_d1_misses = 0, total_d1_accesses = 0; + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) { + unsigned custer_d1_misses = 0, cluster_d1_accesses = 0; + m_cluster[i]->print_cache_stats(fout, cluster_d1_accesses, + custer_d1_misses); + total_d1_misses += custer_d1_misses; + total_d1_accesses += cluster_d1_accesses; + } + fprintf(fout, "total_dl1_misses=%d\n", total_d1_misses); + fprintf(fout, "total_dl1_accesses=%d\n", total_d1_accesses); + fprintf(fout, "total_dl1_miss_rate= %f\n", + (float)total_d1_misses / (float)total_d1_accesses); + /* + fprintf(fout, "THD_INSN_AC: "); + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) + fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn_ac(i)); + fprintf(fout, "\n"); + fprintf(fout, "T_L1_Mss: "); //l1 miss rate per thread + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) + fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i)); + fprintf(fout, "\n"); + fprintf(fout, "T_L1_Mgs: "); //l1 merged miss rate per thread + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) + fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i) - + m_sc[0]->get_thread_n_l1_mrghit_ac(i)); fprintf(fout, "\n"); fprintf(fout, + "T_L1_Acc: "); //l1 access per thread for (unsigned i=0; + i<m_shader_config->n_thread_per_shader; i++) fprintf(fout, "%d ", + m_sc[0]->get_thread_n_l1_access_ac(i)); fprintf(fout, "\n"); - //per warp - int temp =0; - fprintf(fout, "W_L1_Mss: "); //l1 miss rate per warp - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { - temp += m_sc[0]->get_thread_n_l1_mis_ac(i); - if (i%m_shader_config->warp_size == (unsigned)(m_shader_config->warp_size-1)) { - fprintf(fout, "%d ", temp); - temp = 0; - } - } - fprintf(fout, "\n"); - temp=0; - fprintf(fout, "W_L1_Mgs: "); //l1 merged miss rate per warp - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { - temp += (m_sc[0]->get_thread_n_l1_mis_ac(i) - m_sc[0]->get_thread_n_l1_mrghit_ac(i) ); - if (i%m_shader_config->warp_size == (unsigned)(m_shader_config->warp_size-1)) { - fprintf(fout, "%d ", temp); - temp = 0; - } - } - fprintf(fout, "\n"); - temp =0; - fprintf(fout, "W_L1_Acc: "); //l1 access per warp - for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { - temp += m_sc[0]->get_thread_n_l1_access_ac(i); - if (i%m_shader_config->warp_size == (unsigned)(m_shader_config->warp_size-1)) { - fprintf(fout, "%d ", temp); - temp = 0; - } - } - fprintf(fout, "\n"); - */ + //per warp + int temp =0; + fprintf(fout, "W_L1_Mss: "); //l1 miss rate per warp + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { + temp += m_sc[0]->get_thread_n_l1_mis_ac(i); + if (i%m_shader_config->warp_size == + (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp = + 0; + } + } + fprintf(fout, "\n"); + temp=0; + fprintf(fout, "W_L1_Mgs: "); //l1 merged miss rate per warp + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { + temp += (m_sc[0]->get_thread_n_l1_mis_ac(i) - + m_sc[0]->get_thread_n_l1_mrghit_ac(i) ); if (i%m_shader_config->warp_size == + (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp = + 0; + } + } + fprintf(fout, "\n"); + temp =0; + fprintf(fout, "W_L1_Acc: "); //l1 access per warp + for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) { + temp += m_sc[0]->get_thread_n_l1_access_ac(i); + if (i%m_shader_config->warp_size == + (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp = + 0; + } + } + fprintf(fout, "\n"); + */ } -void warp_inst_t::print( FILE *fout ) const -{ - if (empty() ) { - fprintf(fout,"bubble\n" ); - return; - } else - fprintf(fout,"0x%04x ", pc ); - fprintf(fout, "w%02d[", m_warp_id); - for (unsigned j=0; j<m_config->warp_size; j++) - fprintf(fout, "%c", (active(j)?'1':'0') ); - fprintf(fout, "]: "); - m_config->gpgpu_ctx->func_sim->ptx_print_insn( pc, fout ); - fprintf(fout, "\n"); +void warp_inst_t::print(FILE *fout) const { + if (empty()) { + fprintf(fout, "bubble\n"); + return; + } else + fprintf(fout, "0x%04x ", pc); + fprintf(fout, "w%02d[", m_warp_id); + for (unsigned j = 0; j < m_config->warp_size; j++) + fprintf(fout, "%c", (active(j) ? '1' : '0')); + fprintf(fout, "]: "); + m_config->gpgpu_ctx->func_sim->ptx_print_insn(pc, fout); + fprintf(fout, "\n"); } -void shader_core_ctx::incexecstat(warp_inst_t *&inst) -{ - if(inst->mem_op==TEX) - inctex_stat(inst->active_count(),1); +void shader_core_ctx::incexecstat(warp_inst_t *&inst) { + if (inst->mem_op == TEX) inctex_stat(inst->active_count(), 1); - // Latency numbers for next operations are used to scale the power values - // for special operations, according observations from microbenchmarking - // TODO: put these numbers in the xml configuration + // Latency numbers for next operations are used to scale the power values + // for special operations, according observations from microbenchmarking + // TODO: put these numbers in the xml configuration - switch(inst->sp_op){ - case INT__OP: - incialu_stat(inst->active_count(),32); - break; - case INT_MUL_OP: - incimul_stat(inst->active_count(),7.2); - break; - case INT_MUL24_OP: - incimul24_stat(inst->active_count(),4.2); - break; - case INT_MUL32_OP: - incimul32_stat(inst->active_count(),4); - break; - case INT_DIV_OP: - incidiv_stat(inst->active_count(),40); - break; - case FP__OP: - incfpalu_stat(inst->active_count(),1); - break; - case FP_MUL_OP: - incfpmul_stat(inst->active_count(),1.8); - break; - case FP_DIV_OP: - incfpdiv_stat(inst->active_count(),48); - break; - case FP_SQRT_OP: - inctrans_stat(inst->active_count(),25); - break; - case FP_LG_OP: - inctrans_stat(inst->active_count(),35); - break; - case FP_SIN_OP: - inctrans_stat(inst->active_count(),12); - break; - case FP_EXP_OP: - inctrans_stat(inst->active_count(),35); - break; - default: - break; - } + switch (inst->sp_op) { + case INT__OP: + incialu_stat(inst->active_count(), 32); + break; + case INT_MUL_OP: + incimul_stat(inst->active_count(), 7.2); + break; + case INT_MUL24_OP: + incimul24_stat(inst->active_count(), 4.2); + break; + case INT_MUL32_OP: + incimul32_stat(inst->active_count(), 4); + break; + case INT_DIV_OP: + incidiv_stat(inst->active_count(), 40); + break; + case FP__OP: + incfpalu_stat(inst->active_count(), 1); + break; + case FP_MUL_OP: + incfpmul_stat(inst->active_count(), 1.8); + break; + case FP_DIV_OP: + incfpdiv_stat(inst->active_count(), 48); + break; + case FP_SQRT_OP: + inctrans_stat(inst->active_count(), 25); + break; + case FP_LG_OP: + inctrans_stat(inst->active_count(), 35); + break; + case FP_SIN_OP: + inctrans_stat(inst->active_count(), 12); + break; + case FP_EXP_OP: + inctrans_stat(inst->active_count(), 35); + break; + default: + break; + } } -void shader_core_ctx::print_stage(unsigned int stage, FILE *fout ) const -{ - m_pipeline_reg[stage].print(fout); - //m_pipeline_reg[stage].print(fout); +void shader_core_ctx::print_stage(unsigned int stage, FILE *fout) const { + m_pipeline_reg[stage].print(fout); + // m_pipeline_reg[stage].print(fout); } -void shader_core_ctx::display_simt_state(FILE *fout, int mask ) const -{ - if ( (mask & 4) && m_config->model == POST_DOMINATOR ) { - fprintf(fout,"per warp SIMT control-flow state:\n"); - unsigned n = m_config->n_thread_per_shader / m_config->warp_size; - for (unsigned i=0; i < n; i++) { - unsigned nactive = 0; - for (unsigned j=0; j<m_config->warp_size; j++ ) { - unsigned tid = i*m_config->warp_size + j; - int done = ptx_thread_done(tid); - nactive += (ptx_thread_done(tid)?0:1); - if ( done && (mask & 8) ) { - unsigned done_cycle = m_thread[tid]->donecycle(); - if ( done_cycle ) { - printf("\n w%02u:t%03u: done @ cycle %u", i, tid, done_cycle ); - } - } - } - if ( nactive == 0 ) { - continue; +void shader_core_ctx::display_simt_state(FILE *fout, int mask) const { + if ((mask & 4) && m_config->model == POST_DOMINATOR) { + fprintf(fout, "per warp SIMT control-flow state:\n"); + unsigned n = m_config->n_thread_per_shader / m_config->warp_size; + for (unsigned i = 0; i < n; i++) { + unsigned nactive = 0; + for (unsigned j = 0; j < m_config->warp_size; j++) { + unsigned tid = i * m_config->warp_size + j; + int done = ptx_thread_done(tid); + nactive += (ptx_thread_done(tid) ? 0 : 1); + if (done && (mask & 8)) { + unsigned done_cycle = m_thread[tid]->donecycle(); + if (done_cycle) { + printf("\n w%02u:t%03u: done @ cycle %u", i, tid, done_cycle); } - m_simt_stack[i]->print(fout); - } - fprintf(fout,"\n"); + } + } + if (nactive == 0) { + continue; + } + m_simt_stack[i]->print(fout); } + fprintf(fout, "\n"); + } } -void ldst_unit::print(FILE *fout) const -{ - fprintf(fout,"LD/ST unit = "); - m_dispatch_reg->print(fout); - if ( m_mem_rc != NO_RC_FAIL ) { - fprintf(fout," LD/ST stall condition: "); - switch ( m_mem_rc ) { - case BK_CONF: fprintf(fout,"BK_CONF"); break; - case MSHR_RC_FAIL: fprintf(fout,"MSHR_RC_FAIL"); break; - case ICNT_RC_FAIL: fprintf(fout,"ICNT_RC_FAIL"); break; - case COAL_STALL: fprintf(fout,"COAL_STALL"); break; - case WB_ICNT_RC_FAIL: fprintf(fout,"WB_ICNT_RC_FAIL"); break; - case WB_CACHE_RSRV_FAIL: fprintf(fout,"WB_CACHE_RSRV_FAIL"); break; - case N_MEM_STAGE_STALL_TYPE: fprintf(fout,"N_MEM_STAGE_STALL_TYPE"); break; - default: abort(); - } - fprintf(fout,"\n"); - } - fprintf(fout,"LD/ST wb = "); - m_next_wb.print(fout); - fprintf(fout, "Last LD/ST writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n", - m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle ); - fprintf(fout,"Pending register writes:\n"); - std::map<unsigned/*warp_id*/, std::map<unsigned/*regnum*/,unsigned/*count*/> >::const_iterator w; - for( w=m_pending_writes.begin(); w!=m_pending_writes.end(); w++ ) { - unsigned warp_id = w->first; - const std::map<unsigned/*regnum*/,unsigned/*count*/> &warp_info = w->second; - if( warp_info.empty() ) - continue; - fprintf(fout," w%2u : ", warp_id ); - std::map<unsigned/*regnum*/,unsigned/*count*/>::const_iterator r; - for( r=warp_info.begin(); r!=warp_info.end(); ++r ) { - fprintf(fout," %u(%u)", r->first, r->second ); - } - fprintf(fout,"\n"); +void ldst_unit::print(FILE *fout) const { + fprintf(fout, "LD/ST unit = "); + m_dispatch_reg->print(fout); + if (m_mem_rc != NO_RC_FAIL) { + fprintf(fout, " LD/ST stall condition: "); + switch (m_mem_rc) { + case BK_CONF: + fprintf(fout, "BK_CONF"); + break; + case MSHR_RC_FAIL: + fprintf(fout, "MSHR_RC_FAIL"); + break; + case ICNT_RC_FAIL: + fprintf(fout, "ICNT_RC_FAIL"); + break; + case COAL_STALL: + fprintf(fout, "COAL_STALL"); + break; + case WB_ICNT_RC_FAIL: + fprintf(fout, "WB_ICNT_RC_FAIL"); + break; + case WB_CACHE_RSRV_FAIL: + fprintf(fout, "WB_CACHE_RSRV_FAIL"); + break; + case N_MEM_STAGE_STALL_TYPE: + fprintf(fout, "N_MEM_STAGE_STALL_TYPE"); + break; + default: + abort(); } - m_L1C->display_state(fout); - m_L1T->display_state(fout); - if( !m_config->m_L1D_config.disabled() ) - m_L1D->display_state(fout); - fprintf(fout,"LD/ST response FIFO (occupancy = %zu):\n", m_response_fifo.size() ); - for( std::list<mem_fetch*>::const_iterator i=m_response_fifo.begin(); i != m_response_fifo.end(); i++ ) { - const mem_fetch *mf = *i; - mf->print(fout); + fprintf(fout, "\n"); + } + fprintf(fout, "LD/ST wb = "); + m_next_wb.print(fout); + fprintf( + fout, + "Last LD/ST writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n", + m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle); + fprintf(fout, "Pending register writes:\n"); + std::map<unsigned /*warp_id*/, + std::map<unsigned /*regnum*/, unsigned /*count*/> >::const_iterator + w; + for (w = m_pending_writes.begin(); w != m_pending_writes.end(); w++) { + unsigned warp_id = w->first; + const std::map<unsigned /*regnum*/, unsigned /*count*/> &warp_info = + w->second; + if (warp_info.empty()) continue; + fprintf(fout, " w%2u : ", warp_id); + std::map<unsigned /*regnum*/, unsigned /*count*/>::const_iterator r; + for (r = warp_info.begin(); r != warp_info.end(); ++r) { + fprintf(fout, " %u(%u)", r->first, r->second); } + fprintf(fout, "\n"); + } + m_L1C->display_state(fout); + m_L1T->display_state(fout); + if (!m_config->m_L1D_config.disabled()) m_L1D->display_state(fout); + fprintf(fout, "LD/ST response FIFO (occupancy = %zu):\n", + m_response_fifo.size()); + for (std::list<mem_fetch *>::const_iterator i = m_response_fifo.begin(); + i != m_response_fifo.end(); i++) { + const mem_fetch *mf = *i; + mf->print(fout); + } } -void shader_core_ctx::display_pipeline(FILE *fout, int print_mem, int mask ) const -{ - fprintf(fout, "=================================================\n"); - fprintf(fout, "shader %u at cycle %Lu+%Lu (%u threads running)\n", m_sid, - m_gpu->gpu_tot_sim_cycle, m_gpu->gpu_sim_cycle, m_not_completed); - fprintf(fout, "=================================================\n"); - - dump_warp_state(fout); - fprintf(fout,"\n"); +void shader_core_ctx::display_pipeline(FILE *fout, int print_mem, + int mask) const { + fprintf(fout, "=================================================\n"); + fprintf(fout, "shader %u at cycle %Lu+%Lu (%u threads running)\n", m_sid, + m_gpu->gpu_tot_sim_cycle, m_gpu->gpu_sim_cycle, m_not_completed); + fprintf(fout, "=================================================\n"); - m_L1I->display_state(fout); + dump_warp_state(fout); + fprintf(fout, "\n"); - fprintf(fout, "IF/ID = "); - if( !m_inst_fetch_buffer.m_valid ) - fprintf(fout,"bubble\n"); - else { - fprintf(fout,"w%2u : pc = 0x%x, nbytes = %u\n", - m_inst_fetch_buffer.m_warp_id, - m_inst_fetch_buffer.m_pc, - m_inst_fetch_buffer.m_nbytes ); - } - fprintf(fout,"\nibuffer status:\n"); - for( unsigned i=0; i<m_config->max_warps_per_shader; i++) { - if( !m_warp[i].ibuffer_empty() ) - m_warp[i].print_ibuffer(fout); - } - fprintf(fout,"\n"); - display_simt_state(fout,mask); - fprintf(fout, "-------------------------- Scoreboard\n"); - m_scoreboard->printContents(); -/* - fprintf(fout,"ID/OC (SP) = "); - print_stage(ID_OC_SP, fout); - fprintf(fout,"ID/OC (SFU) = "); - print_stage(ID_OC_SFU, fout); - fprintf(fout,"ID/OC (MEM) = "); - print_stage(ID_OC_MEM, fout); -*/ - fprintf(fout, "-------------------------- OP COL\n"); - m_operand_collector.dump(fout); -/* fprintf(fout, "OC/EX (SP) = "); - print_stage(OC_EX_SP, fout); - fprintf(fout, "OC/EX (SFU) = "); - print_stage(OC_EX_SFU, fout); - fprintf(fout, "OC/EX (MEM) = "); - print_stage(OC_EX_MEM, fout); -*/ - fprintf(fout, "-------------------------- Pipe Regs\n"); + m_L1I->display_state(fout); - for (unsigned i = 0; i < N_PIPELINE_STAGES; i++) { - fprintf(fout,"--- %s ---\n",pipeline_stage_name_decode[i]); - print_stage(i,fout);fprintf(fout,"\n"); - } + fprintf(fout, "IF/ID = "); + if (!m_inst_fetch_buffer.m_valid) + fprintf(fout, "bubble\n"); + else { + fprintf(fout, "w%2u : pc = 0x%x, nbytes = %u\n", + m_inst_fetch_buffer.m_warp_id, m_inst_fetch_buffer.m_pc, + m_inst_fetch_buffer.m_nbytes); + } + fprintf(fout, "\nibuffer status:\n"); + for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { + if (!m_warp[i].ibuffer_empty()) m_warp[i].print_ibuffer(fout); + } + fprintf(fout, "\n"); + display_simt_state(fout, mask); + fprintf(fout, "-------------------------- Scoreboard\n"); + m_scoreboard->printContents(); + /* + fprintf(fout,"ID/OC (SP) = "); + print_stage(ID_OC_SP, fout); + fprintf(fout,"ID/OC (SFU) = "); + print_stage(ID_OC_SFU, fout); + fprintf(fout,"ID/OC (MEM) = "); + print_stage(ID_OC_MEM, fout); + */ + fprintf(fout, "-------------------------- OP COL\n"); + m_operand_collector.dump(fout); + /* fprintf(fout, "OC/EX (SP) = "); + print_stage(OC_EX_SP, fout); + fprintf(fout, "OC/EX (SFU) = "); + print_stage(OC_EX_SFU, fout); + fprintf(fout, "OC/EX (MEM) = "); + print_stage(OC_EX_MEM, fout); + */ + fprintf(fout, "-------------------------- Pipe Regs\n"); - fprintf(fout, "-------------------------- Fu\n"); - for( unsigned n=0; n < m_num_function_units; n++ ){ - m_fu[n]->print(fout); - fprintf(fout, "---------------\n"); - } - fprintf(fout, "-------------------------- other:\n"); + for (unsigned i = 0; i < N_PIPELINE_STAGES; i++) { + fprintf(fout, "--- %s ---\n", pipeline_stage_name_decode[i]); + print_stage(i, fout); + fprintf(fout, "\n"); + } - for(unsigned i=0; i<num_result_bus; i++){ - std::string bits = m_result_bus[i]->to_string(); - fprintf(fout, "EX/WB sched[%d]= %s\n", i, bits.c_str() ); - } - fprintf(fout, "EX/WB = "); - print_stage(EX_WB, fout); - fprintf(fout, "\n"); - fprintf(fout, "Last EX/WB writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n", - m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle ); + fprintf(fout, "-------------------------- Fu\n"); + for (unsigned n = 0; n < m_num_function_units; n++) { + m_fu[n]->print(fout); + fprintf(fout, "---------------\n"); + } + fprintf(fout, "-------------------------- other:\n"); - if( m_active_threads.count() <= 2*m_config->warp_size ) { - fprintf(fout,"Active Threads : "); - unsigned last_warp_id = -1; - for(unsigned tid=0; tid < m_active_threads.size(); tid++ ) { - unsigned warp_id = tid/m_config->warp_size; - if( m_active_threads.test(tid) ) { - if( warp_id != last_warp_id ) { - fprintf(fout,"\n warp %u : ", warp_id ); - last_warp_id=warp_id; - } - fprintf(fout,"%u ", tid ); - } - } - } + for (unsigned i = 0; i < num_result_bus; i++) { + std::string bits = m_result_bus[i]->to_string(); + fprintf(fout, "EX/WB sched[%d]= %s\n", i, bits.c_str()); + } + fprintf(fout, "EX/WB = "); + print_stage(EX_WB, fout); + fprintf(fout, "\n"); + fprintf( + fout, + "Last EX/WB writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n", + m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle); + if (m_active_threads.count() <= 2 * m_config->warp_size) { + fprintf(fout, "Active Threads : "); + unsigned last_warp_id = -1; + for (unsigned tid = 0; tid < m_active_threads.size(); tid++) { + unsigned warp_id = tid / m_config->warp_size; + if (m_active_threads.test(tid)) { + if (warp_id != last_warp_id) { + fprintf(fout, "\n warp %u : ", warp_id); + last_warp_id = warp_id; + } + fprintf(fout, "%u ", tid); + } + } + } } -unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const -{ - unsigned threads_per_cta = k.threads_per_cta(); - const class function_info *kernel = k.entry(); - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); +unsigned int shader_core_config::max_cta(const kernel_info_t &k) const { + unsigned threads_per_cta = k.threads_per_cta(); + const class function_info *kernel = k.entry(); + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); - //Limit by n_threads/shader - unsigned int result_thread = n_thread_per_shader / padded_cta_size; + // Limit by n_threads/shader + unsigned int result_thread = n_thread_per_shader / padded_cta_size; - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel); - //Limit by shmem/shader - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; + // Limit by shmem/shader + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; - //Limit by register count, rounded up to multiple of 4. - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); + // Limit by register count, rounded up to multiple of 4. + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / + (padded_cta_size * ((kernel_info->regs + 3) & ~3)); - //Limit by CTA - unsigned int result_cta = max_cta_per_core; + // Limit by CTA + unsigned int result_cta = max_cta_per_core; - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); - static const struct gpgpu_ptx_sim_info* last_kinfo = NULL; - if (last_kinfo != kernel_info) { //Only print out stats if kernel_info struct changes - last_kinfo = kernel_info; - printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf (" threads"); - if (result == result_shmem) printf (" shmem"); - if (result == result_regs) printf (" regs"); - if (result == result_cta) printf (" cta_limit"); - printf ("\n"); - } + static const struct gpgpu_ptx_sim_info *last_kinfo = NULL; + if (last_kinfo != + kernel_info) { // Only print out stats if kernel_info struct changes + last_kinfo = kernel_info; + printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf(" threads"); + if (result == result_shmem) printf(" shmem"); + if (result == result_regs) printf(" regs"); + if (result == result_cta) printf(" cta_limit"); + printf("\n"); + } - //gpu_max_cta_per_shader is limited by number of CTAs if not enough to keep all cores busy - if( k.num_blocks() < result*num_shader() ) { - result = k.num_blocks() / num_shader(); - if (k.num_blocks() % num_shader()) - result++; - } + // gpu_max_cta_per_shader is limited by number of CTAs if not enough to keep + // all cores busy + if (k.num_blocks() < result * num_shader()) { + result = k.num_blocks() / num_shader(); + if (k.num_blocks() % num_shader()) result++; + } - assert( result <= MAX_CTA_PER_SHADER ); - if (result < 1) { - printf ("GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader has.\n"); - if(gpgpu_ignore_resources_limitation) { - printf ("GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore the ERROR!\n"); - return 1; - } - abort(); + assert(result <= MAX_CTA_PER_SHADER); + if (result < 1) { + printf( + "GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader " + "has.\n"); + if (gpgpu_ignore_resources_limitation) { + printf( + "GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore " + "the ERROR!\n"); + return 1; } + abort(); + } - if(adaptive_volta_cache_config && !k.volta_cache_config_set) { - //For Volta, we assign the remaining shared memory to L1 cache - //For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x - unsigned total_shmed = kernel_info->smem * result; - assert(total_shmed >=0 && total_shmed <= gpgpu_shmem_size); - assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared - assert(m_L1D_config.get_nset() == 4); //Volta L1 has four sets - if(total_shmed < gpgpu_shmem_size){ - if(total_shmed == 0) - m_L1D_config.set_assoc(256); //L1 is 128KB ans shd=0 - else if(total_shmed > 0 && total_shmed <= 8192) - m_L1D_config.set_assoc(240); //L1 is 120KB ans shd=8KB - else if(total_shmed > 8192 && total_shmed <= 16384) - m_L1D_config.set_assoc(224); //L1 is 112KB ans shd=16KB - else if(total_shmed > 16384 && total_shmed <= 32768) - m_L1D_config.set_assoc(192); //L1 is 96KB ans shd=32KB - else if(total_shmed > 32768 && total_shmed <= 65536) - m_L1D_config.set_assoc(128); //L1 is 64KB ans shd=64KB - else if(total_shmed > 65536 && total_shmed <= gpgpu_shmem_size) - m_L1D_config.set_assoc(64); //L1 is 32KB and shd=96KB - else - assert(0); - - printf ("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", m_L1D_config.get_total_size_inKB()); - } + if (adaptive_volta_cache_config && !k.volta_cache_config_set) { + // For Volta, we assign the remaining shared memory to L1 cache + // For more info, see + // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x + unsigned total_shmed = kernel_info->smem * result; + assert(total_shmed >= 0 && total_shmed <= gpgpu_shmem_size); + assert(gpgpu_shmem_size == 98304); // Volta has 96 KB shared + assert(m_L1D_config.get_nset() == 4); // Volta L1 has four sets + if (total_shmed < gpgpu_shmem_size) { + if (total_shmed == 0) + m_L1D_config.set_assoc(256); // L1 is 128KB ans shd=0 + else if (total_shmed > 0 && total_shmed <= 8192) + m_L1D_config.set_assoc(240); // L1 is 120KB ans shd=8KB + else if (total_shmed > 8192 && total_shmed <= 16384) + m_L1D_config.set_assoc(224); // L1 is 112KB ans shd=16KB + else if (total_shmed > 16384 && total_shmed <= 32768) + m_L1D_config.set_assoc(192); // L1 is 96KB ans shd=32KB + else if (total_shmed > 32768 && total_shmed <= 65536) + m_L1D_config.set_assoc(128); // L1 is 64KB ans shd=64KB + else if (total_shmed > 65536 && total_shmed <= gpgpu_shmem_size) + m_L1D_config.set_assoc(64); // L1 is 32KB and shd=96KB + else + assert(0); - k.volta_cache_config_set = true; + printf("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", + m_L1D_config.get_total_size_inKB()); } - return result; + k.volta_cache_config_set = true; + } + + return result; } void shader_core_config::set_pipeline_latency() { + // calculate the max latency based on the input - //calculate the max latency based on the input - - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0],&int_latency[1],&int_latency[2], - &int_latency[3],&int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0],&fp_latency[1],&fp_latency[2], - &fp_latency[3],&fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0],&dp_latency[1],&dp_latency[2], - &dp_latency[3],&dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", - &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", - &tensor_latency); - - //all div operation are executed on sfu - //assume that the max latency are dp div or normal sfu_latency - max_sfu_latency = std::max(dp_latency[4],sfu_latency); - //assume that the max operation has the max latency - max_sp_latency = fp_latency[1]; - max_int_latency = int_latency[1]; - max_dp_latency = dp_latency[1]; - max_tensor_core_latency = tensor_latency; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], + &int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], + &fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], + &dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); + // all div operation are executed on sfu + // assume that the max latency are dp div or normal sfu_latency + max_sfu_latency = std::max(dp_latency[4], sfu_latency); + // assume that the max operation has the max latency + max_sp_latency = fp_latency[1]; + max_int_latency = int_latency[1]; + max_dp_latency = dp_latency[1]; + max_tensor_core_latency = tensor_latency; } -void shader_core_ctx::cycle() -{ - if(!isactive() && get_not_completed() == 0) - return; +void shader_core_ctx::cycle() { + if (!isactive() && get_not_completed() == 0) return; - m_stats->shader_cycles[m_sid]++; - writeback(); - execute(); - read_operands(); - issue(); - decode(); - fetch(); + m_stats->shader_cycles[m_sid]++; + writeback(); + execute(); + read_operands(); + issue(); + decode(); + fetch(); } // Flushes all content of the cache to memory -void shader_core_ctx::cache_flush() -{ - m_ldst_unit->flush(); -} +void shader_core_ctx::cache_flush() { m_ldst_unit->flush(); } -void shader_core_ctx::cache_invalidate() -{ - m_ldst_unit->invalidate(); -} +void shader_core_ctx::cache_invalidate() { m_ldst_unit->invalidate(); } // modifiers -std::list<opndcoll_rfu_t::op_t> opndcoll_rfu_t::arbiter_t::allocate_reads() -{ - std::list<op_t> result; // a list of registers that (a) are in different register banks, (b) do not go to the same operand collector +std::list<opndcoll_rfu_t::op_t> opndcoll_rfu_t::arbiter_t::allocate_reads() { + std::list<op_t> + result; // a list of registers that (a) are in different register banks, + // (b) do not go to the same operand collector - int input; - int output; - int _inputs = m_num_banks; - int _outputs = m_num_collectors; - int _square = ( _inputs > _outputs ) ? _inputs : _outputs; - assert(_square > 0); - int _pri = (int)m_last_cu; + int input; + int output; + int _inputs = m_num_banks; + int _outputs = m_num_collectors; + int _square = (_inputs > _outputs) ? _inputs : _outputs; + assert(_square > 0); + int _pri = (int)m_last_cu; - // Clear matching - for ( int i = 0; i < _inputs; ++i ) - _inmatch[i] = -1; - for ( int j = 0; j < _outputs; ++j ) - _outmatch[j] = -1; + // Clear matching + for (int i = 0; i < _inputs; ++i) _inmatch[i] = -1; + for (int j = 0; j < _outputs; ++j) _outmatch[j] = -1; - for( unsigned i=0; i<m_num_banks; i++) { - for( unsigned j=0; j<m_num_collectors; j++) { - assert( i < (unsigned)_inputs ); - assert( j < (unsigned)_outputs ); - _request[i][j] = 0; - } - if( !m_queue[i].empty() ) { - const op_t &op = m_queue[i].front(); - int oc_id = op.get_oc_id(); - assert( i < (unsigned)_inputs ); - assert( oc_id < _outputs ); - _request[i][oc_id] = 1; - } - if( m_allocated_bank[i].is_write() ) { - assert( i < (unsigned)_inputs ); - _inmatch[i] = 0; // write gets priority - } - } + for (unsigned i = 0; i < m_num_banks; i++) { + for (unsigned j = 0; j < m_num_collectors; j++) { + assert(i < (unsigned)_inputs); + assert(j < (unsigned)_outputs); + _request[i][j] = 0; + } + if (!m_queue[i].empty()) { + const op_t &op = m_queue[i].front(); + int oc_id = op.get_oc_id(); + assert(i < (unsigned)_inputs); + assert(oc_id < _outputs); + _request[i][oc_id] = 1; + } + if (m_allocated_bank[i].is_write()) { + assert(i < (unsigned)_inputs); + _inmatch[i] = 0; // write gets priority + } + } - ///// wavefront allocator from booksim... ---> - - // Loop through diagonals of request matrix + ///// wavefront allocator from booksim... ---> - for ( int p = 0; p < _square; ++p ) { - output = ( _pri + p ) % _square; + // Loop through diagonals of request matrix - // Step through the current diagonal - for ( input = 0; input < _inputs; ++input ) { - assert( input < _inputs ); - assert( output < _outputs ); - if ( ( output < _outputs ) && - ( _inmatch[input] == -1 ) && - ( _outmatch[output] == -1 ) && - ( _request[input][output]/*.label != -1*/ ) ) { - // Grant! - _inmatch[input] = output; - _outmatch[output] = input; - } + for (int p = 0; p < _square; ++p) { + output = (_pri + p) % _square; - output = ( output + 1 ) % _square; + // Step through the current diagonal + for (input = 0; input < _inputs; ++input) { + assert(input < _inputs); + assert(output < _outputs); + if ((output < _outputs) && (_inmatch[input] == -1) && + (_outmatch[output] == -1) && + (_request[input][output] /*.label != -1*/)) { + // Grant! + _inmatch[input] = output; + _outmatch[output] = input; } - } - // Round-robin the priority diagonal - _pri = ( _pri + 1 ) % _square; + output = (output + 1) % _square; + } + } - /// <--- end code from booksim + // Round-robin the priority diagonal + _pri = (_pri + 1) % _square; - m_last_cu = _pri; - for( unsigned i=0; i < m_num_banks; i++ ) { - if( _inmatch[i] != -1 ) { - if( !m_allocated_bank[i].is_write() ) { - unsigned bank = (unsigned)i; - op_t &op = m_queue[bank].front(); - result.push_back(op); - m_queue[bank].pop_front(); - } + /// <--- end code from booksim + + m_last_cu = _pri; + for (unsigned i = 0; i < m_num_banks; i++) { + if (_inmatch[i] != -1) { + if (!m_allocated_bank[i].is_write()) { + unsigned bank = (unsigned)i; + op_t &op = m_queue[bank].front(); + result.push_back(op); + m_queue[bank].pop_front(); } - } + } + } - return result; + return result; } -barrier_set_t::barrier_set_t(shader_core_ctx *shader,unsigned max_warps_per_core, unsigned max_cta_per_core, unsigned max_barriers_per_cta, unsigned warp_size) -{ - m_max_warps_per_core = max_warps_per_core; - m_max_cta_per_core = max_cta_per_core; - m_max_barriers_per_cta = max_barriers_per_cta; - m_warp_size = warp_size; - m_shader = shader; - if( max_warps_per_core > WARP_PER_CTA_MAX ) { - printf("ERROR ** increase WARP_PER_CTA_MAX in shader.h from %u to >= %u or warps per cta in gpgpusim.config\n", - WARP_PER_CTA_MAX, max_warps_per_core ); - exit(1); - } - if(max_barriers_per_cta > MAX_BARRIERS_PER_CTA){ - printf("ERROR ** increase MAX_BARRIERS_PER_CTA in abstract_hardware_model.h from %u to >= %u or barriers per cta in gpgpusim.config\n", - MAX_BARRIERS_PER_CTA, max_barriers_per_cta ); - exit(1); - } - m_warp_active.reset(); - m_warp_at_barrier.reset(); - for(unsigned i=0; i<max_barriers_per_cta; i++){ - m_bar_id_to_warps[i].reset(); - } +barrier_set_t::barrier_set_t(shader_core_ctx *shader, + unsigned max_warps_per_core, + unsigned max_cta_per_core, + unsigned max_barriers_per_cta, + unsigned warp_size) { + m_max_warps_per_core = max_warps_per_core; + m_max_cta_per_core = max_cta_per_core; + m_max_barriers_per_cta = max_barriers_per_cta; + m_warp_size = warp_size; + m_shader = shader; + if (max_warps_per_core > WARP_PER_CTA_MAX) { + printf( + "ERROR ** increase WARP_PER_CTA_MAX in shader.h from %u to >= %u or " + "warps per cta in gpgpusim.config\n", + WARP_PER_CTA_MAX, max_warps_per_core); + exit(1); + } + if (max_barriers_per_cta > MAX_BARRIERS_PER_CTA) { + printf( + "ERROR ** increase MAX_BARRIERS_PER_CTA in abstract_hardware_model.h " + "from %u to >= %u or barriers per cta in gpgpusim.config\n", + MAX_BARRIERS_PER_CTA, max_barriers_per_cta); + exit(1); + } + m_warp_active.reset(); + m_warp_at_barrier.reset(); + for (unsigned i = 0; i < max_barriers_per_cta; i++) { + m_bar_id_to_warps[i].reset(); + } } // during cta allocation -void barrier_set_t::allocate_barrier( unsigned cta_id, warp_set_t warps ) -{ - assert( cta_id < m_max_cta_per_core ); - cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); - assert( w == m_cta_to_warps.end() ); // cta should not already be active or allocated barrier resources - m_cta_to_warps[cta_id] = warps; - assert( m_cta_to_warps.size() <= m_max_cta_per_core ); // catch cta's that were not properly deallocated - - m_warp_active |= warps; - m_warp_at_barrier &= ~warps; - for(unsigned i=0; i<m_max_barriers_per_cta; i++){ - m_bar_id_to_warps[i] &=~warps; - } +void barrier_set_t::allocate_barrier(unsigned cta_id, warp_set_t warps) { + assert(cta_id < m_max_cta_per_core); + cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id); + assert(w == m_cta_to_warps.end()); // cta should not already be active or + // allocated barrier resources + m_cta_to_warps[cta_id] = warps; + assert(m_cta_to_warps.size() <= + m_max_cta_per_core); // catch cta's that were not properly deallocated + m_warp_active |= warps; + m_warp_at_barrier &= ~warps; + for (unsigned i = 0; i < m_max_barriers_per_cta; i++) { + m_bar_id_to_warps[i] &= ~warps; + } } // during cta deallocation -void barrier_set_t::deallocate_barrier( unsigned cta_id ) -{ - cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); - if( w == m_cta_to_warps.end() ) - return; - warp_set_t warps = w->second; - warp_set_t at_barrier = warps & m_warp_at_barrier; - assert( at_barrier.any() == false ); // no warps stuck at barrier - warp_set_t active = warps & m_warp_active; - assert( active.any() == false ); // no warps in CTA still running - m_warp_active &= ~warps; - m_warp_at_barrier &= ~warps; +void barrier_set_t::deallocate_barrier(unsigned cta_id) { + cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id); + if (w == m_cta_to_warps.end()) return; + warp_set_t warps = w->second; + warp_set_t at_barrier = warps & m_warp_at_barrier; + assert(at_barrier.any() == false); // no warps stuck at barrier + warp_set_t active = warps & m_warp_active; + assert(active.any() == false); // no warps in CTA still running + m_warp_active &= ~warps; + m_warp_at_barrier &= ~warps; - for(unsigned i=0; i<m_max_barriers_per_cta; i++){ - warp_set_t at_a_specific_barrier = warps & m_bar_id_to_warps[i]; - assert( at_a_specific_barrier.any() == false ); // no warps stuck at barrier - m_bar_id_to_warps[i] &=~warps; - } - m_cta_to_warps.erase(w); + for (unsigned i = 0; i < m_max_barriers_per_cta; i++) { + warp_set_t at_a_specific_barrier = warps & m_bar_id_to_warps[i]; + assert(at_a_specific_barrier.any() == false); // no warps stuck at barrier + m_bar_id_to_warps[i] &= ~warps; + } + m_cta_to_warps.erase(w); } // individual warp hits barrier -void barrier_set_t::warp_reaches_barrier(unsigned cta_id,unsigned warp_id,warp_inst_t* inst) -{ - barrier_type bar_type = inst->bar_type; - unsigned bar_id = inst->bar_id; - unsigned bar_count = inst->bar_count; - assert(bar_id!=(unsigned)-1); - cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); - - if( w == m_cta_to_warps.end() ) { // cta is active - printf("ERROR ** cta_id %u not found in barrier set on cycle %llu+%llu...\n", cta_id, m_shader->get_gpu()->gpu_tot_sim_cycle, m_shader->get_gpu()->gpu_sim_cycle ); - dump(); - abort(); - } - assert( w->second.test(warp_id) == true ); // warp is in cta +void barrier_set_t::warp_reaches_barrier(unsigned cta_id, unsigned warp_id, + warp_inst_t *inst) { + barrier_type bar_type = inst->bar_type; + unsigned bar_id = inst->bar_id; + unsigned bar_count = inst->bar_count; + assert(bar_id != (unsigned)-1); + cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id); - m_bar_id_to_warps[bar_id].set(warp_id); - if(bar_type==SYNC || bar_type==RED){ - m_warp_at_barrier.set(warp_id); - } - warp_set_t warps_in_cta = w->second; - warp_set_t at_barrier = warps_in_cta & m_bar_id_to_warps[bar_id]; - warp_set_t active = warps_in_cta & m_warp_active; - if(bar_count==(unsigned)-1){ - if( at_barrier == active ) { - // all warps have reached barrier, so release waiting warps... - m_bar_id_to_warps[bar_id] &= ~at_barrier; - m_warp_at_barrier &= ~at_barrier; - if(bar_type==RED){ - m_shader->broadcast_barrier_reduction(cta_id, bar_id,at_barrier); - } - } - }else{ - // TODO: check on the hardware if the count should include warp that exited - if ((at_barrier.count() * m_warp_size) == bar_count){ - // required number of warps have reached barrier, so release waiting warps... - m_bar_id_to_warps[bar_id] &= ~at_barrier; - m_warp_at_barrier &= ~at_barrier; - if(bar_type==RED){ - m_shader->broadcast_barrier_reduction(cta_id, bar_id,at_barrier); - } - } + if (w == m_cta_to_warps.end()) { // cta is active + printf( + "ERROR ** cta_id %u not found in barrier set on cycle %llu+%llu...\n", + cta_id, m_shader->get_gpu()->gpu_tot_sim_cycle, + m_shader->get_gpu()->gpu_sim_cycle); + dump(); + abort(); } + assert(w->second.test(warp_id) == true); // warp is in cta - + m_bar_id_to_warps[bar_id].set(warp_id); + if (bar_type == SYNC || bar_type == RED) { + m_warp_at_barrier.set(warp_id); + } + warp_set_t warps_in_cta = w->second; + warp_set_t at_barrier = warps_in_cta & m_bar_id_to_warps[bar_id]; + warp_set_t active = warps_in_cta & m_warp_active; + if (bar_count == (unsigned)-1) { + if (at_barrier == active) { + // all warps have reached barrier, so release waiting warps... + m_bar_id_to_warps[bar_id] &= ~at_barrier; + m_warp_at_barrier &= ~at_barrier; + if (bar_type == RED) { + m_shader->broadcast_barrier_reduction(cta_id, bar_id, at_barrier); + } + } + } else { + // TODO: check on the hardware if the count should include warp that exited + if ((at_barrier.count() * m_warp_size) == bar_count) { + // required number of warps have reached barrier, so release waiting + // warps... + m_bar_id_to_warps[bar_id] &= ~at_barrier; + m_warp_at_barrier &= ~at_barrier; + if (bar_type == RED) { + m_shader->broadcast_barrier_reduction(cta_id, bar_id, at_barrier); + } + } + } } +// warp reaches exit +void barrier_set_t::warp_exit(unsigned warp_id) { + // caller needs to verify all threads in warp are done, e.g., by checking PDOM + // stack to see it has only one entry during exit_impl() + m_warp_active.reset(warp_id); -// warp reaches exit -void barrier_set_t::warp_exit( unsigned warp_id ) -{ - // caller needs to verify all threads in warp are done, e.g., by checking PDOM stack to - // see it has only one entry during exit_impl() - m_warp_active.reset(warp_id); - - // test for barrier release - cta_to_warp_t::iterator w=m_cta_to_warps.begin(); - for (; w != m_cta_to_warps.end(); ++w) { - if (w->second.test(warp_id) == true) break; - } - warp_set_t warps_in_cta = w->second; - warp_set_t active = warps_in_cta & m_warp_active; + // test for barrier release + cta_to_warp_t::iterator w = m_cta_to_warps.begin(); + for (; w != m_cta_to_warps.end(); ++w) { + if (w->second.test(warp_id) == true) break; + } + warp_set_t warps_in_cta = w->second; + warp_set_t active = warps_in_cta & m_warp_active; - for(unsigned i=0; i<m_max_barriers_per_cta; i++){ - warp_set_t at_a_specific_barrier = warps_in_cta & m_bar_id_to_warps[i]; - if( at_a_specific_barrier == active ) { - // all warps have reached barrier, so release waiting warps... - m_bar_id_to_warps[i] &= ~at_a_specific_barrier; - m_warp_at_barrier &= ~at_a_specific_barrier; - } - } + for (unsigned i = 0; i < m_max_barriers_per_cta; i++) { + warp_set_t at_a_specific_barrier = warps_in_cta & m_bar_id_to_warps[i]; + if (at_a_specific_barrier == active) { + // all warps have reached barrier, so release waiting warps... + m_bar_id_to_warps[i] &= ~at_a_specific_barrier; + m_warp_at_barrier &= ~at_a_specific_barrier; + } + } } // assertions -bool barrier_set_t::warp_waiting_at_barrier( unsigned warp_id ) const -{ - return m_warp_at_barrier.test(warp_id); +bool barrier_set_t::warp_waiting_at_barrier(unsigned warp_id) const { + return m_warp_at_barrier.test(warp_id); } -void barrier_set_t::dump() -{ - printf( "barrier set information\n"); - printf( " m_max_cta_per_core = %u\n", m_max_cta_per_core ); - printf( " m_max_warps_per_core = %u\n", m_max_warps_per_core ); - printf( " m_max_barriers_per_cta =%u\n", m_max_barriers_per_cta); - printf( " cta_to_warps:\n"); - - cta_to_warp_t::const_iterator i; - for( i=m_cta_to_warps.begin(); i!=m_cta_to_warps.end(); i++ ) { - unsigned cta_id = i->first; - warp_set_t warps = i->second; - printf(" cta_id %u : %s\n", cta_id, warps.to_string().c_str() ); - } - printf(" warp_active: %s\n", m_warp_active.to_string().c_str() ); - printf(" warp_at_barrier: %s\n", m_warp_at_barrier.to_string().c_str() ); - for( unsigned i=0; i<m_max_barriers_per_cta; i++){ - warp_set_t warps_reached_barrier = m_bar_id_to_warps[i]; - printf(" warp_at_barrier %u: %s\n", i, warps_reached_barrier.to_string().c_str() ); - } - fflush(stdout); -} - -void shader_core_ctx::warp_exit( unsigned warp_id ) -{ - bool done = true; - for ( unsigned i = warp_id*get_config()->warp_size; - i < (warp_id+1)*get_config()->warp_size; - i++ ) { +void barrier_set_t::dump() { + printf("barrier set information\n"); + printf(" m_max_cta_per_core = %u\n", m_max_cta_per_core); + printf(" m_max_warps_per_core = %u\n", m_max_warps_per_core); + printf(" m_max_barriers_per_cta =%u\n", m_max_barriers_per_cta); + printf(" cta_to_warps:\n"); -// if(this->m_thread[i]->m_functional_model_thread_state && this->m_thread[i].m_functional_model_thread_state->donecycle()==0) { -// done = false; -// } + cta_to_warp_t::const_iterator i; + for (i = m_cta_to_warps.begin(); i != m_cta_to_warps.end(); i++) { + unsigned cta_id = i->first; + warp_set_t warps = i->second; + printf(" cta_id %u : %s\n", cta_id, warps.to_string().c_str()); + } + printf(" warp_active: %s\n", m_warp_active.to_string().c_str()); + printf(" warp_at_barrier: %s\n", m_warp_at_barrier.to_string().c_str()); + for (unsigned i = 0; i < m_max_barriers_per_cta; i++) { + warp_set_t warps_reached_barrier = m_bar_id_to_warps[i]; + printf(" warp_at_barrier %u: %s\n", i, + warps_reached_barrier.to_string().c_str()); + } + fflush(stdout); +} +void shader_core_ctx::warp_exit(unsigned warp_id) { + bool done = true; + for (unsigned i = warp_id * get_config()->warp_size; + i < (warp_id + 1) * get_config()->warp_size; i++) { + // if(this->m_thread[i]->m_functional_model_thread_state && + //this->m_thread[i].m_functional_model_thread_state->donecycle()==0) { done + //= false; + // } - if (m_thread[i] && !m_thread[i]->is_done()) done = false; - } - //if (m_warp[warp_id].get_n_completed() == get_config()->warp_size) - //if (this->m_simt_stack[warp_id]->get_num_entries() == 0) - if (done) - m_barriers.warp_exit( warp_id ); + if (m_thread[i] && !m_thread[i]->is_done()) done = false; + } + // if (m_warp[warp_id].get_n_completed() == get_config()->warp_size) + // if (this->m_simt_stack[warp_id]->get_num_entries() == 0) + if (done) m_barriers.warp_exit(warp_id); } -bool shader_core_ctx::check_if_non_released_reduction_barrier(warp_inst_t &inst) -{ - unsigned warp_id = inst.warp_id(); - bool bar_red_op = (inst.op == BARRIER_OP) && (inst.bar_type == RED); - bool non_released_barrier_reduction = false; - bool warp_stucked_at_barrier = warp_waiting_at_barrier(warp_id); - bool single_inst_in_pipeline = (m_warp[warp_id].num_issued_inst_in_pipeline()==1); - non_released_barrier_reduction = single_inst_in_pipeline and warp_stucked_at_barrier and bar_red_op; - printf("non_released_barrier_reduction=%u\n",non_released_barrier_reduction); - return non_released_barrier_reduction; +bool shader_core_ctx::check_if_non_released_reduction_barrier( + warp_inst_t &inst) { + unsigned warp_id = inst.warp_id(); + bool bar_red_op = (inst.op == BARRIER_OP) && (inst.bar_type == RED); + bool non_released_barrier_reduction = false; + bool warp_stucked_at_barrier = warp_waiting_at_barrier(warp_id); + bool single_inst_in_pipeline = + (m_warp[warp_id].num_issued_inst_in_pipeline() == 1); + non_released_barrier_reduction = + single_inst_in_pipeline and warp_stucked_at_barrier and bar_red_op; + printf("non_released_barrier_reduction=%u\n", non_released_barrier_reduction); + return non_released_barrier_reduction; } -bool shader_core_ctx::warp_waiting_at_barrier( unsigned warp_id ) const -{ - return m_barriers.warp_waiting_at_barrier(warp_id); +bool shader_core_ctx::warp_waiting_at_barrier(unsigned warp_id) const { + return m_barriers.warp_waiting_at_barrier(warp_id); } -bool shader_core_ctx::warp_waiting_at_mem_barrier( unsigned warp_id ) -{ - if( !m_warp[warp_id].get_membar() ) - return false; - if( !m_scoreboard->pendingWrites(warp_id) ) { - m_warp[warp_id].clear_membar(); - return false; - } - return true; +bool shader_core_ctx::warp_waiting_at_mem_barrier(unsigned warp_id) { + if (!m_warp[warp_id].get_membar()) return false; + if (!m_scoreboard->pendingWrites(warp_id)) { + m_warp[warp_id].clear_membar(); + return false; + } + return true; } -void shader_core_ctx::set_max_cta( const kernel_info_t &kernel ) -{ - // calculate the max cta count and cta size for local memory address mapping - kernel_max_cta_per_shader = m_config->max_cta(kernel); - unsigned int gpu_cta_size = kernel.threads_per_cta(); - kernel_padded_threads_per_cta = (gpu_cta_size%m_config->warp_size) ? - m_config->warp_size*((gpu_cta_size/m_config->warp_size)+1) : - gpu_cta_size; +void shader_core_ctx::set_max_cta(const kernel_info_t &kernel) { + // calculate the max cta count and cta size for local memory address mapping + kernel_max_cta_per_shader = m_config->max_cta(kernel); + unsigned int gpu_cta_size = kernel.threads_per_cta(); + kernel_padded_threads_per_cta = + (gpu_cta_size % m_config->warp_size) + ? m_config->warp_size * ((gpu_cta_size / m_config->warp_size) + 1) + : gpu_cta_size; } -void shader_core_ctx::decrement_atomic_count( unsigned wid, unsigned n ) -{ - assert( m_warp[wid].get_n_atomic() >= n ); - m_warp[wid].dec_n_atomic(n); +void shader_core_ctx::decrement_atomic_count(unsigned wid, unsigned n) { + assert(m_warp[wid].get_n_atomic() >= n); + m_warp[wid].dec_n_atomic(n); } -void shader_core_ctx::broadcast_barrier_reduction(unsigned cta_id,unsigned bar_id,warp_set_t warps) -{ - for(unsigned i=0; i<m_config->max_warps_per_shader;i++){ - if(warps.test(i)){ - const warp_inst_t * inst = m_warp[i].restore_info_of_last_inst_at_barrier(); - const_cast<warp_inst_t *> (inst)->broadcast_barrier_reduction(inst->get_active_mask()); - } - } +void shader_core_ctx::broadcast_barrier_reduction(unsigned cta_id, + unsigned bar_id, + warp_set_t warps) { + for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { + if (warps.test(i)) { + const warp_inst_t *inst = + m_warp[i].restore_info_of_last_inst_at_barrier(); + const_cast<warp_inst_t *>(inst)->broadcast_barrier_reduction( + inst->get_active_mask()); + } + } } -bool shader_core_ctx::fetch_unit_response_buffer_full() const -{ - return false; -} +bool shader_core_ctx::fetch_unit_response_buffer_full() const { return false; } -void shader_core_ctx::accept_fetch_response( mem_fetch *mf ) -{ - mf->set_status(IN_SHADER_FETCHED,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - m_L1I->fill(mf,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); +void shader_core_ctx::accept_fetch_response(mem_fetch *mf) { + mf->set_status(IN_SHADER_FETCHED, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + m_L1I->fill(mf, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); } -bool shader_core_ctx::ldst_unit_response_buffer_full() const -{ - return m_ldst_unit->response_buffer_full(); +bool shader_core_ctx::ldst_unit_response_buffer_full() const { + return m_ldst_unit->response_buffer_full(); } -void shader_core_ctx::accept_ldst_unit_response(mem_fetch * mf) -{ - m_ldst_unit->fill(mf); +void shader_core_ctx::accept_ldst_unit_response(mem_fetch *mf) { + m_ldst_unit->fill(mf); } -void shader_core_ctx::store_ack( class mem_fetch *mf ) -{ - assert( mf->get_type() == WRITE_ACK || ( m_config->gpgpu_perfect_mem && mf->get_is_write() ) ); - unsigned warp_id = mf->get_wid(); - m_warp[warp_id].dec_store_req(); +void shader_core_ctx::store_ack(class mem_fetch *mf) { + assert(mf->get_type() == WRITE_ACK || + (m_config->gpgpu_perfect_mem && mf->get_is_write())); + unsigned warp_id = mf->get_wid(); + m_warp[warp_id].dec_store_req(); } -void shader_core_ctx::print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ) { - m_ldst_unit->print_cache_stats( fp, dl1_accesses, dl1_misses ); +void shader_core_ctx::print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses) { + m_ldst_unit->print_cache_stats(fp, dl1_accesses, dl1_misses); } -void shader_core_ctx::get_cache_stats(cache_stats &cs){ - // Adds stats from each cache to 'cs' - cs += m_L1I->get_stats(); // Get L1I stats - m_ldst_unit->get_cache_stats(cs); // Get L1D, L1C, L1T stats +void shader_core_ctx::get_cache_stats(cache_stats &cs) { + // Adds stats from each cache to 'cs' + cs += m_L1I->get_stats(); // Get L1I stats + m_ldst_unit->get_cache_stats(cs); // Get L1D, L1C, L1T stats } -void shader_core_ctx::get_L1I_sub_stats(struct cache_sub_stats &css) const{ - if(m_L1I) - m_L1I->get_sub_stats(css); +void shader_core_ctx::get_L1I_sub_stats(struct cache_sub_stats &css) const { + if (m_L1I) m_L1I->get_sub_stats(css); } -void shader_core_ctx::get_L1D_sub_stats(struct cache_sub_stats &css) const{ - m_ldst_unit->get_L1D_sub_stats(css); +void shader_core_ctx::get_L1D_sub_stats(struct cache_sub_stats &css) const { + m_ldst_unit->get_L1D_sub_stats(css); } -void shader_core_ctx::get_L1C_sub_stats(struct cache_sub_stats &css) const{ - m_ldst_unit->get_L1C_sub_stats(css); +void shader_core_ctx::get_L1C_sub_stats(struct cache_sub_stats &css) const { + m_ldst_unit->get_L1C_sub_stats(css); } -void shader_core_ctx::get_L1T_sub_stats(struct cache_sub_stats &css) const{ - m_ldst_unit->get_L1T_sub_stats(css); +void shader_core_ctx::get_L1T_sub_stats(struct cache_sub_stats &css) const { + m_ldst_unit->get_L1T_sub_stats(css); } -void shader_core_ctx::get_icnt_power_stats(long &n_simt_to_mem, long &n_mem_to_simt) const{ - n_simt_to_mem += m_stats->n_simt_to_mem[m_sid]; - n_mem_to_simt += m_stats->n_mem_to_simt[m_sid]; +void shader_core_ctx::get_icnt_power_stats(long &n_simt_to_mem, + long &n_mem_to_simt) const { + n_simt_to_mem += m_stats->n_simt_to_mem[m_sid]; + n_mem_to_simt += m_stats->n_mem_to_simt[m_sid]; } -bool shd_warp_t::functional_done() const -{ - return get_n_completed() == m_warp_size; +bool shd_warp_t::functional_done() const { + return get_n_completed() == m_warp_size; } -bool shd_warp_t::hardware_done() const -{ - return functional_done() && stores_done() && !inst_in_pipeline(); +bool shd_warp_t::hardware_done() const { + return functional_done() && stores_done() && !inst_in_pipeline(); } -bool shd_warp_t::waiting() -{ - if ( functional_done() ) { - // waiting to be initialized with a kernel - return true; - } else if ( m_shader->warp_waiting_at_barrier(m_warp_id) ) { - // waiting for other warps in CTA to reach barrier - return true; - } else if ( m_shader->warp_waiting_at_mem_barrier(m_warp_id) ) { - // waiting for memory barrier - return true; - } else if ( m_n_atomic >0 ) { - // waiting for atomic operation to complete at memory: - // this stall is not required for accurate timing model, but rather we - // stall here since if a call/return instruction occurs in the meantime - // the functional execution of the atomic when it hits DRAM can cause - // the wrong register to be read. - return true; - } - return false; +bool shd_warp_t::waiting() { + if (functional_done()) { + // waiting to be initialized with a kernel + return true; + } else if (m_shader->warp_waiting_at_barrier(m_warp_id)) { + // waiting for other warps in CTA to reach barrier + return true; + } else if (m_shader->warp_waiting_at_mem_barrier(m_warp_id)) { + // waiting for memory barrier + return true; + } else if (m_n_atomic > 0) { + // waiting for atomic operation to complete at memory: + // this stall is not required for accurate timing model, but rather we + // stall here since if a call/return instruction occurs in the meantime + // the functional execution of the atomic when it hits DRAM can cause + // the wrong register to be read. + return true; + } + return false; } -void shd_warp_t::print( FILE *fout ) const -{ - if( !done_exit() ) { - fprintf( fout, "w%02u npc: 0x%04x, done:%c%c%c%c:%2u i:%u s:%u a:%u (done: ", - m_warp_id, - m_next_pc, - (functional_done()?'f':' '), - (stores_done()?'s':' '), - (inst_in_pipeline()?' ':'i'), - (done_exit()?'e':' '), - n_completed, - m_inst_in_pipeline, - m_stores_outstanding, - m_n_atomic ); - for (unsigned i = m_warp_id*m_warp_size; i < (m_warp_id+1)*m_warp_size; i++ ) { - if ( m_shader->ptx_thread_done(i) ) fprintf(fout,"1"); - else fprintf(fout,"0"); - if ( (((i+1)%4) == 0) && (i+1) < (m_warp_id+1)*m_warp_size ) - fprintf(fout,","); - } - fprintf(fout,") "); - fprintf(fout," active=%s", m_active_threads.to_string().c_str() ); - fprintf(fout," last fetched @ %5llu", m_last_fetch); - if( m_imiss_pending ) - fprintf(fout," i-miss pending"); - fprintf(fout,"\n"); +void shd_warp_t::print(FILE *fout) const { + if (!done_exit()) { + fprintf(fout, "w%02u npc: 0x%04x, done:%c%c%c%c:%2u i:%u s:%u a:%u (done: ", + m_warp_id, m_next_pc, (functional_done() ? 'f' : ' '), + (stores_done() ? 's' : ' '), (inst_in_pipeline() ? ' ' : 'i'), + (done_exit() ? 'e' : ' '), n_completed, m_inst_in_pipeline, + m_stores_outstanding, m_n_atomic); + for (unsigned i = m_warp_id * m_warp_size; + i < (m_warp_id + 1) * m_warp_size; i++) { + if (m_shader->ptx_thread_done(i)) + fprintf(fout, "1"); + else + fprintf(fout, "0"); + if ((((i + 1) % 4) == 0) && (i + 1) < (m_warp_id + 1) * m_warp_size) + fprintf(fout, ","); } + fprintf(fout, ") "); + fprintf(fout, " active=%s", m_active_threads.to_string().c_str()); + fprintf(fout, " last fetched @ %5llu", m_last_fetch); + if (m_imiss_pending) fprintf(fout, " i-miss pending"); + fprintf(fout, "\n"); + } } -void shd_warp_t::print_ibuffer( FILE *fout ) const -{ - fprintf(fout," ibuffer[%2u] : ", m_warp_id ); - for( unsigned i=0; i < IBUFFER_SIZE; i++) { - const inst_t *inst = m_ibuffer[i].m_inst; - if( inst ) inst->print_insn(fout); - else if( m_ibuffer[i].m_valid ) - fprintf(fout," <invalid instruction> "); - else fprintf(fout," <empty> "); - } - fprintf(fout,"\n"); +void shd_warp_t::print_ibuffer(FILE *fout) const { + fprintf(fout, " ibuffer[%2u] : ", m_warp_id); + for (unsigned i = 0; i < IBUFFER_SIZE; i++) { + const inst_t *inst = m_ibuffer[i].m_inst; + if (inst) + inst->print_insn(fout); + else if (m_ibuffer[i].m_valid) + fprintf(fout, " <invalid instruction> "); + else + fprintf(fout, " <empty> "); + } + fprintf(fout, "\n"); } -void opndcoll_rfu_t::add_cu_set(unsigned set_id, unsigned num_cu, unsigned num_dispatch){ - m_cus[set_id].reserve(num_cu); //this is necessary to stop pointers in m_cu from being invalid do to a resize; - for (unsigned i = 0; i < num_cu; i++) { - m_cus[set_id].push_back(collector_unit_t()); - m_cu.push_back(&m_cus[set_id].back()); - } - // for now each collector set gets dedicated dispatch units. - for (unsigned i = 0; i < num_dispatch; i++) { - m_dispatch_units.push_back(dispatch_unit_t(&m_cus[set_id])); - } +void opndcoll_rfu_t::add_cu_set(unsigned set_id, unsigned num_cu, + unsigned num_dispatch) { + m_cus[set_id].reserve(num_cu); // this is necessary to stop pointers in m_cu + // from being invalid do to a resize; + for (unsigned i = 0; i < num_cu; i++) { + m_cus[set_id].push_back(collector_unit_t()); + m_cu.push_back(&m_cus[set_id].back()); + } + // for now each collector set gets dedicated dispatch units. + for (unsigned i = 0; i < num_dispatch; i++) { + m_dispatch_units.push_back(dispatch_unit_t(&m_cus[set_id])); + } } - -void opndcoll_rfu_t::add_port(port_vector_t & input, port_vector_t & output, uint_vector_t cu_sets) -{ - //m_num_ports++; - //m_num_collectors += num_collector_units; - //m_input.resize(m_num_ports); - //m_output.resize(m_num_ports); - //m_num_collector_units.resize(m_num_ports); - //m_input[m_num_ports-1]=input_port; - //m_output[m_num_ports-1]=output_port; - //m_num_collector_units[m_num_ports-1]=num_collector_units; - m_in_ports.push_back(input_port_t(input,output,cu_sets)); +void opndcoll_rfu_t::add_port(port_vector_t &input, port_vector_t &output, + uint_vector_t cu_sets) { + // m_num_ports++; + // m_num_collectors += num_collector_units; + // m_input.resize(m_num_ports); + // m_output.resize(m_num_ports); + // m_num_collector_units.resize(m_num_ports); + // m_input[m_num_ports-1]=input_port; + // m_output[m_num_ports-1]=output_port; + // m_num_collector_units[m_num_ports-1]=num_collector_units; + m_in_ports.push_back(input_port_t(input, output, cu_sets)); } -void opndcoll_rfu_t::init( unsigned num_banks, shader_core_ctx *shader ) -{ - m_shader=shader; - m_arbiter.init(m_cu.size(),num_banks); - //for( unsigned n=0; n<m_num_ports;n++ ) - // m_dispatch_units[m_output[n]].init( m_num_collector_units[n] ); - m_num_banks = num_banks; - m_bank_warp_shift = 0; - m_warp_size = shader->get_config()->warp_size; - m_bank_warp_shift = (unsigned)(int) (log(m_warp_size+0.5) / log(2.0)); - assert( (m_bank_warp_shift == 5) || (m_warp_size != 32) ); - - sub_core_model = shader->get_config()->sub_core_model; - m_num_warp_sceds = shader->get_config()->gpgpu_num_sched_per_core; - if(sub_core_model) - assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0); - m_num_banks_per_sched = num_banks / shader->get_config()->gpgpu_num_sched_per_core; - - for( unsigned j=0; j<m_cu.size(); j++) { - m_cu[j]->init(j,num_banks,m_bank_warp_shift,shader->get_config(),this, sub_core_model, m_num_banks_per_sched ); - } - m_initialized=true; - - +void opndcoll_rfu_t::init(unsigned num_banks, shader_core_ctx *shader) { + m_shader = shader; + m_arbiter.init(m_cu.size(), num_banks); + // for( unsigned n=0; n<m_num_ports;n++ ) + // m_dispatch_units[m_output[n]].init( m_num_collector_units[n] ); + m_num_banks = num_banks; + m_bank_warp_shift = 0; + m_warp_size = shader->get_config()->warp_size; + m_bank_warp_shift = (unsigned)(int)(log(m_warp_size + 0.5) / log(2.0)); + assert((m_bank_warp_shift == 5) || (m_warp_size != 32)); + sub_core_model = shader->get_config()->sub_core_model; + m_num_warp_sceds = shader->get_config()->gpgpu_num_sched_per_core; + if (sub_core_model) + assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0); + m_num_banks_per_sched = + num_banks / shader->get_config()->gpgpu_num_sched_per_core; + for (unsigned j = 0; j < m_cu.size(); j++) { + m_cu[j]->init(j, num_banks, m_bank_warp_shift, shader->get_config(), this, + sub_core_model, m_num_banks_per_sched); + } + m_initialized = true; } -int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id) -{ - int bank = regnum; - if (bank_warp_shift) - bank += wid; - if(sub_core_model) { - unsigned bank_num = (bank % banks_per_sched) + (sched_id * banks_per_sched); - assert(bank_num < num_banks); - return bank_num; - } - else - return bank % num_banks; +int register_bank(int regnum, int wid, unsigned num_banks, + unsigned bank_warp_shift, bool sub_core_model, + unsigned banks_per_sched, unsigned sched_id) { + int bank = regnum; + if (bank_warp_shift) bank += wid; + if (sub_core_model) { + unsigned bank_num = (bank % banks_per_sched) + (sched_id * banks_per_sched); + assert(bank_num < num_banks); + return bank_num; + } else + return bank % num_banks; } -bool opndcoll_rfu_t::writeback( warp_inst_t &inst ) -{ - assert( !inst.empty() ); - std::list<unsigned> regs = m_shader->get_regs_written(inst); - for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) { - int reg_num = inst.arch_reg.dst[op]; // this math needs to match that used in function_info::ptx_decode_inst - if( reg_num >= 0 ){ // valid register - unsigned bank = register_bank(reg_num,inst.warp_id(),m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id()); - if( m_arbiter.bank_idle(bank) ) { - m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg_num,m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id())); - inst.arch_reg.dst[op] = -1; - } else { - return false; - } +bool opndcoll_rfu_t::writeback(warp_inst_t &inst) { + assert(!inst.empty()); + std::list<unsigned> regs = m_shader->get_regs_written(inst); + for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) { + int reg_num = inst.arch_reg.dst[op]; // this math needs to match that used + // in function_info::ptx_decode_inst + if (reg_num >= 0) { // valid register + unsigned bank = register_bank(reg_num, inst.warp_id(), m_num_banks, + m_bank_warp_shift, sub_core_model, + m_num_banks_per_sched, inst.get_schd_id()); + if (m_arbiter.bank_idle(bank)) { + m_arbiter.allocate_bank_for_write( + bank, + op_t(&inst, reg_num, m_num_banks, m_bank_warp_shift, sub_core_model, + m_num_banks_per_sched, inst.get_schd_id())); + inst.arch_reg.dst[op] = -1; + } else { + return false; } - } - for(unsigned i=0;i<(unsigned)regs.size();i++){ - if(m_shader->get_config()->gpgpu_clock_gated_reg_file){ - unsigned active_count=0; - for(unsigned i=0;i<m_shader->get_config()->warp_size;i=i+m_shader->get_config()->n_regfile_gating_group){ - for(unsigned j=0;j<m_shader->get_config()->n_regfile_gating_group;j++){ - if(inst.get_active_mask().test(i+j)){ - active_count+=m_shader->get_config()->n_regfile_gating_group; - break; - } - } - } - m_shader->incregfile_writes(active_count); - }else{ - m_shader->incregfile_writes(m_shader->get_config()->warp_size);//inst.active_count()); - } - } - return true; -} - -void opndcoll_rfu_t::dispatch_ready_cu() -{ - for( unsigned p=0; p < m_dispatch_units.size(); ++p ) { - dispatch_unit_t &du = m_dispatch_units[p]; - collector_unit_t *cu = du.find_ready(); - if( cu ) { - for(unsigned i=0;i<(cu->get_num_operands()-cu->get_num_regs());i++){ - if(m_shader->get_config()->gpgpu_clock_gated_reg_file){ - unsigned active_count=0; - for(unsigned i=0;i<m_shader->get_config()->warp_size;i=i+m_shader->get_config()->n_regfile_gating_group){ - for(unsigned j=0;j<m_shader->get_config()->n_regfile_gating_group;j++){ - if(cu->get_active_mask().test(i+j)){ - active_count+=m_shader->get_config()->n_regfile_gating_group; - break; - } - } - } - m_shader->incnon_rf_operands(active_count); - }else{ - m_shader->incnon_rf_operands(m_shader->get_config()->warp_size);//cu->get_active_count()); - } - } - cu->dispatch(); + } + } + for (unsigned i = 0; i < (unsigned)regs.size(); i++) { + if (m_shader->get_config()->gpgpu_clock_gated_reg_file) { + unsigned active_count = 0; + for (unsigned i = 0; i < m_shader->get_config()->warp_size; + i = i + m_shader->get_config()->n_regfile_gating_group) { + for (unsigned j = 0; j < m_shader->get_config()->n_regfile_gating_group; + j++) { + if (inst.get_active_mask().test(i + j)) { + active_count += m_shader->get_config()->n_regfile_gating_group; + break; + } + } } - } + m_shader->incregfile_writes(active_count); + } else { + m_shader->incregfile_writes( + m_shader->get_config()->warp_size); // inst.active_count()); + } + } + return true; } -void opndcoll_rfu_t::allocate_cu( unsigned port_num ) -{ - input_port_t& inp = m_in_ports[port_num]; - for (unsigned i = 0; i < inp.m_in.size(); i++) { - if( (*inp.m_in[i]).has_ready() ) { - //find a free cu - for (unsigned j = 0; j < inp.m_cu_sets.size(); j++) { - std::vector<collector_unit_t> & cu_set = m_cus[inp.m_cu_sets[j]]; - bool allocated = false; - for (unsigned k = 0; k < cu_set.size(); k++) { - if(cu_set[k].is_free()) { - collector_unit_t *cu = &cu_set[k]; - allocated = cu->allocate(inp.m_in[i],inp.m_out[i]); - m_arbiter.add_read_requests(cu); - break; - } +void opndcoll_rfu_t::dispatch_ready_cu() { + for (unsigned p = 0; p < m_dispatch_units.size(); ++p) { + dispatch_unit_t &du = m_dispatch_units[p]; + collector_unit_t *cu = du.find_ready(); + if (cu) { + for (unsigned i = 0; i < (cu->get_num_operands() - cu->get_num_regs()); + i++) { + if (m_shader->get_config()->gpgpu_clock_gated_reg_file) { + unsigned active_count = 0; + for (unsigned i = 0; i < m_shader->get_config()->warp_size; + i = i + m_shader->get_config()->n_regfile_gating_group) { + for (unsigned j = 0; + j < m_shader->get_config()->n_regfile_gating_group; j++) { + if (cu->get_active_mask().test(i + j)) { + active_count += m_shader->get_config()->n_regfile_gating_group; + break; } - if (allocated) break; //cu has been allocated, no need to search more. + } } - break; // can only service a single input, if it failed it will fail for others. - } - } + m_shader->incnon_rf_operands(active_count); + } else { + m_shader->incnon_rf_operands( + m_shader->get_config()->warp_size); // cu->get_active_count()); + } + } + cu->dispatch(); + } + } } -void opndcoll_rfu_t::allocate_reads() -{ - // process read requests that do not have conflicts - std::list<op_t> allocated = m_arbiter.allocate_reads(); - std::map<unsigned,op_t> read_ops; - for( std::list<op_t>::iterator r=allocated.begin(); r!=allocated.end(); r++ ) { - const op_t &rr = *r; - unsigned reg = rr.get_reg(); - unsigned wid = rr.get_wid(); - unsigned bank = register_bank(reg,wid,m_num_banks,m_bank_warp_shift,sub_core_model, m_num_banks_per_sched, rr.get_sid()); - m_arbiter.allocate_for_read(bank,rr); - read_ops[bank] = rr; - } - std::map<unsigned,op_t>::iterator r; - for(r=read_ops.begin();r!=read_ops.end();++r ) { - op_t &op = r->second; - unsigned cu = op.get_oc_id(); - unsigned operand = op.get_operand(); - m_cu[cu]->collect_operand(operand); - if(m_shader->get_config()->gpgpu_clock_gated_reg_file){ - unsigned active_count=0; - for(unsigned i=0;i<m_shader->get_config()->warp_size;i=i+m_shader->get_config()->n_regfile_gating_group){ - for(unsigned j=0;j<m_shader->get_config()->n_regfile_gating_group;j++){ - if(op.get_active_mask().test(i+j)){ - active_count+=m_shader->get_config()->n_regfile_gating_group; - break; - } - } - } - m_shader->incregfile_reads(active_count); - }else{ - m_shader->incregfile_reads(m_shader->get_config()->warp_size);//op.get_active_count()); +void opndcoll_rfu_t::allocate_cu(unsigned port_num) { + input_port_t &inp = m_in_ports[port_num]; + for (unsigned i = 0; i < inp.m_in.size(); i++) { + if ((*inp.m_in[i]).has_ready()) { + // find a free cu + for (unsigned j = 0; j < inp.m_cu_sets.size(); j++) { + std::vector<collector_unit_t> &cu_set = m_cus[inp.m_cu_sets[j]]; + bool allocated = false; + for (unsigned k = 0; k < cu_set.size(); k++) { + if (cu_set[k].is_free()) { + collector_unit_t *cu = &cu_set[k]; + allocated = cu->allocate(inp.m_in[i], inp.m_out[i]); + m_arbiter.add_read_requests(cu); + break; + } + } + if (allocated) break; // cu has been allocated, no need to search more. } + break; // can only service a single input, if it failed it will fail for + // others. + } } -} - -bool opndcoll_rfu_t::collector_unit_t::ready() const -{ - return (!m_free) && m_not_ready.none() && (*m_output_register).has_free(); } -void opndcoll_rfu_t::collector_unit_t::dump(FILE *fp, const shader_core_ctx *shader ) const -{ - if( m_free ) { - fprintf(fp," <free>\n"); - } else { - m_warp->print(fp); - for( unsigned i=0; i < MAX_REG_OPERANDS*2; i++ ) { - if( m_not_ready.test(i) ) { - std::string r = m_src_op[i].get_reg_string(); - fprintf(fp," '%s' not ready\n", r.c_str() ); - } +void opndcoll_rfu_t::allocate_reads() { + // process read requests that do not have conflicts + std::list<op_t> allocated = m_arbiter.allocate_reads(); + std::map<unsigned, op_t> read_ops; + for (std::list<op_t>::iterator r = allocated.begin(); r != allocated.end(); + r++) { + const op_t &rr = *r; + unsigned reg = rr.get_reg(); + unsigned wid = rr.get_wid(); + unsigned bank = + register_bank(reg, wid, m_num_banks, m_bank_warp_shift, sub_core_model, + m_num_banks_per_sched, rr.get_sid()); + m_arbiter.allocate_for_read(bank, rr); + read_ops[bank] = rr; + } + std::map<unsigned, op_t>::iterator r; + for (r = read_ops.begin(); r != read_ops.end(); ++r) { + op_t &op = r->second; + unsigned cu = op.get_oc_id(); + unsigned operand = op.get_operand(); + m_cu[cu]->collect_operand(operand); + if (m_shader->get_config()->gpgpu_clock_gated_reg_file) { + unsigned active_count = 0; + for (unsigned i = 0; i < m_shader->get_config()->warp_size; + i = i + m_shader->get_config()->n_regfile_gating_group) { + for (unsigned j = 0; j < m_shader->get_config()->n_regfile_gating_group; + j++) { + if (op.get_active_mask().test(i + j)) { + active_count += m_shader->get_config()->n_regfile_gating_group; + break; + } + } } - } + m_shader->incregfile_reads(active_count); + } else { + m_shader->incregfile_reads( + m_shader->get_config()->warp_size); // op.get_active_count()); + } + } } -void opndcoll_rfu_t::collector_unit_t::init( unsigned n, - unsigned num_banks, - unsigned log2_warp_size, - const core_config *config, - opndcoll_rfu_t *rfu, - bool sub_core_model, - unsigned banks_per_sched) -{ - m_rfu=rfu; - m_cuid=n; - m_num_banks=num_banks; - assert(m_warp==NULL); - m_warp = new warp_inst_t(config); - m_bank_warp_shift=log2_warp_size; - m_sub_core_model = sub_core_model; - m_num_banks_per_sched = banks_per_sched; +bool opndcoll_rfu_t::collector_unit_t::ready() const { + return (!m_free) && m_not_ready.none() && (*m_output_register).has_free(); } -bool opndcoll_rfu_t::collector_unit_t::allocate( register_set* pipeline_reg_set, register_set* output_reg_set ) -{ - assert(m_free); - assert(m_not_ready.none()); - m_free = false; - m_output_register = output_reg_set; - warp_inst_t **pipeline_reg = pipeline_reg_set->get_ready(); - if( (pipeline_reg) and !((*pipeline_reg)->empty()) ) { - m_warp_id = (*pipeline_reg)->warp_id(); - for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) { - int reg_num = (*pipeline_reg)->arch_reg.src[op]; // this math needs to match that used in function_info::ptx_decode_inst - if( reg_num >= 0 ) { // valid register - m_src_op[op] = op_t( this, op, reg_num, m_num_banks, m_bank_warp_shift, m_sub_core_model, m_num_banks_per_sched, (*pipeline_reg)->get_schd_id() ); - m_not_ready.set(op); - } else - m_src_op[op] = op_t(); +void opndcoll_rfu_t::collector_unit_t::dump( + FILE *fp, const shader_core_ctx *shader) const { + if (m_free) { + fprintf(fp, " <free>\n"); + } else { + m_warp->print(fp); + for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) { + if (m_not_ready.test(i)) { + std::string r = m_src_op[i].get_reg_string(); + fprintf(fp, " '%s' not ready\n", r.c_str()); } - //move_warp(m_warp,*pipeline_reg); - pipeline_reg_set->move_out_to(m_warp); - return true; - } - return false; + } + } } -void opndcoll_rfu_t::collector_unit_t::dispatch() -{ - assert( m_not_ready.none() ); - //move_warp(*m_output_register,m_warp); - m_output_register->move_in(m_warp); - m_free=true; - m_output_register = NULL; - for( unsigned i=0; i<MAX_REG_OPERANDS*2;i++) - m_src_op[i].reset(); +void opndcoll_rfu_t::collector_unit_t::init(unsigned n, unsigned num_banks, + unsigned log2_warp_size, + const core_config *config, + opndcoll_rfu_t *rfu, + bool sub_core_model, + unsigned banks_per_sched) { + m_rfu = rfu; + m_cuid = n; + m_num_banks = num_banks; + assert(m_warp == NULL); + m_warp = new warp_inst_t(config); + m_bank_warp_shift = log2_warp_size; + m_sub_core_model = sub_core_model; + m_num_banks_per_sched = banks_per_sched; } -simt_core_cluster::simt_core_cluster( class gpgpu_sim *gpu, - unsigned cluster_id, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - class memory_stats_t *mstats ) -{ - m_config = config; - m_cta_issue_next_core=m_config->n_simt_cores_per_cluster-1; // this causes first launch to use hw cta 0 - m_cluster_id=cluster_id; - m_gpu = gpu; - m_stats = stats; - m_memory_stats = mstats; - m_core = new shader_core_ctx*[ config->n_simt_cores_per_cluster ]; - for( unsigned i=0; i < config->n_simt_cores_per_cluster; i++ ) { - unsigned sid = m_config->cid_to_sid(i,m_cluster_id); - m_core[i] = new shader_core_ctx(gpu,this,sid,m_cluster_id,config,mem_config,stats); - m_core_sim_order.push_back(i); +bool opndcoll_rfu_t::collector_unit_t::allocate(register_set *pipeline_reg_set, + register_set *output_reg_set) { + assert(m_free); + assert(m_not_ready.none()); + m_free = false; + m_output_register = output_reg_set; + warp_inst_t **pipeline_reg = pipeline_reg_set->get_ready(); + if ((pipeline_reg) and !((*pipeline_reg)->empty())) { + m_warp_id = (*pipeline_reg)->warp_id(); + for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) { + int reg_num = + (*pipeline_reg) + ->arch_reg.src[op]; // this math needs to match that used in + // function_info::ptx_decode_inst + if (reg_num >= 0) { // valid register + m_src_op[op] = op_t(this, op, reg_num, m_num_banks, m_bank_warp_shift, + m_sub_core_model, m_num_banks_per_sched, + (*pipeline_reg)->get_schd_id()); + m_not_ready.set(op); + } else + m_src_op[op] = op_t(); } + // move_warp(m_warp,*pipeline_reg); + pipeline_reg_set->move_out_to(m_warp); + return true; + } + return false; } -void simt_core_cluster::core_cycle() -{ - for( std::list<unsigned>::iterator it = m_core_sim_order.begin(); it != m_core_sim_order.end(); ++it ) { - m_core[*it]->cycle(); - } +void opndcoll_rfu_t::collector_unit_t::dispatch() { + assert(m_not_ready.none()); + // move_warp(*m_output_register,m_warp); + m_output_register->move_in(m_warp); + m_free = true; + m_output_register = NULL; + for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) m_src_op[i].reset(); +} - if (m_config->simt_core_sim_order == 1) { - m_core_sim_order.splice(m_core_sim_order.end(), m_core_sim_order, m_core_sim_order.begin()); - } +simt_core_cluster::simt_core_cluster(class gpgpu_sim *gpu, unsigned cluster_id, + const shader_core_config *config, + const memory_config *mem_config, + shader_core_stats *stats, + class memory_stats_t *mstats) { + m_config = config; + m_cta_issue_next_core = m_config->n_simt_cores_per_cluster - + 1; // this causes first launch to use hw cta 0 + m_cluster_id = cluster_id; + m_gpu = gpu; + m_stats = stats; + m_memory_stats = mstats; + m_core = new shader_core_ctx *[config->n_simt_cores_per_cluster]; + for (unsigned i = 0; i < config->n_simt_cores_per_cluster; i++) { + unsigned sid = m_config->cid_to_sid(i, m_cluster_id); + m_core[i] = new shader_core_ctx(gpu, this, sid, m_cluster_id, config, + mem_config, stats); + m_core_sim_order.push_back(i); + } } -void simt_core_cluster::reinit() -{ - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - m_core[i]->reinit(0,m_config->n_thread_per_shader,true); +void simt_core_cluster::core_cycle() { + for (std::list<unsigned>::iterator it = m_core_sim_order.begin(); + it != m_core_sim_order.end(); ++it) { + m_core[*it]->cycle(); + } + + if (m_config->simt_core_sim_order == 1) { + m_core_sim_order.splice(m_core_sim_order.end(), m_core_sim_order, + m_core_sim_order.begin()); + } } -unsigned simt_core_cluster::max_cta( const kernel_info_t &kernel ) -{ - return m_config->n_simt_cores_per_cluster * m_config->max_cta(kernel); +void simt_core_cluster::reinit() { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + m_core[i]->reinit(0, m_config->n_thread_per_shader, true); } -unsigned simt_core_cluster::get_not_completed() const -{ - unsigned not_completed=0; - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - not_completed += m_core[i]->get_not_completed(); - return not_completed; +unsigned simt_core_cluster::max_cta(const kernel_info_t &kernel) { + return m_config->n_simt_cores_per_cluster * m_config->max_cta(kernel); } -void simt_core_cluster::print_not_completed( FILE *fp ) const -{ - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) { - unsigned not_completed=m_core[i]->get_not_completed(); - unsigned sid=m_config->cid_to_sid(i,m_cluster_id); - fprintf(fp,"%u(%u) ", sid, not_completed ); - } +unsigned simt_core_cluster::get_not_completed() const { + unsigned not_completed = 0; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + not_completed += m_core[i]->get_not_completed(); + return not_completed; } +void simt_core_cluster::print_not_completed(FILE *fp) const { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) { + unsigned not_completed = m_core[i]->get_not_completed(); + unsigned sid = m_config->cid_to_sid(i, m_cluster_id); + fprintf(fp, "%u(%u) ", sid, not_completed); + } +} -float simt_core_cluster::get_current_occupancy( unsigned long long& active, unsigned long long& total ) const { - float aggregate = 0.f; - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) { - aggregate+=m_core[i]->get_current_occupancy( active, total ); - } - return aggregate / m_config->n_simt_cores_per_cluster; +float simt_core_cluster::get_current_occupancy( + unsigned long long &active, unsigned long long &total) const { + float aggregate = 0.f; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) { + aggregate += m_core[i]->get_current_occupancy(active, total); + } + return aggregate / m_config->n_simt_cores_per_cluster; } -unsigned simt_core_cluster::get_n_active_cta() const -{ - unsigned n=0; - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - n += m_core[i]->get_n_active_cta(); - return n; +unsigned simt_core_cluster::get_n_active_cta() const { + unsigned n = 0; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + n += m_core[i]->get_n_active_cta(); + return n; } -unsigned simt_core_cluster::get_n_active_sms() const -{ - unsigned n=0; - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - n += m_core[i]->isactive(); - return n; +unsigned simt_core_cluster::get_n_active_sms() const { + unsigned n = 0; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + n += m_core[i]->isactive(); + return n; } -unsigned simt_core_cluster::issue_block2core() -{ - unsigned num_blocks_issued=0; - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) { - unsigned core = (i+m_cta_issue_next_core+1)%m_config->n_simt_cores_per_cluster; +unsigned simt_core_cluster::issue_block2core() { + unsigned num_blocks_issued = 0; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) { + unsigned core = + (i + m_cta_issue_next_core + 1) % m_config->n_simt_cores_per_cluster; - kernel_info_t * kernel; - //Jin: fetch kernel according to concurrent kernel setting - if(m_config->gpgpu_concurrent_kernel_sm) {//concurrent kernel on sm - //always select latest issued kernel - kernel_info_t *k = m_gpu->select_kernel(); - kernel = k; - } - else { - //first select core kernel, if no more cta, get a new kernel - //only when core completes - kernel = m_core[core]->get_kernel(); - if( !m_gpu->kernel_more_cta_left(kernel) ) { - //wait till current kernel finishes - if(m_core[core]->get_not_completed() == 0) - { - kernel_info_t *k = m_gpu->select_kernel(); - if( k ) - m_core[core]->set_kernel(k); - kernel = k; - } - } + kernel_info_t *kernel; + // Jin: fetch kernel according to concurrent kernel setting + if (m_config->gpgpu_concurrent_kernel_sm) { // concurrent kernel on sm + // always select latest issued kernel + kernel_info_t *k = m_gpu->select_kernel(); + kernel = k; + } else { + // first select core kernel, if no more cta, get a new kernel + // only when core completes + kernel = m_core[core]->get_kernel(); + if (!m_gpu->kernel_more_cta_left(kernel)) { + // wait till current kernel finishes + if (m_core[core]->get_not_completed() == 0) { + kernel_info_t *k = m_gpu->select_kernel(); + if (k) m_core[core]->set_kernel(k); + kernel = k; } + } + } - if( m_gpu->kernel_more_cta_left(kernel) && -// (m_core[core]->get_n_active_cta() < m_config->max_cta(*kernel)) ) { - m_core[core]->can_issue_1block(*kernel)) { - m_core[core]->issue_block2core(*kernel); - num_blocks_issued++; - m_cta_issue_next_core=core; - break; - } + if (m_gpu->kernel_more_cta_left(kernel) && + // (m_core[core]->get_n_active_cta() < + // m_config->max_cta(*kernel)) ) { + m_core[core]->can_issue_1block(*kernel)) { + m_core[core]->issue_block2core(*kernel); + num_blocks_issued++; + m_cta_issue_next_core = core; + break; } - return num_blocks_issued; + } + return num_blocks_issued; } -void simt_core_cluster::cache_flush() -{ - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - m_core[i]->cache_flush(); +void simt_core_cluster::cache_flush() { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + m_core[i]->cache_flush(); } -void simt_core_cluster::cache_invalidate() -{ - for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) - m_core[i]->cache_invalidate(); +void simt_core_cluster::cache_invalidate() { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) + m_core[i]->cache_invalidate(); } -bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write) -{ - unsigned request_size = size; - if (!write) - request_size = READ_PACKET_SIZE; - return ! ::icnt_has_buffer(m_cluster_id, request_size); +bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write) { + unsigned request_size = size; + if (!write) request_size = READ_PACKET_SIZE; + return !::icnt_has_buffer(m_cluster_id, request_size); } -void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf) -{ - // stats - if (mf->get_is_write()) m_stats->made_write_mfs++; - else m_stats->made_read_mfs++; - switch (mf->get_access_type()) { - case CONST_ACC_R: m_stats->gpgpu_n_mem_const++; break; - case TEXTURE_ACC_R: m_stats->gpgpu_n_mem_texture++; break; - case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; break; - //case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; printf("read_global%d\n",m_stats->gpgpu_n_mem_read_global); break; - case GLOBAL_ACC_W: m_stats->gpgpu_n_mem_write_global++; break; - case LOCAL_ACC_R: m_stats->gpgpu_n_mem_read_local++; break; - case LOCAL_ACC_W: m_stats->gpgpu_n_mem_write_local++; break; - case INST_ACC_R: m_stats->gpgpu_n_mem_read_inst++; break; - case L1_WRBK_ACC: m_stats->gpgpu_n_mem_write_global++; break; - case L2_WRBK_ACC: m_stats->gpgpu_n_mem_l2_writeback++; break; - case L1_WR_ALLOC_R: m_stats->gpgpu_n_mem_l1_write_allocate++; break; - case L2_WR_ALLOC_R: m_stats->gpgpu_n_mem_l2_write_allocate++; break; - default: assert(0); - } +void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf) { + // stats + if (mf->get_is_write()) + m_stats->made_write_mfs++; + else + m_stats->made_read_mfs++; + switch (mf->get_access_type()) { + case CONST_ACC_R: + m_stats->gpgpu_n_mem_const++; + break; + case TEXTURE_ACC_R: + m_stats->gpgpu_n_mem_texture++; + break; + case GLOBAL_ACC_R: + m_stats->gpgpu_n_mem_read_global++; + break; + // case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; + // printf("read_global%d\n",m_stats->gpgpu_n_mem_read_global); break; + case GLOBAL_ACC_W: + m_stats->gpgpu_n_mem_write_global++; + break; + case LOCAL_ACC_R: + m_stats->gpgpu_n_mem_read_local++; + break; + case LOCAL_ACC_W: + m_stats->gpgpu_n_mem_write_local++; + break; + case INST_ACC_R: + m_stats->gpgpu_n_mem_read_inst++; + break; + case L1_WRBK_ACC: + m_stats->gpgpu_n_mem_write_global++; + break; + case L2_WRBK_ACC: + m_stats->gpgpu_n_mem_l2_writeback++; + break; + case L1_WR_ALLOC_R: + m_stats->gpgpu_n_mem_l1_write_allocate++; + break; + case L2_WR_ALLOC_R: + m_stats->gpgpu_n_mem_l2_write_allocate++; + break; + default: + assert(0); + } - // The packet size varies depending on the type of request: - // - For write request and atomic request, the packet contains the data - // - For read request (i.e. not write nor atomic), the packet only has control metadata - unsigned int packet_size = mf->size(); - if (!mf->get_is_write() && !mf->isatomic()) { - packet_size = mf->get_ctrl_size(); - } - m_stats->m_outgoing_traffic_stats->record_traffic(mf, packet_size); - unsigned destination = mf->get_sub_partition_id(); - mf->set_status(IN_ICNT_TO_MEM,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - if (!mf->get_is_write() && !mf->isatomic()) - ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void*)mf, mf->get_ctrl_size() ); - else - ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void*)mf, mf->size()); + // The packet size varies depending on the type of request: + // - For write request and atomic request, the packet contains the data + // - For read request (i.e. not write nor atomic), the packet only has control + // metadata + unsigned int packet_size = mf->size(); + if (!mf->get_is_write() && !mf->isatomic()) { + packet_size = mf->get_ctrl_size(); + } + m_stats->m_outgoing_traffic_stats->record_traffic(mf, packet_size); + unsigned destination = mf->get_sub_partition_id(); + mf->set_status(IN_ICNT_TO_MEM, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + if (!mf->get_is_write() && !mf->isatomic()) + ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void *)mf, + mf->get_ctrl_size()); + else + ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void *)mf, + mf->size()); } -void simt_core_cluster::icnt_cycle() -{ - if( !m_response_fifo.empty() ) { - mem_fetch *mf = m_response_fifo.front(); - unsigned cid = m_config->sid_to_cid(mf->get_sid()); - if( mf->get_access_type() == INST_ACC_R ) { - // instruction fetch response - if( !m_core[cid]->fetch_unit_response_buffer_full() ) { - m_response_fifo.pop_front(); - m_core[cid]->accept_fetch_response(mf); - } - } else { - // data response - if( !m_core[cid]->ldst_unit_response_buffer_full() ) { - m_response_fifo.pop_front(); - m_memory_stats->memlatstat_read_done(mf); - m_core[cid]->accept_ldst_unit_response(mf); - } - } +void simt_core_cluster::icnt_cycle() { + if (!m_response_fifo.empty()) { + mem_fetch *mf = m_response_fifo.front(); + unsigned cid = m_config->sid_to_cid(mf->get_sid()); + if (mf->get_access_type() == INST_ACC_R) { + // instruction fetch response + if (!m_core[cid]->fetch_unit_response_buffer_full()) { + m_response_fifo.pop_front(); + m_core[cid]->accept_fetch_response(mf); + } + } else { + // data response + if (!m_core[cid]->ldst_unit_response_buffer_full()) { + m_response_fifo.pop_front(); + m_memory_stats->memlatstat_read_done(mf); + m_core[cid]->accept_ldst_unit_response(mf); + } } - if( m_response_fifo.size() < m_config->n_simt_ejection_buffer_size ) { - mem_fetch *mf = (mem_fetch*) ::icnt_pop(m_cluster_id); - if (!mf) - return; - assert(mf->get_tpc() == m_cluster_id); - assert(mf->get_type() == READ_REPLY || mf->get_type() == WRITE_ACK ); + } + if (m_response_fifo.size() < m_config->n_simt_ejection_buffer_size) { + mem_fetch *mf = (mem_fetch *)::icnt_pop(m_cluster_id); + if (!mf) return; + assert(mf->get_tpc() == m_cluster_id); + assert(mf->get_type() == READ_REPLY || mf->get_type() == WRITE_ACK); - // The packet size varies depending on the type of request: - // - For read request and atomic request, the packet contains the data - // - For write-ack, the packet only has control metadata - unsigned int packet_size = (mf->get_is_write())? mf->get_ctrl_size() : mf->size(); - m_stats->m_incoming_traffic_stats->record_traffic(mf, packet_size); - mf->set_status(IN_CLUSTER_TO_SHADER_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle); - //m_memory_stats->memlatstat_read_done(mf,m_shader_config->max_warps_per_shader); - m_response_fifo.push_back(mf); - m_stats->n_mem_to_simt[m_cluster_id] += mf->get_num_flits(false); - } + // The packet size varies depending on the type of request: + // - For read request and atomic request, the packet contains the data + // - For write-ack, the packet only has control metadata + unsigned int packet_size = + (mf->get_is_write()) ? mf->get_ctrl_size() : mf->size(); + m_stats->m_incoming_traffic_stats->record_traffic(mf, packet_size); + mf->set_status(IN_CLUSTER_TO_SHADER_QUEUE, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle); + // m_memory_stats->memlatstat_read_done(mf,m_shader_config->max_warps_per_shader); + m_response_fifo.push_back(mf); + m_stats->n_mem_to_simt[m_cluster_id] += mf->get_num_flits(false); + } } -void simt_core_cluster::get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ) const -{ - unsigned cid = m_config->sid_to_cid(sid); - m_core[cid]->get_pdom_stack_top_info(tid,pc,rpc); +void simt_core_cluster::get_pdom_stack_top_info(unsigned sid, unsigned tid, + unsigned *pc, + unsigned *rpc) const { + unsigned cid = m_config->sid_to_cid(sid); + m_core[cid]->get_pdom_stack_top_info(tid, pc, rpc); } -void simt_core_cluster::display_pipeline( unsigned sid, FILE *fout, int print_mem, int mask ) -{ - m_core[m_config->sid_to_cid(sid)]->display_pipeline(fout,print_mem,mask); +void simt_core_cluster::display_pipeline(unsigned sid, FILE *fout, + int print_mem, int mask) { + m_core[m_config->sid_to_cid(sid)]->display_pipeline(fout, print_mem, mask); - fprintf(fout,"\n"); - fprintf(fout,"Cluster %u pipeline state\n", m_cluster_id ); - fprintf(fout,"Response FIFO (occupancy = %zu):\n", m_response_fifo.size() ); - for( std::list<mem_fetch*>::const_iterator i=m_response_fifo.begin(); i != m_response_fifo.end(); i++ ) { - const mem_fetch *mf = *i; - mf->print(fout); - } + fprintf(fout, "\n"); + fprintf(fout, "Cluster %u pipeline state\n", m_cluster_id); + fprintf(fout, "Response FIFO (occupancy = %zu):\n", m_response_fifo.size()); + for (std::list<mem_fetch *>::const_iterator i = m_response_fifo.begin(); + i != m_response_fifo.end(); i++) { + const mem_fetch *mf = *i; + mf->print(fout); + } } -void simt_core_cluster::print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ) const { - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[ i ]->print_cache_stats( fp, dl1_accesses, dl1_misses ); - } +void simt_core_cluster::print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses) const { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->print_cache_stats(fp, dl1_accesses, dl1_misses); + } } -void simt_core_cluster::get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const { - long simt_to_mem=0; - long mem_to_simt=0; - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_icnt_power_stats(simt_to_mem, mem_to_simt); - } - n_simt_to_mem = simt_to_mem; - n_mem_to_simt = mem_to_simt; +void simt_core_cluster::get_icnt_stats(long &n_simt_to_mem, + long &n_mem_to_simt) const { + long simt_to_mem = 0; + long mem_to_simt = 0; + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_icnt_power_stats(simt_to_mem, mem_to_simt); + } + n_simt_to_mem = simt_to_mem; + n_mem_to_simt = mem_to_simt; } -void simt_core_cluster::get_cache_stats(cache_stats &cs) const{ - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_cache_stats(cs); - } +void simt_core_cluster::get_cache_stats(cache_stats &cs) const { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_cache_stats(cs); + } } -void simt_core_cluster::get_L1I_sub_stats(struct cache_sub_stats &css) const{ - struct cache_sub_stats temp_css; - struct cache_sub_stats total_css; - temp_css.clear(); - total_css.clear(); - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_L1I_sub_stats(temp_css); - total_css += temp_css; - } - css = total_css; +void simt_core_cluster::get_L1I_sub_stats(struct cache_sub_stats &css) const { + struct cache_sub_stats temp_css; + struct cache_sub_stats total_css; + temp_css.clear(); + total_css.clear(); + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_L1I_sub_stats(temp_css); + total_css += temp_css; + } + css = total_css; } -void simt_core_cluster::get_L1D_sub_stats(struct cache_sub_stats &css) const{ - struct cache_sub_stats temp_css; - struct cache_sub_stats total_css; - temp_css.clear(); - total_css.clear(); - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_L1D_sub_stats(temp_css); - total_css += temp_css; - } - css = total_css; +void simt_core_cluster::get_L1D_sub_stats(struct cache_sub_stats &css) const { + struct cache_sub_stats temp_css; + struct cache_sub_stats total_css; + temp_css.clear(); + total_css.clear(); + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_L1D_sub_stats(temp_css); + total_css += temp_css; + } + css = total_css; } -void simt_core_cluster::get_L1C_sub_stats(struct cache_sub_stats &css) const{ - struct cache_sub_stats temp_css; - struct cache_sub_stats total_css; - temp_css.clear(); - total_css.clear(); - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_L1C_sub_stats(temp_css); - total_css += temp_css; - } - css = total_css; +void simt_core_cluster::get_L1C_sub_stats(struct cache_sub_stats &css) const { + struct cache_sub_stats temp_css; + struct cache_sub_stats total_css; + temp_css.clear(); + total_css.clear(); + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_L1C_sub_stats(temp_css); + total_css += temp_css; + } + css = total_css; } -void simt_core_cluster::get_L1T_sub_stats(struct cache_sub_stats &css) const{ - struct cache_sub_stats temp_css; - struct cache_sub_stats total_css; - temp_css.clear(); - total_css.clear(); - for ( unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i ) { - m_core[i]->get_L1T_sub_stats(temp_css); - total_css += temp_css; - } - css = total_css; +void simt_core_cluster::get_L1T_sub_stats(struct cache_sub_stats &css) const { + struct cache_sub_stats temp_css; + struct cache_sub_stats total_css; + temp_css.clear(); + total_css.clear(); + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) { + m_core[i]->get_L1T_sub_stats(temp_css); + total_css += temp_css; + } + css = total_css; } -void shader_core_ctx::checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) -{ - if(inst.isatomic()) - m_warp[inst.warp_id()].inc_n_atomic(); - if (inst.space.is_local() && (inst.is_load() || inst.is_store())) { - new_addr_type localaddrs[MAX_ACCESSES_PER_INSN_PER_THREAD]; - unsigned num_addrs; - num_addrs = translate_local_memaddr(inst.get_addr(t), tid, m_config->n_simt_clusters*m_config->n_simt_cores_per_cluster, - inst.data_size, (new_addr_type*) localaddrs ); - inst.set_addr(t, (new_addr_type*) localaddrs, num_addrs); - } - if ( ptx_thread_done(tid) ) { - m_warp[inst.warp_id()].set_completed(t); - m_warp[inst.warp_id()].ibuffer_flush(); - } +void shader_core_ctx::checkExecutionStatusAndUpdate(warp_inst_t &inst, + unsigned t, unsigned tid) { + if (inst.isatomic()) m_warp[inst.warp_id()].inc_n_atomic(); + if (inst.space.is_local() && (inst.is_load() || inst.is_store())) { + new_addr_type localaddrs[MAX_ACCESSES_PER_INSN_PER_THREAD]; + unsigned num_addrs; + num_addrs = translate_local_memaddr( + inst.get_addr(t), tid, + m_config->n_simt_clusters * m_config->n_simt_cores_per_cluster, + inst.data_size, (new_addr_type *)localaddrs); + inst.set_addr(t, (new_addr_type *)localaddrs, num_addrs); + } + if (ptx_thread_done(tid)) { + m_warp[inst.warp_id()].set_completed(t); + m_warp[inst.warp_id()].ibuffer_flush(); + } - // PC-Histogram Update - unsigned warp_id = inst.warp_id(); - unsigned pc = inst.pc; - for (unsigned t = 0; t < m_config->warp_size; t++) { - if (inst.active(t)) { - int tid = warp_id * m_config->warp_size + t; - cflog_update_thread_pc(m_sid, tid, pc); - } + // PC-Histogram Update + unsigned warp_id = inst.warp_id(); + unsigned pc = inst.pc; + for (unsigned t = 0; t < m_config->warp_size; t++) { + if (inst.active(t)) { + int tid = warp_id * m_config->warp_size + t; + cflog_update_thread_pc(m_sid, tid, pc); } + } } - diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 667cb2d..1af35cf 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Andrew Turner, -// Ali Bakhoda +// Ali Bakhoda // The University of British Columbia // All rights reserved. // @@ -8,71 +8,71 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef SHADER_H #define SHADER_H +#include <assert.h> +#include <math.h> #include <stdio.h> #include <stdlib.h> -#include <math.h> -#include <assert.h> +#include <algorithm> +#include <bitset> +#include <deque> +#include <list> #include <map> #include <set> -#include <vector> -#include <list> -#include <bitset> #include <utility> -#include <algorithm> -#include <deque> +#include <vector> //#include "../cuda-sim/ptx.tab.h" +#include "../abstract_hardware_model.h" #include "delayqueue.h" -#include "stack.h" #include "dram.h" -#include "../abstract_hardware_model.h" -#include "scoreboard.h" +#include "gpu-cache.h" #include "mem_fetch.h" +#include "scoreboard.h" +#include "stack.h" #include "stats.h" -#include "gpu-cache.h" #include "traffic_breakdown.h" - -#define NO_OP_FLAG 0xFF +#define NO_OP_FLAG 0xFF /* READ_PACKET_SIZE: - bytes: 6 address (flit can specify chanel so this gives up to ~2GB/channel, so good for now), - 2 bytes [shaderid + mshrid](14 bits) + req_size(0-2 bits if req_size variable) - so up to 2^14 = 16384 mshr total + bytes: 6 address (flit can specify chanel so this gives up to ~2GB/channel, + so good for now), 2 bytes [shaderid + mshrid](14 bits) + req_size(0-2 bits + if req_size variable) - so up to 2^14 = 16384 mshr total */ #define READ_PACKET_SIZE 8 -//WRITE_PACKET_SIZE: bytes: 6 address, 2 miscelaneous. +// WRITE_PACKET_SIZE: bytes: 6 address, 2 miscelaneous. #define WRITE_PACKET_SIZE 8 #define WRITE_MASK_SIZE 8 class gpgpu_context; -enum exec_unit_type_t -{ +enum exec_unit_type_t { NONE = 0, SP = 1, SFU = 2, @@ -83,1127 +83,1132 @@ enum exec_unit_type_t }; class thread_ctx_t { -public: - unsigned m_cta_id; // hardware CTA this thread belongs + public: + unsigned m_cta_id; // hardware CTA this thread belongs - // per thread stats (ac stands for accumulative). - unsigned n_insn; - unsigned n_insn_ac; - unsigned n_l1_mis_ac; - unsigned n_l1_mrghit_ac; - unsigned n_l1_access_ac; + // per thread stats (ac stands for accumulative). + unsigned n_insn; + unsigned n_insn_ac; + unsigned n_l1_mis_ac; + unsigned n_l1_mrghit_ac; + unsigned n_l1_access_ac; - bool m_active; + bool m_active; }; class shd_warp_t { -public: - shd_warp_t( class shader_core_ctx *shader, unsigned warp_size) - : m_shader(shader), m_warp_size(warp_size) - { - m_stores_outstanding=0; - m_inst_in_pipeline=0; - reset(); - } - void reset() - { - assert( m_stores_outstanding==0); - assert( m_inst_in_pipeline==0); - m_imiss_pending=false; - m_warp_id=(unsigned)-1; - m_dynamic_warp_id = (unsigned)-1; - n_completed = m_warp_size; - m_n_atomic=0; - m_membar=false; - m_done_exit=true; - m_last_fetch=0; - m_next=0; + public: + shd_warp_t(class shader_core_ctx *shader, unsigned warp_size) + : m_shader(shader), m_warp_size(warp_size) { + m_stores_outstanding = 0; + m_inst_in_pipeline = 0; + reset(); + } + void reset() { + assert(m_stores_outstanding == 0); + assert(m_inst_in_pipeline == 0); + m_imiss_pending = false; + m_warp_id = (unsigned)-1; + m_dynamic_warp_id = (unsigned)-1; + n_completed = m_warp_size; + m_n_atomic = 0; + m_membar = false; + m_done_exit = true; + m_last_fetch = 0; + m_next = 0; - //Jin: cdp support - m_cdp_latency = 0; - m_cdp_dummy = false; - } - void init( address_type start_pc, - unsigned cta_id, - unsigned wid, - const std::bitset<MAX_WARP_SIZE> &active, - unsigned dynamic_warp_id ) - { - m_cta_id=cta_id; - m_warp_id=wid; - m_dynamic_warp_id=dynamic_warp_id; - m_next_pc=start_pc; - assert( n_completed >= active.count() ); - assert( n_completed <= m_warp_size); - n_completed -= active.count(); // active threads are not yet completed - m_active_threads = active; - m_done_exit=false; + // Jin: cdp support + m_cdp_latency = 0; + m_cdp_dummy = false; + } + void init(address_type start_pc, unsigned cta_id, unsigned wid, + const std::bitset<MAX_WARP_SIZE> &active, + unsigned dynamic_warp_id) { + m_cta_id = cta_id; + m_warp_id = wid; + m_dynamic_warp_id = dynamic_warp_id; + m_next_pc = start_pc; + assert(n_completed >= active.count()); + assert(n_completed <= m_warp_size); + n_completed -= active.count(); // active threads are not yet completed + m_active_threads = active; + m_done_exit = false; - //Jin: cdp support - m_cdp_latency = 0; - m_cdp_dummy = false; - } + // Jin: cdp support + m_cdp_latency = 0; + m_cdp_dummy = false; + } - bool functional_done() const; - bool waiting(); // not const due to membar - bool hardware_done() const; + bool functional_done() const; + bool waiting(); // not const due to membar + bool hardware_done() const; - bool done_exit() const { return m_done_exit; } - void set_done_exit() { m_done_exit=true; } + bool done_exit() const { return m_done_exit; } + void set_done_exit() { m_done_exit = true; } - void print( FILE *fout ) const; - void print_ibuffer( FILE *fout ) const; + void print(FILE *fout) const; + void print_ibuffer(FILE *fout) const; - unsigned get_n_completed() const { return n_completed; } - void set_completed( unsigned lane ) - { - assert( m_active_threads.test(lane) ); - m_active_threads.reset(lane); - n_completed++; - } + unsigned get_n_completed() const { return n_completed; } + void set_completed(unsigned lane) { + assert(m_active_threads.test(lane)); + m_active_threads.reset(lane); + n_completed++; + } - void set_last_fetch( unsigned long long sim_cycle ) { m_last_fetch=sim_cycle; } + void set_last_fetch(unsigned long long sim_cycle) { + m_last_fetch = sim_cycle; + } - unsigned get_n_atomic() const { return m_n_atomic; } - void inc_n_atomic() { m_n_atomic++; } - void dec_n_atomic(unsigned n) { m_n_atomic-=n; } + unsigned get_n_atomic() const { return m_n_atomic; } + void inc_n_atomic() { m_n_atomic++; } + void dec_n_atomic(unsigned n) { m_n_atomic -= n; } - void set_membar() { m_membar=true; } - void clear_membar() { m_membar=false; } - bool get_membar() const { return m_membar; } - address_type get_pc() const { return m_next_pc; } - void set_next_pc( address_type pc ) { m_next_pc = pc; } + void set_membar() { m_membar = true; } + void clear_membar() { m_membar = false; } + bool get_membar() const { return m_membar; } + address_type get_pc() const { return m_next_pc; } + void set_next_pc(address_type pc) { m_next_pc = pc; } - void store_info_of_last_inst_at_barrier(const warp_inst_t *pI){ m_inst_at_barrier = *pI;} - warp_inst_t * restore_info_of_last_inst_at_barrier(){ return &m_inst_at_barrier;} + void store_info_of_last_inst_at_barrier(const warp_inst_t *pI) { + m_inst_at_barrier = *pI; + } + warp_inst_t *restore_info_of_last_inst_at_barrier() { + return &m_inst_at_barrier; + } - void ibuffer_fill( unsigned slot, const warp_inst_t *pI ) - { - assert(slot < IBUFFER_SIZE ); - m_ibuffer[slot].m_inst=pI; - m_ibuffer[slot].m_valid=true; - m_next=0; + void ibuffer_fill(unsigned slot, const warp_inst_t *pI) { + assert(slot < IBUFFER_SIZE); + m_ibuffer[slot].m_inst = pI; + m_ibuffer[slot].m_valid = true; + m_next = 0; + } + bool ibuffer_empty() const { + for (unsigned i = 0; i < IBUFFER_SIZE; i++) + if (m_ibuffer[i].m_valid) return false; + return true; + } + void ibuffer_flush() { + for (unsigned i = 0; i < IBUFFER_SIZE; i++) { + if (m_ibuffer[i].m_valid) dec_inst_in_pipeline(); + m_ibuffer[i].m_inst = NULL; + m_ibuffer[i].m_valid = false; } - bool ibuffer_empty() const - { - for( unsigned i=0; i < IBUFFER_SIZE; i++) - if(m_ibuffer[i].m_valid) - return false; - return true; - } - void ibuffer_flush() - { - for(unsigned i=0;i<IBUFFER_SIZE;i++) { - if( m_ibuffer[i].m_valid ) - dec_inst_in_pipeline(); - m_ibuffer[i].m_inst=NULL; - m_ibuffer[i].m_valid=false; - } - } - const warp_inst_t *ibuffer_next_inst() { return m_ibuffer[m_next].m_inst; } - bool ibuffer_next_valid() { return m_ibuffer[m_next].m_valid; } - void ibuffer_free() - { - m_ibuffer[m_next].m_inst = NULL; - m_ibuffer[m_next].m_valid = false; - } - void ibuffer_step() { m_next = (m_next+1)%IBUFFER_SIZE; } + } + const warp_inst_t *ibuffer_next_inst() { return m_ibuffer[m_next].m_inst; } + bool ibuffer_next_valid() { return m_ibuffer[m_next].m_valid; } + void ibuffer_free() { + m_ibuffer[m_next].m_inst = NULL; + m_ibuffer[m_next].m_valid = false; + } + void ibuffer_step() { m_next = (m_next + 1) % IBUFFER_SIZE; } - bool imiss_pending() const { return m_imiss_pending; } - void set_imiss_pending() { m_imiss_pending=true; } - void clear_imiss_pending() { m_imiss_pending=false; } + bool imiss_pending() const { return m_imiss_pending; } + void set_imiss_pending() { m_imiss_pending = true; } + void clear_imiss_pending() { m_imiss_pending = false; } - bool stores_done() const { return m_stores_outstanding == 0; } - void inc_store_req() { m_stores_outstanding++; } - void dec_store_req() - { - assert( m_stores_outstanding > 0 ); - m_stores_outstanding--; - } + bool stores_done() const { return m_stores_outstanding == 0; } + void inc_store_req() { m_stores_outstanding++; } + void dec_store_req() { + assert(m_stores_outstanding > 0); + m_stores_outstanding--; + } - unsigned num_inst_in_buffer() const - { - unsigned count=0; - for(unsigned i=0;i<IBUFFER_SIZE;i++) { - if( m_ibuffer[i].m_valid ) - count++; - } - return count; - } - unsigned num_inst_in_pipeline() const { return m_inst_in_pipeline;} - unsigned num_issued_inst_in_pipeline() const {return (num_inst_in_pipeline()-num_inst_in_buffer());} - bool inst_in_pipeline() const { return m_inst_in_pipeline > 0; } - void inc_inst_in_pipeline() { m_inst_in_pipeline++; } - void dec_inst_in_pipeline() - { - assert( m_inst_in_pipeline > 0 ); - m_inst_in_pipeline--; + unsigned num_inst_in_buffer() const { + unsigned count = 0; + for (unsigned i = 0; i < IBUFFER_SIZE; i++) { + if (m_ibuffer[i].m_valid) count++; } + return count; + } + unsigned num_inst_in_pipeline() const { return m_inst_in_pipeline; } + unsigned num_issued_inst_in_pipeline() const { + return (num_inst_in_pipeline() - num_inst_in_buffer()); + } + bool inst_in_pipeline() const { return m_inst_in_pipeline > 0; } + void inc_inst_in_pipeline() { m_inst_in_pipeline++; } + void dec_inst_in_pipeline() { + assert(m_inst_in_pipeline > 0); + m_inst_in_pipeline--; + } - unsigned get_cta_id() const { return m_cta_id; } + unsigned get_cta_id() const { return m_cta_id; } - unsigned get_dynamic_warp_id() const { return m_dynamic_warp_id; } - unsigned get_warp_id() const { return m_warp_id; } + unsigned get_dynamic_warp_id() const { return m_dynamic_warp_id; } + unsigned get_warp_id() const { return m_warp_id; } -private: - static const unsigned IBUFFER_SIZE=2; - class shader_core_ctx *m_shader; - unsigned m_cta_id; - unsigned m_warp_id; - unsigned m_warp_size; - unsigned m_dynamic_warp_id; + private: + static const unsigned IBUFFER_SIZE = 2; + class shader_core_ctx *m_shader; + unsigned m_cta_id; + unsigned m_warp_id; + unsigned m_warp_size; + unsigned m_dynamic_warp_id; - address_type m_next_pc; - unsigned n_completed; // number of threads in warp completed - std::bitset<MAX_WARP_SIZE> m_active_threads; + address_type m_next_pc; + unsigned n_completed; // number of threads in warp completed + std::bitset<MAX_WARP_SIZE> m_active_threads; - bool m_imiss_pending; - - struct ibuffer_entry { - ibuffer_entry() { m_valid = false; m_inst = NULL; } - const warp_inst_t *m_inst; - bool m_valid; - }; + bool m_imiss_pending; - warp_inst_t m_inst_at_barrier; - ibuffer_entry m_ibuffer[IBUFFER_SIZE]; - unsigned m_next; - - unsigned m_n_atomic; // number of outstanding atomic operations - bool m_membar; // if true, warp is waiting at memory barrier + struct ibuffer_entry { + ibuffer_entry() { + m_valid = false; + m_inst = NULL; + } + const warp_inst_t *m_inst; + bool m_valid; + }; - bool m_done_exit; // true once thread exit has been registered for threads in this warp + warp_inst_t m_inst_at_barrier; + ibuffer_entry m_ibuffer[IBUFFER_SIZE]; + unsigned m_next; - unsigned long long m_last_fetch; + unsigned m_n_atomic; // number of outstanding atomic operations + bool m_membar; // if true, warp is waiting at memory barrier - unsigned m_stores_outstanding; // number of store requests sent but not yet acknowledged - unsigned m_inst_in_pipeline; + bool m_done_exit; // true once thread exit has been registered for threads in + // this warp - //Jin: cdp support -public: - unsigned int m_cdp_latency; - bool m_cdp_dummy; -}; + unsigned long long m_last_fetch; + unsigned m_stores_outstanding; // number of store requests sent but not yet + // acknowledged + unsigned m_inst_in_pipeline; + // Jin: cdp support + public: + unsigned int m_cdp_latency; + bool m_cdp_dummy; +}; -inline unsigned hw_tid_from_wid(unsigned wid, unsigned warp_size, unsigned i){return wid * warp_size + i;}; -inline unsigned wid_from_hw_tid(unsigned tid, unsigned warp_size){return tid/warp_size;}; +inline unsigned hw_tid_from_wid(unsigned wid, unsigned warp_size, unsigned i) { + return wid * warp_size + i; +}; +inline unsigned wid_from_hw_tid(unsigned tid, unsigned warp_size) { + return tid / warp_size; +}; const unsigned WARP_PER_CTA_MAX = 64; typedef std::bitset<WARP_PER_CTA_MAX> warp_set_t; -int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ); +int register_bank(int regnum, int wid, unsigned num_banks, + unsigned bank_warp_shift, bool sub_core_model, + unsigned banks_per_sched, unsigned sched_id); class shader_core_ctx; class shader_core_config; class shader_core_stats; -enum scheduler_prioritization_type -{ - SCHEDULER_PRIORITIZATION_LRR = 0, // Loose Round Robin - SCHEDULER_PRIORITIZATION_SRR, // Strict Round Robin - SCHEDULER_PRIORITIZATION_GTO, // Greedy Then Oldest - SCHEDULER_PRIORITIZATION_GTLRR, // Greedy Then Loose Round Robin - SCHEDULER_PRIORITIZATION_GTY, // Greedy Then Youngest - SCHEDULER_PRIORITIZATION_OLDEST, // Oldest First - SCHEDULER_PRIORITIZATION_YOUNGEST, // Youngest First +enum scheduler_prioritization_type { + SCHEDULER_PRIORITIZATION_LRR = 0, // Loose Round Robin + SCHEDULER_PRIORITIZATION_SRR, // Strict Round Robin + SCHEDULER_PRIORITIZATION_GTO, // Greedy Then Oldest + SCHEDULER_PRIORITIZATION_GTLRR, // Greedy Then Loose Round Robin + SCHEDULER_PRIORITIZATION_GTY, // Greedy Then Youngest + SCHEDULER_PRIORITIZATION_OLDEST, // Oldest First + SCHEDULER_PRIORITIZATION_YOUNGEST, // Youngest First }; // Each of these corresponds to a string value in the gpgpsim.config file // For example - to specify the LRR scheudler the config must contain lrr -enum concrete_scheduler -{ - CONCRETE_SCHEDULER_LRR = 0, - CONCRETE_SCHEDULER_GTO, - CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE, - CONCRETE_SCHEDULER_WARP_LIMITING, - CONCRETE_SCHEDULER_OLDEST_FIRST, - NUM_CONCRETE_SCHEDULERS +enum concrete_scheduler { + CONCRETE_SCHEDULER_LRR = 0, + CONCRETE_SCHEDULER_GTO, + CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE, + CONCRETE_SCHEDULER_WARP_LIMITING, + CONCRETE_SCHEDULER_OLDEST_FIRST, + NUM_CONCRETE_SCHEDULERS }; -class scheduler_unit { //this can be copied freely, so can be used in std containers. -public: - scheduler_unit(shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id) - : m_supervised_warps(), m_stats(stats), m_shader(shader), - m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), - m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_int_out(int_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} - virtual ~scheduler_unit(){} - virtual void add_supervised_warp_id(int i) { - m_supervised_warps.push_back(&warp(i)); - } - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.end(); - } +class scheduler_unit { // this can be copied freely, so can be used in std + // containers. + public: + scheduler_unit(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, register_set *sp_out, + register_set *dp_out, register_set *sfu_out, + register_set *int_out, register_set *tensor_core_out, + register_set *mem_out, int id) + : m_supervised_warps(), + m_stats(stats), + m_shader(shader), + m_scoreboard(scoreboard), + m_simt_stack(simt), + /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), + m_sp_out(sp_out), + m_dp_out(dp_out), + m_sfu_out(sfu_out), + m_int_out(int_out), + m_tensor_core_out(tensor_core_out), + m_mem_out(mem_out), + m_id(id) {} + virtual ~scheduler_unit() {} + virtual void add_supervised_warp_id(int i) { + m_supervised_warps.push_back(&warp(i)); + } + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.end(); + } + + // The core scheduler cycle method is meant to be common between + // all the derived schedulers. The scheduler's behaviour can be + // modified by changing the contents of the m_next_cycle_prioritized_warps + // list. + void cycle(); + // These are some common ordering fucntions that the + // higher order schedulers can take advantage of + template <typename T> + void order_lrr( + typename std::vector<T> &result_list, + const typename std::vector<T> &input_list, + const typename std::vector<T>::const_iterator &last_issued_from_input, + unsigned num_warps_to_add); - // The core scheduler cycle method is meant to be common between - // all the derived schedulers. The scheduler's behaviour can be - // modified by changing the contents of the m_next_cycle_prioritized_warps list. - void cycle(); + enum OrderingType { + // The item that issued last is prioritized first then the sorted result + // of the priority_function + ORDERING_GREEDY_THEN_PRIORITY_FUNC = 0, + // No greedy scheduling based on last to issue. Only the priority function + // determines priority + ORDERED_PRIORITY_FUNC_ONLY, + NUM_ORDERING, + }; + template <typename U> + void order_by_priority( + std::vector<U> &result_list, const typename std::vector<U> &input_list, + const typename std::vector<U>::const_iterator &last_issued_from_input, + unsigned num_warps_to_add, OrderingType age_ordering, + bool (*priority_func)(U lhs, U rhs)); + static bool sort_warps_by_oldest_dynamic_id(shd_warp_t *lhs, shd_warp_t *rhs); - // These are some common ordering fucntions that the - // higher order schedulers can take advantage of - template < typename T > - void order_lrr( typename std::vector< T >& result_list, - const typename std::vector< T >& input_list, - const typename std::vector< T >::const_iterator& last_issued_from_input, - unsigned num_warps_to_add ); - - enum OrderingType - { - // The item that issued last is prioritized first then the sorted result - // of the priority_function - ORDERING_GREEDY_THEN_PRIORITY_FUNC = 0, - // No greedy scheduling based on last to issue. Only the priority function determines - // priority - ORDERED_PRIORITY_FUNC_ONLY, - NUM_ORDERING, - }; - template < typename U > - void order_by_priority( std::vector< U >& result_list, - const typename std::vector< U >& input_list, - const typename std::vector< U >::const_iterator& last_issued_from_input, - unsigned num_warps_to_add, - OrderingType age_ordering, - bool (*priority_func)(U lhs, U rhs) ); - static bool sort_warps_by_oldest_dynamic_id(shd_warp_t* lhs, shd_warp_t* rhs); + // Derived classes can override this function to populate + // m_supervised_warps with their scheduling policies + virtual void order_warps() = 0; - // Derived classes can override this function to populate - // m_supervised_warps with their scheduling policies - virtual void order_warps() = 0; + int get_schd_id() const { return m_id; } - int get_schd_id() const {return m_id;} + protected: + virtual void do_on_warp_issued( + unsigned warp_id, unsigned num_issued, + const std::vector<shd_warp_t *>::const_iterator &prioritized_iter); + inline int get_sid() const; -protected: - virtual void do_on_warp_issued( unsigned warp_id, - unsigned num_issued, - const std::vector< shd_warp_t* >::const_iterator& prioritized_iter ); - inline int get_sid() const; -protected: - shd_warp_t& warp(int i); + protected: + shd_warp_t &warp(int i); - // This is the prioritized warp list that is looped over each cycle to determine - // which warp gets to issue. - std::vector< shd_warp_t* > m_next_cycle_prioritized_warps; - // The m_supervised_warps list is all the warps this scheduler is supposed to - // arbitrate between. This is useful in systems where there is more than - // one warp scheduler. In a single scheduler system, this is simply all - // the warps assigned to this core. - std::vector< shd_warp_t* > m_supervised_warps; - // This is the iterator pointer to the last supervised warp you issued - std::vector< shd_warp_t* >::const_iterator m_last_supervised_issued; - shader_core_stats *m_stats; - shader_core_ctx* m_shader; - // these things should become accessors: but would need a bigger rearchitect of how shader_core_ctx interacts with its parts. - Scoreboard* m_scoreboard; - simt_stack** m_simt_stack; - //warp_inst_t** m_pipeline_reg; - std::vector<shd_warp_t>* m_warp; - register_set* m_sp_out; - register_set* m_dp_out; - register_set* m_sfu_out; - register_set* m_int_out; - register_set* m_tensor_core_out; - register_set* m_mem_out; + // This is the prioritized warp list that is looped over each cycle to + // determine which warp gets to issue. + std::vector<shd_warp_t *> m_next_cycle_prioritized_warps; + // The m_supervised_warps list is all the warps this scheduler is supposed to + // arbitrate between. This is useful in systems where there is more than + // one warp scheduler. In a single scheduler system, this is simply all + // the warps assigned to this core. + std::vector<shd_warp_t *> m_supervised_warps; + // This is the iterator pointer to the last supervised warp you issued + std::vector<shd_warp_t *>::const_iterator m_last_supervised_issued; + shader_core_stats *m_stats; + shader_core_ctx *m_shader; + // these things should become accessors: but would need a bigger rearchitect + // of how shader_core_ctx interacts with its parts. + Scoreboard *m_scoreboard; + simt_stack **m_simt_stack; + // warp_inst_t** m_pipeline_reg; + std::vector<shd_warp_t> *m_warp; + register_set *m_sp_out; + register_set *m_dp_out; + register_set *m_sfu_out; + register_set *m_int_out; + register_set *m_tensor_core_out; + register_set *m_mem_out; - int m_id; + int m_id; }; class lrr_scheduler : public scheduler_unit { -public: - lrr_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} - virtual ~lrr_scheduler () {} - virtual void order_warps (); - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.end(); - } + public: + lrr_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, register_set *sp_out, + register_set *dp_out, register_set *sfu_out, + register_set *int_out, register_set *tensor_core_out, + register_set *mem_out, int id) + : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out, + sfu_out, int_out, tensor_core_out, mem_out, id) {} + virtual ~lrr_scheduler() {} + virtual void order_warps(); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.end(); + } }; class gto_scheduler : public scheduler_unit { -public: - gto_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} - virtual ~gto_scheduler () {} - virtual void order_warps (); - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.begin(); - } - + public: + gto_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, register_set *sp_out, + register_set *dp_out, register_set *sfu_out, + register_set *int_out, register_set *tensor_core_out, + register_set *mem_out, int id) + : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out, + sfu_out, int_out, tensor_core_out, mem_out, id) {} + virtual ~gto_scheduler() {} + virtual void order_warps(); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } }; class oldest_scheduler : public scheduler_unit { -public: - oldest_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} - virtual ~oldest_scheduler () {} - virtual void order_warps (); - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.begin(); - } - + public: + oldest_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, register_set *sp_out, + register_set *dp_out, register_set *sfu_out, + register_set *int_out, register_set *tensor_core_out, + register_set *mem_out, int id) + : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out, + sfu_out, int_out, tensor_core_out, mem_out, id) {} + virtual ~oldest_scheduler() {} + virtual void order_warps(); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } }; class two_level_active_scheduler : public scheduler_unit { -public: - two_level_active_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id, - char* config_str ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ), - m_pending_warps() - { - unsigned inner_level_readin; - unsigned outer_level_readin; - int ret = sscanf( config_str, - "two_level_active:%d:%d:%d", - &m_max_active_warps, - &inner_level_readin, - &outer_level_readin); - assert( 3 == ret ); - m_inner_level_prioritization=(scheduler_prioritization_type)inner_level_readin; - m_outer_level_prioritization=(scheduler_prioritization_type)outer_level_readin; - } - virtual ~two_level_active_scheduler () {} - virtual void order_warps(); - void add_supervised_warp_id(int i) { - if ( m_next_cycle_prioritized_warps.size() < m_max_active_warps ) { - m_next_cycle_prioritized_warps.push_back( &warp(i) ); - } else { - m_pending_warps.push_back(&warp(i)); - } - } - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.begin(); + public: + two_level_active_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, + register_set *sp_out, register_set *dp_out, + register_set *sfu_out, register_set *int_out, + register_set *tensor_core_out, + register_set *mem_out, int id, char *config_str) + : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out, + sfu_out, int_out, tensor_core_out, mem_out, id), + m_pending_warps() { + unsigned inner_level_readin; + unsigned outer_level_readin; + int ret = + sscanf(config_str, "two_level_active:%d:%d:%d", &m_max_active_warps, + &inner_level_readin, &outer_level_readin); + assert(3 == ret); + m_inner_level_prioritization = + (scheduler_prioritization_type)inner_level_readin; + m_outer_level_prioritization = + (scheduler_prioritization_type)outer_level_readin; + } + virtual ~two_level_active_scheduler() {} + virtual void order_warps(); + void add_supervised_warp_id(int i) { + if (m_next_cycle_prioritized_warps.size() < m_max_active_warps) { + m_next_cycle_prioritized_warps.push_back(&warp(i)); + } else { + m_pending_warps.push_back(&warp(i)); } + } + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } -protected: - virtual void do_on_warp_issued( unsigned warp_id, - unsigned num_issued, - const std::vector< shd_warp_t* >::const_iterator& prioritized_iter ); + protected: + virtual void do_on_warp_issued( + unsigned warp_id, unsigned num_issued, + const std::vector<shd_warp_t *>::const_iterator &prioritized_iter); -private: - std::deque< shd_warp_t* > m_pending_warps; - scheduler_prioritization_type m_inner_level_prioritization; - scheduler_prioritization_type m_outer_level_prioritization; - unsigned m_max_active_warps; + private: + std::deque<shd_warp_t *> m_pending_warps; + scheduler_prioritization_type m_inner_level_prioritization; + scheduler_prioritization_type m_outer_level_prioritization; + unsigned m_max_active_warps; }; // Static Warp Limiting Scheduler class swl_scheduler : public scheduler_unit { -public: - swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, - Scoreboard* scoreboard, simt_stack** simt, - std::vector<shd_warp_t>* warp, - register_set* sp_out, - register_set* dp_out, - register_set* sfu_out, - register_set* int_out, - register_set* tensor_core_out, - register_set* mem_out, - int id, - char* config_string ); - virtual ~swl_scheduler () {} - virtual void order_warps (); - virtual void done_adding_supervised_warps() { - m_last_supervised_issued = m_supervised_warps.begin(); - } + public: + swl_scheduler(shader_core_stats *stats, shader_core_ctx *shader, + Scoreboard *scoreboard, simt_stack **simt, + std::vector<shd_warp_t> *warp, register_set *sp_out, + register_set *dp_out, register_set *sfu_out, + register_set *int_out, register_set *tensor_core_out, + register_set *mem_out, int id, char *config_string); + virtual ~swl_scheduler() {} + virtual void order_warps(); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } -protected: - scheduler_prioritization_type m_prioritization; - unsigned m_num_warps_to_limit; + protected: + scheduler_prioritization_type m_prioritization; + unsigned m_num_warps_to_limit; }; +class opndcoll_rfu_t { // operand collector based register file unit + public: + // constructors + opndcoll_rfu_t() { + m_num_banks = 0; + m_shader = NULL; + m_initialized = false; + } + void add_cu_set(unsigned cu_set, unsigned num_cu, unsigned num_dispatch); + typedef std::vector<register_set *> port_vector_t; + typedef std::vector<unsigned int> uint_vector_t; + void add_port(port_vector_t &input, port_vector_t &ouput, + uint_vector_t cu_sets); + void init(unsigned num_banks, shader_core_ctx *shader); + // modifiers + bool writeback(warp_inst_t &warp); -class opndcoll_rfu_t { // operand collector based register file unit -public: - // constructors - opndcoll_rfu_t() - { - m_num_banks=0; - m_shader=NULL; - m_initialized=false; - } - void add_cu_set(unsigned cu_set, unsigned num_cu, unsigned num_dispatch); - typedef std::vector<register_set*> port_vector_t; - typedef std::vector<unsigned int> uint_vector_t; - void add_port( port_vector_t & input, port_vector_t & ouput, uint_vector_t cu_sets); - void init( unsigned num_banks, shader_core_ctx *shader ); - - // modifiers - bool writeback( warp_inst_t &warp ); + void step() { + dispatch_ready_cu(); + allocate_reads(); + for (unsigned p = 0; p < m_in_ports.size(); p++) allocate_cu(p); + process_banks(); + } - void step() - { - dispatch_ready_cu(); - allocate_reads(); - for( unsigned p = 0 ; p < m_in_ports.size(); p++ ) - allocate_cu( p ); - process_banks(); - } - - void dump( FILE *fp ) const - { - fprintf(fp,"\n"); - fprintf(fp,"Operand Collector State:\n"); - for( unsigned n=0; n < m_cu.size(); n++ ) { - fprintf(fp," CU-%2u: ", n); - m_cu[n]->dump(fp,m_shader); - } - m_arbiter.dump(fp); - } - - shader_core_ctx *shader_core() { return m_shader; } + void dump(FILE *fp) const { + fprintf(fp, "\n"); + fprintf(fp, "Operand Collector State:\n"); + for (unsigned n = 0; n < m_cu.size(); n++) { + fprintf(fp, " CU-%2u: ", n); + m_cu[n]->dump(fp, m_shader); + } + m_arbiter.dump(fp); + } -private: + shader_core_ctx *shader_core() { return m_shader; } - void process_banks() - { - m_arbiter.reset_alloction(); - } + private: + void process_banks() { m_arbiter.reset_alloction(); } - void dispatch_ready_cu(); - void allocate_cu( unsigned port ); - void allocate_reads(); + void dispatch_ready_cu(); + void allocate_cu(unsigned port); + void allocate_reads(); - // types + // types - class collector_unit_t; + class collector_unit_t; - class op_t { + class op_t { public: + op_t() { m_valid = false; } + op_t(collector_unit_t *cu, unsigned op, unsigned reg, unsigned num_banks, + unsigned bank_warp_shift, bool sub_core_model, + unsigned banks_per_sched, unsigned sched_id) { + m_valid = true; + m_warp = NULL; + m_cu = cu; + m_operand = op; + m_register = reg; + m_shced_id = sched_id; + m_bank = register_bank(reg, cu->get_warp_id(), num_banks, bank_warp_shift, + sub_core_model, banks_per_sched, sched_id); + } + op_t(const warp_inst_t *warp, unsigned reg, unsigned num_banks, + unsigned bank_warp_shift, bool sub_core_model, + unsigned banks_per_sched, unsigned sched_id) { + m_valid = true; + m_warp = warp; + m_register = reg; + m_cu = NULL; + m_operand = -1; + m_shced_id = sched_id; + m_bank = register_bank(reg, warp->warp_id(), num_banks, bank_warp_shift, + sub_core_model, banks_per_sched, sched_id); + } - op_t() { m_valid = false; } - op_t( collector_unit_t *cu, unsigned op, unsigned reg, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ) - { - m_valid = true; - m_warp=NULL; - m_cu = cu; - m_operand = op; - m_register = reg; - m_shced_id = sched_id; - m_bank = register_bank(reg,cu->get_warp_id(),num_banks,bank_warp_shift, sub_core_model, banks_per_sched, sched_id); - } - op_t( const warp_inst_t *warp, unsigned reg, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ) - { - m_valid=true; - m_warp=warp; - m_register=reg; - m_cu=NULL; - m_operand = -1; - m_shced_id = sched_id; - m_bank = register_bank(reg,warp->warp_id(),num_banks,bank_warp_shift, sub_core_model, banks_per_sched, sched_id); - } + // accessors + bool valid() const { return m_valid; } + unsigned get_reg() const { + assert(m_valid); + return m_register; + } + unsigned get_wid() const { + if (m_warp) + return m_warp->warp_id(); + else if (m_cu) + return m_cu->get_warp_id(); + else + abort(); + } + unsigned get_sid() const { return m_shced_id; } + unsigned get_active_count() const { + if (m_warp) + return m_warp->active_count(); + else if (m_cu) + return m_cu->get_active_count(); + else + abort(); + } + const active_mask_t &get_active_mask() { + if (m_warp) + return m_warp->get_active_mask(); + else if (m_cu) + return m_cu->get_active_mask(); + else + abort(); + } + unsigned get_sp_op() const { + if (m_warp) + return m_warp->sp_op; + else if (m_cu) + return m_cu->get_sp_op(); + else + abort(); + } + unsigned get_oc_id() const { return m_cu->get_id(); } + unsigned get_bank() const { return m_bank; } + unsigned get_operand() const { return m_operand; } + void dump(FILE *fp) const { + if (m_cu) + fprintf(fp, " <R%u, CU:%u, w:%02u> ", m_register, m_cu->get_id(), + m_cu->get_warp_id()); + else if (!m_warp->empty()) + fprintf(fp, " <R%u, wid:%02u> ", m_register, m_warp->warp_id()); + } + std::string get_reg_string() const { + char buffer[64]; + snprintf(buffer, 64, "R%u", m_register); + return std::string(buffer); + } - // accessors - bool valid() const { return m_valid; } - unsigned get_reg() const - { - assert( m_valid ); - return m_register; - } - unsigned get_wid() const - { - if( m_warp ) return m_warp->warp_id(); - else if( m_cu ) return m_cu->get_warp_id(); - else abort(); - } - unsigned get_sid() const - { - return m_shced_id; - } - unsigned get_active_count() const - { - if( m_warp ) return m_warp->active_count(); - else if( m_cu ) return m_cu->get_active_count(); - else abort(); - } - const active_mask_t & get_active_mask() - { - if( m_warp ) return m_warp->get_active_mask(); - else if( m_cu ) return m_cu->get_active_mask(); - else abort(); - } - unsigned get_sp_op() const - { - if( m_warp ) return m_warp->sp_op; - else if( m_cu ) return m_cu->get_sp_op(); - else abort(); - } - unsigned get_oc_id() const { return m_cu->get_id(); } - unsigned get_bank() const { return m_bank; } - unsigned get_operand() const { return m_operand; } - void dump(FILE *fp) const - { - if(m_cu) - fprintf(fp," <R%u, CU:%u, w:%02u> ", m_register,m_cu->get_id(),m_cu->get_warp_id()); - else if( !m_warp->empty() ) - fprintf(fp," <R%u, wid:%02u> ", m_register,m_warp->warp_id() ); - } - std::string get_reg_string() const - { - char buffer[64]; - snprintf(buffer,64,"R%u", m_register); - return std::string(buffer); - } + // modifiers + void reset() { m_valid = false; } - // modifiers - void reset() { m_valid = false; } private: - bool m_valid; - collector_unit_t *m_cu; - const warp_inst_t *m_warp; - unsigned m_operand; // operand offset in instruction. e.g., add r1,r2,r3; r2 is oprd 0, r3 is 1 (r1 is dst) - unsigned m_register; - unsigned m_bank; - unsigned m_shced_id; //scheduler id that has issued this inst - }; + bool m_valid; + collector_unit_t *m_cu; + const warp_inst_t *m_warp; + unsigned m_operand; // operand offset in instruction. e.g., add r1,r2,r3; + // r2 is oprd 0, r3 is 1 (r1 is dst) + unsigned m_register; + unsigned m_bank; + unsigned m_shced_id; // scheduler id that has issued this inst + }; - enum alloc_t { - NO_ALLOC, - READ_ALLOC, - WRITE_ALLOC, - }; + enum alloc_t { + NO_ALLOC, + READ_ALLOC, + WRITE_ALLOC, + }; - class allocation_t { + class allocation_t { public: - allocation_t() { m_allocation = NO_ALLOC; } - bool is_read() const { return m_allocation==READ_ALLOC; } - bool is_write() const {return m_allocation==WRITE_ALLOC; } - bool is_free() const {return m_allocation==NO_ALLOC; } - void dump(FILE *fp) const { - if( m_allocation == NO_ALLOC ) { fprintf(fp,"<free>"); } - else if( m_allocation == READ_ALLOC ) { fprintf(fp,"rd: "); m_op.dump(fp); } - else if( m_allocation == WRITE_ALLOC ) { fprintf(fp,"wr: "); m_op.dump(fp); } - fprintf(fp,"\n"); + allocation_t() { m_allocation = NO_ALLOC; } + bool is_read() const { return m_allocation == READ_ALLOC; } + bool is_write() const { return m_allocation == WRITE_ALLOC; } + bool is_free() const { return m_allocation == NO_ALLOC; } + void dump(FILE *fp) const { + if (m_allocation == NO_ALLOC) { + fprintf(fp, "<free>"); + } else if (m_allocation == READ_ALLOC) { + fprintf(fp, "rd: "); + m_op.dump(fp); + } else if (m_allocation == WRITE_ALLOC) { + fprintf(fp, "wr: "); + m_op.dump(fp); } - void alloc_read( const op_t &op ) { assert(is_free()); m_allocation=READ_ALLOC; m_op=op; } - void alloc_write( const op_t &op ) { assert(is_free()); m_allocation=WRITE_ALLOC; m_op=op; } - void reset() { m_allocation = NO_ALLOC; } + fprintf(fp, "\n"); + } + void alloc_read(const op_t &op) { + assert(is_free()); + m_allocation = READ_ALLOC; + m_op = op; + } + void alloc_write(const op_t &op) { + assert(is_free()); + m_allocation = WRITE_ALLOC; + m_op = op; + } + void reset() { m_allocation = NO_ALLOC; } + private: - enum alloc_t m_allocation; - op_t m_op; - }; + enum alloc_t m_allocation; + op_t m_op; + }; - class arbiter_t { + class arbiter_t { public: - // constructors - arbiter_t() - { - m_queue=NULL; - m_allocated_bank=NULL; - m_allocator_rr_head=NULL; - _inmatch=NULL; - _outmatch=NULL; - _request=NULL; - m_last_cu=0; - } - void init( unsigned num_cu, unsigned num_banks ) - { - assert(num_cu > 0); - assert(num_banks > 0); - m_num_collectors = num_cu; - m_num_banks = num_banks; - _inmatch = new int[ m_num_banks ]; - _outmatch = new int[ m_num_collectors ]; - _request = new int*[ m_num_banks ]; - for(unsigned i=0; i<m_num_banks;i++) - _request[i] = new int[m_num_collectors]; - m_queue = new std::list<op_t>[num_banks]; - m_allocated_bank = new allocation_t[num_banks]; - m_allocator_rr_head = new unsigned[num_cu]; - for( unsigned n=0; n<num_cu;n++ ) - m_allocator_rr_head[n] = n%num_banks; - reset_alloction(); - } + // constructors + arbiter_t() { + m_queue = NULL; + m_allocated_bank = NULL; + m_allocator_rr_head = NULL; + _inmatch = NULL; + _outmatch = NULL; + _request = NULL; + m_last_cu = 0; + } + void init(unsigned num_cu, unsigned num_banks) { + assert(num_cu > 0); + assert(num_banks > 0); + m_num_collectors = num_cu; + m_num_banks = num_banks; + _inmatch = new int[m_num_banks]; + _outmatch = new int[m_num_collectors]; + _request = new int *[m_num_banks]; + for (unsigned i = 0; i < m_num_banks; i++) + _request[i] = new int[m_num_collectors]; + m_queue = new std::list<op_t>[num_banks]; + m_allocated_bank = new allocation_t[num_banks]; + m_allocator_rr_head = new unsigned[num_cu]; + for (unsigned n = 0; n < num_cu; n++) + m_allocator_rr_head[n] = n % num_banks; + reset_alloction(); + } - // accessors - void dump(FILE *fp) const - { - fprintf(fp,"\n"); - fprintf(fp," Arbiter State:\n"); - fprintf(fp," requests:\n"); - for( unsigned b=0; b<m_num_banks; b++ ) { - fprintf(fp," bank %u : ", b ); - std::list<op_t>::const_iterator o = m_queue[b].begin(); - for(; o != m_queue[b].end(); o++ ) { - o->dump(fp); - } - fprintf(fp,"\n"); - } - fprintf(fp," grants:\n"); - for(unsigned b=0;b<m_num_banks;b++) { - fprintf(fp," bank %u : ", b ); - m_allocated_bank[b].dump(fp); - } - fprintf(fp,"\n"); + // accessors + void dump(FILE *fp) const { + fprintf(fp, "\n"); + fprintf(fp, " Arbiter State:\n"); + fprintf(fp, " requests:\n"); + for (unsigned b = 0; b < m_num_banks; b++) { + fprintf(fp, " bank %u : ", b); + std::list<op_t>::const_iterator o = m_queue[b].begin(); + for (; o != m_queue[b].end(); o++) { + o->dump(fp); + } + fprintf(fp, "\n"); + } + fprintf(fp, " grants:\n"); + for (unsigned b = 0; b < m_num_banks; b++) { + fprintf(fp, " bank %u : ", b); + m_allocated_bank[b].dump(fp); } + fprintf(fp, "\n"); + } - // modifiers - std::list<op_t> allocate_reads(); + // modifiers + std::list<op_t> allocate_reads(); - void add_read_requests( collector_unit_t *cu ) - { - const op_t *src = cu->get_operands(); - for( unsigned i=0; i<MAX_REG_OPERANDS*2; i++) { - const op_t &op = src[i]; - if( op.valid() ) { - unsigned bank = op.get_bank(); - m_queue[bank].push_back(op); - } - } - } - bool bank_idle( unsigned bank ) const - { - return m_allocated_bank[bank].is_free(); - } - void allocate_bank_for_write( unsigned bank, const op_t &op ) - { - assert( bank < m_num_banks ); - m_allocated_bank[bank].alloc_write(op); - } - void allocate_for_read( unsigned bank, const op_t &op ) - { - assert( bank < m_num_banks ); - m_allocated_bank[bank].alloc_read(op); - } - void reset_alloction() - { - for( unsigned b=0; b < m_num_banks; b++ ) - m_allocated_bank[b].reset(); + void add_read_requests(collector_unit_t *cu) { + const op_t *src = cu->get_operands(); + for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) { + const op_t &op = src[i]; + if (op.valid()) { + unsigned bank = op.get_bank(); + m_queue[bank].push_back(op); + } } + } + bool bank_idle(unsigned bank) const { + return m_allocated_bank[bank].is_free(); + } + void allocate_bank_for_write(unsigned bank, const op_t &op) { + assert(bank < m_num_banks); + m_allocated_bank[bank].alloc_write(op); + } + void allocate_for_read(unsigned bank, const op_t &op) { + assert(bank < m_num_banks); + m_allocated_bank[bank].alloc_read(op); + } + void reset_alloction() { + for (unsigned b = 0; b < m_num_banks; b++) m_allocated_bank[b].reset(); + } private: - unsigned m_num_banks; - unsigned m_num_collectors; + unsigned m_num_banks; + unsigned m_num_collectors; - allocation_t *m_allocated_bank; // bank # -> register that wins - std::list<op_t> *m_queue; + allocation_t *m_allocated_bank; // bank # -> register that wins + std::list<op_t> *m_queue; - unsigned *m_allocator_rr_head; // cu # -> next bank to check for request (rr-arb) - unsigned m_last_cu; // first cu to check while arb-ing banks (rr) + unsigned * + m_allocator_rr_head; // cu # -> next bank to check for request (rr-arb) + unsigned m_last_cu; // first cu to check while arb-ing banks (rr) - int *_inmatch; - int *_outmatch; - int **_request; - }; + int *_inmatch; + int *_outmatch; + int **_request; + }; - class input_port_t { + class input_port_t { public: - input_port_t(port_vector_t & input, port_vector_t & output, uint_vector_t cu_sets) - : m_in(input),m_out(output), m_cu_sets(cu_sets) - { - assert(input.size() == output.size()); - assert(not m_cu_sets.empty()); - } - //private: - port_vector_t m_in,m_out; - uint_vector_t m_cu_sets; - }; + input_port_t(port_vector_t &input, port_vector_t &output, + uint_vector_t cu_sets) + : m_in(input), m_out(output), m_cu_sets(cu_sets) { + assert(input.size() == output.size()); + assert(not m_cu_sets.empty()); + } + // private: + port_vector_t m_in, m_out; + uint_vector_t m_cu_sets; + }; - class collector_unit_t { + class collector_unit_t { public: - // constructors - collector_unit_t() - { - m_free = true; - m_warp = NULL; - m_output_register = NULL; - m_src_op = new op_t[MAX_REG_OPERANDS*2]; - m_not_ready.reset(); - m_warp_id = -1; - m_num_banks = 0; - m_bank_warp_shift = 0; - } - // accessors - bool ready() const; - const op_t *get_operands() const { return m_src_op; } - void dump(FILE *fp, const shader_core_ctx *shader ) const; + // constructors + collector_unit_t() { + m_free = true; + m_warp = NULL; + m_output_register = NULL; + m_src_op = new op_t[MAX_REG_OPERANDS * 2]; + m_not_ready.reset(); + m_warp_id = -1; + m_num_banks = 0; + m_bank_warp_shift = 0; + } + // accessors + bool ready() const; + const op_t *get_operands() const { return m_src_op; } + void dump(FILE *fp, const shader_core_ctx *shader) const; - unsigned get_warp_id() const { return m_warp_id; } - unsigned get_active_count() const { return m_warp->active_count(); } - const active_mask_t & get_active_mask() const { return m_warp->get_active_mask(); } - unsigned get_sp_op() const { return m_warp->sp_op; } - unsigned get_id() const { return m_cuid; } // returns CU hw id + unsigned get_warp_id() const { return m_warp_id; } + unsigned get_active_count() const { return m_warp->active_count(); } + const active_mask_t &get_active_mask() const { + return m_warp->get_active_mask(); + } + unsigned get_sp_op() const { return m_warp->sp_op; } + unsigned get_id() const { return m_cuid; } // returns CU hw id - // modifiers - void init(unsigned n, - unsigned num_banks, - unsigned log2_warp_size, - const core_config *config, - opndcoll_rfu_t *rfu, - bool m_sub_core_model, - unsigned num_banks_per_sched); - bool allocate( register_set* pipeline_reg, register_set* output_reg ); + // modifiers + void init(unsigned n, unsigned num_banks, unsigned log2_warp_size, + const core_config *config, opndcoll_rfu_t *rfu, + bool m_sub_core_model, unsigned num_banks_per_sched); + bool allocate(register_set *pipeline_reg, register_set *output_reg); - void collect_operand( unsigned op ) - { - m_not_ready.reset(op); - } - unsigned get_num_operands() const{ - return m_warp->get_num_operands(); - } - unsigned get_num_regs() const{ - return m_warp->get_num_regs(); - } - void dispatch(); - bool is_free(){return m_free;} + void collect_operand(unsigned op) { m_not_ready.reset(op); } + unsigned get_num_operands() const { return m_warp->get_num_operands(); } + unsigned get_num_regs() const { return m_warp->get_num_regs(); } + void dispatch(); + bool is_free() { return m_free; } private: - bool m_free; - unsigned m_cuid; // collector unit hw id - unsigned m_warp_id; - warp_inst_t *m_warp; - register_set* m_output_register; // pipeline register to issue to when ready - op_t *m_src_op; - std::bitset<MAX_REG_OPERANDS*2> m_not_ready; - unsigned m_num_banks; - unsigned m_bank_warp_shift; - opndcoll_rfu_t *m_rfu; - - unsigned m_num_banks_per_sched; - bool m_sub_core_model; + bool m_free; + unsigned m_cuid; // collector unit hw id + unsigned m_warp_id; + warp_inst_t *m_warp; + register_set + *m_output_register; // pipeline register to issue to when ready + op_t *m_src_op; + std::bitset<MAX_REG_OPERANDS * 2> m_not_ready; + unsigned m_num_banks; + unsigned m_bank_warp_shift; + opndcoll_rfu_t *m_rfu; - }; + unsigned m_num_banks_per_sched; + bool m_sub_core_model; + }; - class dispatch_unit_t { + class dispatch_unit_t { public: - dispatch_unit_t(std::vector<collector_unit_t>* cus) - { - m_last_cu=0; - m_collector_units=cus; - m_num_collectors = (*cus).size(); - m_next_cu=0; - } + dispatch_unit_t(std::vector<collector_unit_t> *cus) { + m_last_cu = 0; + m_collector_units = cus; + m_num_collectors = (*cus).size(); + m_next_cu = 0; + } - collector_unit_t *find_ready() - { - for( unsigned n=0; n < m_num_collectors; n++ ) { - unsigned c=(m_last_cu+n+1)%m_num_collectors; - if( (*m_collector_units)[c].ready() ) { - m_last_cu=c; - return &((*m_collector_units)[c]); - } - } - return NULL; + collector_unit_t *find_ready() { + for (unsigned n = 0; n < m_num_collectors; n++) { + unsigned c = (m_last_cu + n + 1) % m_num_collectors; + if ((*m_collector_units)[c].ready()) { + m_last_cu = c; + return &((*m_collector_units)[c]); + } } + return NULL; + } private: - unsigned m_num_collectors; - std::vector<collector_unit_t>* m_collector_units; - unsigned m_last_cu; // dispatch ready cu's rr - unsigned m_next_cu; // for initialization - }; + unsigned m_num_collectors; + std::vector<collector_unit_t> *m_collector_units; + unsigned m_last_cu; // dispatch ready cu's rr + unsigned m_next_cu; // for initialization + }; - // opndcoll_rfu_t data members - bool m_initialized; + // opndcoll_rfu_t data members + bool m_initialized; - unsigned m_num_collector_sets; - //unsigned m_num_collectors; - unsigned m_num_banks; - unsigned m_bank_warp_shift; - unsigned m_warp_size; - std::vector<collector_unit_t *> m_cu; - arbiter_t m_arbiter; + unsigned m_num_collector_sets; + // unsigned m_num_collectors; + unsigned m_num_banks; + unsigned m_bank_warp_shift; + unsigned m_warp_size; + std::vector<collector_unit_t *> m_cu; + arbiter_t m_arbiter; - unsigned m_num_banks_per_sched; - unsigned m_num_warp_sceds; - bool sub_core_model; + unsigned m_num_banks_per_sched; + unsigned m_num_warp_sceds; + bool sub_core_model; - //unsigned m_num_ports; - //std::vector<warp_inst_t**> m_input; - //std::vector<warp_inst_t**> m_output; - //std::vector<unsigned> m_num_collector_units; - //warp_inst_t **m_alu_port; + // unsigned m_num_ports; + // std::vector<warp_inst_t**> m_input; + // std::vector<warp_inst_t**> m_output; + // std::vector<unsigned> m_num_collector_units; + // warp_inst_t **m_alu_port; - std::vector<input_port_t> m_in_ports; - typedef std::map<unsigned /* collector set */, std::vector<collector_unit_t> /*collector sets*/ > cu_sets_t; - cu_sets_t m_cus; - std::vector<dispatch_unit_t> m_dispatch_units; + std::vector<input_port_t> m_in_ports; + typedef std::map<unsigned /* collector set */, + std::vector<collector_unit_t> /*collector sets*/> + cu_sets_t; + cu_sets_t m_cus; + std::vector<dispatch_unit_t> m_dispatch_units; - //typedef std::map<warp_inst_t**/*port*/,dispatch_unit_t> port_to_du_t; - //port_to_du_t m_dispatch_units; - //std::map<warp_inst_t**,std::list<collector_unit_t*> > m_free_cu; - shader_core_ctx *m_shader; + // typedef std::map<warp_inst_t**/*port*/,dispatch_unit_t> port_to_du_t; + // port_to_du_t m_dispatch_units; + // std::map<warp_inst_t**,std::list<collector_unit_t*> > m_free_cu; + shader_core_ctx *m_shader; }; class barrier_set_t { -public: - barrier_set_t(shader_core_ctx * shader, unsigned max_warps_per_core, unsigned max_cta_per_core, unsigned max_barriers_per_cta, unsigned warp_size); - - // during cta allocation - void allocate_barrier( unsigned cta_id, warp_set_t warps ); - - // during cta deallocation - void deallocate_barrier( unsigned cta_id ); - - typedef std::map<unsigned, warp_set_t > cta_to_warp_t; - typedef std::map<unsigned, warp_set_t > bar_id_to_warp_t; /*set of warps reached a specific barrier id*/ + public: + barrier_set_t(shader_core_ctx *shader, unsigned max_warps_per_core, + unsigned max_cta_per_core, unsigned max_barriers_per_cta, + unsigned warp_size); + // during cta allocation + void allocate_barrier(unsigned cta_id, warp_set_t warps); - // individual warp hits barrier - void warp_reaches_barrier( unsigned cta_id, unsigned warp_id, warp_inst_t* inst); + // during cta deallocation + void deallocate_barrier(unsigned cta_id); + typedef std::map<unsigned, warp_set_t> cta_to_warp_t; + typedef std::map<unsigned, warp_set_t> + bar_id_to_warp_t; /*set of warps reached a specific barrier id*/ - // warp reaches exit - void warp_exit( unsigned warp_id ); + // individual warp hits barrier + void warp_reaches_barrier(unsigned cta_id, unsigned warp_id, + warp_inst_t *inst); - // assertions - bool warp_waiting_at_barrier( unsigned warp_id ) const; + // warp reaches exit + void warp_exit(unsigned warp_id); - // debug - void dump(); + // assertions + bool warp_waiting_at_barrier(unsigned warp_id) const; -private: - unsigned m_max_cta_per_core; - unsigned m_max_warps_per_core; - unsigned m_max_barriers_per_cta; - unsigned m_warp_size; - cta_to_warp_t m_cta_to_warps; - bar_id_to_warp_t m_bar_id_to_warps; - warp_set_t m_warp_active; - warp_set_t m_warp_at_barrier; - shader_core_ctx *m_shader; + // debug + void dump(); + private: + unsigned m_max_cta_per_core; + unsigned m_max_warps_per_core; + unsigned m_max_barriers_per_cta; + unsigned m_warp_size; + cta_to_warp_t m_cta_to_warps; + bar_id_to_warp_t m_bar_id_to_warps; + warp_set_t m_warp_active; + warp_set_t m_warp_at_barrier; + shader_core_ctx *m_shader; }; struct insn_latency_info { - unsigned pc; - unsigned long latency; + unsigned pc; + unsigned long latency; }; struct ifetch_buffer_t { - ifetch_buffer_t() { m_valid=false; } + ifetch_buffer_t() { m_valid = false; } - ifetch_buffer_t( address_type pc, unsigned nbytes, unsigned warp_id ) - { - m_valid=true; - m_pc=pc; - m_nbytes=nbytes; - m_warp_id=warp_id; - } + ifetch_buffer_t(address_type pc, unsigned nbytes, unsigned warp_id) { + m_valid = true; + m_pc = pc; + m_nbytes = nbytes; + m_warp_id = warp_id; + } - bool m_valid; - address_type m_pc; - unsigned m_nbytes; - unsigned m_warp_id; + bool m_valid; + address_type m_pc; + unsigned m_nbytes; + unsigned m_warp_id; }; class shader_core_config; class simd_function_unit { -public: - simd_function_unit( const shader_core_config *config ); - ~simd_function_unit() { delete m_dispatch_reg; } + public: + simd_function_unit(const shader_core_config *config); + ~simd_function_unit() { delete m_dispatch_reg; } - // modifiers - virtual void issue( register_set& source_reg ) { source_reg.move_out_to(m_dispatch_reg); occupied.set(m_dispatch_reg->latency);} - virtual void cycle() = 0; - virtual void active_lanes_in_pipeline() = 0; + // modifiers + virtual void issue(register_set &source_reg) { + source_reg.move_out_to(m_dispatch_reg); + occupied.set(m_dispatch_reg->latency); + } + virtual void cycle() = 0; + virtual void active_lanes_in_pipeline() = 0; - // accessors - virtual unsigned clock_multiplier() const { return 1; } - virtual bool can_issue( const warp_inst_t &inst ) const { return m_dispatch_reg->empty() && !occupied.test(inst.latency); } - virtual bool stallable() const = 0; - virtual void print( FILE *fp ) const - { - fprintf(fp,"%s dispatch= ", m_name.c_str() ); - m_dispatch_reg->print(fp); - } - const char* get_name() { - return m_name.c_str(); - } -protected: - std::string m_name; - const shader_core_config *m_config; - warp_inst_t *m_dispatch_reg; - static const unsigned MAX_ALU_LATENCY = 512; - std::bitset<MAX_ALU_LATENCY> occupied; + // accessors + virtual unsigned clock_multiplier() const { return 1; } + virtual bool can_issue(const warp_inst_t &inst) const { + return m_dispatch_reg->empty() && !occupied.test(inst.latency); + } + virtual bool stallable() const = 0; + virtual void print(FILE *fp) const { + fprintf(fp, "%s dispatch= ", m_name.c_str()); + m_dispatch_reg->print(fp); + } + const char *get_name() { return m_name.c_str(); } + + protected: + std::string m_name; + const shader_core_config *m_config; + warp_inst_t *m_dispatch_reg; + static const unsigned MAX_ALU_LATENCY = 512; + std::bitset<MAX_ALU_LATENCY> occupied; }; class pipelined_simd_unit : public simd_function_unit { -public: - pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency, shader_core_ctx *core ); + public: + pipelined_simd_unit(register_set *result_port, + const shader_core_config *config, unsigned max_latency, + shader_core_ctx *core); - //modifiers - virtual void cycle(); - virtual void issue( register_set& source_reg ); - virtual unsigned get_active_lanes_in_pipeline(); + // modifiers + virtual void cycle(); + virtual void issue(register_set &source_reg); + virtual unsigned get_active_lanes_in_pipeline(); - virtual void active_lanes_in_pipeline() = 0; -/* - virtual void issue( register_set& source_reg ) - { - //move_warp(m_dispatch_reg,source_reg); - //source_reg.move_out_to(m_dispatch_reg); - simd_function_unit::issue(source_reg); - } -*/ - // accessors - virtual bool stallable() const { return false; } - virtual bool can_issue( const warp_inst_t &inst ) const - { - return simd_function_unit::can_issue(inst); - } - virtual void print(FILE *fp) const - { - simd_function_unit::print(fp); - for( int s=m_pipeline_depth-1; s>=0; s-- ) { - if( !m_pipeline_reg[s]->empty() ) { - fprintf(fp," %s[%2d] ", m_name.c_str(), s ); - m_pipeline_reg[s]->print(fp); - } - } + virtual void active_lanes_in_pipeline() = 0; + /* + virtual void issue( register_set& source_reg ) + { + //move_warp(m_dispatch_reg,source_reg); + //source_reg.move_out_to(m_dispatch_reg); + simd_function_unit::issue(source_reg); + } + */ + // accessors + virtual bool stallable() const { return false; } + virtual bool can_issue(const warp_inst_t &inst) const { + return simd_function_unit::can_issue(inst); + } + virtual void print(FILE *fp) const { + simd_function_unit::print(fp); + for (int s = m_pipeline_depth - 1; s >= 0; s--) { + if (!m_pipeline_reg[s]->empty()) { + fprintf(fp, " %s[%2d] ", m_name.c_str(), s); + m_pipeline_reg[s]->print(fp); + } } -protected: - unsigned m_pipeline_depth; - warp_inst_t **m_pipeline_reg; - register_set *m_result_port; - class shader_core_ctx *m_core; + } - unsigned active_insts_in_pipeline; + protected: + unsigned m_pipeline_depth; + warp_inst_t **m_pipeline_reg; + register_set *m_result_port; + class shader_core_ctx *m_core; + unsigned active_insts_in_pipeline; }; -class sfu : public pipelined_simd_unit -{ -public: - sfu( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case SFU_OP: break; - case ALU_SFU_OP: break; - case DP_OP: break; //for compute <= 29 (i..e Fermi and GT200) - default: return false; - } - return pipelined_simd_unit::can_issue(inst); +class sfu : public pipelined_simd_unit { + public: + sfu(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core); + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case SFU_OP: + break; + case ALU_SFU_OP: + break; + case DP_OP: + break; // for compute <= 29 (i..e Fermi and GT200) + default: + return false; } - virtual void active_lanes_in_pipeline(); - virtual void issue( register_set& source_reg ); + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue(register_set &source_reg); }; -class dp_unit : public pipelined_simd_unit -{ -public: - dp_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case DP_OP: break; - default: return false; - } - return pipelined_simd_unit::can_issue(inst); +class dp_unit : public pipelined_simd_unit { + public: + dp_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core); + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case DP_OP: + break; + default: + return false; } - virtual void active_lanes_in_pipeline(); - virtual void issue( register_set& source_reg ); + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue(register_set &source_reg); }; -class tensor_core : public pipelined_simd_unit -{ -public: - tensor_core( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case TENSOR_CORE_OP: break; - default: return false; - } - return pipelined_simd_unit::can_issue(inst); +class tensor_core : public pipelined_simd_unit { + public: + tensor_core(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core); + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case TENSOR_CORE_OP: + break; + default: + return false; } - virtual void active_lanes_in_pipeline(); - virtual void issue( register_set& source_reg ); + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue(register_set &source_reg); }; - -class int_unit : public pipelined_simd_unit -{ -public: - int_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case SFU_OP: return false; - case LOAD_OP: return false; - case TENSOR_CORE_LOAD_OP: return false; - case STORE_OP: return false; - case TENSOR_CORE_STORE_OP: return false; - case MEMORY_BARRIER_OP: return false; - case SP_OP: return false; - case DP_OP: return false; - default: break; - } - return pipelined_simd_unit::can_issue(inst); +class int_unit : public pipelined_simd_unit { + public: + int_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core); + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case SFU_OP: + return false; + case LOAD_OP: + return false; + case TENSOR_CORE_LOAD_OP: + return false; + case STORE_OP: + return false; + case TENSOR_CORE_STORE_OP: + return false; + case MEMORY_BARRIER_OP: + return false; + case SP_OP: + return false; + case DP_OP: + return false; + default: + break; } - virtual void active_lanes_in_pipeline(); - virtual void issue( register_set& source_reg ); + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue(register_set &source_reg); }; -class sp_unit : public pipelined_simd_unit -{ -public: - sp_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case SFU_OP: return false; - case LOAD_OP: return false; - case TENSOR_CORE_LOAD_OP: return false; - case STORE_OP: return false; - case TENSOR_CORE_STORE_OP: return false; - case MEMORY_BARRIER_OP: return false; - case DP_OP: return false; - default: break; - } - return pipelined_simd_unit::can_issue(inst); +class sp_unit : public pipelined_simd_unit { + public: + sp_unit(register_set *result_port, const shader_core_config *config, + shader_core_ctx *core); + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case SFU_OP: + return false; + case LOAD_OP: + return false; + case TENSOR_CORE_LOAD_OP: + return false; + case STORE_OP: + return false; + case TENSOR_CORE_STORE_OP: + return false; + case MEMORY_BARRIER_OP: + return false; + case DP_OP: + return false; + default: + break; } - virtual void active_lanes_in_pipeline(); - virtual void issue( register_set& source_reg ); + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue(register_set &source_reg); }; class simt_core_cluster; @@ -1211,947 +1216,1060 @@ class shader_memory_interface; class shader_core_mem_fetch_allocator; class cache_t; -class ldst_unit: public pipelined_simd_unit { -public: - ldst_unit( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - class shader_core_stats *stats, - unsigned sid, unsigned tpc ); +class ldst_unit : public pipelined_simd_unit { + public: + ldst_unit(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, class shader_core_stats *stats, + unsigned sid, unsigned tpc); - // modifiers - virtual void issue( register_set &inst ); - virtual void cycle(); - - void fill( mem_fetch *mf ); - void flush(); - void invalidate(); - void writeback(); + // modifiers + virtual void issue(register_set &inst); + virtual void cycle(); - // accessors - virtual unsigned clock_multiplier() const; + void fill(mem_fetch *mf); + void flush(); + void invalidate(); + void writeback(); - virtual bool can_issue( const warp_inst_t &inst ) const - { - switch(inst.op) { - case LOAD_OP: break; - case TENSOR_CORE_LOAD_OP: break; - case STORE_OP: break; - case TENSOR_CORE_STORE_OP: break; - case MEMORY_BARRIER_OP: break; - default: return false; - } - return m_dispatch_reg->empty(); + // accessors + virtual unsigned clock_multiplier() const; + + virtual bool can_issue(const warp_inst_t &inst) const { + switch (inst.op) { + case LOAD_OP: + break; + case TENSOR_CORE_LOAD_OP: + break; + case STORE_OP: + break; + case TENSOR_CORE_STORE_OP: + break; + case MEMORY_BARRIER_OP: + break; + default: + return false; } + return m_dispatch_reg->empty(); + } - virtual void active_lanes_in_pipeline(); - virtual bool stallable() const { return true; } - bool response_buffer_full() const; - void print(FILE *fout) const; - void print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ); - void get_cache_stats(unsigned &read_accesses, unsigned &write_accesses, unsigned &read_misses, unsigned &write_misses, unsigned cache_type); - void get_cache_stats(cache_stats &cs); + virtual void active_lanes_in_pipeline(); + virtual bool stallable() const { return true; } + bool response_buffer_full() const; + void print(FILE *fout) const; + void print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses); + void get_cache_stats(unsigned &read_accesses, unsigned &write_accesses, + unsigned &read_misses, unsigned &write_misses, + unsigned cache_type); + void get_cache_stats(cache_stats &cs); - void get_L1D_sub_stats(struct cache_sub_stats &css) const; - void get_L1C_sub_stats(struct cache_sub_stats &css) const; - void get_L1T_sub_stats(struct cache_sub_stats &css) const; + void get_L1D_sub_stats(struct cache_sub_stats &css) const; + void get_L1C_sub_stats(struct cache_sub_stats &css) const; + void get_L1T_sub_stats(struct cache_sub_stats &css) const; -protected: - ldst_unit( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - unsigned sid, - unsigned tpc, - l1_cache* new_l1d_cache ); - void init( mem_fetch_interface *icnt, - shader_core_mem_fetch_allocator *mf_allocator, - shader_core_ctx *core, - opndcoll_rfu_t *operand_collector, - Scoreboard *scoreboard, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - unsigned sid, - unsigned tpc ); + protected: + ldst_unit(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + unsigned sid, unsigned tpc, l1_cache *new_l1d_cache); + void init(mem_fetch_interface *icnt, + shader_core_mem_fetch_allocator *mf_allocator, + shader_core_ctx *core, opndcoll_rfu_t *operand_collector, + Scoreboard *scoreboard, const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + unsigned sid, unsigned tpc); -protected: - bool shared_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type); - bool constant_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type); - bool texture_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type); - bool memory_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type); + protected: + bool shared_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type); + bool constant_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type); + bool texture_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type); + bool memory_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, + mem_stage_access_type &fail_type); - virtual mem_stage_stall_type process_cache_access( cache_t* cache, - new_addr_type address, - warp_inst_t &inst, - std::list<cache_event>& events, - mem_fetch *mf, - enum cache_request_status status ); - mem_stage_stall_type process_memory_access_queue( cache_t *cache, warp_inst_t &inst ); - mem_stage_stall_type process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ); + virtual mem_stage_stall_type process_cache_access( + cache_t *cache, new_addr_type address, warp_inst_t &inst, + std::list<cache_event> &events, mem_fetch *mf, + enum cache_request_status status); + mem_stage_stall_type process_memory_access_queue(cache_t *cache, + warp_inst_t &inst); + mem_stage_stall_type process_memory_access_queue_l1cache(l1_cache *cache, + warp_inst_t &inst); - const memory_config *m_memory_config; - class mem_fetch_interface *m_icnt; - shader_core_mem_fetch_allocator *m_mf_allocator; - class shader_core_ctx *m_core; - unsigned m_sid; - unsigned m_tpc; + const memory_config *m_memory_config; + class mem_fetch_interface *m_icnt; + shader_core_mem_fetch_allocator *m_mf_allocator; + class shader_core_ctx *m_core; + unsigned m_sid; + unsigned m_tpc; - tex_cache *m_L1T; // texture cache - read_only_cache *m_L1C; // constant cache - l1_cache *m_L1D; // data cache - std::map<unsigned/*warp_id*/, std::map<unsigned/*regnum*/,unsigned/*count*/> > m_pending_writes; - std::list<mem_fetch*> m_response_fifo; - opndcoll_rfu_t *m_operand_collector; - Scoreboard *m_scoreboard; + tex_cache *m_L1T; // texture cache + read_only_cache *m_L1C; // constant cache + l1_cache *m_L1D; // data cache + std::map<unsigned /*warp_id*/, + std::map<unsigned /*regnum*/, unsigned /*count*/>> + m_pending_writes; + std::list<mem_fetch *> m_response_fifo; + opndcoll_rfu_t *m_operand_collector; + Scoreboard *m_scoreboard; - mem_fetch *m_next_global; - warp_inst_t m_next_wb; - unsigned m_writeback_arb; // round-robin arbiter for writeback contention between L1T, L1C, shared - unsigned m_num_writeback_clients; + mem_fetch *m_next_global; + warp_inst_t m_next_wb; + unsigned m_writeback_arb; // round-robin arbiter for writeback contention + // between L1T, L1C, shared + unsigned m_num_writeback_clients; - enum mem_stage_stall_type m_mem_rc; + enum mem_stage_stall_type m_mem_rc; - shader_core_stats *m_stats; + shader_core_stats *m_stats; - // for debugging - unsigned long long m_last_inst_gpu_sim_cycle; - unsigned long long m_last_inst_gpu_tot_sim_cycle; + // for debugging + unsigned long long m_last_inst_gpu_sim_cycle; + unsigned long long m_last_inst_gpu_tot_sim_cycle; - std::vector<std::deque<mem_fetch* >> l1_latency_queue; - void L1_latency_queue_cycle(); + std::vector<std::deque<mem_fetch *>> l1_latency_queue; + void L1_latency_queue_cycle(); }; enum pipeline_stage_name_t { - ID_OC_SP=0, - ID_OC_DP, - ID_OC_INT, - ID_OC_SFU, - ID_OC_MEM, - OC_EX_SP, - OC_EX_DP, - OC_EX_INT, - OC_EX_SFU, - OC_EX_MEM, - EX_WB, - ID_OC_TENSOR_CORE, - OC_EX_TENSOR_CORE, - N_PIPELINE_STAGES - }; - -const char* const pipeline_stage_name_decode[] = { - "ID_OC_SP", - "ID_OC_DP", - "ID_OC_INT", - "ID_OC_SFU", - "ID_OC_MEM", - "OC_EX_SP", - "OC_EX_DP", - "OC_EX_INT", - "OC_EX_SFU", - "OC_EX_MEM", - "EX_WB", - "ID_OC_TENSOR_CORE", - "OC_EX_TENSOR_CORE", - "N_PIPELINE_STAGES" + ID_OC_SP = 0, + ID_OC_DP, + ID_OC_INT, + ID_OC_SFU, + ID_OC_MEM, + OC_EX_SP, + OC_EX_DP, + OC_EX_INT, + OC_EX_SFU, + OC_EX_MEM, + EX_WB, + ID_OC_TENSOR_CORE, + OC_EX_TENSOR_CORE, + N_PIPELINE_STAGES }; -class shader_core_config : public core_config -{ - public: - shader_core_config(gpgpu_context* ctx):core_config(ctx){ - pipeline_widths_string = NULL; - gpgpu_ctx = ctx; +const char *const pipeline_stage_name_decode[] = { + "ID_OC_SP", "ID_OC_DP", "ID_OC_INT", "ID_OC_SFU", + "ID_OC_MEM", "OC_EX_SP", "OC_EX_DP", "OC_EX_INT", + "OC_EX_SFU", "OC_EX_MEM", "EX_WB", "ID_OC_TENSOR_CORE", + "OC_EX_TENSOR_CORE", "N_PIPELINE_STAGES"}; + +class shader_core_config : public core_config { + public: + shader_core_config(gpgpu_context *ctx) : core_config(ctx) { + pipeline_widths_string = NULL; + gpgpu_ctx = ctx; + } + + void init() { + int ntok = sscanf(gpgpu_shader_core_pipeline_opt, "%d:%d", + &n_thread_per_shader, &warp_size); + if (ntok != 2) { + printf( + "GPGPU-Sim uArch: error while parsing configuration string " + "gpgpu_shader_core_pipeline_opt\n"); + abort(); } - void init() - { - int ntok = sscanf(gpgpu_shader_core_pipeline_opt,"%d:%d", - &n_thread_per_shader, - &warp_size); - if(ntok != 2) { - printf("GPGPU-Sim uArch: error while parsing configuration string gpgpu_shader_core_pipeline_opt\n"); - abort(); - } + char *toks = new char[100]; + char *tokd = toks; + strcpy(toks, pipeline_widths_string); - char* toks = new char[100]; - char* tokd = toks; - strcpy(toks,pipeline_widths_string); - - toks = strtok(toks,","); + toks = strtok(toks, ","); - /* Removing the tensorcore pipeline while reading the config files if the tensor core is not available. - If we won't remove it, old regression will be broken. - So to support the legacy config files it's best to handle in this way. - */ - int num_config_to_read= N_PIPELINE_STAGES - 2 * (!gpgpu_tensor_core_avail); + /* Removing the tensorcore pipeline while reading the config files if the + tensor core is not available. If we won't remove it, old regression will + be broken. So to support the legacy config files it's best to handle in + this way. + */ + int num_config_to_read = N_PIPELINE_STAGES - 2 * (!gpgpu_tensor_core_avail); - for (int i = 0; i < num_config_to_read; i++) { - assert(toks); - ntok = sscanf(toks,"%d", &pipe_widths[i]); - assert(ntok == 1); - toks = strtok(NULL,","); - } + for (int i = 0; i < num_config_to_read; i++) { + assert(toks); + ntok = sscanf(toks, "%d", &pipe_widths[i]); + assert(ntok == 1); + toks = strtok(NULL, ","); + } - delete[] tokd; - - if (n_thread_per_shader > MAX_THREAD_PER_SM) { - printf("GPGPU-Sim uArch: Error ** increase MAX_THREAD_PER_SM in abstract_hardware_model.h from %u to %u\n", - MAX_THREAD_PER_SM, n_thread_per_shader); - abort(); - } - max_warps_per_shader = n_thread_per_shader/warp_size; - assert( !(n_thread_per_shader % warp_size) ); + delete[] tokd; - set_pipeline_latency(); - - m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone); - m_L1T_config.init(m_L1T_config.m_config_string,FuncCachePreferNone); - m_L1C_config.init(m_L1C_config.m_config_string,FuncCachePreferNone); - m_L1D_config.init(m_L1D_config.m_config_string,FuncCachePreferNone); - gpgpu_cache_texl1_linesize = m_L1T_config.get_line_sz(); - gpgpu_cache_constl1_linesize = m_L1C_config.get_line_sz(); - m_valid = true; + if (n_thread_per_shader > MAX_THREAD_PER_SM) { + printf( + "GPGPU-Sim uArch: Error ** increase MAX_THREAD_PER_SM in " + "abstract_hardware_model.h from %u to %u\n", + MAX_THREAD_PER_SM, n_thread_per_shader); + abort(); } - void reg_options(class OptionParser * opp ); - unsigned max_cta( const kernel_info_t &k ) const; - unsigned num_shader() const { return n_simt_clusters*n_simt_cores_per_cluster; } - unsigned sid_to_cluster( unsigned sid ) const { return sid / n_simt_cores_per_cluster; } - unsigned sid_to_cid( unsigned sid ) const { return sid % n_simt_cores_per_cluster; } - unsigned cid_to_sid( unsigned cid, unsigned cluster_id ) const { return cluster_id*n_simt_cores_per_cluster + cid; } - void set_pipeline_latency(); + max_warps_per_shader = n_thread_per_shader / warp_size; + assert(!(n_thread_per_shader % warp_size)); - // backward pointer - class gpgpu_context* gpgpu_ctx; -// data - char *gpgpu_shader_core_pipeline_opt; - bool gpgpu_perfect_mem; - bool gpgpu_clock_gated_reg_file; - bool gpgpu_clock_gated_lanes; - enum divergence_support_t model; - unsigned n_thread_per_shader; - unsigned n_regfile_gating_group; - unsigned max_warps_per_shader; - unsigned max_cta_per_core; //Limit on number of concurrent CTAs in shader core - unsigned max_barriers_per_cta; - char * gpgpu_scheduler_string; - unsigned gpgpu_shmem_per_block; - unsigned gpgpu_registers_per_block; - char* pipeline_widths_string; - int pipe_widths[N_PIPELINE_STAGES]; + set_pipeline_latency(); - mutable cache_config m_L1I_config; - mutable cache_config m_L1T_config; - mutable cache_config m_L1C_config; - mutable l1d_cache_config m_L1D_config; + m_L1I_config.init(m_L1I_config.m_config_string, FuncCachePreferNone); + m_L1T_config.init(m_L1T_config.m_config_string, FuncCachePreferNone); + m_L1C_config.init(m_L1C_config.m_config_string, FuncCachePreferNone); + m_L1D_config.init(m_L1D_config.m_config_string, FuncCachePreferNone); + gpgpu_cache_texl1_linesize = m_L1T_config.get_line_sz(); + gpgpu_cache_constl1_linesize = m_L1C_config.get_line_sz(); + m_valid = true; + } + void reg_options(class OptionParser *opp); + unsigned max_cta(const kernel_info_t &k) const; + unsigned num_shader() const { + return n_simt_clusters * n_simt_cores_per_cluster; + } + unsigned sid_to_cluster(unsigned sid) const { + return sid / n_simt_cores_per_cluster; + } + unsigned sid_to_cid(unsigned sid) const { + return sid % n_simt_cores_per_cluster; + } + unsigned cid_to_sid(unsigned cid, unsigned cluster_id) const { + return cluster_id * n_simt_cores_per_cluster + cid; + } + void set_pipeline_latency(); - bool gpgpu_dwf_reg_bankconflict; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // data + char *gpgpu_shader_core_pipeline_opt; + bool gpgpu_perfect_mem; + bool gpgpu_clock_gated_reg_file; + bool gpgpu_clock_gated_lanes; + enum divergence_support_t model; + unsigned n_thread_per_shader; + unsigned n_regfile_gating_group; + unsigned max_warps_per_shader; + unsigned + max_cta_per_core; // Limit on number of concurrent CTAs in shader core + unsigned max_barriers_per_cta; + char *gpgpu_scheduler_string; + unsigned gpgpu_shmem_per_block; + unsigned gpgpu_registers_per_block; + char *pipeline_widths_string; + int pipe_widths[N_PIPELINE_STAGES]; - unsigned gpgpu_num_sched_per_core; - int gpgpu_max_insn_issue_per_warp; - bool gpgpu_dual_issue_diff_exec_units; + mutable cache_config m_L1I_config; + mutable cache_config m_L1T_config; + mutable cache_config m_L1C_config; + mutable l1d_cache_config m_L1D_config; - //op collector - bool enable_specialized_operand_collector; - int gpgpu_operand_collector_num_units_sp; - int gpgpu_operand_collector_num_units_dp; - int gpgpu_operand_collector_num_units_sfu; - int gpgpu_operand_collector_num_units_tensor_core; - int gpgpu_operand_collector_num_units_mem; - int gpgpu_operand_collector_num_units_gen; - int gpgpu_operand_collector_num_units_int; + bool gpgpu_dwf_reg_bankconflict; - unsigned int gpgpu_operand_collector_num_in_ports_sp; - unsigned int gpgpu_operand_collector_num_in_ports_dp; - unsigned int gpgpu_operand_collector_num_in_ports_sfu; - unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; - unsigned int gpgpu_operand_collector_num_in_ports_mem; - unsigned int gpgpu_operand_collector_num_in_ports_gen; - unsigned int gpgpu_operand_collector_num_in_ports_int; + unsigned gpgpu_num_sched_per_core; + int gpgpu_max_insn_issue_per_warp; + bool gpgpu_dual_issue_diff_exec_units; - unsigned int gpgpu_operand_collector_num_out_ports_sp; - unsigned int gpgpu_operand_collector_num_out_ports_dp; - unsigned int gpgpu_operand_collector_num_out_ports_sfu; - unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; - unsigned int gpgpu_operand_collector_num_out_ports_mem; - unsigned int gpgpu_operand_collector_num_out_ports_gen; - unsigned int gpgpu_operand_collector_num_out_ports_int; + // op collector + bool enable_specialized_operand_collector; + int gpgpu_operand_collector_num_units_sp; + int gpgpu_operand_collector_num_units_dp; + int gpgpu_operand_collector_num_units_sfu; + int gpgpu_operand_collector_num_units_tensor_core; + int gpgpu_operand_collector_num_units_mem; + int gpgpu_operand_collector_num_units_gen; + int gpgpu_operand_collector_num_units_int; - int gpgpu_num_sp_units; - int gpgpu_tensor_core_avail; - int gpgpu_num_dp_units; - int gpgpu_num_sfu_units; - int gpgpu_num_tensor_core_units; - int gpgpu_num_mem_units; - int gpgpu_num_int_units; + unsigned int gpgpu_operand_collector_num_in_ports_sp; + unsigned int gpgpu_operand_collector_num_in_ports_dp; + unsigned int gpgpu_operand_collector_num_in_ports_sfu; + unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; + unsigned int gpgpu_operand_collector_num_in_ports_mem; + unsigned int gpgpu_operand_collector_num_in_ports_gen; + unsigned int gpgpu_operand_collector_num_in_ports_int; - //Shader core resources - unsigned gpgpu_shader_registers; - int gpgpu_warpdistro_shader; - int gpgpu_warp_issue_shader; - unsigned gpgpu_num_reg_banks; - bool gpgpu_reg_bank_use_warp_id; - bool gpgpu_local_mem_map; - bool gpgpu_ignore_resources_limitation; - bool sub_core_model; - - unsigned max_sp_latency; - unsigned max_int_latency; - unsigned max_sfu_latency; - unsigned max_dp_latency; - unsigned max_tensor_core_latency; - - unsigned n_simt_cores_per_cluster; - unsigned n_simt_clusters; - unsigned n_simt_ejection_buffer_size; - unsigned ldst_unit_response_queue_size; + unsigned int gpgpu_operand_collector_num_out_ports_sp; + unsigned int gpgpu_operand_collector_num_out_ports_dp; + unsigned int gpgpu_operand_collector_num_out_ports_sfu; + unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; + unsigned int gpgpu_operand_collector_num_out_ports_mem; + unsigned int gpgpu_operand_collector_num_out_ports_gen; + unsigned int gpgpu_operand_collector_num_out_ports_int; - int simt_core_sim_order; - - unsigned smem_latency; + int gpgpu_num_sp_units; + int gpgpu_tensor_core_avail; + int gpgpu_num_dp_units; + int gpgpu_num_sfu_units; + int gpgpu_num_tensor_core_units; + int gpgpu_num_mem_units; + int gpgpu_num_int_units; - unsigned mem2device(unsigned memid) const { return memid + n_simt_clusters; } + // Shader core resources + unsigned gpgpu_shader_registers; + int gpgpu_warpdistro_shader; + int gpgpu_warp_issue_shader; + unsigned gpgpu_num_reg_banks; + bool gpgpu_reg_bank_use_warp_id; + bool gpgpu_local_mem_map; + bool gpgpu_ignore_resources_limitation; + bool sub_core_model; - //Jin: concurrent kernel on sm - bool gpgpu_concurrent_kernel_sm; + unsigned max_sp_latency; + unsigned max_int_latency; + unsigned max_sfu_latency; + unsigned max_dp_latency; + unsigned max_tensor_core_latency; - bool adpative_volta_cache_config; + unsigned n_simt_cores_per_cluster; + unsigned n_simt_clusters; + unsigned n_simt_ejection_buffer_size; + unsigned ldst_unit_response_queue_size; + int simt_core_sim_order; + + unsigned smem_latency; + + unsigned mem2device(unsigned memid) const { return memid + n_simt_clusters; } + + // Jin: concurrent kernel on sm + bool gpgpu_concurrent_kernel_sm; + + bool adpative_volta_cache_config; }; struct shader_core_stats_pod { + void * + shader_core_stats_pod_start[0]; // DO NOT MOVE FROM THE TOP - spaceless + // pointer to the start of this structure + unsigned long long *shader_cycles; + unsigned *m_num_sim_insn; // number of scalar thread instructions committed + // by this shader core + unsigned *m_num_sim_winsn; // number of warp instructions committed by this + // shader core + unsigned *m_last_num_sim_insn; + unsigned *m_last_num_sim_winsn; + unsigned * + m_num_decoded_insn; // number of instructions decoded by this shader core + float *m_pipeline_duty_cycle; + unsigned *m_num_FPdecoded_insn; + unsigned *m_num_INTdecoded_insn; + unsigned *m_num_storequeued_insn; + unsigned *m_num_loadqueued_insn; + unsigned *m_num_ialu_acesses; + unsigned *m_num_fp_acesses; + unsigned *m_num_imul_acesses; + unsigned *m_num_tex_inst; + unsigned *m_num_fpmul_acesses; + unsigned *m_num_idiv_acesses; + unsigned *m_num_fpdiv_acesses; + unsigned *m_num_sp_acesses; + unsigned *m_num_sfu_acesses; + unsigned *m_num_tensor_core_acesses; + unsigned *m_num_trans_acesses; + unsigned *m_num_mem_acesses; + unsigned *m_num_sp_committed; + unsigned *m_num_tlb_hits; + unsigned *m_num_tlb_accesses; + unsigned *m_num_sfu_committed; + unsigned *m_num_tensor_core_committed; + unsigned *m_num_mem_committed; + unsigned *m_read_regfile_acesses; + unsigned *m_write_regfile_acesses; + unsigned *m_non_rf_operands; + unsigned *m_num_imul24_acesses; + unsigned *m_num_imul32_acesses; + unsigned *m_active_sp_lanes; + unsigned *m_active_sfu_lanes; + unsigned *m_active_tensor_core_lanes; + unsigned *m_active_fu_lanes; + unsigned *m_active_fu_mem_lanes; + unsigned *m_n_diverge; // number of divergence occurring in this shader + unsigned gpgpu_n_load_insn; + unsigned gpgpu_n_store_insn; + unsigned gpgpu_n_shmem_insn; + unsigned gpgpu_n_sstarr_insn; + unsigned gpgpu_n_tex_insn; + unsigned gpgpu_n_const_insn; + unsigned gpgpu_n_param_insn; + unsigned gpgpu_n_shmem_bkconflict; + unsigned gpgpu_n_cache_bkconflict; + int gpgpu_n_intrawarp_mshr_merge; + unsigned gpgpu_n_cmem_portconflict; + unsigned gpu_stall_shd_mem_breakdown[N_MEM_STAGE_ACCESS_TYPE] + [N_MEM_STAGE_STALL_TYPE]; + unsigned gpu_reg_bank_conflict_stalls; + unsigned *shader_cycle_distro; + unsigned *last_shader_cycle_distro; + unsigned *num_warps_issuable; + unsigned gpgpu_n_stall_shd_mem; + unsigned *single_issue_nums; + unsigned *dual_issue_nums; - void* shader_core_stats_pod_start[0]; // DO NOT MOVE FROM THE TOP - spaceless pointer to the start of this structure - unsigned long long *shader_cycles; - unsigned *m_num_sim_insn; // number of scalar thread instructions committed by this shader core - unsigned *m_num_sim_winsn; // number of warp instructions committed by this shader core - unsigned *m_last_num_sim_insn; - unsigned *m_last_num_sim_winsn; - unsigned *m_num_decoded_insn; // number of instructions decoded by this shader core - float *m_pipeline_duty_cycle; - unsigned *m_num_FPdecoded_insn; - unsigned *m_num_INTdecoded_insn; - unsigned *m_num_storequeued_insn; - unsigned *m_num_loadqueued_insn; - unsigned *m_num_ialu_acesses; - unsigned *m_num_fp_acesses; - unsigned *m_num_imul_acesses; - unsigned *m_num_tex_inst; - unsigned *m_num_fpmul_acesses; - unsigned *m_num_idiv_acesses; - unsigned *m_num_fpdiv_acesses; - unsigned *m_num_sp_acesses; - unsigned *m_num_sfu_acesses; - unsigned *m_num_tensor_core_acesses; - unsigned *m_num_trans_acesses; - unsigned *m_num_mem_acesses; - unsigned *m_num_sp_committed; - unsigned *m_num_tlb_hits; - unsigned *m_num_tlb_accesses; - unsigned *m_num_sfu_committed; - unsigned *m_num_tensor_core_committed; - unsigned *m_num_mem_committed; - unsigned *m_read_regfile_acesses; - unsigned *m_write_regfile_acesses; - unsigned *m_non_rf_operands; - unsigned *m_num_imul24_acesses; - unsigned *m_num_imul32_acesses; - unsigned *m_active_sp_lanes; - unsigned *m_active_sfu_lanes; - unsigned *m_active_tensor_core_lanes; - unsigned *m_active_fu_lanes; - unsigned *m_active_fu_mem_lanes; - unsigned *m_n_diverge; // number of divergence occurring in this shader - unsigned gpgpu_n_load_insn; - unsigned gpgpu_n_store_insn; - unsigned gpgpu_n_shmem_insn; - unsigned gpgpu_n_sstarr_insn; - unsigned gpgpu_n_tex_insn; - unsigned gpgpu_n_const_insn; - unsigned gpgpu_n_param_insn; - unsigned gpgpu_n_shmem_bkconflict; - unsigned gpgpu_n_cache_bkconflict; - int gpgpu_n_intrawarp_mshr_merge; - unsigned gpgpu_n_cmem_portconflict; - unsigned gpu_stall_shd_mem_breakdown[N_MEM_STAGE_ACCESS_TYPE][N_MEM_STAGE_STALL_TYPE]; - unsigned gpu_reg_bank_conflict_stalls; - unsigned *shader_cycle_distro; - unsigned *last_shader_cycle_distro; - unsigned *num_warps_issuable; - unsigned gpgpu_n_stall_shd_mem; - unsigned* single_issue_nums; - unsigned* dual_issue_nums; + // memory access classification + int gpgpu_n_mem_read_local; + int gpgpu_n_mem_write_local; + int gpgpu_n_mem_texture; + int gpgpu_n_mem_const; + int gpgpu_n_mem_read_global; + int gpgpu_n_mem_write_global; + int gpgpu_n_mem_read_inst; - //memory access classification - int gpgpu_n_mem_read_local; - int gpgpu_n_mem_write_local; - int gpgpu_n_mem_texture; - int gpgpu_n_mem_const; - int gpgpu_n_mem_read_global; - int gpgpu_n_mem_write_global; - int gpgpu_n_mem_read_inst; - - int gpgpu_n_mem_l2_writeback; - int gpgpu_n_mem_l1_write_allocate; - int gpgpu_n_mem_l2_write_allocate; + int gpgpu_n_mem_l2_writeback; + int gpgpu_n_mem_l1_write_allocate; + int gpgpu_n_mem_l2_write_allocate; - unsigned made_write_mfs; - unsigned made_read_mfs; + unsigned made_write_mfs; + unsigned made_read_mfs; - unsigned *gpgpu_n_shmem_bank_access; - long *n_simt_to_mem; // Interconnect power stats - long *n_mem_to_simt; + unsigned *gpgpu_n_shmem_bank_access; + long *n_simt_to_mem; // Interconnect power stats + long *n_mem_to_simt; }; class shader_core_stats : public shader_core_stats_pod { -public: - shader_core_stats( const shader_core_config *config ) - { - m_config = config; - shader_core_stats_pod *pod = reinterpret_cast< shader_core_stats_pod * > ( this->shader_core_stats_pod_start ); - memset(pod,0,sizeof(shader_core_stats_pod)); - shader_cycles=(unsigned long long *) calloc(config->num_shader(),sizeof(unsigned long long )); - m_num_sim_insn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_sim_winsn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_last_num_sim_winsn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_last_num_sim_insn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_pipeline_duty_cycle=(float*) calloc(config->num_shader(),sizeof(float)); - m_num_decoded_insn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_FPdecoded_insn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_storequeued_insn=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_loadqueued_insn=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_INTdecoded_insn = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_ialu_acesses = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_fp_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_tex_inst= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_imul_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_imul24_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_imul32_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_fpmul_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_idiv_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_fpdiv_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_sp_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_sfu_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_tensor_core_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_trans_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_mem_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_sp_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_tlb_hits=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_tlb_accesses=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_active_sp_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_active_sfu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_active_tensor_core_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_active_fu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_active_fu_mem_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_sfu_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_tensor_core_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_num_mem_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_read_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_write_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_non_rf_operands=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - m_n_diverge = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); - shader_cycle_distro = (unsigned*) calloc(config->warp_size+3, sizeof(unsigned)); - last_shader_cycle_distro = (unsigned*) calloc(m_config->warp_size+3, sizeof(unsigned)); - single_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core,sizeof(unsigned)); - dual_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core, sizeof(unsigned)); + public: + shader_core_stats(const shader_core_config *config) { + m_config = config; + shader_core_stats_pod *pod = reinterpret_cast<shader_core_stats_pod *>( + this->shader_core_stats_pod_start); + memset(pod, 0, sizeof(shader_core_stats_pod)); + shader_cycles = (unsigned long long *)calloc(config->num_shader(), + sizeof(unsigned long long)); + m_num_sim_insn = (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_sim_winsn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_last_num_sim_winsn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_last_num_sim_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_pipeline_duty_cycle = + (float *)calloc(config->num_shader(), sizeof(float)); + m_num_decoded_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_FPdecoded_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_storequeued_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_loadqueued_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_INTdecoded_insn = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_ialu_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_fp_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_tex_inst = (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_imul_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_imul24_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_imul32_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_fpmul_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_idiv_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_fpdiv_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_sp_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_sfu_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_tensor_core_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_trans_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_mem_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_sp_committed = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_tlb_hits = (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_tlb_accesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_active_sp_lanes = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_active_sfu_lanes = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_active_tensor_core_lanes = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_active_fu_lanes = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_active_fu_mem_lanes = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_sfu_committed = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_tensor_core_committed = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_num_mem_committed = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_read_regfile_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_write_regfile_acesses = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_non_rf_operands = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + m_n_diverge = (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + shader_cycle_distro = + (unsigned *)calloc(config->warp_size + 3, sizeof(unsigned)); + last_shader_cycle_distro = + (unsigned *)calloc(m_config->warp_size + 3, sizeof(unsigned)); + single_issue_nums = + (unsigned *)calloc(config->gpgpu_num_sched_per_core, sizeof(unsigned)); + dual_issue_nums = + (unsigned *)calloc(config->gpgpu_num_sched_per_core, sizeof(unsigned)); - n_simt_to_mem = (long *)calloc(config->num_shader(), sizeof(long)); - n_mem_to_simt = (long *)calloc(config->num_shader(), sizeof(long)); + n_simt_to_mem = (long *)calloc(config->num_shader(), sizeof(long)); + n_mem_to_simt = (long *)calloc(config->num_shader(), sizeof(long)); - m_outgoing_traffic_stats = new traffic_breakdown("coretomem"); - m_incoming_traffic_stats = new traffic_breakdown("memtocore"); + m_outgoing_traffic_stats = new traffic_breakdown("coretomem"); + m_incoming_traffic_stats = new traffic_breakdown("memtocore"); - gpgpu_n_shmem_bank_access = (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); + gpgpu_n_shmem_bank_access = + (unsigned *)calloc(config->num_shader(), sizeof(unsigned)); - m_shader_dynamic_warp_issue_distro.resize( config->num_shader() ); - m_shader_warp_slot_issue_distro.resize( config->num_shader() ); - } + m_shader_dynamic_warp_issue_distro.resize(config->num_shader()); + m_shader_warp_slot_issue_distro.resize(config->num_shader()); + } - ~shader_core_stats() - { - delete m_outgoing_traffic_stats; - delete m_incoming_traffic_stats; - free(m_num_sim_insn); - free(m_num_sim_winsn); - free(m_n_diverge); - free(shader_cycle_distro); - free(last_shader_cycle_distro); - } + ~shader_core_stats() { + delete m_outgoing_traffic_stats; + delete m_incoming_traffic_stats; + free(m_num_sim_insn); + free(m_num_sim_winsn); + free(m_n_diverge); + free(shader_cycle_distro); + free(last_shader_cycle_distro); + } - void new_grid() - { - } + void new_grid() {} - void event_warp_issued( unsigned s_id, unsigned warp_id, unsigned num_issued, unsigned dynamic_warp_id ); + void event_warp_issued(unsigned s_id, unsigned warp_id, unsigned num_issued, + unsigned dynamic_warp_id); - void visualizer_print( gzFile visualizer_file ); + void visualizer_print(gzFile visualizer_file); - void print( FILE *fout ) const; + void print(FILE *fout) const; - const std::vector< std::vector<unsigned> >& get_dynamic_warp_issue() const - { - return m_shader_dynamic_warp_issue_distro; - } + const std::vector<std::vector<unsigned>> &get_dynamic_warp_issue() const { + return m_shader_dynamic_warp_issue_distro; + } - const std::vector< std::vector<unsigned> >& get_warp_slot_issue() const - { - return m_shader_warp_slot_issue_distro; - } + const std::vector<std::vector<unsigned>> &get_warp_slot_issue() const { + return m_shader_warp_slot_issue_distro; + } -private: - const shader_core_config *m_config; + private: + const shader_core_config *m_config; - traffic_breakdown *m_outgoing_traffic_stats; // core to memory partitions - traffic_breakdown *m_incoming_traffic_stats; // memory partition to core + traffic_breakdown *m_outgoing_traffic_stats; // core to memory partitions + traffic_breakdown *m_incoming_traffic_stats; // memory partition to core - // Counts the instructions issued for each dynamic warp. - std::vector< std::vector<unsigned> > m_shader_dynamic_warp_issue_distro; - std::vector<unsigned> m_last_shader_dynamic_warp_issue_distro; - std::vector< std::vector<unsigned> > m_shader_warp_slot_issue_distro; - std::vector<unsigned> m_last_shader_warp_slot_issue_distro; + // Counts the instructions issued for each dynamic warp. + std::vector<std::vector<unsigned>> m_shader_dynamic_warp_issue_distro; + std::vector<unsigned> m_last_shader_dynamic_warp_issue_distro; + std::vector<std::vector<unsigned>> m_shader_warp_slot_issue_distro; + std::vector<unsigned> m_last_shader_warp_slot_issue_distro; - friend class power_stat_t; - friend class shader_core_ctx; - friend class ldst_unit; - friend class simt_core_cluster; - friend class scheduler_unit; - friend class TwoLevelScheduler; - friend class LooseRoundRobbinScheduler; + friend class power_stat_t; + friend class shader_core_ctx; + friend class ldst_unit; + friend class simt_core_cluster; + friend class scheduler_unit; + friend class TwoLevelScheduler; + friend class LooseRoundRobbinScheduler; }; class memory_config; class shader_core_mem_fetch_allocator : public mem_fetch_allocator { -public: - shader_core_mem_fetch_allocator( unsigned core_id, unsigned cluster_id, const memory_config *config ) - { - m_core_id = core_id; - m_cluster_id = cluster_id; - m_memory_config = config; - } - mem_fetch *alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle ) const; - mem_fetch *alloc( const warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle ) const - { - warp_inst_t inst_copy = inst; - mem_fetch *mf = new mem_fetch(access, - &inst_copy, - access.is_write()?WRITE_PACKET_SIZE:READ_PACKET_SIZE, - inst.warp_id(), - m_core_id, - m_cluster_id, - m_memory_config, - cycle); - return mf; - } + public: + shader_core_mem_fetch_allocator(unsigned core_id, unsigned cluster_id, + const memory_config *config) { + m_core_id = core_id; + m_cluster_id = cluster_id; + m_memory_config = config; + } + mem_fetch *alloc(new_addr_type addr, mem_access_type type, unsigned size, + bool wr, unsigned long long cycle) const; + mem_fetch *alloc(const warp_inst_t &inst, const mem_access_t &access, + unsigned long long cycle) const { + warp_inst_t inst_copy = inst; + mem_fetch *mf = new mem_fetch( + access, &inst_copy, + access.is_write() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, + inst.warp_id(), m_core_id, m_cluster_id, m_memory_config, cycle); + return mf; + } -private: - unsigned m_core_id; - unsigned m_cluster_id; - const memory_config *m_memory_config; + private: + unsigned m_core_id; + unsigned m_cluster_id; + const memory_config *m_memory_config; }; class shader_core_ctx : public core_t { -public: - // creator: - shader_core_ctx( class gpgpu_sim *gpu, - class simt_core_cluster *cluster, - unsigned shader_id, - unsigned tpc_id, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats ); + public: + // creator: + shader_core_ctx(class gpgpu_sim *gpu, class simt_core_cluster *cluster, + unsigned shader_id, unsigned tpc_id, + const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats); -// used by simt_core_cluster: - // modifiers - void cycle(); - void reinit(unsigned start_thread, unsigned end_thread, bool reset_not_completed ); - void issue_block2core( class kernel_info_t &kernel ); + // used by simt_core_cluster: + // modifiers + void cycle(); + void reinit(unsigned start_thread, unsigned end_thread, + bool reset_not_completed); + void issue_block2core(class kernel_info_t &kernel); - void cache_flush(); - void cache_invalidate(); - void accept_fetch_response( mem_fetch *mf ); - void accept_ldst_unit_response( class mem_fetch * mf ); - void broadcast_barrier_reduction(unsigned cta_id, unsigned bar_id,warp_set_t warps); - void set_kernel( kernel_info_t *k ) - { - assert(k); - m_kernel=k; -// k->inc_running(); - printf("GPGPU-Sim uArch: Shader %d bind to kernel %u \'%s\'\n", m_sid, m_kernel->get_uid(), - m_kernel->name().c_str() ); - } - - // accessors - bool fetch_unit_response_buffer_full() const; - bool ldst_unit_response_buffer_full() const; - unsigned get_not_completed() const { return m_not_completed; } - unsigned get_n_active_cta() const { return m_n_active_cta; } - unsigned isactive() const {if(m_n_active_cta>0) return 1; else return 0;} - kernel_info_t *get_kernel() { return m_kernel; } - unsigned get_sid() const {return m_sid;} + void cache_flush(); + void cache_invalidate(); + void accept_fetch_response(mem_fetch *mf); + void accept_ldst_unit_response(class mem_fetch *mf); + void broadcast_barrier_reduction(unsigned cta_id, unsigned bar_id, + warp_set_t warps); + void set_kernel(kernel_info_t *k) { + assert(k); + m_kernel = k; + // k->inc_running(); + printf("GPGPU-Sim uArch: Shader %d bind to kernel %u \'%s\'\n", m_sid, + m_kernel->get_uid(), m_kernel->name().c_str()); + } -// used by functional simulation: - // modifiers - virtual void warp_exit( unsigned warp_id ); - - // accessors - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const; - void get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsigned *rpc ) const; - float get_current_occupancy( unsigned long long & active, unsigned long long & total ) const; + // accessors + bool fetch_unit_response_buffer_full() const; + bool ldst_unit_response_buffer_full() const; + unsigned get_not_completed() const { return m_not_completed; } + unsigned get_n_active_cta() const { return m_n_active_cta; } + unsigned isactive() const { + if (m_n_active_cta > 0) + return 1; + else + return 0; + } + kernel_info_t *get_kernel() { return m_kernel; } + unsigned get_sid() const { return m_sid; } -// used by pipeline timing model components: - // modifiers - void mem_instruction_stats(const warp_inst_t &inst); - void decrement_atomic_count( unsigned wid, unsigned n ); - void inc_store_req( unsigned warp_id) { m_warp[warp_id].inc_store_req(); } - void dec_inst_in_pipeline( unsigned warp_id ) { m_warp[warp_id].dec_inst_in_pipeline(); } // also used in writeback() - void store_ack( class mem_fetch *mf ); - bool warp_waiting_at_mem_barrier( unsigned warp_id ); - void set_max_cta( const kernel_info_t &kernel ); - void warp_inst_complete(const warp_inst_t &inst); - - // accessors - std::list<unsigned> get_regs_written( const inst_t &fvt ) const; - const shader_core_config *get_config() const { return m_config; } - void print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ); + // used by functional simulation: + // modifiers + virtual void warp_exit(unsigned warp_id); + + // accessors + virtual bool warp_waiting_at_barrier(unsigned warp_id) const; + void get_pdom_stack_top_info(unsigned tid, unsigned *pc, unsigned *rpc) const; + float get_current_occupancy(unsigned long long &active, + unsigned long long &total) const; + + // used by pipeline timing model components: + // modifiers + void mem_instruction_stats(const warp_inst_t &inst); + void decrement_atomic_count(unsigned wid, unsigned n); + void inc_store_req(unsigned warp_id) { m_warp[warp_id].inc_store_req(); } + void dec_inst_in_pipeline(unsigned warp_id) { + m_warp[warp_id].dec_inst_in_pipeline(); + } // also used in writeback() + void store_ack(class mem_fetch *mf); + bool warp_waiting_at_mem_barrier(unsigned warp_id); + void set_max_cta(const kernel_info_t &kernel); + void warp_inst_complete(const warp_inst_t &inst); - void get_cache_stats(cache_stats &cs); - void get_L1I_sub_stats(struct cache_sub_stats &css) const; - void get_L1D_sub_stats(struct cache_sub_stats &css) const; - void get_L1C_sub_stats(struct cache_sub_stats &css) const; - void get_L1T_sub_stats(struct cache_sub_stats &css) const; + // accessors + std::list<unsigned> get_regs_written(const inst_t &fvt) const; + const shader_core_config *get_config() const { return m_config; } + void print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses); - void get_icnt_power_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; + void get_cache_stats(cache_stats &cs); + void get_L1I_sub_stats(struct cache_sub_stats &css) const; + void get_L1D_sub_stats(struct cache_sub_stats &css) const; + void get_L1C_sub_stats(struct cache_sub_stats &css) const; + void get_L1T_sub_stats(struct cache_sub_stats &css) const; -// debug: - void display_simt_state(FILE *fout, int mask ) const; - void display_pipeline( FILE *fout, int print_mem, int mask3bit ) const; + void get_icnt_power_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; - void incload_stat() {m_stats->m_num_loadqueued_insn[m_sid]++;} - void incstore_stat() {m_stats->m_num_storequeued_insn[m_sid]++;} - void incialu_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_ialu_acesses[m_sid]=m_stats->m_num_ialu_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_ialu_acesses[m_sid]=m_stats->m_num_ialu_acesses[m_sid]+active_count*latency; - } - } - void inctex_stat(unsigned active_count,double latency){ - m_stats->m_num_tex_inst[m_sid]=m_stats->m_num_tex_inst[m_sid]+active_count*latency; + // debug: + void display_simt_state(FILE *fout, int mask) const; + void display_pipeline(FILE *fout, int print_mem, int mask3bit) const; + + void incload_stat() { m_stats->m_num_loadqueued_insn[m_sid]++; } + void incstore_stat() { m_stats->m_num_storequeued_insn[m_sid]++; } + void incialu_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_ialu_acesses[m_sid] = + m_stats->m_num_ialu_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_ialu_acesses[m_sid] = + m_stats->m_num_ialu_acesses[m_sid] + active_count * latency; + } + } + void inctex_stat(unsigned active_count, double latency) { + m_stats->m_num_tex_inst[m_sid] = + m_stats->m_num_tex_inst[m_sid] + active_count * latency; + } + void incimul_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_imul_acesses[m_sid] = + m_stats->m_num_imul_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_imul_acesses[m_sid] = + m_stats->m_num_imul_acesses[m_sid] + active_count * latency; + } + } + void incimul24_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_imul24_acesses[m_sid] = + m_stats->m_num_imul24_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_imul24_acesses[m_sid] = + m_stats->m_num_imul24_acesses[m_sid] + active_count * latency; + } + } + void incimul32_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_imul32_acesses[m_sid] = + m_stats->m_num_imul32_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_sfu(active_count, latency); + } else { + m_stats->m_num_imul32_acesses[m_sid] = + m_stats->m_num_imul32_acesses[m_sid] + active_count * latency; + } + // printf("Int_Mul -- Active_count: %d\n",active_count); + } + void incidiv_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_idiv_acesses[m_sid] = + m_stats->m_num_idiv_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_sfu(active_count, latency); + } else { + m_stats->m_num_idiv_acesses[m_sid] = + m_stats->m_num_idiv_acesses[m_sid] + active_count * latency; + } + } + void incfpalu_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_fp_acesses[m_sid] = + m_stats->m_num_fp_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_fp_acesses[m_sid] = + m_stats->m_num_fp_acesses[m_sid] + active_count * latency; + } + } + void incfpmul_stat(unsigned active_count, double latency) { + // printf("FP MUL stat increament\n"); + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_fpmul_acesses[m_sid] = + m_stats->m_num_fpmul_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_fpmul_acesses[m_sid] = + m_stats->m_num_fpmul_acesses[m_sid] + active_count * latency; } - void incimul_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_imul_acesses[m_sid]=m_stats->m_num_imul_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_imul_acesses[m_sid]=m_stats->m_num_imul_acesses[m_sid]+active_count*latency; - } - } - void incimul24_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_imul24_acesses[m_sid]=m_stats->m_num_imul24_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_imul24_acesses[m_sid]=m_stats->m_num_imul24_acesses[m_sid]+active_count*latency; - } - } - void incimul32_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_imul32_acesses[m_sid]=m_stats->m_num_imul32_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_sfu(active_count, latency); - }else{ - m_stats->m_num_imul32_acesses[m_sid]=m_stats->m_num_imul32_acesses[m_sid]+active_count*latency; - } - //printf("Int_Mul -- Active_count: %d\n",active_count); - } - void incidiv_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_idiv_acesses[m_sid]=m_stats->m_num_idiv_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_sfu(active_count, latency); - }else { - m_stats->m_num_idiv_acesses[m_sid]=m_stats->m_num_idiv_acesses[m_sid]+active_count*latency; - } - } - void incfpalu_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_fp_acesses[m_sid]=m_stats->m_num_fp_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_fp_acesses[m_sid]=m_stats->m_num_fp_acesses[m_sid]+active_count*latency; - } - } - void incfpmul_stat(unsigned active_count,double latency) { - // printf("FP MUL stat increament\n"); - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_fpmul_acesses[m_sid]=m_stats->m_num_fpmul_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_fpmul_acesses[m_sid]=m_stats->m_num_fpmul_acesses[m_sid]+active_count*latency; - } - } - void incfpdiv_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_fpdiv_acesses[m_sid]=m_stats->m_num_fpdiv_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_sfu(active_count, latency); - }else { - m_stats->m_num_fpdiv_acesses[m_sid]=m_stats->m_num_fpdiv_acesses[m_sid]+active_count*latency; - } - } - void inctrans_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_trans_acesses[m_sid]=m_stats->m_num_trans_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_sfu(active_count, latency); - }else{ - m_stats->m_num_trans_acesses[m_sid]=m_stats->m_num_trans_acesses[m_sid]+active_count*latency; - } - } + } + void incfpdiv_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_fpdiv_acesses[m_sid] = + m_stats->m_num_fpdiv_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_sfu(active_count, latency); + } else { + m_stats->m_num_fpdiv_acesses[m_sid] = + m_stats->m_num_fpdiv_acesses[m_sid] + active_count * latency; + } + } + void inctrans_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_trans_acesses[m_sid] = + m_stats->m_num_trans_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_sfu(active_count, latency); + } else { + m_stats->m_num_trans_acesses[m_sid] = + m_stats->m_num_trans_acesses[m_sid] + active_count * latency; + } + } + + void incsfu_stat(unsigned active_count, double latency) { + m_stats->m_num_sfu_acesses[m_sid] = + m_stats->m_num_sfu_acesses[m_sid] + active_count * latency; + } + void incsp_stat(unsigned active_count, double latency) { + m_stats->m_num_sp_acesses[m_sid] = + m_stats->m_num_sp_acesses[m_sid] + active_count * latency; + } + void incmem_stat(unsigned active_count, double latency) { + if (m_config->gpgpu_clock_gated_lanes == false) { + m_stats->m_num_mem_acesses[m_sid] = + m_stats->m_num_mem_acesses[m_sid] + active_count * latency + + inactive_lanes_accesses_nonsfu(active_count, latency); + } else { + m_stats->m_num_mem_acesses[m_sid] = + m_stats->m_num_mem_acesses[m_sid] + active_count * latency; + } + } + void incexecstat(warp_inst_t *&inst); + + void incregfile_reads(unsigned active_count) { + m_stats->m_read_regfile_acesses[m_sid] = + m_stats->m_read_regfile_acesses[m_sid] + active_count; + } + void incregfile_writes(unsigned active_count) { + m_stats->m_write_regfile_acesses[m_sid] = + m_stats->m_write_regfile_acesses[m_sid] + active_count; + } + void incnon_rf_operands(unsigned active_count) { + m_stats->m_non_rf_operands[m_sid] = + m_stats->m_non_rf_operands[m_sid] + active_count; + } + + void incspactivelanes_stat(unsigned active_count) { + m_stats->m_active_sp_lanes[m_sid] = + m_stats->m_active_sp_lanes[m_sid] + active_count; + } + void incsfuactivelanes_stat(unsigned active_count) { + m_stats->m_active_sfu_lanes[m_sid] = + m_stats->m_active_sfu_lanes[m_sid] + active_count; + } + void incfuactivelanes_stat(unsigned active_count) { + m_stats->m_active_fu_lanes[m_sid] = + m_stats->m_active_fu_lanes[m_sid] + active_count; + } + void incfumemactivelanes_stat(unsigned active_count) { + m_stats->m_active_fu_mem_lanes[m_sid] = + m_stats->m_active_fu_mem_lanes[m_sid] + active_count; + } - void incsfu_stat(unsigned active_count,double latency) {m_stats->m_num_sfu_acesses[m_sid]=m_stats->m_num_sfu_acesses[m_sid]+active_count*latency;} - void incsp_stat(unsigned active_count,double latency) {m_stats->m_num_sp_acesses[m_sid]=m_stats->m_num_sp_acesses[m_sid]+active_count*latency;} - void incmem_stat(unsigned active_count,double latency) { - if(m_config->gpgpu_clock_gated_lanes==false){ - m_stats->m_num_mem_acesses[m_sid]=m_stats->m_num_mem_acesses[m_sid]+active_count*latency - + inactive_lanes_accesses_nonsfu(active_count, latency); - }else { - m_stats->m_num_mem_acesses[m_sid]=m_stats->m_num_mem_acesses[m_sid]+active_count*latency; - } - } - void incexecstat(warp_inst_t *&inst); + void inc_simt_to_mem(unsigned n_flits) { + m_stats->n_simt_to_mem[m_sid] += n_flits; + } + bool check_if_non_released_reduction_barrier(warp_inst_t &inst); - void incregfile_reads(unsigned active_count) {m_stats->m_read_regfile_acesses[m_sid]=m_stats->m_read_regfile_acesses[m_sid]+active_count;} - void incregfile_writes(unsigned active_count){m_stats->m_write_regfile_acesses[m_sid]=m_stats->m_write_regfile_acesses[m_sid]+active_count;} - void incnon_rf_operands(unsigned active_count){m_stats->m_non_rf_operands[m_sid]=m_stats->m_non_rf_operands[m_sid]+active_count;} + private: + unsigned inactive_lanes_accesses_sfu(unsigned active_count, double latency) { + return (((32 - active_count) >> 1) * latency) + + (((32 - active_count) >> 3) * latency) + + (((32 - active_count) >> 3) * latency); + } + unsigned inactive_lanes_accesses_nonsfu(unsigned active_count, + double latency) { + return (((32 - active_count) >> 1) * latency); + } - void incspactivelanes_stat(unsigned active_count) {m_stats->m_active_sp_lanes[m_sid]=m_stats->m_active_sp_lanes[m_sid]+active_count;} - void incsfuactivelanes_stat(unsigned active_count) {m_stats->m_active_sfu_lanes[m_sid]=m_stats->m_active_sfu_lanes[m_sid]+active_count;} - void incfuactivelanes_stat(unsigned active_count) {m_stats->m_active_fu_lanes[m_sid]=m_stats->m_active_fu_lanes[m_sid]+active_count;} - void incfumemactivelanes_stat(unsigned active_count) {m_stats->m_active_fu_mem_lanes[m_sid]=m_stats->m_active_fu_mem_lanes[m_sid]+active_count;} + int test_res_bus(int latency); + void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread, + unsigned ctaid, int cta_size, unsigned kernel_id); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid); + address_type next_pc(int tid) const; + void fetch(); + void register_cta_thread_exit(unsigned cta_num, kernel_info_t *kernel); - void inc_simt_to_mem(unsigned n_flits){ m_stats->n_simt_to_mem[m_sid] += n_flits; } - bool check_if_non_released_reduction_barrier(warp_inst_t &inst); + void decode(); - private: - unsigned inactive_lanes_accesses_sfu(unsigned active_count,double latency){ - return ( ((32-active_count)>>1)*latency) + ( ((32-active_count)>>3)*latency) + ( ((32-active_count)>>3)*latency); - } - unsigned inactive_lanes_accesses_nonsfu(unsigned active_count,double latency){ - return ( ((32-active_count)>>1)*latency); - } + void issue(); + friend class scheduler_unit; // this is needed to use private issue warp. + friend class TwoLevelScheduler; + friend class LooseRoundRobbinScheduler; + void issue_warp(register_set &warp, const warp_inst_t *pI, + const active_mask_t &active_mask, unsigned warp_id, + unsigned sch_id); + void func_exec_inst(warp_inst_t &inst); - int test_res_bus(int latency); - void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid); - address_type next_pc( int tid ) const; - void fetch(); - void register_cta_thread_exit(unsigned cta_num, kernel_info_t * kernel ); + // Returns numbers of addresses in translated_addrs + unsigned translate_local_memaddr(address_type localaddr, unsigned tid, + unsigned num_shader, unsigned datasize, + new_addr_type *translated_addrs); - void decode(); - - void issue(); - friend class scheduler_unit; //this is needed to use private issue warp. - friend class TwoLevelScheduler; - friend class LooseRoundRobbinScheduler; - void issue_warp( register_set& warp, const warp_inst_t *pI, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id ); - void func_exec_inst( warp_inst_t &inst ); + void read_operands(); - // Returns numbers of addresses in translated_addrs - unsigned translate_local_memaddr( address_type localaddr, unsigned tid, unsigned num_shader, unsigned datasize, new_addr_type* translated_addrs ); + void execute(); - void read_operands(); - - void execute(); - - void writeback(); - - // used in display_pipeline(): - void dump_warp_state( FILE *fout ) const; - void print_stage(unsigned int stage, FILE *fout) const; - unsigned long long m_last_inst_gpu_sim_cycle; - unsigned long long m_last_inst_gpu_tot_sim_cycle; + void writeback(); - // general information - unsigned m_sid; // shader id - unsigned m_tpc; // texture processor cluster id (aka, node id when using interconnect concentration) - const shader_core_config *m_config; - const memory_config *m_memory_config; - class simt_core_cluster *m_cluster; + // used in display_pipeline(): + void dump_warp_state(FILE *fout) const; + void print_stage(unsigned int stage, FILE *fout) const; + unsigned long long m_last_inst_gpu_sim_cycle; + unsigned long long m_last_inst_gpu_tot_sim_cycle; - // statistics - shader_core_stats *m_stats; + // general information + unsigned m_sid; // shader id + unsigned m_tpc; // texture processor cluster id (aka, node id when using + // interconnect concentration) + const shader_core_config *m_config; + const memory_config *m_memory_config; + class simt_core_cluster *m_cluster; - // CTA scheduling / hardware thread allocation - unsigned m_n_active_cta; // number of Cooperative Thread Arrays (blocks) currently running on this shader. - unsigned m_cta_status[MAX_CTA_PER_SHADER]; // CTAs status - unsigned m_not_completed; // number of threads to be completed (==0 when all thread on this core completed) - std::bitset<MAX_THREAD_PER_SM> m_active_threads; - - // thread contexts - thread_ctx_t *m_threadState; - - // interconnect interface - mem_fetch_interface *m_icnt; - shader_core_mem_fetch_allocator *m_mem_fetch_allocator; - - // fetch - read_only_cache *m_L1I; // instruction cache - int m_last_warp_fetched; + // statistics + shader_core_stats *m_stats; - // decode/dispatch - std::vector<shd_warp_t> m_warp; // per warp information array - barrier_set_t m_barriers; - ifetch_buffer_t m_inst_fetch_buffer; - std::vector<register_set> m_pipeline_reg; - Scoreboard *m_scoreboard; - opndcoll_rfu_t m_operand_collector; - int m_active_warps; + // CTA scheduling / hardware thread allocation + unsigned m_n_active_cta; // number of Cooperative Thread Arrays (blocks) + // currently running on this shader. + unsigned m_cta_status[MAX_CTA_PER_SHADER]; // CTAs status + unsigned m_not_completed; // number of threads to be completed (==0 when all + // thread on this core completed) + std::bitset<MAX_THREAD_PER_SM> m_active_threads; - //schedule - std::vector<scheduler_unit*> schedulers; + // thread contexts + thread_ctx_t *m_threadState; - //issue - unsigned int Issue_Prio; + // interconnect interface + mem_fetch_interface *m_icnt; + shader_core_mem_fetch_allocator *m_mem_fetch_allocator; - // execute - unsigned m_num_function_units; - std::vector<pipeline_stage_name_t> m_dispatch_port; - std::vector<pipeline_stage_name_t> m_issue_port; - std::vector<simd_function_unit*> m_fu; // stallable pipelines should be last in this array - ldst_unit *m_ldst_unit; - static const unsigned MAX_ALU_LATENCY = 512; - unsigned num_result_bus; - std::vector< std::bitset<MAX_ALU_LATENCY>* > m_result_bus; + // fetch + read_only_cache *m_L1I; // instruction cache + int m_last_warp_fetched; - // used for local address mapping with single kernel launch - unsigned kernel_max_cta_per_shader; - unsigned kernel_padded_threads_per_cta; - // Used for handing out dynamic warp_ids to new warps. - // the differnece between a warp_id and a dynamic_warp_id - // is that the dynamic_warp_id is a running number unique to every warp - // run on this shader, where the warp_id is the static warp slot. - unsigned m_dynamic_warp_id; + // decode/dispatch + std::vector<shd_warp_t> m_warp; // per warp information array + barrier_set_t m_barriers; + ifetch_buffer_t m_inst_fetch_buffer; + std::vector<register_set> m_pipeline_reg; + Scoreboard *m_scoreboard; + opndcoll_rfu_t m_operand_collector; + int m_active_warps; - //Jin: concurrent kernels on a sm -public: - bool can_issue_1block(kernel_info_t & kernel); - bool occupy_shader_resource_1block(kernel_info_t & kernel, bool occupy); - void release_shader_resource_1block(unsigned hw_ctaid, kernel_info_t & kernel); - int find_available_hwtid(unsigned int cta_size, bool occupy); -private: - unsigned int m_occupied_n_threads; - unsigned int m_occupied_shmem; - unsigned int m_occupied_regs; - unsigned int m_occupied_ctas; - std::bitset<MAX_THREAD_PER_SM> m_occupied_hwtid; - std::map<unsigned int, unsigned int> m_occupied_cta_to_hwtid; + // schedule + std::vector<scheduler_unit *> schedulers; + // issue + unsigned int Issue_Prio; + // execute + unsigned m_num_function_units; + std::vector<pipeline_stage_name_t> m_dispatch_port; + std::vector<pipeline_stage_name_t> m_issue_port; + std::vector<simd_function_unit *> + m_fu; // stallable pipelines should be last in this array + ldst_unit *m_ldst_unit; + static const unsigned MAX_ALU_LATENCY = 512; + unsigned num_result_bus; + std::vector<std::bitset<MAX_ALU_LATENCY> *> m_result_bus; + + // used for local address mapping with single kernel launch + unsigned kernel_max_cta_per_shader; + unsigned kernel_padded_threads_per_cta; + // Used for handing out dynamic warp_ids to new warps. + // the differnece between a warp_id and a dynamic_warp_id + // is that the dynamic_warp_id is a running number unique to every warp + // run on this shader, where the warp_id is the static warp slot. + unsigned m_dynamic_warp_id; + + // Jin: concurrent kernels on a sm + public: + bool can_issue_1block(kernel_info_t &kernel); + bool occupy_shader_resource_1block(kernel_info_t &kernel, bool occupy); + void release_shader_resource_1block(unsigned hw_ctaid, kernel_info_t &kernel); + int find_available_hwtid(unsigned int cta_size, bool occupy); + + private: + unsigned int m_occupied_n_threads; + unsigned int m_occupied_shmem; + unsigned int m_occupied_regs; + unsigned int m_occupied_ctas; + std::bitset<MAX_THREAD_PER_SM> m_occupied_hwtid; + std::map<unsigned int, unsigned int> m_occupied_cta_to_hwtid; }; class simt_core_cluster { -public: - simt_core_cluster( class gpgpu_sim *gpu, - unsigned cluster_id, - const shader_core_config *config, - const memory_config *mem_config, - shader_core_stats *stats, - memory_stats_t *mstats ); + public: + simt_core_cluster(class gpgpu_sim *gpu, unsigned cluster_id, + const shader_core_config *config, + const memory_config *mem_config, shader_core_stats *stats, + memory_stats_t *mstats); - void core_cycle(); - void icnt_cycle(); + void core_cycle(); + void icnt_cycle(); - void reinit(); - unsigned issue_block2core(); - void cache_flush(); - void cache_invalidate(); - bool icnt_injection_buffer_full(unsigned size, bool write); - void icnt_inject_request_packet(class mem_fetch *mf); + void reinit(); + unsigned issue_block2core(); + void cache_flush(); + void cache_invalidate(); + bool icnt_injection_buffer_full(unsigned size, bool write); + void icnt_inject_request_packet(class mem_fetch *mf); + // for perfect memory interface + bool response_queue_full() { + return (m_response_fifo.size() >= m_config->n_simt_ejection_buffer_size); + } + void push_response_fifo(class mem_fetch *mf) { + m_response_fifo.push_back(mf); + } - // for perfect memory interface - bool response_queue_full() { - return ( m_response_fifo.size() >= m_config->n_simt_ejection_buffer_size ); - } - void push_response_fifo(class mem_fetch *mf) { - m_response_fifo.push_back(mf); - } - - void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ) const; - unsigned max_cta( const kernel_info_t &kernel ); - unsigned get_not_completed() const; - void print_not_completed( FILE *fp ) const; - unsigned get_n_active_cta() const; - unsigned get_n_active_sms() const; - gpgpu_sim *get_gpu() { return m_gpu; } + void get_pdom_stack_top_info(unsigned sid, unsigned tid, unsigned *pc, + unsigned *rpc) const; + unsigned max_cta(const kernel_info_t &kernel); + unsigned get_not_completed() const; + void print_not_completed(FILE *fp) const; + unsigned get_n_active_cta() const; + unsigned get_n_active_sms() const; + gpgpu_sim *get_gpu() { return m_gpu; } - void display_pipeline( unsigned sid, FILE *fout, int print_mem, int mask ); - void print_cache_stats( FILE *fp, unsigned& dl1_accesses, unsigned& dl1_misses ) const; + void display_pipeline(unsigned sid, FILE *fout, int print_mem, int mask); + void print_cache_stats(FILE *fp, unsigned &dl1_accesses, + unsigned &dl1_misses) const; - void get_cache_stats(cache_stats &cs) const; - void get_L1I_sub_stats(struct cache_sub_stats &css) const; - void get_L1D_sub_stats(struct cache_sub_stats &css) const; - void get_L1C_sub_stats(struct cache_sub_stats &css) const; - void get_L1T_sub_stats(struct cache_sub_stats &css) const; + void get_cache_stats(cache_stats &cs) const; + void get_L1I_sub_stats(struct cache_sub_stats &css) const; + void get_L1D_sub_stats(struct cache_sub_stats &css) const; + void get_L1C_sub_stats(struct cache_sub_stats &css) const; + void get_L1T_sub_stats(struct cache_sub_stats &css) const; - void get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; - float get_current_occupancy( unsigned long long& active, unsigned long long & total ) const; + void get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; + float get_current_occupancy(unsigned long long &active, + unsigned long long &total) const; -private: - unsigned m_cluster_id; - gpgpu_sim *m_gpu; - const shader_core_config *m_config; - shader_core_stats *m_stats; - memory_stats_t *m_memory_stats; - shader_core_ctx **m_core; + private: + unsigned m_cluster_id; + gpgpu_sim *m_gpu; + const shader_core_config *m_config; + shader_core_stats *m_stats; + memory_stats_t *m_memory_stats; + shader_core_ctx **m_core; - unsigned m_cta_issue_next_core; - std::list<unsigned> m_core_sim_order; - std::list<mem_fetch*> m_response_fifo; + unsigned m_cta_issue_next_core; + std::list<unsigned> m_core_sim_order; + std::list<mem_fetch *> m_response_fifo; }; class shader_memory_interface : public mem_fetch_interface { -public: - shader_memory_interface( shader_core_ctx *core, simt_core_cluster *cluster ) { m_core=core; m_cluster=cluster; } - virtual bool full( unsigned size, bool write ) const - { - return m_cluster->icnt_injection_buffer_full(size,write); - } - virtual void push(mem_fetch *mf) - { - m_core->inc_simt_to_mem(mf->get_num_flits(true)); - m_cluster->icnt_inject_request_packet(mf); - } -private: - shader_core_ctx *m_core; - simt_core_cluster *m_cluster; + public: + shader_memory_interface(shader_core_ctx *core, simt_core_cluster *cluster) { + m_core = core; + m_cluster = cluster; + } + virtual bool full(unsigned size, bool write) const { + return m_cluster->icnt_injection_buffer_full(size, write); + } + virtual void push(mem_fetch *mf) { + m_core->inc_simt_to_mem(mf->get_num_flits(true)); + m_cluster->icnt_inject_request_packet(mf); + } + + private: + shader_core_ctx *m_core; + simt_core_cluster *m_cluster; }; class perfect_memory_interface : public mem_fetch_interface { -public: - perfect_memory_interface( shader_core_ctx *core, simt_core_cluster *cluster ) { m_core=core; m_cluster=cluster; } - virtual bool full( unsigned size, bool write) const - { - return m_cluster->response_queue_full(); - } - virtual void push(mem_fetch *mf) - { - if ( mf && mf->isatomic() ) - mf->do_atomic(); // execute atomic inside the "memory subsystem" - m_core->inc_simt_to_mem(mf->get_num_flits(true)); - m_cluster->push_response_fifo(mf); - } -private: - shader_core_ctx *m_core; - simt_core_cluster *m_cluster; -}; + public: + perfect_memory_interface(shader_core_ctx *core, simt_core_cluster *cluster) { + m_core = core; + m_cluster = cluster; + } + virtual bool full(unsigned size, bool write) const { + return m_cluster->response_queue_full(); + } + virtual void push(mem_fetch *mf) { + if (mf && mf->isatomic()) + mf->do_atomic(); // execute atomic inside the "memory subsystem" + m_core->inc_simt_to_mem(mf->get_num_flits(true)); + m_cluster->push_response_fifo(mf); + } + private: + shader_core_ctx *m_core; + simt_core_cluster *m_cluster; +}; inline int scheduler_unit::get_sid() const { return m_shader->get_sid(); } diff --git a/src/gpgpu-sim/shader_trace.h b/src/gpgpu-sim/shader_trace.h index ac4e894..e7486d8 100644 --- a/src/gpgpu-sim/shader_trace.h +++ b/src/gpgpu-sim/shader_trace.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Tim Rogers -// George L. Yuan, Andrew Turner, Inderpreet Singh +// George L. Yuan, Andrew Turner, Inderpreet Singh // The University of British Columbia // All rights reserved. // @@ -8,67 +8,73 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef __SHADER_TRACE_H__ #define __SHADER_TRACE_H__ #include "../trace.h" - #if TRACING_ON #define SHADER_PRINT_STR SIM_PRINT_STR "Core %d - " #define SCHED_PRINT_STR SHADER_PRINT_STR "Scheduler %d - " -#define SHADER_DTRACE(x) (DTRACE(x) && (Trace::sampling_core == get_sid()\ - || Trace::sampling_core == -1)) +#define SHADER_DTRACE(x) \ + (DTRACE(x) && \ + (Trace::sampling_core == get_sid() || Trace::sampling_core == -1)) // Intended to be called from inside components of a shader core. // Depends on a get_sid() function -#define SHADER_DPRINTF(x, ...) do {\ - if (SHADER_DTRACE(x)) {\ - printf( SHADER_PRINT_STR,\ - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::x],\ - get_sid() );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define SHADER_DPRINTF(x, ...) \ + do { \ + if (SHADER_DTRACE(x)) { \ + printf(SHADER_PRINT_STR, \ + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::x], get_sid()); \ + printf(__VA_ARGS__); \ + } \ + } while (0) // Intended to be called from inside a scheduler_unit. // Depends on a m_id member -#define SCHED_DPRINTF(...) do {\ - if (SHADER_DTRACE(WARP_SCHEDULER)) {\ - printf( SCHED_PRINT_STR,\ - m_shader->get_gpu()->gpu_sim_cycle + m_shader->get_gpu()->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::WARP_SCHEDULER],\ - get_sid(),\ - m_id );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define SCHED_DPRINTF(...) \ + do { \ + if (SHADER_DTRACE(WARP_SCHEDULER)) { \ + printf(SCHED_PRINT_STR, \ + m_shader->get_gpu()->gpu_sim_cycle + \ + m_shader->get_gpu()->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::WARP_SCHEDULER], get_sid(), \ + m_id); \ + printf(__VA_ARGS__); \ + } \ + } while (0) #else -#define SHADER_DTRACE(x) (false) -#define SHADER_DPRINTF(x, ...) do {} while (0) -#define SCHED_DPRINTF(x, ...) do {} while (0) +#define SHADER_DTRACE(x) (false) +#define SHADER_DPRINTF(x, ...) \ + do { \ + } while (0) +#define SCHED_DPRINTF(x, ...) \ + do { \ + } while (0) #endif diff --git a/src/gpgpu-sim/stack.cc b/src/gpgpu-sim/stack.cc index e77fc06..ab9fcbc 100644 --- a/src/gpgpu-sim/stack.cc +++ b/src/gpgpu-sim/stack.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Ivan Sham, +// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Ivan Sham, // Wilson W.L. Fung // All rights reserved. // @@ -7,82 +7,74 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "stack.h" -#include <stdlib.h> #include <assert.h> +#include <stdlib.h> void push_stack(Stack *S, address_type val) { - assert(S->top < S->max_size); - S->v[S->top] = val; - (S->top)++; - + assert(S->top < S->max_size); + S->v[S->top] = val; + (S->top)++; } address_type pop_stack(Stack *S) { - (S->top)--; - return(S->v[S->top]); + (S->top)--; + return (S->v[S->top]); } address_type top_stack(Stack *S) { - assert(S->top >= 1); - return(S->v[S->top - 1]); + assert(S->top >= 1); + return (S->v[S->top - 1]); } -Stack* new_stack(int size) { - Stack* S; - S = (Stack*)malloc(sizeof(Stack)); - S->max_size = size; - S->top = 0; - S->v = (address_type*)calloc(size, sizeof(address_type)); - return S; +Stack *new_stack(int size) { + Stack *S; + S = (Stack *)malloc(sizeof(Stack)); + S->max_size = size; + S->top = 0; + S->v = (address_type *)calloc(size, sizeof(address_type)); + return S; } void free_stack(Stack *S) { - free(S->v); - free(S); + free(S->v); + free(S); } -int size_stack(Stack *S) { - return S->top; -} +int size_stack(Stack *S) { return S->top; } -int full_stack(Stack *S) { - return S->top >= S->max_size; -} +int full_stack(Stack *S) { return S->top >= S->max_size; } -int empty_stack(Stack *S) { - return S->top == 0; -} +int empty_stack(Stack *S) { return S->top == 0; } int element_exist_stack(Stack *S, address_type value) { - int i; - for (i = 0; i < S->top; ++i) { - if (value == S->v[i]) { - return 1; - } - } - return 0; + int i; + for (i = 0; i < S->top; ++i) { + if (value == S->v[i]) { + return 1; + } + } + return 0; } -void reset_stack(Stack *S) { - S->top = 0; -} +void reset_stack(Stack *S) { S->top = 0; } diff --git a/src/gpgpu-sim/stack.h b/src/gpgpu-sim/stack.h index 54b6621..c884680 100644 --- a/src/gpgpu-sim/stack.h +++ b/src/gpgpu-sim/stack.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef _MY_STACK_ #define _MY_STACK_ @@ -31,19 +32,19 @@ #include "../abstract_hardware_model.h" typedef struct { - address_type *v; - int max_size; - int top; + address_type *v; + int max_size; + int top; } Stack; void push_stack(Stack *S, address_type val); address_type pop_stack(Stack *S); address_type top_stack(Stack *S); -Stack* new_stack(int size); +Stack *new_stack(int size); void free_stack(Stack *S); int size_stack(Stack *S); int full_stack(Stack *S); int empty_stack(Stack *S); int element_exist_stack(Stack *S, address_type value); void reset_stack(Stack *S); -#endif // _MY_STACK_ +#endif // _MY_STACK_ diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc index 35a4cc3..6fafaa6 100644 --- a/src/gpgpu-sim/stat-tool.cc +++ b/src/gpgpu-sim/stat-tool.cc @@ -7,180 +7,179 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "stat-tool.h" +#include <assert.h> #include <stdio.h> #include <stdlib.h> -#include <assert.h> #include <zlib.h> -#include <string> +#include <algorithm> #include <list> -#include <vector> #include <map> -#include <algorithm> #include <string> +#include <vector> #include "../../libcuda/gpgpu_context.h" //////////////////////////////////////////////////////////////////////////////// -static unsigned long long min_snap_shot_interval = 0; -static unsigned long long next_snap_shot_cycle = 0; -static std::list<snap_shot_trigger*> list_ss_trigger; +static unsigned long long min_snap_shot_interval = 0; +static unsigned long long next_snap_shot_cycle = 0; +static std::list<snap_shot_trigger *> list_ss_trigger; -void add_snap_shot_trigger (snap_shot_trigger* ss_trigger) -{ - // quick optimization assuming that all snap shot intervals are perfect multiples of each other - if (min_snap_shot_interval == 0 || min_snap_shot_interval > ss_trigger->get_interval()) { - min_snap_shot_interval = ss_trigger->get_interval(); - next_snap_shot_cycle = min_snap_shot_interval; // assume that snap shots haven't started yet - } - list_ss_trigger.push_back(ss_trigger); +void add_snap_shot_trigger(snap_shot_trigger *ss_trigger) { + // quick optimization assuming that all snap shot intervals are perfect + // multiples of each other + if (min_snap_shot_interval == 0 || + min_snap_shot_interval > ss_trigger->get_interval()) { + min_snap_shot_interval = ss_trigger->get_interval(); + next_snap_shot_cycle = + min_snap_shot_interval; // assume that snap shots haven't started yet + } + list_ss_trigger.push_back(ss_trigger); } -void remove_snap_shot_trigger (snap_shot_trigger* ss_trigger) -{ - list_ss_trigger.remove(ss_trigger); +void remove_snap_shot_trigger(snap_shot_trigger *ss_trigger) { + list_ss_trigger.remove(ss_trigger); } -void try_snap_shot (unsigned long long current_cycle) -{ - if (min_snap_shot_interval == 0) return; - if (current_cycle != next_snap_shot_cycle) return; - - std::list<snap_shot_trigger*>::iterator ss_trigger_iter = list_ss_trigger.begin(); - for(; ss_trigger_iter != list_ss_trigger.end(); ++ss_trigger_iter) { - (*ss_trigger_iter)->snap_shot(current_cycle); // WF: should be try_snap_shot - } - next_snap_shot_cycle = current_cycle + min_snap_shot_interval; // WF: stateful testing, maybe bad +void try_snap_shot(unsigned long long current_cycle) { + if (min_snap_shot_interval == 0) return; + if (current_cycle != next_snap_shot_cycle) return; + + std::list<snap_shot_trigger *>::iterator ss_trigger_iter = + list_ss_trigger.begin(); + for (; ss_trigger_iter != list_ss_trigger.end(); ++ss_trigger_iter) { + (*ss_trigger_iter) + ->snap_shot(current_cycle); // WF: should be try_snap_shot + } + next_snap_shot_cycle = + current_cycle + + min_snap_shot_interval; // WF: stateful testing, maybe bad } //////////////////////////////////////////////////////////////////////////////// - -static unsigned long long spill_interval = 0; -static unsigned long long next_spill_cycle = 0; -static std::list<spill_log_interface*> list_spill_log; -void add_spill_log (spill_log_interface* spill_log) -{ - list_spill_log.push_back(spill_log); +static unsigned long long spill_interval = 0; +static unsigned long long next_spill_cycle = 0; +static std::list<spill_log_interface *> list_spill_log; + +void add_spill_log(spill_log_interface *spill_log) { + list_spill_log.push_back(spill_log); } -void remove_spill_log (spill_log_interface* spill_log) -{ - list_spill_log.remove(spill_log); +void remove_spill_log(spill_log_interface *spill_log) { + list_spill_log.remove(spill_log); } -void set_spill_interval (unsigned long long interval) -{ - spill_interval = interval; - next_spill_cycle = spill_interval; +void set_spill_interval(unsigned long long interval) { + spill_interval = interval; + next_spill_cycle = spill_interval; } -void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle) -{ - if (!final && spill_interval == 0) return; - if (!final && current_cycle <= next_spill_cycle) return; +void spill_log_to_file(FILE *fout, int final, + unsigned long long current_cycle) { + if (!final && spill_interval == 0) return; + if (!final && current_cycle <= next_spill_cycle) return; - fprintf(fout, "\n"); // ensure that the spill occurs at a new line - std::list<spill_log_interface*>::iterator i_spill_log = list_spill_log.begin(); - for(; i_spill_log != list_spill_log.end(); ++i_spill_log) { - (*i_spill_log)->spill(fout, final); - } - fflush(fout); + fprintf(fout, "\n"); // ensure that the spill occurs at a new line + std::list<spill_log_interface *>::iterator i_spill_log = + list_spill_log.begin(); + for (; i_spill_log != list_spill_log.end(); ++i_spill_log) { + (*i_spill_log)->spill(fout, final); + } + fflush(fout); - next_spill_cycle = current_cycle + spill_interval; // WF: stateful testing, maybe bad + next_spill_cycle = + current_cycle + spill_interval; // WF: stateful testing, maybe bad } //////////////////////////////////////////////////////////////////////////////// static int n_thread_CFloggers = 0; -static thread_CFlocality** thread_CFlogger = NULL; +static thread_CFlocality **thread_CFlogger = NULL; -void create_thread_CFlogger(gpgpu_context* ctx, int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval) -{ - destroy_thread_CFlogger(); - - n_thread_CFloggers = n_loggers; - thread_CFlogger = new thread_CFlocality*[n_loggers]; +void create_thread_CFlogger(gpgpu_context *ctx, int n_loggers, int n_threads, + address_type start_pc, + unsigned long long logging_interval) { + destroy_thread_CFlogger(); + + n_thread_CFloggers = n_loggers; + thread_CFlogger = new thread_CFlocality *[n_loggers]; - std::string name_tpl("CFLog"); - char buffer[32]; - for (int i = 0; i < n_thread_CFloggers; i++) { - snprintf(buffer, 32, "%02d", i); - thread_CFlogger[i] = new thread_CFlocality( ctx, name_tpl + buffer, logging_interval, n_threads, start_pc); - if (logging_interval != 0) { - add_snap_shot_trigger(thread_CFlogger[i]); - add_spill_log(thread_CFlogger[i]); - } - } + std::string name_tpl("CFLog"); + char buffer[32]; + for (int i = 0; i < n_thread_CFloggers; i++) { + snprintf(buffer, 32, "%02d", i); + thread_CFlogger[i] = new thread_CFlocality( + ctx, name_tpl + buffer, logging_interval, n_threads, start_pc); + if (logging_interval != 0) { + add_snap_shot_trigger(thread_CFlogger[i]); + add_spill_log(thread_CFlogger[i]); + } + } } -void destroy_thread_CFlogger( ) -{ - if (thread_CFlogger != NULL) { - for (int i = 0; i < n_thread_CFloggers; i++) { - remove_snap_shot_trigger(thread_CFlogger[i]); - remove_spill_log(thread_CFlogger[i]); - delete thread_CFlogger[i]; - } - delete [] thread_CFlogger; - thread_CFlogger = NULL; - } +void destroy_thread_CFlogger() { + if (thread_CFlogger != NULL) { + for (int i = 0; i < n_thread_CFloggers; i++) { + remove_snap_shot_trigger(thread_CFlogger[i]); + remove_spill_log(thread_CFlogger[i]); + delete thread_CFlogger[i]; + } + delete[] thread_CFlogger; + thread_CFlogger = NULL; + } } -void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ) -{ - if (thread_CFlogger == NULL) return; // this means no visualizer output - if (thread_id < 0) return; - thread_CFlogger[logger_id]->update_thread_pc(thread_id, pc); +void cflog_update_thread_pc(int logger_id, int thread_id, address_type pc) { + if (thread_CFlogger == NULL) return; // this means no visualizer output + if (thread_id < 0) return; + thread_CFlogger[logger_id]->update_thread_pc(thread_id, pc); } -// deprecated -void cflog_snapshot( int logger_id, unsigned long long cycle ) -{ - thread_CFlogger[logger_id]->snap_shot(cycle); +// deprecated +void cflog_snapshot(int logger_id, unsigned long long cycle) { + thread_CFlogger[logger_id]->snap_shot(cycle); } -void cflog_print(FILE *fout) -{ - if (thread_CFlogger == NULL) return; // this means no visualizer output - for (int i = 0; i < n_thread_CFloggers; i++) { - thread_CFlogger[i]->print_histo(fout); - } +void cflog_print(FILE *fout) { + if (thread_CFlogger == NULL) return; // this means no visualizer output + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_histo(fout); + } } -void cflog_visualizer_print(FILE *fout) -{ - if (thread_CFlogger == NULL) return; // this means no visualizer output - for (int i = 0; i < n_thread_CFloggers; i++) { - thread_CFlogger[i]->print_visualizer(fout); - } +void cflog_visualizer_print(FILE *fout) { + if (thread_CFlogger == NULL) return; // this means no visualizer output + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_visualizer(fout); + } } -void cflog_visualizer_gzprint(gzFile fout) -{ - if (thread_CFlogger == NULL) return; // this means no visualizer output - for (int i = 0; i < n_thread_CFloggers; i++) { - thread_CFlogger[i]->print_visualizer(fout); - } +void cflog_visualizer_gzprint(gzFile fout) { + if (thread_CFlogger == NULL) return; // this means no visualizer output + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_visualizer(fout); + } } //////////////////////////////////////////////////////////////////////////////// @@ -189,26 +188,23 @@ int insn_warp_occ_logger::s_ids = 0; static std::vector<insn_warp_occ_logger> iwo_logger; -void insn_warp_occ_create( int n_loggers, int simd_width ) -{ - iwo_logger.clear(); - iwo_logger.assign(n_loggers, insn_warp_occ_logger(simd_width)); - for (unsigned i = 0; i < iwo_logger.size(); i++) { - iwo_logger[i].set_id(i); - } +void insn_warp_occ_create(int n_loggers, int simd_width) { + iwo_logger.clear(); + iwo_logger.assign(n_loggers, insn_warp_occ_logger(simd_width)); + for (unsigned i = 0; i < iwo_logger.size(); i++) { + iwo_logger[i].set_id(i); + } } -void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ) -{ - if (warp_occ <= 0) return; - iwo_logger[logger_id].log(pc, warp_occ); +void insn_warp_occ_log(int logger_id, address_type pc, int warp_occ) { + if (warp_occ <= 0) return; + iwo_logger[logger_id].log(pc, warp_occ); } -void insn_warp_occ_print( FILE *fout ) -{ - for (unsigned i = 0; i < iwo_logger.size(); i++) { - iwo_logger[i].print(fout); - } +void insn_warp_occ_print(FILE *fout) { + for (unsigned i = 0; i < iwo_logger.size(); i++) { + iwo_logger[i].print(fout); + } } //////////////////////////////////////////////////////////////////////////////// @@ -221,36 +217,33 @@ int linear_histogram_logger::s_ids = 0; static std::vector<linear_histogram_logger> s_warp_occ_logger; -void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long logging_interval) -{ - // simd_width + 1 to include the case with full warp - s_warp_occ_logger.assign(n_loggers, - linear_histogram_logger(simd_width + 1, logging_interval, "ShdrWarpOcc")); - for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { - s_warp_occ_logger[i].set_id(i); - add_snap_shot_trigger(&(s_warp_occ_logger[i])); - add_spill_log(&(s_warp_occ_logger[i])); - } +void shader_warp_occ_create(int n_loggers, int simd_width, + unsigned long long logging_interval) { + // simd_width + 1 to include the case with full warp + s_warp_occ_logger.assign( + n_loggers, + linear_histogram_logger(simd_width + 1, logging_interval, "ShdrWarpOcc")); + for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { + s_warp_occ_logger[i].set_id(i); + add_snap_shot_trigger(&(s_warp_occ_logger[i])); + add_spill_log(&(s_warp_occ_logger[i])); + } } -void shader_warp_occ_log( int logger_id, int warp_occ) -{ - s_warp_occ_logger[logger_id].log(warp_occ); +void shader_warp_occ_log(int logger_id, int warp_occ) { + s_warp_occ_logger[logger_id].log(warp_occ); } -void shader_warp_occ_snapshot( int logger_id, unsigned long long current_cycle) -{ - s_warp_occ_logger[logger_id].snap_shot(current_cycle); +void shader_warp_occ_snapshot(int logger_id, unsigned long long current_cycle) { + s_warp_occ_logger[logger_id].snap_shot(current_cycle); } -void shader_warp_occ_print( FILE *fout ) -{ - for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { - s_warp_occ_logger[i].print(fout); - } +void shader_warp_occ_print(FILE *fout) { + for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { + s_warp_occ_logger[i].print(fout); + } } - ///////////////////////////////////////////////////////////////////////////////////// // per-shadercore memory-access logger ///////////////////////////////////////////////////////////////////////////////////// @@ -259,105 +252,115 @@ static int s_mem_acc_logger_n_dram = 0; static int s_mem_acc_logger_n_bank = 0; static std::vector<linear_histogram_logger> s_mem_acc_logger; -void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long logging_interval) -{ - // (n_bank + 1) to space data out; 2x to separate read and write - s_mem_acc_logger.assign(n_loggers, - linear_histogram_logger(2 * n_dram * (n_bank + 1), logging_interval, "ShdrMemAcc")); +void shader_mem_acc_create(int n_loggers, int n_dram, int n_bank, + unsigned long long logging_interval) { + // (n_bank + 1) to space data out; 2x to separate read and write + s_mem_acc_logger.assign( + n_loggers, linear_histogram_logger(2 * n_dram * (n_bank + 1), + logging_interval, "ShdrMemAcc")); - s_mem_acc_logger_n_dram = n_dram; - s_mem_acc_logger_n_bank = n_bank; - for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { - s_mem_acc_logger[i].set_id(i); - add_snap_shot_trigger(&(s_mem_acc_logger[i])); - add_spill_log(&(s_mem_acc_logger[i])); - } + s_mem_acc_logger_n_dram = n_dram; + s_mem_acc_logger_n_bank = n_bank; + for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { + s_mem_acc_logger[i].set_id(i); + add_snap_shot_trigger(&(s_mem_acc_logger[i])); + add_spill_log(&(s_mem_acc_logger[i])); + } } -void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw) -{ - if (s_mem_acc_logger_n_dram == 0) return; - int write_offset = 0; - switch(rw) { - case 'r': write_offset = 0; break; - case 'w': write_offset = (s_mem_acc_logger_n_bank + 1) * s_mem_acc_logger_n_dram; break; - default: assert(0); break; - } - s_mem_acc_logger[logger_id].log(dram_id * s_mem_acc_logger_n_bank + bank + write_offset); +void shader_mem_acc_log(int logger_id, int dram_id, int bank, char rw) { + if (s_mem_acc_logger_n_dram == 0) return; + int write_offset = 0; + switch (rw) { + case 'r': + write_offset = 0; + break; + case 'w': + write_offset = (s_mem_acc_logger_n_bank + 1) * s_mem_acc_logger_n_dram; + break; + default: + assert(0); + break; + } + s_mem_acc_logger[logger_id].log(dram_id * s_mem_acc_logger_n_bank + bank + + write_offset); } -void shader_mem_acc_snapshot( int logger_id, unsigned long long current_cycle) -{ - s_mem_acc_logger[logger_id].snap_shot(current_cycle); +void shader_mem_acc_snapshot(int logger_id, unsigned long long current_cycle) { + s_mem_acc_logger[logger_id].snap_shot(current_cycle); } -void shader_mem_acc_print( FILE *fout ) -{ - for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { - s_mem_acc_logger[i].print(fout); - } +void shader_mem_acc_print(FILE *fout) { + for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { + s_mem_acc_logger[i].print(fout); + } } - ///////////////////////////////////////////////////////////////////////////////////// // per-shadercore memory-latency logger ///////////////////////////////////////////////////////////////////////////////////// static bool s_mem_lat_logger_used = false; -static int s_mem_lat_logger_nbins = 48; // up to 2^24 = 16M +static int s_mem_lat_logger_nbins = 48; // up to 2^24 = 16M static std::vector<linear_histogram_logger> s_mem_lat_logger; -void shader_mem_lat_create( int n_loggers, unsigned long long logging_interval) -{ - s_mem_lat_logger.assign(n_loggers, - linear_histogram_logger(s_mem_lat_logger_nbins, logging_interval, "ShdrMemLat")); +void shader_mem_lat_create(int n_loggers, unsigned long long logging_interval) { + s_mem_lat_logger.assign( + n_loggers, linear_histogram_logger(s_mem_lat_logger_nbins, + logging_interval, "ShdrMemLat")); + + for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { + s_mem_lat_logger[i].set_id(i); + add_snap_shot_trigger(&(s_mem_lat_logger[i])); + add_spill_log(&(s_mem_lat_logger[i])); + } - for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { - s_mem_lat_logger[i].set_id(i); - add_snap_shot_trigger(&(s_mem_lat_logger[i])); - add_spill_log(&(s_mem_lat_logger[i])); - } - - s_mem_lat_logger_used = true; + s_mem_lat_logger_used = true; } -void shader_mem_lat_log( int logger_id, int latency) -{ - if (s_mem_lat_logger_used == false) return; - if (latency > (1<<(s_mem_lat_logger_nbins/2))) assert(0); // guard for out of bound bin - assert(latency > 0); - - int latency_bin; - - int bin; // LOG_2(latency) - int v = latency; - register unsigned int shift; +void shader_mem_lat_log(int logger_id, int latency) { + if (s_mem_lat_logger_used == false) return; + if (latency > (1 << (s_mem_lat_logger_nbins / 2))) + assert(0); // guard for out of bound bin + assert(latency > 0); - bin = (v > 0xFFFF) << 4; v >>= bin; - shift = (v > 0xFF ) << 3; v >>= shift; bin |= shift; - shift = (v > 0xF ) << 2; v >>= shift; bin |= shift; - shift = (v > 0x3 ) << 1; v >>= shift; bin |= shift; - bin |= (v >> 1); - latency_bin = 2 * bin; - if (bin > 0) { - latency_bin += ((latency & (1 << (bin - 1))) != 0)? 1 : 0; // approx. for LOG_sqrt2(latency) - } + int latency_bin; - s_mem_lat_logger[logger_id].log(latency_bin); -} + int bin; // LOG_2(latency) + int v = latency; + register unsigned int shift; -void shader_mem_lat_snapshot( int logger_id, unsigned long long current_cycle) -{ - s_mem_lat_logger[logger_id].snap_shot(current_cycle); + bin = (v > 0xFFFF) << 4; + v >>= bin; + shift = (v > 0xFF) << 3; + v >>= shift; + bin |= shift; + shift = (v > 0xF) << 2; + v >>= shift; + bin |= shift; + shift = (v > 0x3) << 1; + v >>= shift; + bin |= shift; + bin |= (v >> 1); + latency_bin = 2 * bin; + if (bin > 0) { + latency_bin += ((latency & (1 << (bin - 1))) != 0) + ? 1 + : 0; // approx. for LOG_sqrt2(latency) + } + + s_mem_lat_logger[logger_id].log(latency_bin); } -void shader_mem_lat_print( FILE *fout ) -{ - for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { - s_mem_lat_logger[i].print(fout); - } +void shader_mem_lat_snapshot(int logger_id, unsigned long long current_cycle) { + s_mem_lat_logger[logger_id].snap_shot(current_cycle); } +void shader_mem_lat_print(FILE *fout) { + for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { + s_mem_lat_logger[i].print(fout); + } +} ///////////////////////////////////////////////////////////////////////////////////// // per-shadercore cache-miss logger @@ -366,425 +369,393 @@ void shader_mem_lat_print( FILE *fout ) static int s_cache_access_logger_n_types = 0; static std::vector<linear_histogram_logger> s_cache_access_logger; -enum cache_access_logger_types { - NORMALS, TEXTURE, CONSTANT, INSTRUCTION -}; +enum cache_access_logger_types { NORMALS, TEXTURE, CONSTANT, INSTRUCTION }; int get_shader_normal_cache_id() { return NORMALS; } int get_shader_texture_cache_id() { return TEXTURE; } int get_shader_constant_cache_id() { return CONSTANT; } int get_shader_instruction_cache_id() { return INSTRUCTION; } -void shader_cache_access_create( int n_loggers, int n_types, unsigned long long logging_interval) -{ - // There are different type of cache (x2 for recording accesses and misses) - s_cache_access_logger.assign(n_loggers, - linear_histogram_logger(n_types * 2, logging_interval, "ShdrCacheMiss")); +void shader_cache_access_create(int n_loggers, int n_types, + unsigned long long logging_interval) { + // There are different type of cache (x2 for recording accesses and misses) + s_cache_access_logger.assign( + n_loggers, + linear_histogram_logger(n_types * 2, logging_interval, "ShdrCacheMiss")); - s_cache_access_logger_n_types = n_types; - for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { - s_cache_access_logger[i].set_id(i); - add_snap_shot_trigger(&(s_cache_access_logger[i])); - add_spill_log(&(s_cache_access_logger[i])); - } + s_cache_access_logger_n_types = n_types; + for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { + s_cache_access_logger[i].set_id(i); + add_snap_shot_trigger(&(s_cache_access_logger[i])); + add_spill_log(&(s_cache_access_logger[i])); + } } -void shader_cache_access_log( int logger_id, int type, int miss) -{ - if (s_cache_access_logger_n_types == 0) return; - if (logger_id < 0) return; - assert(type == NORMALS || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); - assert(miss == 0 || miss == 1); - - s_cache_access_logger[logger_id].log(2 * type + miss); -} +void shader_cache_access_log(int logger_id, int type, int miss) { + if (s_cache_access_logger_n_types == 0) return; + if (logger_id < 0) return; + assert(type == NORMALS || type == TEXTURE || type == CONSTANT || + type == INSTRUCTION); + assert(miss == 0 || miss == 1); -void shader_cache_access_unlog( int logger_id, int type, int miss) -{ - if (s_cache_access_logger_n_types == 0) return; - if (logger_id < 0) return; - assert(type == NORMALS || type == TEXTURE || type == CONSTANT || type == INSTRUCTION); - assert(miss == 0 || miss == 1); - - s_cache_access_logger[logger_id].unlog(2 * type + miss); + s_cache_access_logger[logger_id].log(2 * type + miss); } -void shader_cache_access_print( FILE *fout ) -{ - for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { - s_cache_access_logger[i].print(fout); - } +void shader_cache_access_unlog(int logger_id, int type, int miss) { + if (s_cache_access_logger_n_types == 0) return; + if (logger_id < 0) return; + assert(type == NORMALS || type == TEXTURE || type == CONSTANT || + type == INSTRUCTION); + assert(miss == 0 || miss == 1); + + s_cache_access_logger[logger_id].unlog(2 * type + miss); } +void shader_cache_access_print(FILE *fout) { + for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { + s_cache_access_logger[i].print(fout); + } +} ///////////////////////////////////////////////////////////////////////////////////// -// per-shadercore CTA count logger (only make sense with gpgpu_spread_blocks_across_cores) +// per-shadercore CTA count logger (only make sense with +// gpgpu_spread_blocks_across_cores) ///////////////////////////////////////////////////////////////////////////////////// static linear_histogram_logger *s_CTA_count_logger = NULL; -void shader_CTA_count_create( int n_shaders, unsigned long long logging_interval) -{ - // only need one logger to track all the shaders - if (s_CTA_count_logger != NULL) delete s_CTA_count_logger; - s_CTA_count_logger = new linear_histogram_logger(n_shaders, logging_interval, "ShdrCTACount", false); +void shader_CTA_count_create(int n_shaders, + unsigned long long logging_interval) { + // only need one logger to track all the shaders + if (s_CTA_count_logger != NULL) delete s_CTA_count_logger; + s_CTA_count_logger = new linear_histogram_logger(n_shaders, logging_interval, + "ShdrCTACount", false); - s_CTA_count_logger->set_id(-1); - if (logging_interval != 0) { - add_snap_shot_trigger(s_CTA_count_logger); - add_spill_log(s_CTA_count_logger); - } + s_CTA_count_logger->set_id(-1); + if (logging_interval != 0) { + add_snap_shot_trigger(s_CTA_count_logger); + add_spill_log(s_CTA_count_logger); + } } -void shader_CTA_count_log( int shader_id, int nCTAadded ) -{ - if (s_CTA_count_logger == NULL) return; - - for (int i = 0; i < nCTAadded; i++) { - s_CTA_count_logger->log(shader_id); - } -} +void shader_CTA_count_log(int shader_id, int nCTAadded) { + if (s_CTA_count_logger == NULL) return; -void shader_CTA_count_unlog( int shader_id, int nCTAdone ) -{ - if (s_CTA_count_logger == NULL) return; - - for (int i = 0; i < nCTAdone; i++) { - s_CTA_count_logger->unlog(shader_id); - } + for (int i = 0; i < nCTAadded; i++) { + s_CTA_count_logger->log(shader_id); + } } -void shader_CTA_count_print( FILE *fout ) -{ - if (s_CTA_count_logger == NULL) return; - s_CTA_count_logger->print(fout); +void shader_CTA_count_unlog(int shader_id, int nCTAdone) { + if (s_CTA_count_logger == NULL) return; + + for (int i = 0; i < nCTAdone; i++) { + s_CTA_count_logger->unlog(shader_id); + } } -void shader_CTA_count_visualizer_print( FILE *fout ) -{ - if (s_CTA_count_logger == NULL) return; - s_CTA_count_logger->print_visualizer(fout); +void shader_CTA_count_print(FILE *fout) { + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print(fout); } -void shader_CTA_count_visualizer_gzprint( gzFile fout ) -{ - if (s_CTA_count_logger == NULL) return; - s_CTA_count_logger->print_visualizer(fout); +void shader_CTA_count_visualizer_print(FILE *fout) { + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print_visualizer(fout); } +void shader_CTA_count_visualizer_gzprint(gzFile fout) { + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print_visualizer(fout); +} //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// -thread_insn_span::thread_insn_span(unsigned long long cycle, gpgpu_context* ctx) - : m_cycle(cycle), +thread_insn_span::thread_insn_span(unsigned long long cycle, gpgpu_context *ctx) + : m_cycle(cycle), #if (tr1_hash_map_ismap == 1) - m_insn_span_count() -#else - m_insn_span_count(32*1024) + m_insn_span_count() +#else + m_insn_span_count(32 * 1024) #endif { - gpgpu_ctx = ctx; + gpgpu_ctx = ctx; } -thread_insn_span::~thread_insn_span() { } - -thread_insn_span::thread_insn_span(const thread_insn_span& other, gpgpu_context* ctx) - : m_cycle(other.m_cycle), - m_insn_span_count(other.m_insn_span_count) -{ - gpgpu_ctx = ctx; +thread_insn_span::~thread_insn_span() {} + +thread_insn_span::thread_insn_span(const thread_insn_span &other, + gpgpu_context *ctx) + : m_cycle(other.m_cycle), m_insn_span_count(other.m_insn_span_count) { + gpgpu_ctx = ctx; } - -thread_insn_span& thread_insn_span::operator=(const thread_insn_span& other) -{ - printf("thread_insn_span& operator=\n"); - if (this != &other) { - m_insn_span_count = other.m_insn_span_count; - m_cycle = other.m_cycle; - } - return *this; + +thread_insn_span &thread_insn_span::operator=(const thread_insn_span &other) { + printf("thread_insn_span& operator=\n"); + if (this != &other) { + m_insn_span_count = other.m_insn_span_count; + m_cycle = other.m_cycle; + } + return *this; } - -thread_insn_span& thread_insn_span::operator+=(const thread_insn_span& other) -{ - span_count_map::const_iterator i_sc = other.m_insn_span_count.begin(); - for (; i_sc != other.m_insn_span_count.end(); ++i_sc) { - m_insn_span_count[i_sc->first] += i_sc->second; - } - return *this; + +thread_insn_span &thread_insn_span::operator+=(const thread_insn_span &other) { + span_count_map::const_iterator i_sc = other.m_insn_span_count.begin(); + for (; i_sc != other.m_insn_span_count.end(); ++i_sc) { + m_insn_span_count[i_sc->first] += i_sc->second; + } + return *this; } - -void thread_insn_span::set_span( address_type pc ) -{ - if( ((int)pc) >= 0 ) - m_insn_span_count[pc] += 1; + +void thread_insn_span::set_span(address_type pc) { + if (((int)pc) >= 0) m_insn_span_count[pc] += 1; } - -void thread_insn_span::reset(unsigned long long cycle) -{ - m_cycle = cycle; - m_insn_span_count.clear(); + +void thread_insn_span::reset(unsigned long long cycle) { + m_cycle = cycle; + m_insn_span_count.clear(); } - -void thread_insn_span::print_span(FILE *fout) const -{ - fprintf(fout, "%d: ", (int)m_cycle); - span_count_map::const_iterator i_sc = m_insn_span_count.begin(); - for (; i_sc != m_insn_span_count.end(); ++i_sc) { - fprintf(fout, "%d ", i_sc->first); - } - fprintf(fout, "\n"); + +void thread_insn_span::print_span(FILE *fout) const { + fprintf(fout, "%d: ", (int)m_cycle); + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + fprintf(fout, "%d ", i_sc->first); + } + fprintf(fout, "\n"); } -void thread_insn_span::print_histo(FILE *fout) const -{ - fprintf(fout, "%d:", (int)m_cycle); - span_count_map::const_iterator i_sc = m_insn_span_count.begin(); - for (; i_sc != m_insn_span_count.end(); ++i_sc) { - fprintf(fout, "%d ", i_sc->second); - } - fprintf(fout, "\n"); +void thread_insn_span::print_histo(FILE *fout) const { + fprintf(fout, "%d:", (int)m_cycle); + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + fprintf(fout, "%d ", i_sc->second); + } + fprintf(fout, "\n"); } -void thread_insn_span::print_sparse_histo(FILE *fout) const -{ - int n_printed_entries = 0; - span_count_map::const_iterator i_sc = m_insn_span_count.begin(); - for (; i_sc != m_insn_span_count.end(); ++i_sc) { - unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); - fprintf(fout, "%u %d ", ptx_lineno, i_sc->second); - n_printed_entries++; - } - if (n_printed_entries == 0) { - fprintf(fout, "0 0 "); - } - fprintf(fout, "\n"); +void thread_insn_span::print_sparse_histo(FILE *fout) const { + int n_printed_entries = 0; + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); + fprintf(fout, "%u %d ", ptx_lineno, i_sc->second); + n_printed_entries++; + } + if (n_printed_entries == 0) { + fprintf(fout, "0 0 "); + } + fprintf(fout, "\n"); } -void thread_insn_span::print_sparse_histo(gzFile fout) const -{ - int n_printed_entries = 0; - span_count_map::const_iterator i_sc = m_insn_span_count.begin(); - for (; i_sc != m_insn_span_count.end(); ++i_sc) { - unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); - gzprintf(fout, "%u %d ", ptx_lineno, i_sc->second); - n_printed_entries++; - } - if (n_printed_entries == 0) { - gzprintf(fout, "0 0 "); - } - gzprintf(fout, "\n"); +void thread_insn_span::print_sparse_histo(gzFile fout) const { + int n_printed_entries = 0; + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + unsigned ptx_lineno = gpgpu_ctx->translate_pc_to_ptxlineno(i_sc->first); + gzprintf(fout, "%u %d ", ptx_lineno, i_sc->second); + n_printed_entries++; + } + if (n_printed_entries == 0) { + gzprintf(fout, "0 0 "); + } + gzprintf(fout, "\n"); } //////////////////////////////////////////////////////////////////////////////// -thread_CFlocality::thread_CFlocality( gpgpu_context* ctx, std::string name, - unsigned long long snap_shot_interval, - int nthreads, - address_type start_pc, - unsigned long long start_cycle) - : snap_shot_trigger(snap_shot_interval), m_name(name), - m_nthreads(nthreads), m_thread_pc(nthreads, start_pc), m_cycle(start_cycle), - m_thd_span(start_cycle, ctx) -{ - std::fill(m_thread_pc.begin(), m_thread_pc.end(), -1); // so that hw thread with no work assigned will not clobber results +thread_CFlocality::thread_CFlocality(gpgpu_context *ctx, std::string name, + unsigned long long snap_shot_interval, + int nthreads, address_type start_pc, + unsigned long long start_cycle) + : snap_shot_trigger(snap_shot_interval), + m_name(name), + m_nthreads(nthreads), + m_thread_pc(nthreads, start_pc), + m_cycle(start_cycle), + m_thd_span(start_cycle, ctx) { + std::fill( + m_thread_pc.begin(), m_thread_pc.end(), + -1); // so that hw thread with no work assigned will not clobber results } - -thread_CFlocality::~thread_CFlocality() -{ -} - -void thread_CFlocality::update_thread_pc( int thread_id, address_type pc ) -{ - m_thread_pc[thread_id] = pc; - m_thd_span.set_span(pc); + +thread_CFlocality::~thread_CFlocality() {} + +void thread_CFlocality::update_thread_pc(int thread_id, address_type pc) { + m_thread_pc[thread_id] = pc; + m_thd_span.set_span(pc); } - -void thread_CFlocality::snap_shot(unsigned long long current_cycle) -{ - m_thd_span_archive.push_back(m_thd_span); - m_thd_span.reset(current_cycle); - for (int i = 0; i < (int)m_thread_pc.size(); i++) { - m_thd_span.set_span(m_thread_pc[i]); - } + +void thread_CFlocality::snap_shot(unsigned long long current_cycle) { + m_thd_span_archive.push_back(m_thd_span); + m_thd_span.reset(current_cycle); + for (int i = 0; i < (int)m_thread_pc.size(); i++) { + m_thd_span.set_span(m_thread_pc[i]); + } } - -void thread_CFlocality::spill(FILE *fout, bool final) -{ - std::list<thread_insn_span>::iterator lit = m_thd_span_archive.begin(); - for (; lit != m_thd_span_archive.end(); lit = m_thd_span_archive.erase(lit) ) { - fprintf(fout, "%s-", m_name.c_str()); - lit->print_histo(fout); - } - assert( m_thd_span_archive.empty() ); - if (final) { - fprintf(fout, "%s-", m_name.c_str()); - m_thd_span.print_histo(fout); - } + +void thread_CFlocality::spill(FILE *fout, bool final) { + std::list<thread_insn_span>::iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); lit = m_thd_span_archive.erase(lit)) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_histo(fout); + } + assert(m_thd_span_archive.empty()); + if (final) { + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_histo(fout); + } } - - -void thread_CFlocality::print_visualizer(FILE *fout) -{ - fprintf(fout, "%s: ", m_name.c_str()); - if (m_thd_span_archive.empty()) { - - // visualizer do no require snap_shots - m_thd_span.print_sparse_histo(fout); - - // clean the thread span - m_thd_span.reset(0); - for (int i = 0; i < (int)m_thread_pc.size(); i++) - m_thd_span.set_span(m_thread_pc[i]); - } else { - assert(0); // TODO: implement fall back so that visualizer can work with snap shots - } + +void thread_CFlocality::print_visualizer(FILE *fout) { + fprintf(fout, "%s: ", m_name.c_str()); + if (m_thd_span_archive.empty()) { + // visualizer do no require snap_shots + m_thd_span.print_sparse_histo(fout); + + // clean the thread span + m_thd_span.reset(0); + for (int i = 0; i < (int)m_thread_pc.size(); i++) + m_thd_span.set_span(m_thread_pc[i]); + } else { + assert(0); // TODO: implement fall back so that visualizer can work with + // snap shots + } } - -void thread_CFlocality::print_visualizer(gzFile fout) -{ - gzprintf(fout, "%s: ", m_name.c_str()); - if (m_thd_span_archive.empty()) { - - // visualizer do no require snap_shots - m_thd_span.print_sparse_histo(fout); - - // clean the thread span - m_thd_span.reset(0); - for (int i = 0; i < (int)m_thread_pc.size(); i++) { - m_thd_span.set_span(m_thread_pc[i]); - } - } else { - assert(0); // TODO: implement fall back so that visualizer can work with snap shots - } + +void thread_CFlocality::print_visualizer(gzFile fout) { + gzprintf(fout, "%s: ", m_name.c_str()); + if (m_thd_span_archive.empty()) { + // visualizer do no require snap_shots + m_thd_span.print_sparse_histo(fout); + + // clean the thread span + m_thd_span.reset(0); + for (int i = 0; i < (int)m_thread_pc.size(); i++) { + m_thd_span.set_span(m_thread_pc[i]); + } + } else { + assert(0); // TODO: implement fall back so that visualizer can work with + // snap shots + } } - -void thread_CFlocality::print_span(FILE *fout) const -{ - std::list<thread_insn_span>::const_iterator lit = m_thd_span_archive.begin(); - for (; lit != m_thd_span_archive.end(); ++lit) { - fprintf(fout, "%s-", m_name.c_str()); - lit->print_span(fout); - } - fprintf(fout, "%s-", m_name.c_str()); - m_thd_span.print_span(fout); + +void thread_CFlocality::print_span(FILE *fout) const { + std::list<thread_insn_span>::const_iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); ++lit) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_span(fout); + } + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_span(fout); } -void thread_CFlocality::print_histo(FILE *fout) const -{ - std::list<thread_insn_span>::const_iterator lit = m_thd_span_archive.begin(); - for (; lit != m_thd_span_archive.end(); ++lit) { - fprintf(fout, "%s-", m_name.c_str()); - lit->print_histo(fout); - } - fprintf(fout, "%s-", m_name.c_str()); - m_thd_span.print_histo(fout); +void thread_CFlocality::print_histo(FILE *fout) const { + std::list<thread_insn_span>::const_iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); ++lit) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_histo(fout); + } + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_histo(fout); } //////////////////////////////////////////////////////////////////////////////// -linear_histogram_logger::linear_histogram_logger(int n_bins, - unsigned long long snap_shot_interval, - const char *name, - bool reset_at_snap_shot, - unsigned long long start_cycle ) - : snap_shot_trigger(snap_shot_interval), - m_n_bins(n_bins), - m_curr_lin_hist(m_n_bins, start_cycle), - m_lin_hist_archive(), - m_cycle(start_cycle), - m_reset_at_snap_shot(reset_at_snap_shot), - m_name(name), - m_id(s_ids++) -{ -} +linear_histogram_logger::linear_histogram_logger( + int n_bins, unsigned long long snap_shot_interval, const char *name, + bool reset_at_snap_shot, unsigned long long start_cycle) + : snap_shot_trigger(snap_shot_interval), + m_n_bins(n_bins), + m_curr_lin_hist(m_n_bins, start_cycle), + m_lin_hist_archive(), + m_cycle(start_cycle), + m_reset_at_snap_shot(reset_at_snap_shot), + m_name(name), + m_id(s_ids++) {} -linear_histogram_logger::linear_histogram_logger(const linear_histogram_logger& other) - : snap_shot_trigger(other.get_interval()), - m_n_bins(other.m_n_bins), - m_curr_lin_hist(m_n_bins, other.m_cycle), - m_lin_hist_archive(), - m_cycle(other.m_cycle), - m_reset_at_snap_shot(other.m_reset_at_snap_shot), - m_name(other.m_name), - m_id(s_ids++) -{ -} +linear_histogram_logger::linear_histogram_logger( + const linear_histogram_logger &other) + : snap_shot_trigger(other.get_interval()), + m_n_bins(other.m_n_bins), + m_curr_lin_hist(m_n_bins, other.m_cycle), + m_lin_hist_archive(), + m_cycle(other.m_cycle), + m_reset_at_snap_shot(other.m_reset_at_snap_shot), + m_name(other.m_name), + m_id(s_ids++) {} -linear_histogram_logger::~linear_histogram_logger() -{ - remove_snap_shot_trigger(this); - remove_spill_log(this); -} - -void linear_histogram_logger::snap_shot(unsigned long long current_cycle) { - m_lin_hist_archive.push_back(m_curr_lin_hist); - if (m_reset_at_snap_shot) { - m_curr_lin_hist.reset(current_cycle); - } else { - m_curr_lin_hist.set_cycle(current_cycle); - } +linear_histogram_logger::~linear_histogram_logger() { + remove_snap_shot_trigger(this); + remove_spill_log(this); } - -void linear_histogram_logger::spill(FILE *fout, bool final) -{ - std::list<linear_histogram_snapshot>::iterator iter = m_lin_hist_archive.begin(); - for (; iter != m_lin_hist_archive.end(); iter = m_lin_hist_archive.erase(iter) ) { - fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0); - iter->print(fout); - fprintf(fout, "\n"); - } - assert( m_lin_hist_archive.empty() ); - if (final) { - fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0); - m_curr_lin_hist.print(fout); - fprintf(fout, "\n"); - } + +void linear_histogram_logger::snap_shot(unsigned long long current_cycle) { + m_lin_hist_archive.push_back(m_curr_lin_hist); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(current_cycle); + } else { + m_curr_lin_hist.set_cycle(current_cycle); + } } - -void linear_histogram_logger::print(FILE *fout) const -{ - std::list<linear_histogram_snapshot>::const_iterator iter = m_lin_hist_archive.begin(); - for (; iter != m_lin_hist_archive.end(); ++iter) { - fprintf(fout, "%s%02d-", m_name.c_str(), m_id); - iter->print(fout); - fprintf(fout, "\n"); - } - fprintf(fout, "%s%02d-", m_name.c_str(), m_id); - m_curr_lin_hist.print(fout); - fprintf(fout, "\n"); + +void linear_histogram_logger::spill(FILE *fout, bool final) { + std::list<linear_histogram_snapshot>::iterator iter = + m_lin_hist_archive.begin(); + for (; iter != m_lin_hist_archive.end(); + iter = m_lin_hist_archive.erase(iter)) { + fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0) ? m_id : 0); + iter->print(fout); + fprintf(fout, "\n"); + } + assert(m_lin_hist_archive.empty()); + if (final) { + fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0) ? m_id : 0); + m_curr_lin_hist.print(fout); + fprintf(fout, "\n"); + } } -void linear_histogram_logger::print_visualizer(FILE *fout) -{ - assert(m_lin_hist_archive.empty()); // don't support snapshot for now - fprintf(fout, "%s", m_name.c_str()); - if (m_id >= 0) { - fprintf(fout, "%02d: ", m_id); - } else { - fprintf(fout, ": "); - } - m_curr_lin_hist.print_visualizer(fout); - fprintf(fout, "\n"); - if (m_reset_at_snap_shot) { - m_curr_lin_hist.reset(0); - } +void linear_histogram_logger::print(FILE *fout) const { + std::list<linear_histogram_snapshot>::const_iterator iter = + m_lin_hist_archive.begin(); + for (; iter != m_lin_hist_archive.end(); ++iter) { + fprintf(fout, "%s%02d-", m_name.c_str(), m_id); + iter->print(fout); + fprintf(fout, "\n"); + } + fprintf(fout, "%s%02d-", m_name.c_str(), m_id); + m_curr_lin_hist.print(fout); + fprintf(fout, "\n"); } -void linear_histogram_logger::print_visualizer(gzFile fout) -{ - assert(m_lin_hist_archive.empty()); // don't support snapshot for now - gzprintf(fout, "%s", m_name.c_str()); - if (m_id >= 0) { - gzprintf(fout, "%02d: ", m_id); - } else { - gzprintf(fout, ": "); - } - m_curr_lin_hist.print_visualizer(fout); - gzprintf(fout, "\n"); - if (m_reset_at_snap_shot) { - m_curr_lin_hist.reset(0); - } +void linear_histogram_logger::print_visualizer(FILE *fout) { + assert(m_lin_hist_archive.empty()); // don't support snapshot for now + fprintf(fout, "%s", m_name.c_str()); + if (m_id >= 0) { + fprintf(fout, "%02d: ", m_id); + } else { + fprintf(fout, ": "); + } + m_curr_lin_hist.print_visualizer(fout); + fprintf(fout, "\n"); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(0); + } } +void linear_histogram_logger::print_visualizer(gzFile fout) { + assert(m_lin_hist_archive.empty()); // don't support snapshot for now + gzprintf(fout, "%s", m_name.c_str()); + if (m_id >= 0) { + gzprintf(fout, "%02d: ", m_id); + } else { + gzprintf(fout, ": "); + } + m_curr_lin_hist.print_visualizer(fout); + gzprintf(fout, "\n"); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(0); + } +} diff --git a/src/gpgpu-sim/stat-tool.h b/src/gpgpu-sim/stat-tool.h index 67b3923..3a291be 100644 --- a/src/gpgpu-sim/stat-tool.h +++ b/src/gpgpu-sim/stat-tool.h @@ -7,72 +7,77 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef STAT_TOOL_H #define STAT_TOOL_H #include "../abstract_hardware_model.h" -#include "histogram.h" #include "../tr1_hash_map.h" +#include "histogram.h" #include <stdio.h> #include <zlib.h> class gpgpu_context; ///////////////////////////////////////////////////////////////////////////////////// -// logger snapshot trigger: -// - automate the snap_shot part of loggers to avoid modifying simulation loop everytime +// logger snapshot trigger: +// - automate the snap_shot part of loggers to avoid modifying simulation loop +// everytime // a new time-dependent stat is added ///////////////////////////////////////////////////////////////////////////////////// class snap_shot_trigger { -public: - snap_shot_trigger(unsigned long long interval) : m_snap_shot_interval(interval) {} - virtual ~snap_shot_trigger() {} - - void try_snap_shot(unsigned long long current_cycle) { - if ((current_cycle % m_snap_shot_interval == 0) && current_cycle != 0) { - snap_shot(current_cycle); - } - } - - virtual void snap_shot(unsigned long long current_cycle) = 0; + public: + snap_shot_trigger(unsigned long long interval) + : m_snap_shot_interval(interval) {} + virtual ~snap_shot_trigger() {} - const unsigned long long & get_interval() const { return m_snap_shot_interval;} + void try_snap_shot(unsigned long long current_cycle) { + if ((current_cycle % m_snap_shot_interval == 0) && current_cycle != 0) { + snap_shot(current_cycle); + } + } -protected: - unsigned long long m_snap_shot_interval; -}; + virtual void snap_shot(unsigned long long current_cycle) = 0; + + const unsigned long long &get_interval() const { + return m_snap_shot_interval; + } + protected: + unsigned long long m_snap_shot_interval; +}; ///////////////////////////////////////////////////////////////////////////////////// -// spill log interface: -// - unified interface to spill log to file to avoid infinite memory usage for logging +// spill log interface: +// - unified interface to spill log to file to avoid infinite memory usage for +// logging ///////////////////////////////////////////////////////////////////////////////////// class spill_log_interface { -public: - spill_log_interface() {} - virtual ~spill_log_interface() {} - - virtual void spill(FILE *fout, bool final) = 0; + public: + spill_log_interface() {} + virtual ~spill_log_interface() {} + + virtual void spill(FILE *fout, bool final) = 0; }; ///////////////////////////////////////////////////////////////////////////////////// @@ -80,51 +85,53 @@ public: ///////////////////////////////////////////////////////////////////////////////////// class thread_insn_span { -public: - thread_insn_span(unsigned long long cycle, gpgpu_context* ctx); - thread_insn_span(const thread_insn_span& other, gpgpu_context* ctx); - ~thread_insn_span(); - - thread_insn_span& operator=(const thread_insn_span& other); - thread_insn_span& operator+=(const thread_insn_span& other); - void set_span( address_type pc ); - void reset(unsigned long long cycle); - - void print_span(FILE *fout) const; - void print_histo(FILE *fout) const; - void print_sparse_histo(FILE *fout) const; - void print_sparse_histo(gzFile fout) const; + public: + thread_insn_span(unsigned long long cycle, gpgpu_context *ctx); + thread_insn_span(const thread_insn_span &other, gpgpu_context *ctx); + ~thread_insn_span(); + + thread_insn_span &operator=(const thread_insn_span &other); + thread_insn_span &operator+=(const thread_insn_span &other); + void set_span(address_type pc); + void reset(unsigned long long cycle); + + void print_span(FILE *fout) const; + void print_histo(FILE *fout) const; + void print_sparse_histo(FILE *fout) const; + void print_sparse_histo(gzFile fout) const; -private: - gpgpu_context* gpgpu_ctx; - typedef tr1_hash_map<address_type, int> span_count_map; - unsigned long long m_cycle; - span_count_map m_insn_span_count; + private: + gpgpu_context *gpgpu_ctx; + typedef tr1_hash_map<address_type, int> span_count_map; + unsigned long long m_cycle; + span_count_map m_insn_span_count; }; class thread_CFlocality : public snap_shot_trigger, public spill_log_interface { -public: - thread_CFlocality(gpgpu_context* ctx, std::string name, unsigned long long snap_shot_interval, - int nthreads, address_type start_pc, unsigned long long start_cycle = 0); - ~thread_CFlocality(); - - void update_thread_pc( int thread_id, address_type pc ); - void snap_shot(unsigned long long current_cycle); - void spill(FILE *fout, bool final); - - void print_visualizer(FILE *fout); - void print_visualizer(gzFile fout); - void print_span(FILE *fout) const; - void print_histo(FILE *fout) const; -private: - std::string m_name; + public: + thread_CFlocality(gpgpu_context *ctx, std::string name, + unsigned long long snap_shot_interval, int nthreads, + address_type start_pc, unsigned long long start_cycle = 0); + ~thread_CFlocality(); - int m_nthreads; - std::vector<address_type> m_thread_pc; - - unsigned long long m_cycle; - thread_insn_span m_thd_span; - std::list<thread_insn_span> m_thd_span_archive; + void update_thread_pc(int thread_id, address_type pc); + void snap_shot(unsigned long long current_cycle); + void spill(FILE *fout, bool final); + + void print_visualizer(FILE *fout); + void print_visualizer(gzFile fout); + void print_span(FILE *fout) const; + void print_histo(FILE *fout) const; + + private: + std::string m_name; + + int m_nthreads; + std::vector<address_type> m_thread_pc; + + unsigned long long m_cycle; + thread_insn_span m_thd_span; + std::list<thread_insn_span> m_thd_span_archive; }; ///////////////////////////////////////////////////////////////////////////////////// @@ -132,194 +139,188 @@ private: ///////////////////////////////////////////////////////////////////////////////////// class insn_warp_occ_logger { -public: - insn_warp_occ_logger(int simd_width) - : m_simd_width(simd_width), - m_insn_warp_occ(1,linear_histogram(1, "", m_simd_width)), + public: + insn_warp_occ_logger(int simd_width) + : m_simd_width(simd_width), + m_insn_warp_occ(1, linear_histogram(1, "", m_simd_width)), m_id(s_ids++) {} - - insn_warp_occ_logger(const insn_warp_occ_logger& other) - : m_simd_width(other.m_simd_width), - m_insn_warp_occ(other.m_insn_warp_occ.size(), linear_histogram(1, "", m_simd_width)), + + insn_warp_occ_logger(const insn_warp_occ_logger &other) + : m_simd_width(other.m_simd_width), + m_insn_warp_occ(other.m_insn_warp_occ.size(), + linear_histogram(1, "", m_simd_width)), m_id(s_ids++) {} - - ~insn_warp_occ_logger() {} - insn_warp_occ_logger& operator=(const insn_warp_occ_logger& p) { - printf("insn_warp_occ_logger Operator= called: %02d \n", m_id); - assert(0); - return *this; - } - - void set_id(int id) { m_id = id; } - - void log(address_type pc, int warp_occ) { - if( pc >= m_insn_warp_occ.size() ) - m_insn_warp_occ.resize(2*pc, linear_histogram(1, "", m_simd_width)); - m_insn_warp_occ[pc].add2bin(warp_occ - 1); - } - - void print(FILE *fout) const - { - for (unsigned i = 0; i < m_insn_warp_occ.size(); i++) { - fprintf(fout, "InsnWarpOcc%02d-%d", m_id, i); - m_insn_warp_occ[i].fprint(fout); - fprintf(fout, "\n"); - } - } + ~insn_warp_occ_logger() {} -private: + insn_warp_occ_logger &operator=(const insn_warp_occ_logger &p) { + printf("insn_warp_occ_logger Operator= called: %02d \n", m_id); + assert(0); + return *this; + } - int m_simd_width; - std::vector<linear_histogram> m_insn_warp_occ; - int m_id; - static int s_ids; -}; + void set_id(int id) { m_id = id; } + void log(address_type pc, int warp_occ) { + if (pc >= m_insn_warp_occ.size()) + m_insn_warp_occ.resize(2 * pc, linear_histogram(1, "", m_simd_width)); + m_insn_warp_occ[pc].add2bin(warp_occ - 1); + } + + void print(FILE *fout) const { + for (unsigned i = 0; i < m_insn_warp_occ.size(); i++) { + fprintf(fout, "InsnWarpOcc%02d-%d", m_id, i); + m_insn_warp_occ[i].fprint(fout); + fprintf(fout, "\n"); + } + } + + private: + int m_simd_width; + std::vector<linear_histogram> m_insn_warp_occ; + int m_id; + static int s_ids; +}; ///////////////////////////////////////////////////////////////////////////////////// // generic linear histogram logger ///////////////////////////////////////////////////////////////////////////////////// class linear_histogram_snapshot { -public: - linear_histogram_snapshot(int n_bins, unsigned long long cycle) - : m_cycle(cycle), - m_linear_histogram(n_bins,0) - { } - - linear_histogram_snapshot(const linear_histogram_snapshot& other) - : m_cycle(other.m_cycle), - m_linear_histogram(other.m_linear_histogram) - { } - - ~linear_histogram_snapshot() { } - - void addsample(int pos) { - assert((size_t)pos < m_linear_histogram.size()); - m_linear_histogram[pos] += 1; - } - - void subsample(int pos) { - assert((size_t)pos < m_linear_histogram.size()); - m_linear_histogram[pos] -= 1; - } - - void reset(unsigned long long cycle) { - m_cycle = cycle; - m_linear_histogram.assign(m_linear_histogram.size(), 0); - } - - void set_cycle(unsigned long long cycle) { m_cycle = cycle; } - - void print(FILE *fout) const { - fprintf(fout, "%d = ", (int)m_cycle); - for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { - fprintf(fout, "%d ", m_linear_histogram[i]); - } - } + public: + linear_histogram_snapshot(int n_bins, unsigned long long cycle) + : m_cycle(cycle), m_linear_histogram(n_bins, 0) {} + + linear_histogram_snapshot(const linear_histogram_snapshot &other) + : m_cycle(other.m_cycle), m_linear_histogram(other.m_linear_histogram) {} + + ~linear_histogram_snapshot() {} + + void addsample(int pos) { + assert((size_t)pos < m_linear_histogram.size()); + m_linear_histogram[pos] += 1; + } + + void subsample(int pos) { + assert((size_t)pos < m_linear_histogram.size()); + m_linear_histogram[pos] -= 1; + } + + void reset(unsigned long long cycle) { + m_cycle = cycle; + m_linear_histogram.assign(m_linear_histogram.size(), 0); + } + + void set_cycle(unsigned long long cycle) { m_cycle = cycle; } + + void print(FILE *fout) const { + fprintf(fout, "%d = ", (int)m_cycle); + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + fprintf(fout, "%d ", m_linear_histogram[i]); + } + } - void print_visualizer(FILE *fout) const { - for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { - fprintf(fout, "%d ", m_linear_histogram[i]); - } - } + void print_visualizer(FILE *fout) const { + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + fprintf(fout, "%d ", m_linear_histogram[i]); + } + } - void print_visualizer(gzFile fout) const { - for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { - gzprintf(fout, "%d ", m_linear_histogram[i]); - } - } + void print_visualizer(gzFile fout) const { + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + gzprintf(fout, "%d ", m_linear_histogram[i]); + } + } -private: - unsigned long long m_cycle; - std::vector<int> m_linear_histogram; + private: + unsigned long long m_cycle; + std::vector<int> m_linear_histogram; }; -class linear_histogram_logger : public snap_shot_trigger, public spill_log_interface { -public: - linear_histogram_logger(int n_bins, - unsigned long long snap_shot_interval, - const char *name, - bool reset_at_snap_shot = true, - unsigned long long start_cycle = 0); - linear_histogram_logger(const linear_histogram_logger& other); - - ~linear_histogram_logger(); - - void set_id(int id) { m_id = id; } - void log(int pos) { m_curr_lin_hist.addsample(pos); } - void unlog(int pos) { m_curr_lin_hist.subsample(pos); } - void snap_shot(unsigned long long current_cycle); - void spill(FILE *fout, bool final); +class linear_histogram_logger : public snap_shot_trigger, + public spill_log_interface { + public: + linear_histogram_logger(int n_bins, unsigned long long snap_shot_interval, + const char *name, bool reset_at_snap_shot = true, + unsigned long long start_cycle = 0); + linear_histogram_logger(const linear_histogram_logger &other); - void print(FILE *fout) const; - void print_visualizer(FILE *fout); - void print_visualizer(gzFile fout); + ~linear_histogram_logger(); -private: - int m_n_bins; - linear_histogram_snapshot m_curr_lin_hist; - std::list<linear_histogram_snapshot> m_lin_hist_archive; - unsigned long long m_cycle; - bool m_reset_at_snap_shot; - std::string m_name; - int m_id; - static int s_ids; + void set_id(int id) { m_id = id; } + void log(int pos) { m_curr_lin_hist.addsample(pos); } + void unlog(int pos) { m_curr_lin_hist.subsample(pos); } + void snap_shot(unsigned long long current_cycle); + void spill(FILE *fout, bool final); + + void print(FILE *fout) const; + void print_visualizer(FILE *fout); + void print_visualizer(gzFile fout); + + private: + int m_n_bins; + linear_histogram_snapshot m_curr_lin_hist; + std::list<linear_histogram_snapshot> m_lin_hist_archive; + unsigned long long m_cycle; + bool m_reset_at_snap_shot; + std::string m_name; + int m_id; + static int s_ids; }; -void try_snap_shot (unsigned long long current_cycle); -void set_spill_interval (unsigned long long interval); -void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle); +void try_snap_shot(unsigned long long current_cycle); +void set_spill_interval(unsigned long long interval); +void spill_log_to_file(FILE *fout, int final, unsigned long long current_cycle); -void create_thread_CFlogger(gpgpu_context* ctx, int n_loggers, int n_threads, address_type start_pc, unsigned long long logging_interval); -void destroy_thread_CFlogger( ); -void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ); -void cflog_snapshot( int logger_id, unsigned long long cycle ); +void create_thread_CFlogger(gpgpu_context *ctx, int n_loggers, int n_threads, + address_type start_pc, + unsigned long long logging_interval); +void destroy_thread_CFlogger(); +void cflog_update_thread_pc(int logger_id, int thread_id, address_type pc); +void cflog_snapshot(int logger_id, unsigned long long cycle); void cflog_print(FILE *fout); void cflog_print_path_expression(FILE *fout); void cflog_visualizer_print(FILE *fout); void cflog_visualizer_gzprint(gzFile fout); -void insn_warp_occ_create( int n_loggers, int simd_width ); -void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ ); -void insn_warp_occ_print( FILE *fout ); - - -void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long logging_interval ); -void shader_warp_occ_log( int logger_id, int warp_occ ); -void shader_warp_occ_snapshot( int logger_id, unsigned long long current_cycle ); -void shader_warp_occ_print( FILE *fout ); +void insn_warp_occ_create(int n_loggers, int simd_width); +void insn_warp_occ_log(int logger_id, address_type pc, int warp_occ); +void insn_warp_occ_print(FILE *fout); +void shader_warp_occ_create(int n_loggers, int simd_width, + unsigned long long logging_interval); +void shader_warp_occ_log(int logger_id, int warp_occ); +void shader_warp_occ_snapshot(int logger_id, unsigned long long current_cycle); +void shader_warp_occ_print(FILE *fout); -void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long logging_interval ); -void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw ); -void shader_mem_acc_snapshot( int logger_id, unsigned long long current_cycle ); -void shader_mem_acc_print( FILE *fout ); - - -void shader_mem_lat_create( int n_loggers, unsigned long long logging_interval ); -void shader_mem_lat_log( int logger_id, int latency ); -void shader_mem_lat_snapshot( int logger_id, unsigned long long current_cycle ); -void shader_mem_lat_print( FILE *fout ); +void shader_mem_acc_create(int n_loggers, int n_dram, int n_bank, + unsigned long long logging_interval); +void shader_mem_acc_log(int logger_id, int dram_id, int bank, char rw); +void shader_mem_acc_snapshot(int logger_id, unsigned long long current_cycle); +void shader_mem_acc_print(FILE *fout); +void shader_mem_lat_create(int n_loggers, unsigned long long logging_interval); +void shader_mem_lat_log(int logger_id, int latency); +void shader_mem_lat_snapshot(int logger_id, unsigned long long current_cycle); +void shader_mem_lat_print(FILE *fout); int get_shader_normal_cache_id(); int get_shader_texture_cache_id(); int get_shader_constant_cache_id(); int get_shader_instruction_cache_id(); -void shader_cache_access_create( int n_loggers, int n_types, unsigned long long logging_interval ); -void shader_cache_access_log( int logger_id, int type, int miss); -void shader_cache_access_unlog( int logger_id, int type, int miss); -void shader_cache_access_print( FILE *fout ); - +void shader_cache_access_create(int n_loggers, int n_types, + unsigned long long logging_interval); +void shader_cache_access_log(int logger_id, int type, int miss); +void shader_cache_access_unlog(int logger_id, int type, int miss); +void shader_cache_access_print(FILE *fout); -void shader_CTA_count_create( int n_shaders, unsigned long long logging_interval); -void shader_CTA_count_log( int shader_id, int nCTAadded ); -void shader_CTA_count_unlog( int shader_id, int nCTAdone ); -void shader_CTA_count_resetnow( ); -void shader_CTA_count_print( FILE *fout ); -void shader_CTA_count_visualizer_print( FILE *fout ); +void shader_CTA_count_create(int n_shaders, + unsigned long long logging_interval); +void shader_CTA_count_log(int shader_id, int nCTAadded); +void shader_CTA_count_unlog(int shader_id, int nCTAdone); +void shader_CTA_count_resetnow(); +void shader_CTA_count_print(FILE *fout); +void shader_CTA_count_visualizer_print(FILE *fout); void shader_CTA_count_visualizer_gzprint(gzFile fout); #endif /* CFLOGGER_H */ diff --git a/src/gpgpu-sim/stats.h b/src/gpgpu-sim/stats.h index 6a50f05..4fb7016 100644 --- a/src/gpgpu-sim/stats.h +++ b/src/gpgpu-sim/stats.h @@ -7,54 +7,50 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef STATS_INCLUDED #define STATS_INCLUDED enum mem_stage_access_type { - C_MEM, - T_MEM, - S_MEM, - G_MEM_LD, - L_MEM_LD, - G_MEM_ST, - L_MEM_ST, - N_MEM_STAGE_ACCESS_TYPE -}; -enum tlb_request_status { - TLB_HIT = 0, - TLB_READY, - TLB_PENDING + C_MEM, + T_MEM, + S_MEM, + G_MEM_LD, + L_MEM_LD, + G_MEM_ST, + L_MEM_ST, + N_MEM_STAGE_ACCESS_TYPE }; +enum tlb_request_status { TLB_HIT = 0, TLB_READY, TLB_PENDING }; enum mem_stage_stall_type { - NO_RC_FAIL = 0, - BK_CONF, - MSHR_RC_FAIL, - ICNT_RC_FAIL, - COAL_STALL, - TLB_STALL, - DATA_PORT_STALL, - WB_ICNT_RC_FAIL, - WB_CACHE_RSRV_FAIL, - N_MEM_STAGE_STALL_TYPE + NO_RC_FAIL = 0, + BK_CONF, + MSHR_RC_FAIL, + ICNT_RC_FAIL, + COAL_STALL, + TLB_STALL, + DATA_PORT_STALL, + WB_ICNT_RC_FAIL, + WB_CACHE_RSRV_FAIL, + N_MEM_STAGE_STALL_TYPE }; - #endif diff --git a/src/gpgpu-sim/traffic_breakdown.cc b/src/gpgpu-sim/traffic_breakdown.cc index 32f0d30..5cc7725 100644 --- a/src/gpgpu-sim/traffic_breakdown.cc +++ b/src/gpgpu-sim/traffic_breakdown.cc @@ -1,51 +1,54 @@ -#include "traffic_breakdown.h" -#include "mem_fetch.h" +#include "traffic_breakdown.h" +#include "mem_fetch.h" -void traffic_breakdown::print(FILE* fout) -{ - for (traffic_stat_t::const_iterator i_stat = m_stats.begin(); i_stat != m_stats.end(); i_stat++) { - unsigned int byte_transferred = 0; - for (traffic_class_t::const_iterator i_class = i_stat->second.begin(); i_class != i_stat->second.end(); i_class++) { - byte_transferred += i_class->first * i_class->second; // byte/packet x #packets - } - fprintf(fout, "traffic_breakdown_%s[%s] = %u {", m_network_name.c_str(), i_stat->first.c_str(), byte_transferred); - for (traffic_class_t::const_iterator i_class = i_stat->second.begin(); i_class != i_stat->second.end(); i_class++) { - fprintf(fout, "%u:%u,", i_class->first, i_class->second); - } - fprintf(fout, "}\n"); - } +void traffic_breakdown::print(FILE* fout) { + for (traffic_stat_t::const_iterator i_stat = m_stats.begin(); + i_stat != m_stats.end(); i_stat++) { + unsigned int byte_transferred = 0; + for (traffic_class_t::const_iterator i_class = i_stat->second.begin(); + i_class != i_stat->second.end(); i_class++) { + byte_transferred += + i_class->first * i_class->second; // byte/packet x #packets + } + fprintf(fout, "traffic_breakdown_%s[%s] = %u {", m_network_name.c_str(), + i_stat->first.c_str(), byte_transferred); + for (traffic_class_t::const_iterator i_class = i_stat->second.begin(); + i_class != i_stat->second.end(); i_class++) { + fprintf(fout, "%u:%u,", i_class->first, i_class->second); + } + fprintf(fout, "}\n"); + } } -void traffic_breakdown::record_traffic(class mem_fetch * mf, unsigned int size) -{ - m_stats[classify_memfetch(mf)][size] += 1; +void traffic_breakdown::record_traffic(class mem_fetch* mf, unsigned int size) { + m_stats[classify_memfetch(mf)][size] += 1; } -std::string traffic_breakdown::classify_memfetch(class mem_fetch * mf) -{ - std::string traffic_name; +std::string traffic_breakdown::classify_memfetch(class mem_fetch* mf) { + std::string traffic_name; - enum mem_access_type access_type = mf->get_access_type(); + enum mem_access_type access_type = mf->get_access_type(); - switch (access_type) { - case CONST_ACC_R: - case TEXTURE_ACC_R: - case GLOBAL_ACC_W: - case LOCAL_ACC_R: - case LOCAL_ACC_W: - case INST_ACC_R: - case L1_WRBK_ACC: - case L2_WRBK_ACC: - case L1_WR_ALLOC_R: - case L2_WR_ALLOC_R: - traffic_name = mem_access_type_str(access_type); - break; - case GLOBAL_ACC_R: - // check for global atomic operation - traffic_name = (mf->isatomic())? "GLOBAL_ATOMIC" : mem_access_type_str(GLOBAL_ACC_R); - break; - default: assert(0 && "Unknown traffic type"); - } - return traffic_name; + switch (access_type) { + case CONST_ACC_R: + case TEXTURE_ACC_R: + case GLOBAL_ACC_W: + case LOCAL_ACC_R: + case LOCAL_ACC_W: + case INST_ACC_R: + case L1_WRBK_ACC: + case L2_WRBK_ACC: + case L1_WR_ALLOC_R: + case L2_WR_ALLOC_R: + traffic_name = mem_access_type_str(access_type); + break; + case GLOBAL_ACC_R: + // check for global atomic operation + traffic_name = (mf->isatomic()) ? "GLOBAL_ATOMIC" + : mem_access_type_str(GLOBAL_ACC_R); + break; + default: + assert(0 && "Unknown traffic type"); + } + return traffic_name; } - diff --git a/src/gpgpu-sim/traffic_breakdown.h b/src/gpgpu-sim/traffic_breakdown.h index c9b8df5..a898519 100644 --- a/src/gpgpu-sim/traffic_breakdown.h +++ b/src/gpgpu-sim/traffic_breakdown.h @@ -1,37 +1,35 @@ -#pragma once +#pragma once #include <stdio.h> #include <map> -#include <string> +#include <string> // Breakdown traffic through the network according to category -class traffic_breakdown -{ -public: - traffic_breakdown(const std::string &network_name) - : m_network_name(network_name) - { } +class traffic_breakdown { + public: + traffic_breakdown(const std::string& network_name) + : m_network_name(network_name) {} - // print the stats - void print(FILE* fout); + // print the stats + void print(FILE* fout); - // record the amount and type of traffic introduced by this mem_fetch object - void record_traffic(class mem_fetch * mf, unsigned int size); + // record the amount and type of traffic introduced by this mem_fetch object + void record_traffic(class mem_fetch* mf, unsigned int size); -protected: + protected: + std::string m_network_name; - std::string m_network_name; + /// helper functions to identify the type of traffic sent + std::string classify_memfetch(class mem_fetch* mf); - /// helper functions to identify the type of traffic sent - std::string classify_memfetch(class mem_fetch * mf); + /// helper functions to identify the size of traffic sent + unsigned int packet_size(class mem_fetch* mf); - /// helper functions to identify the size of traffic sent - unsigned int packet_size(class mem_fetch * mf); + typedef std::string + mf_packet_type; // use string so that it remains extensible + typedef unsigned int mf_packet_size; + typedef std::map<mf_packet_size, unsigned int> traffic_class_t; + typedef std::map<mf_packet_type, traffic_class_t> traffic_stat_t; - typedef std::string mf_packet_type; // use string so that it remains extensible - typedef unsigned int mf_packet_size; - typedef std::map < mf_packet_size, unsigned int > traffic_class_t; - typedef std::map < mf_packet_type, traffic_class_t > traffic_stat_t; - - traffic_stat_t m_stats; -}; + traffic_stat_t m_stats; +}; diff --git a/src/gpgpu-sim/visualizer.cc b/src/gpgpu-sim/visualizer.cc index dcb1175..a832d61 100644 --- a/src/gpgpu-sim/visualizer.cc +++ b/src/gpgpu-sim/visualizer.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, +// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, // The University of British Columbia // All rights reserved. // @@ -7,378 +7,384 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "visualizer.h" +#include "../option_parser.h" #include "gpu-sim.h" #include "l2cache.h" -#include "shader.h" -#include "../option_parser.h" #include "mem_latency_stat.h" #include "power_stat.h" +#include "shader.h" //#include "../../../mcpat/processor.h" -#include "stat-tool.h" #include "gpu-cache.h" +#include "stat-tool.h" -#include <time.h> #include <string.h> +#include <time.h> #include <zlib.h> static void time_vector_print_interval2gzfile(gzFile outfile); -void gpgpu_sim::visualizer_printstat() -{ - gzFile visualizer_file = NULL; // gzFile is basically a pointer to a struct, so it is fine to initialize it as NULL - if ( !m_config.g_visualizer_enabled ) - return; +void gpgpu_sim::visualizer_printstat() { + gzFile visualizer_file = NULL; // gzFile is basically a pointer to a struct, + // so it is fine to initialize it as NULL + if (!m_config.g_visualizer_enabled) return; - // clean the content of the visualizer log if it is the first time, otherwise attach at the end - static bool visualizer_first_printstat = true; + // clean the content of the visualizer log if it is the first time, otherwise + // attach at the end + static bool visualizer_first_printstat = true; - visualizer_file = gzopen(m_config.g_visualizer_filename, (visualizer_first_printstat)? "w" : "a"); - if (visualizer_file == NULL) { - printf("error - could not open visualizer trace file.\n"); - exit(1); - } - gzsetparams(visualizer_file, m_config.g_visualizer_zlevel, Z_DEFAULT_STRATEGY); - visualizer_first_printstat = false; - - cflog_visualizer_gzprint(visualizer_file); - shader_CTA_count_visualizer_gzprint(visualizer_file); + visualizer_file = gzopen(m_config.g_visualizer_filename, + (visualizer_first_printstat) ? "w" : "a"); + if (visualizer_file == NULL) { + printf("error - could not open visualizer trace file.\n"); + exit(1); + } + gzsetparams(visualizer_file, m_config.g_visualizer_zlevel, + Z_DEFAULT_STRATEGY); + visualizer_first_printstat = false; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) - m_memory_partition_unit[i]->visualizer_print(visualizer_file); - m_shader_stats->visualizer_print(visualizer_file); - m_memory_stats->visualizer_print(visualizer_file); - m_power_stats->visualizer_print(visualizer_file); - //proc->visualizer_print(visualizer_file); - // other parameters for graphing - gzprintf(visualizer_file, "globalcyclecount: %lld\n", gpu_sim_cycle); - gzprintf(visualizer_file, "globalinsncount: %lld\n", gpu_sim_insn); - gzprintf(visualizer_file, "globaltotinsncount: %lld\n", gpu_tot_sim_insn); + cflog_visualizer_gzprint(visualizer_file); + shader_CTA_count_visualizer_gzprint(visualizer_file); - time_vector_print_interval2gzfile(visualizer_file); + for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) + m_memory_partition_unit[i]->visualizer_print(visualizer_file); + m_shader_stats->visualizer_print(visualizer_file); + m_memory_stats->visualizer_print(visualizer_file); + m_power_stats->visualizer_print(visualizer_file); + // proc->visualizer_print(visualizer_file); + // other parameters for graphing + gzprintf(visualizer_file, "globalcyclecount: %lld\n", gpu_sim_cycle); + gzprintf(visualizer_file, "globalinsncount: %lld\n", gpu_sim_insn); + gzprintf(visualizer_file, "globaltotinsncount: %lld\n", gpu_tot_sim_insn); - gzclose(visualizer_file); -/* - gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(0)); - gzprintf(visualizer_file, "\n"); - gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(0)); - gzprintf(visualizer_file, "\n"); - gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(0)); - gzprintf(visualizer_file, "\n"); - gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1_windowed_cache_miss_rate(1)); - gzprintf(visualizer_file, "\n"); - gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1tex_windowed_cache_miss_rate(1)); - gzprintf(visualizer_file, "\n"); - gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: "); - for (unsigned i=0;i<m_n_shader;i++) - gzprintf(visualizer_file, "%0.4f ", m_sc[i]->L1const_windowed_cache_miss_rate(1)); - gzprintf(visualizer_file, "\n"); - // reset for next interval - for (unsigned i=0;i<m_n_shader;i++) - m_sc[i]->new_cache_window(); -*/ + time_vector_print_interval2gzfile(visualizer_file); + + gzclose(visualizer_file); + /* + gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: "); + for (unsigned i=0;i<m_n_shader;i++) + gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1_windowed_cache_miss_rate(0)); gzprintf(visualizer_file, "\n"); + gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: "); + for (unsigned i=0;i<m_n_shader;i++) + gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1tex_windowed_cache_miss_rate(0)); gzprintf(visualizer_file, + "\n"); gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: "); for + (unsigned i=0;i<m_n_shader;i++) gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1const_windowed_cache_miss_rate(0)); gzprintf(visualizer_file, + "\n"); gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: "); + for (unsigned i=0;i<m_n_shader;i++) + gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1_windowed_cache_miss_rate(1)); gzprintf(visualizer_file, "\n"); + gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: "); + for (unsigned i=0;i<m_n_shader;i++) + gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1tex_windowed_cache_miss_rate(1)); gzprintf(visualizer_file, + "\n"); gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: "); for + (unsigned i=0;i<m_n_shader;i++) gzprintf(visualizer_file, "%0.4f ", + m_sc[i]->L1const_windowed_cache_miss_rate(1)); gzprintf(visualizer_file, + "\n"); + // reset for next interval + for (unsigned i=0;i<m_n_shader;i++) + m_sc[i]->new_cache_window(); + */ } -#include <list> -#include <vector> #include <iostream> +#include <list> #include <map> -#include"../gpgpu-sim/shader.h" +#include <vector> +#include "../gpgpu-sim/shader.h" class my_time_vector { -private: - std::map< unsigned int, std::vector<long int> > ld_time_map; - std::map< unsigned int, std::vector<long int> > st_time_map; - unsigned ld_vector_size; - unsigned st_vector_size; - std::vector<double> ld_time_dist; - std::vector<double> st_time_dist; + private: + std::map<unsigned int, std::vector<long int> > ld_time_map; + std::map<unsigned int, std::vector<long int> > st_time_map; + unsigned ld_vector_size; + unsigned st_vector_size; + std::vector<double> ld_time_dist; + std::vector<double> st_time_dist; + + std::vector<double> overal_ld_time_dist; + std::vector<double> overal_st_time_dist; + int overal_ld_count; + int overal_st_count; - std::vector<double> overal_ld_time_dist; - std::vector<double> overal_st_time_dist; - int overal_ld_count; - int overal_st_count; + public: + my_time_vector(int ld_size, int st_size) { + ld_vector_size = ld_size; + st_vector_size = st_size; + ld_time_dist.resize(ld_size); + st_time_dist.resize(st_size); + overal_ld_time_dist.resize(ld_size); + overal_st_time_dist.resize(st_size); + overal_ld_count = 0; + overal_st_count = 0; + } + void update_ld(unsigned int uid, unsigned int slot, long int time) { + if (ld_time_map.find(uid) != ld_time_map.end()) { + ld_time_map[uid][slot] = time; + } else if (slot < NUM_MEM_REQ_STAT) { + std::vector<long int> time_vec; + time_vec.resize(ld_vector_size); + time_vec[slot] = time; + ld_time_map[uid] = time_vec; + } else { + // It's a merged mshr! forget it + } + } + void update_st(unsigned int uid, unsigned int slot, long int time) { + if (st_time_map.find(uid) != st_time_map.end()) { + st_time_map[uid][slot] = time; + } else { + std::vector<long int> time_vec; + time_vec.resize(st_vector_size); + time_vec[slot] = time; + st_time_map[uid] = time_vec; + } + } + void check_ld_update(unsigned int uid, unsigned int slot, long int latency) { + if (ld_time_map.find(uid) != ld_time_map.end()) { + int our_latency = + ld_time_map[uid][slot] - ld_time_map[uid][IN_ICNT_TO_MEM]; + assert(our_latency == latency); + } else if (slot < NUM_MEM_REQ_STAT) { + abort(); + } + } + void check_st_update(unsigned int uid, unsigned int slot, long int latency) { + if (st_time_map.find(uid) != st_time_map.end()) { + int our_latency = + st_time_map[uid][slot] - st_time_map[uid][IN_ICNT_TO_MEM]; + assert(our_latency == latency); + } else { + abort(); + } + } -public: - my_time_vector(int ld_size,int st_size){ - ld_vector_size = ld_size; - st_vector_size = st_size; - ld_time_dist.resize(ld_size); - st_time_dist.resize(st_size); - overal_ld_time_dist.resize(ld_size); - overal_st_time_dist.resize(st_size); - overal_ld_count = 0; - overal_st_count= 0; - } - void update_ld(unsigned int uid,unsigned int slot, long int time) { - if ( ld_time_map.find( uid )!=ld_time_map.end() ) { - ld_time_map[uid][slot]=time; - } else if (slot < NUM_MEM_REQ_STAT ) { - std::vector<long int> time_vec; - time_vec.resize(ld_vector_size); - time_vec[slot] = time; - ld_time_map[uid] = time_vec; - } else { - //It's a merged mshr! forget it + private: + void calculate_ld_dist(void) { + unsigned i, first; + long int last_update, diff; + int finished_count = 0; + ld_time_dist.clear(); + ld_time_dist.resize(ld_vector_size); + std::map<unsigned int, std::vector<long int> >::iterator iter, iter_temp; + iter = ld_time_map.begin(); + while (iter != ld_time_map.end()) { + last_update = 0; + first = -1; + if (!iter->second[IN_SHADER_FETCHED]) { + // this request is not done yet skip it! + ++iter; + continue; } - } - void update_st(unsigned int uid,unsigned int slot, long int time) { - if ( st_time_map.find( uid )!=st_time_map.end() ) { - st_time_map[uid][slot]=time; - } else { - std::vector<long int> time_vec; - time_vec.resize(st_vector_size); - time_vec[slot] = time; - st_time_map[uid] = time_vec; - } - } - void check_ld_update(unsigned int uid,unsigned int slot, long int latency) { - if ( ld_time_map.find( uid )!=ld_time_map.end() ) { - int our_latency = ld_time_map[uid][slot] - ld_time_map[uid][IN_ICNT_TO_MEM]; - assert( our_latency == latency); - } else if (slot < NUM_MEM_REQ_STAT ) { - abort(); + while (!last_update) { + first++; + assert(first < iter->second.size()); + last_update = iter->second[first]; } - } - void check_st_update(unsigned int uid,unsigned int slot, long int latency) { - if ( st_time_map.find( uid )!=st_time_map.end() ) { - int our_latency = st_time_map[uid][slot] - st_time_map[uid][IN_ICNT_TO_MEM]; - assert( our_latency == latency); - } else { - abort(); - } - } -private: - void calculate_ld_dist(void) { - unsigned i,first; - long int last_update,diff; - int finished_count=0; - ld_time_dist.clear(); - ld_time_dist.resize(ld_vector_size); - std::map< unsigned int, std::vector<long int> >::iterator iter, iter_temp; - iter =ld_time_map.begin() ; - while (iter != ld_time_map.end()) { - last_update=0; - first=-1; - if (!iter->second[IN_SHADER_FETCHED]) { - //this request is not done yet skip it! - ++iter; - continue; - } - while ( !last_update ) { - first++; - assert( first < iter->second.size() ); - last_update = iter->second[first]; - } - for ( i=first;i<ld_vector_size;i++ ) { - diff = iter->second[i] - last_update; - if ( diff>0 ) { - ld_time_dist[i]+=diff; - last_update = iter->second[i]; - } - } - iter_temp = iter; - iter++; - ld_time_map.erase(iter_temp); - finished_count++; + for (i = first; i < ld_vector_size; i++) { + diff = iter->second[i] - last_update; + if (diff > 0) { + ld_time_dist[i] += diff; + last_update = iter->second[i]; + } } - if ( finished_count ) { - for ( i=0;i<ld_vector_size;i++ ) { - overal_ld_time_dist[i] = (overal_ld_time_dist[i]*overal_ld_count + ld_time_dist[i]) / (overal_ld_count + finished_count); - } - overal_ld_count += finished_count; - for ( i=0;i<ld_vector_size;i++ ) { - ld_time_dist[i]/=finished_count; - } + iter_temp = iter; + iter++; + ld_time_map.erase(iter_temp); + finished_count++; + } + if (finished_count) { + for (i = 0; i < ld_vector_size; i++) { + overal_ld_time_dist[i] = + (overal_ld_time_dist[i] * overal_ld_count + ld_time_dist[i]) / + (overal_ld_count + finished_count); } - } - - void calculate_st_dist(void) { - unsigned i,first; - long int last_update,diff; - int finished_count=0; - st_time_dist.clear(); - st_time_dist.resize(st_vector_size); - std::map< unsigned int, std::vector<long int> >::iterator iter,iter_temp; - iter =st_time_map.begin() ; - while ( iter != st_time_map.end() ) { - last_update=0; - first=-1; - if (!iter->second[IN_SHADER_FETCHED]) { - //this request is not done yet skip it! - ++iter; - continue; - } - while ( !last_update ) { - first++; - assert( first < iter->second.size() ); - last_update = iter->second[first]; - } + overal_ld_count += finished_count; + for (i = 0; i < ld_vector_size; i++) { + ld_time_dist[i] /= finished_count; + } + } + } - for ( i=first;i<st_vector_size;i++ ) { - diff = iter->second[i] - last_update; - if ( diff>0 ) { - st_time_dist[i]+=diff; - last_update = iter->second[i]; - } - } - iter_temp = iter; - iter++; - st_time_map.erase(iter_temp); - finished_count++; + void calculate_st_dist(void) { + unsigned i, first; + long int last_update, diff; + int finished_count = 0; + st_time_dist.clear(); + st_time_dist.resize(st_vector_size); + std::map<unsigned int, std::vector<long int> >::iterator iter, iter_temp; + iter = st_time_map.begin(); + while (iter != st_time_map.end()) { + last_update = 0; + first = -1; + if (!iter->second[IN_SHADER_FETCHED]) { + // this request is not done yet skip it! + ++iter; + continue; } - if ( finished_count ) { - for ( i=0;i<st_vector_size;i++ ) { - overal_st_time_dist[i] = (overal_st_time_dist[i]*overal_st_count + st_time_dist[i]) / (overal_st_count + finished_count); - } - overal_st_count += finished_count; - for ( i=0;i<st_vector_size;i++ ) { - st_time_dist[i]/=finished_count; - } + while (!last_update) { + first++; + assert(first < iter->second.size()); + last_update = iter->second[first]; } - } -public: - void clear_time_map_vectors(void) { - ld_time_map.clear(); - st_time_map.clear(); - } - void print_all_ld(void) { - unsigned i; - std::map< unsigned int, std::vector<long int> >::iterator iter; - for ( iter =ld_time_map.begin() ; iter != ld_time_map.end(); ++iter ) { - std::cout<<"ld_uid"<<iter->first; - for ( i=0;i<ld_vector_size;i++ ) { - std::cout<<" "<<iter->second[i]; - } - std::cout<< std::endl; + for (i = first; i < st_vector_size; i++) { + diff = iter->second[i] - last_update; + if (diff > 0) { + st_time_dist[i] += diff; + last_update = iter->second[i]; + } } - } - - void print_all_st(void) { - unsigned i; - std::map< unsigned int, std::vector<long int> >::iterator iter; - - for ( iter =st_time_map.begin() ; iter != st_time_map.end(); ++iter ) { - std::cout<<"st_uid"<<iter->first; - for ( i=0;i<st_vector_size;i++ ) { - std::cout<<" "<<iter->second[i]; - } - std::cout<<std::endl; + iter_temp = iter; + iter++; + st_time_map.erase(iter_temp); + finished_count++; + } + if (finished_count) { + for (i = 0; i < st_vector_size; i++) { + overal_st_time_dist[i] = + (overal_st_time_dist[i] * overal_st_count + st_time_dist[i]) / + (overal_st_count + finished_count); + } + overal_st_count += finished_count; + for (i = 0; i < st_vector_size; i++) { + st_time_dist[i] /= finished_count; } - } + } + } - void calculate_dist() { - calculate_ld_dist(); - calculate_st_dist(); - } - void print_dist(void) { - unsigned i; - calculate_dist(); - std::cout << "LD_mem_lat_dist " ; - for ( i=0;i<ld_vector_size;i++ ) { - std::cout <<" "<<(int)overal_ld_time_dist[i]; + public: + void clear_time_map_vectors(void) { + ld_time_map.clear(); + st_time_map.clear(); + } + void print_all_ld(void) { + unsigned i; + std::map<unsigned int, std::vector<long int> >::iterator iter; + for (iter = ld_time_map.begin(); iter != ld_time_map.end(); ++iter) { + std::cout << "ld_uid" << iter->first; + for (i = 0; i < ld_vector_size; i++) { + std::cout << " " << iter->second[i]; } std::cout << std::endl; - std::cout << "ST_mem_lat_dist " ; - for ( i=0;i<st_vector_size;i++ ) { - std::cout <<" "<<(int)overal_st_time_dist[i]; + } + } + + void print_all_st(void) { + unsigned i; + std::map<unsigned int, std::vector<long int> >::iterator iter; + + for (iter = st_time_map.begin(); iter != st_time_map.end(); ++iter) { + std::cout << "st_uid" << iter->first; + for (i = 0; i < st_vector_size; i++) { + std::cout << " " << iter->second[i]; } std::cout << std::endl; - } - void print_to_file(FILE *outfile) { - unsigned i; - calculate_dist(); - fprintf (outfile,"LDmemlatdist:") ; - for ( i=0;i<ld_vector_size;i++ ) { - fprintf (outfile," %d", (int)ld_time_dist[i]); - } - fprintf (outfile,"\n") ; - fprintf (outfile,"STmemlatdist:") ; - for ( i=0;i<st_vector_size;i++ ) { - fprintf (outfile," %d", (int)st_time_dist[i]); - } - fprintf (outfile,"\n") ; - } - void print_to_gzfile(gzFile outfile) { - unsigned i; - calculate_dist(); - gzprintf (outfile,"LDmemlatdist:") ; - for ( i=0;i<ld_vector_size;i++ ) { - gzprintf (outfile," %d", (int)ld_time_dist[i]); - } - gzprintf (outfile,"\n") ; - gzprintf (outfile,"STmemlatdist:") ; - for ( i=0;i<st_vector_size;i++ ) { - gzprintf (outfile," %d", (int)st_time_dist[i]); - } - gzprintf (outfile,"\n") ; - } + } + } + + void calculate_dist() { + calculate_ld_dist(); + calculate_st_dist(); + } + void print_dist(void) { + unsigned i; + calculate_dist(); + std::cout << "LD_mem_lat_dist "; + for (i = 0; i < ld_vector_size; i++) { + std::cout << " " << (int)overal_ld_time_dist[i]; + } + std::cout << std::endl; + std::cout << "ST_mem_lat_dist "; + for (i = 0; i < st_vector_size; i++) { + std::cout << " " << (int)overal_st_time_dist[i]; + } + std::cout << std::endl; + } + void print_to_file(FILE* outfile) { + unsigned i; + calculate_dist(); + fprintf(outfile, "LDmemlatdist:"); + for (i = 0; i < ld_vector_size; i++) { + fprintf(outfile, " %d", (int)ld_time_dist[i]); + } + fprintf(outfile, "\n"); + fprintf(outfile, "STmemlatdist:"); + for (i = 0; i < st_vector_size; i++) { + fprintf(outfile, " %d", (int)st_time_dist[i]); + } + fprintf(outfile, "\n"); + } + void print_to_gzfile(gzFile outfile) { + unsigned i; + calculate_dist(); + gzprintf(outfile, "LDmemlatdist:"); + for (i = 0; i < ld_vector_size; i++) { + gzprintf(outfile, " %d", (int)ld_time_dist[i]); + } + gzprintf(outfile, "\n"); + gzprintf(outfile, "STmemlatdist:"); + for (i = 0; i < st_vector_size; i++) { + gzprintf(outfile, " %d", (int)st_time_dist[i]); + } + gzprintf(outfile, "\n"); + } }; -my_time_vector* g_my_time_vector; +my_time_vector* g_my_time_vector; void time_vector_create(int size) { - g_my_time_vector = new my_time_vector(size,size); -} - - -void time_vector_print(void) { - g_my_time_vector->print_dist(); + g_my_time_vector = new my_time_vector(size, size); } +void time_vector_print(void) { g_my_time_vector->print_dist(); } + void time_vector_print_interval2gzfile(gzFile outfile) { - g_my_time_vector->print_to_gzfile(outfile); + g_my_time_vector->print_to_gzfile(outfile); } #include "../gpgpu-sim/mem_fetch.h" -void time_vector_update(unsigned int uid,int slot ,long int cycle,int type) { - if ( (type == READ_REQUEST) || (type == READ_REPLY) ) { - g_my_time_vector->update_ld( uid, slot,cycle); - } else if ( (type == WRITE_REQUEST) || (type == WRITE_ACK) ) { - g_my_time_vector->update_st( uid, slot,cycle); - } else { - abort(); - } +void time_vector_update(unsigned int uid, int slot, long int cycle, int type) { + if ((type == READ_REQUEST) || (type == READ_REPLY)) { + g_my_time_vector->update_ld(uid, slot, cycle); + } else if ((type == WRITE_REQUEST) || (type == WRITE_ACK)) { + g_my_time_vector->update_st(uid, slot, cycle); + } else { + abort(); + } } -void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type) -{ - if ( (type == READ_REQUEST) || (type == READ_REPLY) ) { - g_my_time_vector->check_ld_update( uid, slot, latency ); - } else if ( (type == WRITE_REQUEST) || (type == WRITE_ACK) ) { - g_my_time_vector->check_st_update( uid, slot, latency ); - } else { - abort(); - } +void check_time_vector_update(unsigned int uid, int slot, long int latency, + int type) { + if ((type == READ_REQUEST) || (type == READ_REPLY)) { + g_my_time_vector->check_ld_update(uid, slot, latency); + } else if ((type == WRITE_REQUEST) || (type == WRITE_ACK)) { + g_my_time_vector->check_st_update(uid, slot, latency); + } else { + abort(); + } } diff --git a/src/gpgpu-sim/visualizer.h b/src/gpgpu-sim/visualizer.h index e8ab8bf..cd18cc1 100644 --- a/src/gpgpu-sim/visualizer.h +++ b/src/gpgpu-sim/visualizer.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef VISUALIZER_H_INCLUDED #define VISUALIZER_H_INCLUDED @@ -33,7 +34,8 @@ void time_vector_create(int size); void time_vector_print(void); -void time_vector_update(unsigned int uid,int slot ,long int cycle,int type); -void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type); +void time_vector_update(unsigned int uid, int slot, long int cycle, int type); +void check_time_vector_update(unsigned int uid, int slot, long int latency, + int type); #endif diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 846773d..b7b7f34 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -7,274 +7,285 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "gpgpusim_entrypoint.h" #include <stdio.h> -#include "option_parser.h" +#include "../libcuda/gpgpu_context.h" #include "cuda-sim/cuda-sim.h" #include "cuda-sim/ptx_ir.h" #include "cuda-sim/ptx_parser.h" #include "gpgpu-sim/gpu-sim.h" #include "gpgpu-sim/icnt_wrapper.h" +#include "option_parser.h" #include "stream_manager.h" -#include "../libcuda/gpgpu_context.h" -#define MAX(a,b) (((a)>(b))?(a):(b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) static int sg_argc = 3; -static const char *sg_argv[] = {"", "-config","gpgpusim.config"}; - +static const char *sg_argv[] = {"", "-config", "gpgpusim.config"}; -void * gpgpu_sim_thread_sequential(void * ctx_ptr) -{ - gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; - // at most one kernel running at a time - bool done; - do { - sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); - done = true; - if( ctx->the_gpgpusim->g_the_gpu->get_more_cta_left() ) { - done = false; - ctx->the_gpgpusim->g_the_gpu->init(); - while( ctx->the_gpgpusim->g_the_gpu->active() ) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); - } - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); +void *gpgpu_sim_thread_sequential(void *ctx_ptr) { + gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; + // at most one kernel running at a time + bool done; + do { + sem_wait(&(ctx->the_gpgpusim->g_sim_signal_start)); + done = true; + if (ctx->the_gpgpusim->g_the_gpu->get_more_cta_left()) { + done = false; + ctx->the_gpgpusim->g_the_gpu->init(); + while (ctx->the_gpgpusim->g_the_gpu->active()) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); } - sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); - } while(!done); - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); + } + sem_post(&(ctx->the_gpgpusim->g_sim_signal_finish)); + } while (!done); + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } - - -static void termination_callback() -{ - printf("GPGPU-Sim: *** exit detected ***\n"); - fflush(stdout); +static void termination_callback() { + printf("GPGPU-Sim: *** exit detected ***\n"); + fflush(stdout); } -void *gpgpu_sim_thread_concurrent(void * ctx_ptr) -{ - gpgpu_context * ctx = (gpgpu_context *)ctx_ptr; - atexit(termination_callback); - // concurrent kernel execution simulation thread +void *gpgpu_sim_thread_concurrent(void *ctx_ptr) { + gpgpu_context *ctx = (gpgpu_context *)ctx_ptr; + atexit(termination_callback); + // concurrent kernel execution simulation thread + do { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim: *** simulation thread starting and spinning waiting for " + "work ***\n"); + fflush(stdout); + } + while (ctx->the_gpgpusim->g_stream_manager->empty_protected() && + !ctx->the_gpgpusim->g_sim_done) + ; + if (g_debug_execution >= 3) { + printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); + ctx->the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = true; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + bool active = false; + bool sim_cycles = false; + ctx->the_gpgpusim->g_the_gpu->init(); do { - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: *** simulation thread starting and spinning waiting for work ***\n"); - fflush(stdout); - } - while( ctx->the_gpgpusim->g_stream_manager->empty_protected() && !ctx->the_gpgpusim->g_sim_done ) - ; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); - ctx->the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); - } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = true; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - bool active = false; - bool sim_cycles = false; - ctx->the_gpgpusim->g_the_gpu->init(); - do { - // check if a kernel has completed - // launch operation on device if one is pending and can be run - - // Need to break this loop when a kernel completes. This was a - // source of non-deterministic behaviour in GPGPU-Sim (bug 147). - // If another stream operation is available, g_the_gpu remains active, - // causing this loop to not break. If the next operation happens to be - // another kernel, the gpu is not re-initialized and the inter-kernel - // behaviour may be incorrect. Check that a kernel has finished and - // no other kernel is currently running. - if(ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && !ctx->the_gpgpusim->g_the_gpu->active()) - break; - - //functional simulation - if( ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { - kernel_info_t * kernel = ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); - assert(kernel); - ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func(*kernel); - ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); - } + // check if a kernel has completed + // launch operation on device if one is pending and can be run - //performance simulation - if( ctx->the_gpgpusim->g_the_gpu->active() ) { - ctx->the_gpgpusim->g_the_gpu->cycle(); - sim_cycles = true; - ctx->the_gpgpusim->g_the_gpu->deadlock_check(); - }else { - if(ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()){ - ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); - ctx->the_gpgpusim->g_sim_done = true; - ctx->the_gpgpusim->break_limit = true; - } - } + // Need to break this loop when a kernel completes. This was a + // source of non-deterministic behaviour in GPGPU-Sim (bug 147). + // If another stream operation is available, g_the_gpu remains active, + // causing this loop to not break. If the next operation happens to be + // another kernel, the gpu is not re-initialized and the inter-kernel + // behaviour may be incorrect. Check that a kernel has finished and + // no other kernel is currently running. + if (ctx->the_gpgpusim->g_stream_manager->operation(&sim_cycles) && + !ctx->the_gpgpusim->g_the_gpu->active()) + break; - active=ctx->the_gpgpusim->g_the_gpu->active() || !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); + // functional simulation + if (ctx->the_gpgpusim->g_the_gpu->is_functional_sim()) { + kernel_info_t *kernel = + ctx->the_gpgpusim->g_the_gpu->get_functional_kernel(); + assert(kernel); + ctx->the_gpgpusim->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func( + *kernel); + ctx->the_gpgpusim->g_the_gpu->finish_functional_sim(kernel); + } - } while( active && !ctx->the_gpgpusim->g_sim_done); - if(g_debug_execution >= 3) { - printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); - fflush(stdout); - } - if(sim_cycles) { - ctx->the_gpgpusim->g_the_gpu->print_stats(); - ctx->the_gpgpusim->g_the_gpu->update_stats(); - ctx->print_simulation_time(); + // performance simulation + if (ctx->the_gpgpusim->g_the_gpu->active()) { + ctx->the_gpgpusim->g_the_gpu->cycle(); + sim_cycles = true; + ctx->the_gpgpusim->g_the_gpu->deadlock_check(); + } else { + if (ctx->the_gpgpusim->g_the_gpu->cycle_insn_cta_max_hit()) { + ctx->the_gpgpusim->g_stream_manager->stop_all_running_kernels(); + ctx->the_gpgpusim->g_sim_done = true; + ctx->the_gpgpusim->break_limit = true; } - pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); - ctx->the_gpgpusim->g_sim_active = false; - pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); - } while( !ctx->the_gpgpusim->g_sim_done ); + } - printf("GPGPU-Sim: *** simulation thread exiting ***\n"); - fflush(stdout); + active = ctx->the_gpgpusim->g_the_gpu->active() || + !(ctx->the_gpgpusim->g_stream_manager->empty_protected()); - if(ctx->the_gpgpusim->break_limit) { - printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n"); - exit(1); + } while (active && !ctx->the_gpgpusim->g_sim_done); + if (g_debug_execution >= 3) { + printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n"); + fflush(stdout); } + if (sim_cycles) { + ctx->the_gpgpusim->g_the_gpu->print_stats(); + ctx->the_gpgpusim->g_the_gpu->update_stats(); + ctx->print_simulation_time(); + } + pthread_mutex_lock(&(ctx->the_gpgpusim->g_sim_lock)); + ctx->the_gpgpusim->g_sim_active = false; + pthread_mutex_unlock(&(ctx->the_gpgpusim->g_sim_lock)); + } while (!ctx->the_gpgpusim->g_sim_done); + + printf("GPGPU-Sim: *** simulation thread exiting ***\n"); + fflush(stdout); + + if (ctx->the_gpgpusim->break_limit) { + printf( + "GPGPU-Sim: ** break due to reaching the maximum cycles (or " + "instructions) **\n"); + exit(1); + } - sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); - return NULL; + sem_post(&(ctx->the_gpgpusim->g_sim_signal_exit)); + return NULL; } -void gpgpu_context::synchronize() -{ - printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); - the_gpgpusim->g_stream_manager->print(stdout); - fflush(stdout); -// sem_wait(&g_sim_signal_finish); - bool done = false; - do { - pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); - done = ( the_gpgpusim->g_stream_manager->empty() && !the_gpgpusim->g_sim_active ) || the_gpgpusim->g_sim_done; - pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); - } while (!done); - printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); - fflush(stdout); -// sem_post(&g_sim_signal_start); +void gpgpu_context::synchronize() { + printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n"); + the_gpgpusim->g_stream_manager->print(stdout); + fflush(stdout); + // sem_wait(&g_sim_signal_finish); + bool done = false; + do { + pthread_mutex_lock(&(the_gpgpusim->g_sim_lock)); + done = (the_gpgpusim->g_stream_manager->empty() && + !the_gpgpusim->g_sim_active) || + the_gpgpusim->g_sim_done; + pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock)); + } while (!done); + printf("GPGPU-Sim: detected inactive GPU simulation thread\n"); + fflush(stdout); + // sem_post(&g_sim_signal_start); } -void gpgpu_context::exit_simulation() -{ - the_gpgpusim->g_sim_done=true; - printf("GPGPU-Sim: exit_simulation called\n"); - fflush(stdout); - sem_wait(&(the_gpgpusim->g_sim_signal_exit)); - printf("GPGPU-Sim: simulation thread signaled exit\n"); - fflush(stdout); +void gpgpu_context::exit_simulation() { + the_gpgpusim->g_sim_done = true; + printf("GPGPU-Sim: exit_simulation called\n"); + fflush(stdout); + sem_wait(&(the_gpgpusim->g_sim_signal_exit)); + printf("GPGPU-Sim: simulation thread signaled exit\n"); + fflush(stdout); } -gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() -{ - srand(1); - print_splash(); - func_sim->read_sim_environment_variables(); - ptx_parser->read_parser_environment_variables(); - option_parser_t opp = option_parser_create(); +gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() { + srand(1); + print_splash(); + func_sim->read_sim_environment_variables(); + ptx_parser->read_parser_environment_variables(); + option_parser_t opp = option_parser_create(); - ptx_reg_options(opp); - func_sim->ptx_opcocde_latency_options(opp); + ptx_reg_options(opp); + func_sim->ptx_opcocde_latency_options(opp); - icnt_reg_options(opp); - the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); - the_gpgpusim->g_the_gpu_config->reg_options(opp); // register GPU microrachitecture options + icnt_reg_options(opp); + the_gpgpusim->g_the_gpu_config = new gpgpu_sim_config(this); + the_gpgpusim->g_the_gpu_config->reg_options( + opp); // register GPU microrachitecture options - option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options - fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); - option_parser_print(opp, stdout); - // Set the Numeric locale to a standard locale where a decimal point is a "dot" not a "comma" - // so it does the parsing correctly independent of the system environment variables - assert(setlocale(LC_NUMERIC,"C")); - the_gpgpusim->g_the_gpu_config->init(); + option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options + fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n"); + option_parser_print(opp, stdout); + // Set the Numeric locale to a standard locale where a decimal point is a + // "dot" not a "comma" so it does the parsing correctly independent of the + // system environment variables + assert(setlocale(LC_NUMERIC, "C")); + the_gpgpusim->g_the_gpu_config->init(); - the_gpgpusim->g_the_gpu = new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); - the_gpgpusim->g_stream_manager = new stream_manager((the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); + the_gpgpusim->g_the_gpu = + new gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this); + the_gpgpusim->g_stream_manager = new stream_manager( + (the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking); - the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); + the_gpgpusim->g_simulation_starttime = time((time_t *)NULL); - sem_init(&(the_gpgpusim->g_sim_signal_start),0,0); - sem_init(&(the_gpgpusim->g_sim_signal_finish),0,0); - sem_init(&(the_gpgpusim->g_sim_signal_exit),0,0); + sem_init(&(the_gpgpusim->g_sim_signal_start), 0, 0); + sem_init(&(the_gpgpusim->g_sim_signal_finish), 0, 0); + sem_init(&(the_gpgpusim->g_sim_signal_exit), 0, 0); - return the_gpgpusim->g_the_gpu; + return the_gpgpusim->g_the_gpu; } -void gpgpu_context::start_sim_thread(int api) -{ - if( the_gpgpusim->g_sim_done ) { - the_gpgpusim->g_sim_done = false; - if( api == 1 ) { - pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_concurrent,(void *)this); - } else { - pthread_create(&(the_gpgpusim->g_simulation_thread),NULL,gpgpu_sim_thread_sequential,(void *)this); - } +void gpgpu_context::start_sim_thread(int api) { + if (the_gpgpusim->g_sim_done) { + the_gpgpusim->g_sim_done = false; + if (api == 1) { + pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, + gpgpu_sim_thread_concurrent, (void *)this); + } else { + pthread_create(&(the_gpgpusim->g_simulation_thread), NULL, + gpgpu_sim_thread_sequential, (void *)this); } + } } -void gpgpu_context::print_simulation_time() -{ - time_t current_time, difference, d, h, m, s; - current_time = time((time_t *)NULL); - difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); +void gpgpu_context::print_simulation_time() { + time_t current_time, difference, d, h, m, s; + current_time = time((time_t *)NULL); + difference = MAX(current_time - the_gpgpusim->g_simulation_starttime, 1); - d = difference/(3600*24); - h = difference/3600 - 24*d; - m = difference/60 - 60*(h + 24*d); - s = difference - 60*(m + 60*(h + 24*d)); + d = difference / (3600 * 24); + h = difference / 3600 - 24 * d; + m = difference / 60 - 60 * (h + 24 * d); + s = difference - 60 * (m + 60 * (h + 24 * d)); - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference) ); - const unsigned cycles_per_sec = (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); - printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec ); - printf("gpgpu_silicon_slowdown = %ux\n", the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); - fflush(stdout); + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)d, (unsigned)h, (unsigned)m, (unsigned)s, (unsigned)difference); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_insn / difference)); + const unsigned cycles_per_sec = + (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference); + printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec); + printf("gpgpu_silicon_slowdown = %ux\n", + the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec); + fflush(stdout); } -int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf( kernel_info_t *grid ) -{ - the_gpgpusim->g_the_gpu->launch(grid); - sem_post(&(the_gpgpusim->g_sim_signal_start)); - sem_wait(&(the_gpgpusim->g_sim_signal_finish)); - return 0; +int gpgpu_context::gpgpu_opencl_ptx_sim_main_perf(kernel_info_t *grid) { + the_gpgpusim->g_the_gpu->launch(grid); + sem_post(&(the_gpgpusim->g_sim_signal_start)); + sem_wait(&(the_gpgpusim->g_sim_signal_finish)); + return 0; } //! Functional simulation of OpenCL /*! * This function call the CUDA PTX functional simulator */ -int cuda_sim::gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ) -{ - //calling the CUDA PTX simulator, sending the kernel by reference and a flag set to true, - //the flag used by the function to distinguish OpenCL calls from the CUDA simulation calls which - //it is needed by the called function to not register the exit the exit of OpenCL kernel as it doesn't register entering in the first place as the CUDA kernels does - gpgpu_cuda_ptx_sim_main_func( *grid, true ); - return 0; +int cuda_sim::gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid) { + // calling the CUDA PTX simulator, sending the kernel by reference and a flag + // set to true, the flag used by the function to distinguish OpenCL calls from + // the CUDA simulation calls which it is needed by the called function to not + // register the exit the exit of OpenCL kernel as it doesn't register entering + // in the first place as the CUDA kernels does + gpgpu_cuda_ptx_sim_main_func(*grid, true); + return 0; } diff --git a/src/gpgpusim_entrypoint.h b/src/gpgpusim_entrypoint.h index 9f408df..48ef836 100644 --- a/src/gpgpusim_entrypoint.h +++ b/src/gpgpusim_entrypoint.h @@ -7,73 +7,72 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef GPGPUSIM_ENTRYPOINT_H_INCLUDED #define GPGPUSIM_ENTRYPOINT_H_INCLUDED -#include "abstract_hardware_model.h" #include <pthread.h> #include <semaphore.h> #include <time.h> +#include "abstract_hardware_model.h" -//extern time_t g_simulation_starttime; +// extern time_t g_simulation_starttime; class gpgpu_context; class GPGPUsim_ctx { - public: - GPGPUsim_ctx(gpgpu_context* ctx) { - g_sim_active = false; - g_sim_done = true; - break_limit = false; - g_sim_lock = PTHREAD_MUTEX_INITIALIZER; - - g_the_gpu_config=NULL; - g_the_gpu=NULL; - g_stream_manager=NULL; - the_cude_device=NULL; - the_context=NULL; - gpgpu_ctx = ctx; - } - - //struct gpgpu_ptx_sim_arg *grid_params; + public: + GPGPUsim_ctx(gpgpu_context *ctx) { + g_sim_active = false; + g_sim_done = true; + break_limit = false; + g_sim_lock = PTHREAD_MUTEX_INITIALIZER; - sem_t g_sim_signal_start; - sem_t g_sim_signal_finish; - sem_t g_sim_signal_exit; - time_t g_simulation_starttime; - pthread_t g_simulation_thread; + g_the_gpu_config = NULL; + g_the_gpu = NULL; + g_stream_manager = NULL; + the_cude_device = NULL; + the_context = NULL; + gpgpu_ctx = ctx; + } - class gpgpu_sim_config *g_the_gpu_config; - class gpgpu_sim *g_the_gpu; - class stream_manager *g_stream_manager; + // struct gpgpu_ptx_sim_arg *grid_params; - struct _cuda_device_id *the_cude_device; - struct CUctx_st* the_context; - gpgpu_context* gpgpu_ctx; + sem_t g_sim_signal_start; + sem_t g_sim_signal_finish; + sem_t g_sim_signal_exit; + time_t g_simulation_starttime; + pthread_t g_simulation_thread; + class gpgpu_sim_config *g_the_gpu_config; + class gpgpu_sim *g_the_gpu; + class stream_manager *g_stream_manager; - pthread_mutex_t g_sim_lock; - bool g_sim_active; - bool g_sim_done; - bool break_limit; + struct _cuda_device_id *the_cude_device; + struct CUctx_st *the_context; + gpgpu_context *gpgpu_ctx; + pthread_mutex_t g_sim_lock; + bool g_sim_active; + bool g_sim_done; + bool break_limit; }; #endif diff --git a/src/option_parser.cc b/src/option_parser.cc index 7d747f0..c32c952 100644 --- a/src/option_parser.cc +++ b/src/option_parser.cc @@ -7,541 +7,546 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include "option_parser.h" +#include <assert.h> #include <stdio.h> #include <stdlib.h> -#include <assert.h> -#include <string> -#include <iostream> -#include <iomanip> -#include <sstream> +#include <string.h> #include <fstream> -#include <vector> +#include <iomanip> +#include <iostream> #include <list> #include <map> -#include <string.h> -#include "option_parser.h" - +#include <sstream> +#include <string> +#include <vector> using namespace std; // A generic option registry regardless of data type -class OptionRegistryInterface -{ -public: - OptionRegistryInterface(const string optionName, const string optionDesc) - : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) - {} +class OptionRegistryInterface { + public: + OptionRegistryInterface(const string optionName, const string optionDesc) + : m_optionName(optionName), m_optionDesc(optionDesc), m_isParsed(false) {} - virtual ~OptionRegistryInterface() {} + virtual ~OptionRegistryInterface() {} - const string& GetName() { return m_optionName; } - const string& GetDesc() { return m_optionDesc; } - const bool isParsed() { return m_isParsed; } - virtual string toString() = 0; - virtual bool fromString(const string str) = 0; - virtual bool isFlag() = 0; - virtual bool assignDefault(const char *str) = 0; + const string &GetName() { return m_optionName; } + const string &GetDesc() { return m_optionDesc; } + const bool isParsed() { return m_isParsed; } + virtual string toString() = 0; + virtual bool fromString(const string str) = 0; + virtual bool isFlag() = 0; + virtual bool assignDefault(const char *str) = 0; -protected: - string m_optionName; - string m_optionDesc; - bool m_isParsed; // true if the target variable has been updated by fromString() + protected: + string m_optionName; + string m_optionDesc; + bool m_isParsed; // true if the target variable has been updated by + // fromString() }; // Template for option registry - class T = specify data type of the option template <class T> -class OptionRegistry : public OptionRegistryInterface -{ -public: - OptionRegistry(const string name, const string desc, T &variable) - : OptionRegistryInterface(name, desc), m_variable(variable) - {} +class OptionRegistry : public OptionRegistryInterface { + public: + OptionRegistry(const string name, const string desc, T &variable) + : OptionRegistryInterface(name, desc), m_variable(variable) {} - virtual ~OptionRegistry() {} + virtual ~OptionRegistry() {} - virtual string toString() - { - stringstream ss; - ss << m_variable; - return ss.str(); - } + virtual string toString() { + stringstream ss; + ss << m_variable; + return ss.str(); + } - virtual bool fromString(const string str) - { - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - ss << setbase(10); - if (str.size() > 1 && str[0] == '0') { - if (str.size() > 2 && str[1] == 'x') { - ss.ignore(2); - ss << setbase(16); - } else { - ss.ignore(1); - ss << setbase(8); - } - } - try { - ss >> m_variable; - } catch (exception &e) { - return false; + virtual bool fromString(const string str) { + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + ss << setbase(10); + if (str.size() > 1 && str[0] == '0') { + if (str.size() > 2 && str[1] == 'x') { + ss.ignore(2); + ss << setbase(16); + } else { + ss.ignore(1); + ss << setbase(8); } - m_isParsed = true; - return true; - } + } + try { + ss >> m_variable; + } catch (exception &e) { + return false; + } + m_isParsed = true; + return true; + } - virtual bool isFlag() { return false; } - virtual bool assignDefault(const char *str) { return fromString(str); } + virtual bool isFlag() { return false; } + virtual bool assignDefault(const char *str) { return fromString(str); } - operator T() - { - return m_variable; - } + operator T() { return m_variable; } -private: - T &m_variable; + private: + T &m_variable; }; // specialized parser for string-type options -template<> -bool OptionRegistry<string>::fromString(const string str) -{ - m_variable = str; - m_isParsed = true; - return true; +template <> +bool OptionRegistry<string>::fromString(const string str) { + m_variable = str; + m_isParsed = true; + return true; } // specialized parser for c-string type options -template<> -bool OptionRegistry<char *>::fromString(const string str) -{ - m_variable = new char[str.size() + 1]; - strcpy(m_variable, str.c_str()); - m_isParsed = true; - return true; +template <> +bool OptionRegistry<char *>::fromString(const string str) { + m_variable = new char[str.size() + 1]; + strcpy(m_variable, str.c_str()); + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template<> -bool OptionRegistry<char *>::assignDefault(const char *str) -{ - m_variable = const_cast<char *>(str); // c-string options are not meant to be edited anyway - m_isParsed = true; - return true; +template <> +bool OptionRegistry<char *>::assignDefault(const char *str) { + m_variable = const_cast<char *>( + str); // c-string options are not meant to be edited anyway + m_isParsed = true; + return true; } // specialized default assignment for c-string type option to allow NULL default -template<> -string OptionRegistry<char *>::toString() -{ - stringstream ss; - if (m_variable != NULL) { - ss << m_variable; - } else { - ss << "NULL"; - } - return ss.str(); +template <> +string OptionRegistry<char *>::toString() { + stringstream ss; + if (m_variable != NULL) { + ss << m_variable; + } else { + ss << "NULL"; + } + return ss.str(); } -// specialized parser for boolean options -template<> -bool OptionRegistry<bool>::fromString(const string str) -{ - int value = 1; - bool parsed = true; - stringstream ss(str); - ss.exceptions(stringstream::failbit | stringstream::badbit); - try { - ss >> value; - } catch (stringstream::failure &ep) { - parsed = false; - } - assert(value == 0 or value == 1); // sanity check for boolean options (it can only be 1 or 0) - m_variable = (value != 0); - m_isParsed = true; - return parsed; +// specialized parser for boolean options +template <> +bool OptionRegistry<bool>::fromString(const string str) { + int value = 1; + bool parsed = true; + stringstream ss(str); + ss.exceptions(stringstream::failbit | stringstream::badbit); + try { + ss >> value; + } catch (stringstream::failure &ep) { + parsed = false; + } + assert(value == 0 or + value == + 1); // sanity check for boolean options (it can only be 1 or 0) + m_variable = (value != 0); + m_isParsed = true; + return parsed; } // specializing a flag query function to identify boolean option -template<> -bool OptionRegistry<bool>::isFlag() { return true; } - -// class holding a collection of options and parse them from command line/configfile -class OptionParser -{ -public: - OptionParser() {} - ~OptionParser() - { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { - delete (*i_option); - } - } +template <> +bool OptionRegistry<bool>::isFlag() { + return true; +} - template<class T> - void Register(const string optionName, const string optionDesc, T &optionVariable, const char *optionDefault) - { - OptionRegistry<T> *p_option = new OptionRegistry<T>(optionName, optionDesc, optionVariable); - m_optionReg.push_back(p_option); - m_optionMap[optionName] = p_option; - p_option->assignDefault(optionDefault); - } +// class holding a collection of options and parse them from command +// line/configfile +class OptionParser { + public: + OptionParser() {} + ~OptionParser() { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); + ++i_option) { + delete (*i_option); + } + } - void ParseCommandLine(int argc, const char * const argv[]) - { - for (int i = 1; i < argc; i++) { - OptionMap::iterator i_option; - bool optionFound = false; + template <class T> + void Register(const string optionName, const string optionDesc, + T &optionVariable, const char *optionDefault) { + OptionRegistry<T> *p_option = + new OptionRegistry<T>(optionName, optionDesc, optionVariable); + m_optionReg.push_back(p_option); + m_optionMap[optionName] = p_option; + p_option->assignDefault(optionDefault); + } - i_option = m_optionMap.find(argv[i]); - if (i_option != m_optionMap.end()) { - const char *argstr = (i + 1 < argc)? argv[i + 1] : ""; - OptionRegistryInterface *p_option = i_option->second; - if (p_option->isFlag()) { - if (p_option->fromString(argstr) == true) { - i += 1; - } - } else { - if (p_option->fromString(argstr) == false) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for option '%s'.\n", argstr, argv[i]); - exit(1); - } - i += 1; - } - optionFound = true; - } else if (string(argv[i]) == "-config") { - if (i + 1 >= argc) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Missing filename for option '-config'.\n"); - exit(1); - } + void ParseCommandLine(int argc, const char *const argv[]) { + for (int i = 1; i < argc; i++) { + OptionMap::iterator i_option; + bool optionFound = false; - ParseFile(argv[i + 1]); + i_option = m_optionMap.find(argv[i]); + if (i_option != m_optionMap.end()) { + const char *argstr = (i + 1 < argc) ? argv[i + 1] : ""; + OptionRegistryInterface *p_option = i_option->second; + if (p_option->isFlag()) { + if (p_option->fromString(argstr) == true) { i += 1; - optionFound = true; - } - if (optionFound == false) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", argv[i]); + } + } else { + if (p_option->fromString(argstr) == false) { + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: Cannot parse value '%s' for " + "option '%s'.\n", + argstr, argv[i]); exit(1); - } - } - } - - - void ParseFile(const char *filename) { - ifstream inputFile; - stringstream args; + } + i += 1; + } + optionFound = true; + } else if (string(argv[i]) == "-config") { + if (i + 1 >= argc) { + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: Missing filename for option " + "'-config'.\n"); + exit(1); + } - // open config file, stream every line into a continuous buffer - // get rid of comments in the process - inputFile.open(filename); - if (!inputFile.good()) { - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", filename); - exit(1); + ParseFile(argv[i + 1]); + i += 1; + optionFound = true; } - while (inputFile.good()) { - string line; - getline(inputFile, line); - size_t commentStart = line.find_first_of("#"); - if (commentStart != line.npos) { - line.erase(commentStart); - } - args << line << ' '; + if (optionFound == false) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Unknown Option: '%s' \n", + argv[i]); + exit(1); } - inputFile.close(); + } + } - ParseStringStream(args); - } + void ParseFile(const char *filename) { + ifstream inputFile; + stringstream args; - // parse the given string as tokens separated by a set of given delimiters - void ParseString(string inputString, const string delimiters = string(" ;")) { - // convert all delimiter characters into whitespaces - for (unsigned t = 0; t < inputString.size(); t++) { - for (unsigned d = 0; d < delimiters.size(); d++) { - if (inputString[t] == delimiters.at(d)) { - inputString[t] = ' '; - break; - } - } + // open config file, stream every line into a continuous buffer + // get rid of comments in the process + inputFile.open(filename); + if (!inputFile.good()) { + fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: Cannot open config file '%s'\n", + filename); + exit(1); + } + while (inputFile.good()) { + string line; + getline(inputFile, line); + size_t commentStart = line.find_first_of("#"); + if (commentStart != line.npos) { + line.erase(commentStart); } - stringstream args(inputString); - ParseStringStream(args); - } + args << line << ' '; + } + inputFile.close(); - // parse the given stringstream as whitespace-separated tokens. drain the stream in the process - void ParseStringStream(stringstream &args) { - // extract non-whitespace string tokens - vector<char*> argv; - argv.push_back(new char[6]); - strcpy(argv[0], "dummy"); - while (args.good()) { - string argNew; - args >> argNew; + ParseStringStream(args); + } - if (argNew.size() == 0) continue; // this is probably the last token + // parse the given string as tokens separated by a set of given delimiters + void ParseString(string inputString, const string delimiters = string(" ;")) { + // convert all delimiter characters into whitespaces + for (unsigned t = 0; t < inputString.size(); t++) { + for (unsigned d = 0; d < delimiters.size(); d++) { + if (inputString[t] == delimiters.at(d)) { + inputString[t] = ' '; + break; + } + } + } + stringstream args(inputString); + ParseStringStream(args); + } - if (argNew[0] == '"') { - while (args.good() && argNew[argNew.size()-1] != '"') { - string argCont; - args >> argCont; - argNew += " " + argCont; - } - argNew.erase(0,1); - argNew.erase(argNew.size()-1); - } + // parse the given stringstream as whitespace-separated tokens. drain the + // stream in the process + void ParseStringStream(stringstream &args) { + // extract non-whitespace string tokens + vector<char *> argv; + argv.push_back(new char[6]); + strcpy(argv[0], "dummy"); + while (args.good()) { + string argNew; + args >> argNew; - char *c_argNew = new char[argNew.size() + 1]; - strcpy(c_argNew, argNew.c_str()); - argv.push_back(c_argNew); - } + if (argNew.size() == 0) continue; // this is probably the last token - // pass the string token into normal commandline parser - char **targv = (char**)calloc(argv.size(), sizeof(char*)); - for( unsigned k=0; k < argv.size(); k++ ) - targv[k] = argv[k]; - ParseCommandLine(argv.size(), targv); - free(targv); - for (size_t i = 0; i < argv.size(); i++) { - delete[] argv[i]; + if (argNew[0] == '"') { + while (args.good() && argNew[argNew.size() - 1] != '"') { + string argCont; + args >> argCont; + argNew += " " + argCont; + } + argNew.erase(0, 1); + argNew.erase(argNew.size() - 1); } - } - void Print(FILE *fout) - { - OptionCollection::iterator i_option; - for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); ++i_option) { - stringstream sout; - if ((*i_option)->isParsed() == false) { - cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" << (*i_option)->GetName() << "'\n"; - assert(0); - } - sout << setw(20) << left << (*i_option)->GetName() << " "; - sout << setw(20) << right << (*i_option)->toString() << " # "; - sout << left << (*i_option)->GetDesc(); - sout << std::endl; - fprintf(fout, "%s", sout.str().c_str()); + char *c_argNew = new char[argNew.size() + 1]; + strcpy(c_argNew, argNew.c_str()); + argv.push_back(c_argNew); + } + + // pass the string token into normal commandline parser + char **targv = (char **)calloc(argv.size(), sizeof(char *)); + for (unsigned k = 0; k < argv.size(); k++) targv[k] = argv[k]; + ParseCommandLine(argv.size(), targv); + free(targv); + for (size_t i = 0; i < argv.size(); i++) { + delete[] argv[i]; + } + } + + void Print(FILE *fout) { + OptionCollection::iterator i_option; + for (i_option = m_optionReg.begin(); i_option != m_optionReg.end(); + ++i_option) { + stringstream sout; + if ((*i_option)->isParsed() == false) { + cerr << "\n\nGPGPU-Sim ** ERROR: Missing option '" + << (*i_option)->GetName() << "'\n"; + assert(0); } - } + sout << setw(20) << left << (*i_option)->GetName() << " "; + sout << setw(20) << right << (*i_option)->toString() << " # "; + sout << left << (*i_option)->GetDesc(); + sout << std::endl; + fprintf(fout, "%s", sout.str().c_str()); + } + } -private: - typedef list<OptionRegistryInterface*> OptionCollection; - OptionCollection m_optionReg; - typedef map<string, OptionRegistryInterface*> OptionMap; - OptionMap m_optionMap; + private: + typedef list<OptionRegistryInterface *> OptionCollection; + OptionCollection m_optionReg; + typedef map<string, OptionRegistryInterface *> OptionMap; + OptionMap m_optionMap; }; #include "option_parser.h" -option_parser_t option_parser_create() -{ - OptionParser *p_opr = new OptionParser(); - return reinterpret_cast<option_parser_t>(p_opr); +option_parser_t option_parser_create() { + OptionParser *p_opr = new OptionParser(); + return reinterpret_cast<option_parser_t>(p_opr); } -void option_parser_destroy(option_parser_t opp) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - delete p_opr; +void option_parser_destroy(option_parser_t opp) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + delete p_opr; } -void option_parser_register(option_parser_t opp, - const char *name, - enum option_dtype type, - void *variable, - const char *desc, - const char *defaultvalue) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - switch (type) { - case OPT_INT32: p_opr->Register<int>(name, desc, *(int*)variable, defaultvalue); break; - case OPT_UINT32: p_opr->Register<unsigned int>(name, desc, *(unsigned int*)variable, defaultvalue); break; - case OPT_INT64: p_opr->Register<long long>(name, desc, *(long long*)variable, defaultvalue); break; - case OPT_UINT64: p_opr->Register<unsigned long long>(name, desc, *(unsigned long long*)variable, defaultvalue); break; - case OPT_BOOL: p_opr->Register<bool>(name, desc, *(bool*)variable, defaultvalue); break; - case OPT_FLOAT: p_opr->Register<float>(name, desc, *(float*)variable, defaultvalue); break; - case OPT_DOUBLE: p_opr->Register<double>(name, desc, *(double*)variable, defaultvalue); break; - case OPT_CHAR: p_opr->Register<char>(name, desc, *(char*)variable, defaultvalue); break; - case OPT_CSTR: p_opr->Register<char*>(name, desc, *(char**)variable, defaultvalue); break; - default: - fprintf(stderr, "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", type); - exit(1); - break; - } +void option_parser_register(option_parser_t opp, const char *name, + enum option_dtype type, void *variable, + const char *desc, const char *defaultvalue) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + switch (type) { + case OPT_INT32: + p_opr->Register<int>(name, desc, *(int *)variable, defaultvalue); + break; + case OPT_UINT32: + p_opr->Register<unsigned int>(name, desc, *(unsigned int *)variable, + defaultvalue); + break; + case OPT_INT64: + p_opr->Register<long long>(name, desc, *(long long *)variable, + defaultvalue); + break; + case OPT_UINT64: + p_opr->Register<unsigned long long>( + name, desc, *(unsigned long long *)variable, defaultvalue); + break; + case OPT_BOOL: + p_opr->Register<bool>(name, desc, *(bool *)variable, defaultvalue); + break; + case OPT_FLOAT: + p_opr->Register<float>(name, desc, *(float *)variable, defaultvalue); + break; + case OPT_DOUBLE: + p_opr->Register<double>(name, desc, *(double *)variable, defaultvalue); + break; + case OPT_CHAR: + p_opr->Register<char>(name, desc, *(char *)variable, defaultvalue); + break; + case OPT_CSTR: + p_opr->Register<char *>(name, desc, *(char **)variable, defaultvalue); + break; + default: + fprintf(stderr, + "\n\nGPGPU-Sim ** ERROR: option data type (%d) not supported!\n", + type); + exit(1); + break; + } } -void option_parser_cmdline(option_parser_t opp, - int argc, const char *argv[]) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - return p_opr->ParseCommandLine(argc,argv); - +void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + return p_opr->ParseCommandLine(argc, argv); } - -void option_parser_cfgfile(option_parser_t opp, - const char *filename) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - p_opr->ParseFile(filename); +void option_parser_cfgfile(option_parser_t opp, const char *filename) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + p_opr->ParseFile(filename); } void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, - const char *delimiters) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - p_opr->ParseString(inputstring, delimiters); + const char *inputstring, + const char *delimiters) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + p_opr->ParseString(inputstring, delimiters); } -void option_parser_print(option_parser_t opp, - FILE *fout) -{ - OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); - p_opr->Print(fout); +void option_parser_print(option_parser_t opp, FILE *fout) { + OptionParser *p_opr = reinterpret_cast<OptionParser *>(opp); + p_opr->Print(fout); } - - // #define UNIT_TEST #ifdef UNIT_TEST -class testtype -{ -public: - int idata; - float fdata; - string sdata; - unsigned long long ulldata; - bool bdata; - unsigned int boolint; - char * coption; +class testtype { + public: + int idata; + float fdata; + string sdata; + unsigned long long ulldata; + bool bdata; + unsigned int boolint; + char *coption; - testtype() - : idata(0), - fdata(0.0f), - sdata(""), - ulldata(0), - bdata(false) - { } + testtype() : idata(0), fdata(0.0f), sdata(""), ulldata(0), bdata(false) {} }; +int cppinterfacetest(int argc, const char *argv[]) { + testtype c; + OptionParser optionparser; + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); -int cppinterfacetest(int argc, const char *argv[]) -{ - testtype c; - OptionParser optionparser; - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - - optionparser.Register<int>("-idata", "integer data", c.idata, "-456"); - optionparser.Register<float>("-fdata", "floating point data", c.fdata, "0.001"); - optionparser.Register<string>("-sdata", "first string data", c.sdata, "hellow"); - optionparser.Register<unsigned long long>("-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); - optionparser.Register<bool>("-someflag", "first flag", c.bdata, "0"); - optionparser.Register<bool>("-otherflag", "second flag", (bool&)c.boolint, "1"); - optionparser.Register<char *>("-coption", "char * data", c.coption, NULL); + optionparser.Register<int>("-idata", "integer data", c.idata, "-456"); + optionparser.Register<float>("-fdata", "floating point data", c.fdata, + "0.001"); + optionparser.Register<string>("-sdata", "first string data", c.sdata, + "hellow"); + optionparser.Register<unsigned long long>( + "-ulldata", "unsigned long long data", c.ulldata, "0x123456789abcdef1"); + optionparser.Register<bool>("-someflag", "first flag", c.bdata, "0"); + optionparser.Register<bool>("-otherflag", "second flag", (bool &)c.boolint, + "1"); + optionparser.Register<char *>("-coption", "char * data", c.coption, NULL); - cout << "Default: \n"; - optionparser.Print(stdout); + cout << "Default: \n"; + optionparser.Print(stdout); - optionparser.ParseCommandLine(argc, argv); + optionparser.ParseCommandLine(argc, argv); - cout << "Commandline Parse Results: \n"; - optionparser.Print(stdout); + cout << "Commandline Parse Results: \n"; + optionparser.Print(stdout); - optionparser.ParseFile("test.config"); - cout << "File Parse Results: \n"; - optionparser.Print(stdout); - cout << c.sdata << ' ' << c.idata << endl; + optionparser.ParseFile("test.config"); + cout << "File Parse Results: \n"; + optionparser.Print(stdout); + cout << c.sdata << ' ' << c.idata << endl; - return 0; + return 0; } -int cinterfacetest(int argc, const char *argv[]) -{ - testtype c; - option_parser_t opp = option_parser_create(); - c.idata = 123; - c.fdata = 3249586.333; - c.sdata = string("haha"); - char *otherstr; +int cinterfacetest(int argc, const char *argv[]) { + testtype c; + option_parser_t opp = option_parser_create(); + c.idata = 123; + c.fdata = 3249586.333; + c.sdata = string("haha"); + char *otherstr; - option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", "-456"); - option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, "floating point data", "0.001"); - option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, "first string data", "hellow"); - option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, "unsigend long long data", "0x123456789abcdef1"); - option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", "0"); - option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", "1"); - option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", NULL); + option_parser_register(opp, "-idata", OPT_INT32, &c.idata, "integer data", + "-456"); + option_parser_register(opp, "-fdata", OPT_FLOAT, &c.fdata, + "floating point data", "0.001"); + option_parser_register(opp, "-sdata", OPT_CSTR, &otherstr, + "first string data", "hellow"); + option_parser_register(opp, "-ulldata", OPT_UINT64, &c.ulldata, + "unsigend long long data", "0x123456789abcdef1"); + option_parser_register(opp, "-someflag", OPT_BOOL, &c.bdata, "first flag", + "0"); + option_parser_register(opp, "-otherflag", OPT_BOOL, &c.boolint, "second flag", + "1"); + option_parser_register(opp, "-coption", OPT_CSTR, &c.coption, "char * data", + NULL); - printf("Default: \n"); - option_parser_print(opp, stdout); + printf("Default: \n"); + option_parser_print(opp, stdout); - option_parser_cmdline(opp, argc, argv); + option_parser_cmdline(opp, argc, argv); - printf("Commandline Parse Results: \n"); - option_parser_print(opp, stdout); + printf("Commandline Parse Results: \n"); + option_parser_print(opp, stdout); - option_parser_cfgfile(opp, "test.config"); - printf("File Parse Results: \n"); - option_parser_print(opp, stdout); - printf("%s %d\n", otherstr, c.idata); + option_parser_cfgfile(opp, "test.config"); + printf("File Parse Results: \n"); + option_parser_print(opp, stdout); + printf("%s %d\n", otherstr, c.idata); - option_parser_destroy(opp); + option_parser_destroy(opp); - return 0; + return 0; } -int stringparsertest() -{ - int tABC; - int tDEF; - char tMode; - char *tName; +int stringparsertest() { + int tABC; + int tDEF; + char tMode; + char *tName; - option_parser_t opp = option_parser_create(); - option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); - option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); - option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); - option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); + option_parser_t opp = option_parser_create(); + option_parser_register(opp, "ABC", OPT_INT32, &tABC, "tABC", "34"); + option_parser_register(opp, "DEF", OPT_INT32, &tDEF, "tDEF", "-56"); + option_parser_register(opp, "Mode", OPT_CHAR, &tMode, "tMode", "P"); + option_parser_register(opp, "Name", OPT_CSTR, &tName, "tName", "Cache"); - option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", " ;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); + option_parser_delimited_string(opp, "ABC 1111; DEF 88; Mode A; Name out", + " ;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); - option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", " =;"); - printf("String Parse Results: \n"); - option_parser_print(opp, stdout); + option_parser_delimited_string(opp, "Name=dram;DEF=702;Mode=B;ABC=-9573;", + " =;"); + printf("String Parse Results: \n"); + option_parser_print(opp, stdout); - return 0; + return 0; } -int main(int argc, const char *argv[]) -{ - cppinterfacetest(argc,argv); - cinterfacetest(argc,argv); - stringparsertest(); +int main(int argc, const char *argv[]) { + cppinterfacetest(argc, argv); + cinterfacetest(argc, argv); + stringparsertest(); - return 0; + return 0; } #endif - diff --git a/src/option_parser.h b/src/option_parser.h index 1f7f96c..f69cdbb 100644 --- a/src/option_parser.h +++ b/src/option_parser.h @@ -7,44 +7,44 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include <stdio.h> #include <stdlib.h> - // pointer to C++ class typedef class OptionParser *option_parser_t; // data type of the option enum option_dtype { - OPT_INT32, - OPT_UINT32, - OPT_INT64, - OPT_UINT64, - OPT_BOOL, - OPT_FLOAT, - OPT_DOUBLE, - OPT_CHAR, - OPT_CSTR + OPT_INT32, + OPT_UINT32, + OPT_INT64, + OPT_UINT64, + OPT_BOOL, + OPT_FLOAT, + OPT_DOUBLE, + OPT_CHAR, + OPT_CSTR }; // create and destroy option parser @@ -52,28 +52,19 @@ option_parser_t option_parser_create(); void option_parser_destroy(option_parser_t opp); // register new option -void option_parser_register(option_parser_t opp, - const char *name, - enum option_dtype type, - void *variable, - const char *desc, - const char *defaultvalue); +void option_parser_register(option_parser_t opp, const char *name, + enum option_dtype type, void *variable, + const char *desc, const char *defaultvalue); // parse command line -void option_parser_cmdline(option_parser_t opp, - int argc, const char *argv[]); - - +void option_parser_cmdline(option_parser_t opp, int argc, const char *argv[]); // parse config file -void option_parser_cfgfile(option_parser_t opp, - const char *filename); +void option_parser_cfgfile(option_parser_t opp, const char *filename); // parse a delimited string void option_parser_delimited_string(option_parser_t opp, - const char *inputstring, + const char *inputstring, const char *delimiters); // print options -void option_parser_print(option_parser_t opp, - FILE *fout); - +void option_parser_print(option_parser_t opp, FILE *fout); diff --git a/src/statwrapper.cc b/src/statwrapper.cc index e273e78..cda25b6 100644 --- a/src/statwrapper.cc +++ b/src/statwrapper.cc @@ -1,48 +1,33 @@ -//a Wraper function for stats class -#include "intersim2/stats.hpp" +// a Wraper function for stats class #include <stdio.h> +#include "intersim2/stats.hpp" -Stats* StatCreate (const char * name, double bin_size, int num_bins) { - Stats* newstat = new Stats(NULL,name,bin_size,num_bins); - newstat->Clear (); - return newstat; +Stats *StatCreate(const char *name, double bin_size, int num_bins) { + Stats *newstat = new Stats(NULL, name, bin_size, num_bins); + newstat->Clear(); + return newstat; } -void StatClear(void * st) -{ - ((Stats *)st)->Clear(); -} +void StatClear(void *st) { ((Stats *)st)->Clear(); } -void StatAddSample (void * st, int val) -{ - ((Stats *)st)->AddSample(val); -} +void StatAddSample(void *st, int val) { ((Stats *)st)->AddSample(val); } -double StatAverage(void * st) -{ - return((Stats *)st)->Average(); -} +double StatAverage(void *st) { return ((Stats *)st)->Average(); } -double StatMax(void * st) -{ - return((Stats *)st)->Max(); -} +double StatMax(void *st) { return ((Stats *)st)->Max(); } -double StatMin(void * st) -{ - return((Stats *)st)->Min(); -} +double StatMin(void *st) { return ((Stats *)st)->Min(); } -void StatDisp (void * st) -{ - printf ("Stats for "); - ((Stats *)st)->DisplayHierarchy(); -// if (((Stats *)st)->NeverUsed()) { -// printf (" was never updated!\n"); -// } else { - printf("Min %f Max %f Average %f \n",((Stats *)st)->Min(),((Stats *)st)->Max(),StatAverage(st)); - ((Stats *)st)->Display(); -// } +void StatDisp(void *st) { + printf("Stats for "); + ((Stats *)st)->DisplayHierarchy(); + // if (((Stats *)st)->NeverUsed()) { + // printf (" was never updated!\n"); + // } else { + printf("Min %f Max %f Average %f \n", ((Stats *)st)->Min(), + ((Stats *)st)->Max(), StatAverage(st)); + ((Stats *)st)->Display(); + // } } #if 0 @@ -55,5 +40,3 @@ int main () StatDisp(mytest); } #endif - - diff --git a/src/statwrapper.h b/src/statwrapper.h index 65c2ab9..4c00447 100644 --- a/src/statwrapper.h +++ b/src/statwrapper.h @@ -1,12 +1,12 @@ #ifndef STAT_WRAPER_H #define STAT_WRAPER_H -class Stats* StatCreate (const char * name, double bin_size, int num_bins) ; -void StatClear(void * st); -void StatAddSample (void * st, int val); -double StatAverage(void * st) ; -double StatMax(void * st) ; -double StatMin(void * st) ; -void StatDisp (void * st); +class Stats* StatCreate(const char* name, double bin_size, int num_bins); +void StatClear(void* st); +void StatAddSample(void* st, int val); +double StatAverage(void* st); +double StatMax(void* st); +double StatMin(void* st); +void StatDisp(void* st); #endif diff --git a/src/stream_manager.cc b/src/stream_manager.cc index be3dd71..48a1edc 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -7,482 +7,463 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "stream_manager.h" -#include "gpgpusim_entrypoint.h" +#include "../libcuda/gpgpu_context.h" #include "cuda-sim/cuda-sim.h" #include "gpgpu-sim/gpu-sim.h" -#include "../libcuda/gpgpu_context.h" +#include "gpgpusim_entrypoint.h" unsigned CUstream_st::sm_next_stream_uid = 0; -CUstream_st::CUstream_st() -{ - m_pending = false; - m_uid = sm_next_stream_uid++; - pthread_mutex_init(&m_lock,NULL); -} - -bool CUstream_st::empty() -{ - pthread_mutex_lock(&m_lock); - bool empty = m_operations.empty(); - pthread_mutex_unlock(&m_lock); - return empty; +CUstream_st::CUstream_st() { + m_pending = false; + m_uid = sm_next_stream_uid++; + pthread_mutex_init(&m_lock, NULL); } -bool CUstream_st::busy() -{ - pthread_mutex_lock(&m_lock); - bool pending = m_pending; - pthread_mutex_unlock(&m_lock); - return pending; +bool CUstream_st::empty() { + pthread_mutex_lock(&m_lock); + bool empty = m_operations.empty(); + pthread_mutex_unlock(&m_lock); + return empty; } -void CUstream_st::synchronize() -{ - // called by host thread - bool done=false; - do{ - pthread_mutex_lock(&m_lock); - done = m_operations.empty(); - pthread_mutex_unlock(&m_lock); - } while ( !done ); +bool CUstream_st::busy() { + pthread_mutex_lock(&m_lock); + bool pending = m_pending; + pthread_mutex_unlock(&m_lock); + return pending; } -void CUstream_st::push( const stream_operation &op ) -{ - // called by host thread +void CUstream_st::synchronize() { + // called by host thread + bool done = false; + do { pthread_mutex_lock(&m_lock); - m_operations.push_back( op ); + done = m_operations.empty(); pthread_mutex_unlock(&m_lock); + } while (!done); } -void CUstream_st::record_next_done() -{ - // called by gpu thread - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_operations.pop_front(); - m_pending=false; - pthread_mutex_unlock(&m_lock); +void CUstream_st::push(const stream_operation &op) { + // called by host thread + pthread_mutex_lock(&m_lock); + m_operations.push_back(op); + pthread_mutex_unlock(&m_lock); } - -stream_operation CUstream_st::next() -{ - // called by gpu thread - pthread_mutex_lock(&m_lock); - m_pending = true; - stream_operation result = m_operations.front(); - pthread_mutex_unlock(&m_lock); - return result; +void CUstream_st::record_next_done() { + // called by gpu thread + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_operations.pop_front(); + m_pending = false; + pthread_mutex_unlock(&m_lock); } -void CUstream_st::cancel_front() -{ - pthread_mutex_lock(&m_lock); - assert(m_pending); - m_pending = false; - pthread_mutex_unlock(&m_lock); - +stream_operation CUstream_st::next() { + // called by gpu thread + pthread_mutex_lock(&m_lock); + m_pending = true; + stream_operation result = m_operations.front(); + pthread_mutex_unlock(&m_lock); + return result; } -void CUstream_st::print(FILE *fp) -{ - pthread_mutex_lock(&m_lock); - fprintf(fp,"GPGPU-Sim API: stream %u has %zu operations\n", m_uid, m_operations.size() ); - std::list<stream_operation>::iterator i; - unsigned n=0; - for( i=m_operations.begin(); i!=m_operations.end(); i++ ) { - stream_operation &op = *i; - fprintf(fp,"GPGPU-Sim API: %u : ", n++); - op.print(fp); - fprintf(fp,"\n"); - } - pthread_mutex_unlock(&m_lock); +void CUstream_st::cancel_front() { + pthread_mutex_lock(&m_lock); + assert(m_pending); + m_pending = false; + pthread_mutex_unlock(&m_lock); } +void CUstream_st::print(FILE *fp) { + pthread_mutex_lock(&m_lock); + fprintf(fp, "GPGPU-Sim API: stream %u has %zu operations\n", m_uid, + m_operations.size()); + std::list<stream_operation>::iterator i; + unsigned n = 0; + for (i = m_operations.begin(); i != m_operations.end(); i++) { + stream_operation &op = *i; + fprintf(fp, "GPGPU-Sim API: %u : ", n++); + op.print(fp); + fprintf(fp, "\n"); + } + pthread_mutex_unlock(&m_lock); +} -bool stream_operation::do_operation( gpgpu_sim *gpu ) -{ - if( is_noop() ) - return true; +bool stream_operation::do_operation(gpgpu_sim *gpu) { + if (is_noop()) return true; - assert(!m_done && m_stream); - if(g_debug_execution >= 3) - printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid() ); - switch( m_type ) { + assert(!m_done && m_stream); + if (g_debug_execution >= 3) + printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid()); + switch (m_type) { case stream_memcpy_host_to_device: - if(g_debug_execution >= 3) - printf("memcpy host-to-device\n"); - gpu->memcpy_to_gpu(m_device_address_dst,m_host_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy host-to-device\n"); + gpu->memcpy_to_gpu(m_device_address_dst, m_host_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_host: - if(g_debug_execution >= 3) - printf("memcpy device-to-host\n"); - gpu->memcpy_from_gpu(m_host_address_dst,m_device_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy device-to-host\n"); + gpu->memcpy_from_gpu(m_host_address_dst, m_device_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_device_to_device: - if(g_debug_execution >= 3) - printf("memcpy device-to-device\n"); - gpu->memcpy_gpu_to_gpu(m_device_address_dst,m_device_address_src,m_cnt); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy device-to-device\n"); + gpu->memcpy_gpu_to_gpu(m_device_address_dst, m_device_address_src, m_cnt); + m_stream->record_next_done(); + break; case stream_memcpy_to_symbol: - if(g_debug_execution >= 3) - printf("memcpy to symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_src,m_cnt,m_offset,1,gpu); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy to symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( + m_symbol, m_host_address_src, m_cnt, m_offset, 1, gpu); + m_stream->record_next_done(); + break; case stream_memcpy_from_symbol: - if(g_debug_execution >= 3) - printf("memcpy from symbol\n"); - gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_dst,m_cnt,m_offset,0,gpu); - m_stream->record_next_done(); - break; + if (g_debug_execution >= 3) printf("memcpy from symbol\n"); + gpu->gpgpu_ctx->func_sim->gpgpu_ptx_sim_memcpy_symbol( + m_symbol, m_host_address_dst, m_cnt, m_offset, 0, gpu); + m_stream->record_next_done(); + break; case stream_kernel_launch: - if( m_sim_mode ) { //Functional Sim - if(g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); - m_kernel->print_parent_info(); - } - gpu->set_cache_config(m_kernel->name()); - gpu->functional_launch( m_kernel ); + if (m_sim_mode) { // Functional Sim + if (g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str()); + m_kernel->print_parent_info(); } - else { //Performance Sim - if( gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { - if(g_debug_execution >= 3) { - printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() ); - m_kernel->print_parent_info(); - } - gpu->set_cache_config(m_kernel->name()); - gpu->launch( m_kernel ); - } - else { - if(m_kernel->m_launch_latency) - m_kernel->m_launch_latency--; - if(g_debug_execution >= 3) - printf("kernel %d: \'%s\', latency %u not ready to transfer to GPU hardware scheduler\n", - m_kernel->get_uid(), m_kernel->name().c_str(), m_kernel->m_launch_latency); - return false; - } + gpu->set_cache_config(m_kernel->name()); + gpu->functional_launch(m_kernel); + } else { // Performance Sim + if (gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) { + if (g_debug_execution >= 3) { + printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str()); + m_kernel->print_parent_info(); + } + gpu->set_cache_config(m_kernel->name()); + gpu->launch(m_kernel); + } else { + if (m_kernel->m_launch_latency) m_kernel->m_launch_latency--; + if (g_debug_execution >= 3) + printf( + "kernel %d: \'%s\', latency %u not ready to transfer to GPU " + "hardware scheduler\n", + m_kernel->get_uid(), m_kernel->name().c_str(), + m_kernel->m_launch_latency); + return false; } - break; + } + break; case stream_event: { - printf("event update\n"); - time_t wallclock = time((time_t *)NULL); - m_event->update( gpu->gpu_tot_sim_cycle, wallclock ); - m_stream->record_next_done(); - } - break; + printf("event update\n"); + time_t wallclock = time((time_t *)NULL); + m_event->update(gpu->gpu_tot_sim_cycle, wallclock); + m_stream->record_next_done(); + } break; case stream_wait_event: { - //only allows next op to go if event is done - //otherwise stays in the stream queue - printf("stream wait event processing...\n"); - if(m_event->done()) - printf("stream wait event done\n"); - m_stream->record_next_done(); - } - break; + // only allows next op to go if event is done + // otherwise stays in the stream queue + printf("stream wait event processing...\n"); + if (m_event->done()) printf("stream wait event done\n"); + m_stream->record_next_done(); + } break; default: - abort(); - } - m_done=true; - fflush(stdout); - return true; + abort(); + } + m_done = true; + fflush(stdout); + return true; } -void stream_operation::print( FILE *fp ) const -{ - fprintf(fp," stream operation " ); - switch( m_type ) { - case stream_event: fprintf(fp,"event"); break; - case stream_kernel_launch: fprintf(fp,"kernel"); break; - case stream_memcpy_device_to_device: fprintf(fp,"memcpy device-to-device"); break; - case stream_memcpy_device_to_host: fprintf(fp,"memcpy device-to-host"); break; - case stream_memcpy_host_to_device: fprintf(fp,"memcpy host-to-device"); break; - case stream_memcpy_to_symbol: fprintf(fp,"memcpy to symbol"); break; - case stream_memcpy_from_symbol: fprintf(fp,"memcpy from symbol"); break; - case stream_no_op: fprintf(fp,"no-op"); break; - } +void stream_operation::print(FILE *fp) const { + fprintf(fp, " stream operation "); + switch (m_type) { + case stream_event: + fprintf(fp, "event"); + break; + case stream_kernel_launch: + fprintf(fp, "kernel"); + break; + case stream_memcpy_device_to_device: + fprintf(fp, "memcpy device-to-device"); + break; + case stream_memcpy_device_to_host: + fprintf(fp, "memcpy device-to-host"); + break; + case stream_memcpy_host_to_device: + fprintf(fp, "memcpy host-to-device"); + break; + case stream_memcpy_to_symbol: + fprintf(fp, "memcpy to symbol"); + break; + case stream_memcpy_from_symbol: + fprintf(fp, "memcpy from symbol"); + break; + case stream_no_op: + fprintf(fp, "no-op"); + break; + } } -stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) -{ - m_gpu = gpu; - m_service_stream_zero = false; - m_cuda_launch_blocking = cuda_launch_blocking; - pthread_mutex_init(&m_lock,NULL); +stream_manager::stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking) { + m_gpu = gpu; + m_service_stream_zero = false; + m_cuda_launch_blocking = cuda_launch_blocking; + pthread_mutex_init(&m_lock, NULL); } -bool stream_manager::operation( bool * sim) -{ - bool check=check_finished_kernel(); - pthread_mutex_lock(&m_lock); -// if(check)m_gpu->print_stats(); - stream_operation op =front(); - if(!op.do_operation( m_gpu )) //not ready to execute - { - //cancel operation - if( op.is_kernel() ) { - unsigned grid_uid = op.get_kernel()->get_uid(); - m_grid_id_to_stream.erase(grid_uid); - } - op.get_stream()->cancel_front(); - +bool stream_manager::operation(bool *sim) { + bool check = check_finished_kernel(); + pthread_mutex_lock(&m_lock); + // if(check)m_gpu->print_stats(); + stream_operation op = front(); + if (!op.do_operation(m_gpu)) // not ready to execute + { + // cancel operation + if (op.is_kernel()) { + unsigned grid_uid = op.get_kernel()->get_uid(); + m_grid_id_to_stream.erase(grid_uid); } - pthread_mutex_unlock(&m_lock); - //pthread_mutex_lock(&m_lock); - // simulate a clock cycle on the GPU - return check; + op.get_stream()->cancel_front(); + } + pthread_mutex_unlock(&m_lock); + // pthread_mutex_lock(&m_lock); + // simulate a clock cycle on the GPU + return check; } -bool stream_manager::check_finished_kernel() -{ - unsigned grid_uid = m_gpu->finished_kernel(); - bool check=register_finished_kernel(grid_uid); - return check; +bool stream_manager::check_finished_kernel() { + unsigned grid_uid = m_gpu->finished_kernel(); + bool check = register_finished_kernel(grid_uid); + return check; } -bool stream_manager::register_finished_kernel(unsigned grid_uid) -{ - // called by gpu simulation thread - if(grid_uid > 0){ - CUstream_st *stream = m_grid_id_to_stream[grid_uid]; - kernel_info_t *kernel = stream->front().get_kernel(); - assert( grid_uid == kernel->get_uid() ); +bool stream_manager::register_finished_kernel(unsigned grid_uid) { + // called by gpu simulation thread + if (grid_uid > 0) { + CUstream_st *stream = m_grid_id_to_stream[grid_uid]; + kernel_info_t *kernel = stream->front().get_kernel(); + assert(grid_uid == kernel->get_uid()); - //Jin: should check children kernels for CDP - if(kernel->is_finished()) { -// std::ofstream kernel_stat("kernel_stat.txt", std::ofstream::out | std::ofstream::app); -// kernel_stat<< " kernel " << grid_uid << ": " << kernel->name(); -// if(kernel->get_parent()) -// kernel_stat << ", parent " << kernel->get_parent()->get_uid() << -// ", launch " << kernel->launch_cycle; -// kernel_stat<< ", start " << kernel->start_cycle << -// ", end " << kernel->end_cycle << ", retire " << gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; -// printf("kernel %d finishes, retires from stream %d\n", grid_uid, stream->get_uid()); -// kernel_stat.flush(); -// kernel_stat.close(); - stream->record_next_done(); - m_grid_id_to_stream.erase(grid_uid); - kernel->notify_parent_finished(); - delete kernel; - return true; - } + // Jin: should check children kernels for CDP + if (kernel->is_finished()) { + // std::ofstream kernel_stat("kernel_stat.txt", + // std::ofstream::out | std::ofstream::app); kernel_stat<< " + // kernel " << grid_uid << ": " << kernel->name(); + // if(kernel->get_parent()) + // kernel_stat << ", parent " << + // kernel->get_parent()->get_uid() << + // ", launch " << kernel->launch_cycle; + // kernel_stat<< ", start " << kernel->start_cycle << + // ", end " << kernel->end_cycle << ", retire " << + // gpu_sim_cycle + gpu_tot_sim_cycle << "\n"; + // printf("kernel %d finishes, retires from stream %d\n", + // grid_uid, stream->get_uid()); kernel_stat.flush(); + // kernel_stat.close(); + stream->record_next_done(); + m_grid_id_to_stream.erase(grid_uid); + kernel->notify_parent_finished(); + delete kernel; + return true; } + } - return false; + return false; } -void stream_manager::stop_all_running_kernels(){ - pthread_mutex_lock(&m_lock); +void stream_manager::stop_all_running_kernels() { + pthread_mutex_lock(&m_lock); - // Signal m_gpu to stop all running kernels - m_gpu->stop_all_running_kernels(); + // Signal m_gpu to stop all running kernels + m_gpu->stop_all_running_kernels(); - // Clean up all streams waiting on running kernels - int count=0; - while(check_finished_kernel()){ - count++; - } + // Clean up all streams waiting on running kernels + int count = 0; + while (check_finished_kernel()) { + count++; + } - // If any kernels completed, print out the current stats - if(count > 0) - m_gpu->print_stats(); + // If any kernels completed, print out the current stats + if (count > 0) m_gpu->print_stats(); - pthread_mutex_unlock(&m_lock); + pthread_mutex_unlock(&m_lock); } -stream_operation stream_manager::front() -{ - // called by gpu simulation thread - stream_operation result; -// if( concurrent_streams_empty() ) - m_service_stream_zero = true; - if( m_service_stream_zero ) { - if( !m_stream_zero.empty() && !m_stream_zero.busy() ) { - result = m_stream_zero.next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = &m_stream_zero; - } - } else { - m_service_stream_zero = false; - } +stream_operation stream_manager::front() { + // called by gpu simulation thread + stream_operation result; + // if( concurrent_streams_empty() ) + m_service_stream_zero = true; + if (m_service_stream_zero) { + if (!m_stream_zero.empty() && !m_stream_zero.busy()) { + result = m_stream_zero.next(); + if (result.is_kernel()) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = &m_stream_zero; + } + } else { + m_service_stream_zero = false; } - - if(!m_service_stream_zero) - { - std::list<struct CUstream_st*>::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++) { - CUstream_st *stream = *s; - if( !stream->busy() && !stream->empty() ) { - result = stream->next(); - if( result.is_kernel() ) { - unsigned grid_id = result.get_kernel()->get_uid(); - m_grid_id_to_stream[grid_id] = stream; - } - break; - } + } + + if (!m_service_stream_zero) { + std::list<struct CUstream_st *>::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + CUstream_st *stream = *s; + if (!stream->busy() && !stream->empty()) { + result = stream->next(); + if (result.is_kernel()) { + unsigned grid_id = result.get_kernel()->get_uid(); + m_grid_id_to_stream[grid_id] = stream; } + break; + } } - return result; + } + return result; } -void stream_manager::add_stream( struct CUstream_st *stream ) -{ - // called by host thread - pthread_mutex_lock(&m_lock); - m_streams.push_back(stream); - pthread_mutex_unlock(&m_lock); +void stream_manager::add_stream(struct CUstream_st *stream) { + // called by host thread + pthread_mutex_lock(&m_lock); + m_streams.push_back(stream); + pthread_mutex_unlock(&m_lock); } -void stream_manager::destroy_stream( CUstream_st *stream ) -{ - // called by host thread - pthread_mutex_lock(&m_lock); - while( !stream->empty() ) - ; - std::list<CUstream_st *>::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++ ) { - if( *s == stream ) { - m_streams.erase(s); - break; - } +void stream_manager::destroy_stream(CUstream_st *stream) { + // called by host thread + pthread_mutex_lock(&m_lock); + while (!stream->empty()) + ; + std::list<CUstream_st *>::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + if (*s == stream) { + m_streams.erase(s); + break; } - delete stream; - pthread_mutex_unlock(&m_lock); + } + delete stream; + pthread_mutex_unlock(&m_lock); } -bool stream_manager::concurrent_streams_empty() -{ - bool result = true; - if (m_streams.empty()) - return true; - // called by gpu simulation thread - std::list<struct CUstream_st *>::iterator s; - for( s=m_streams.begin(); s!=m_streams.end();++s ) { - struct CUstream_st *stream = *s; - if( !stream->empty() ) { - //stream->print(stdout); - result = false; - break; - } +bool stream_manager::concurrent_streams_empty() { + bool result = true; + if (m_streams.empty()) return true; + // called by gpu simulation thread + std::list<struct CUstream_st *>::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); ++s) { + struct CUstream_st *stream = *s; + if (!stream->empty()) { + // stream->print(stdout); + result = false; + break; } - return result; + } + return result; } -bool stream_manager::empty_protected() -{ - bool result = true; - pthread_mutex_lock(&m_lock); - if( !concurrent_streams_empty() ) - result = false; - if( !m_stream_zero.empty() ) - result = false; - pthread_mutex_unlock(&m_lock); - return result; +bool stream_manager::empty_protected() { + bool result = true; + pthread_mutex_lock(&m_lock); + if (!concurrent_streams_empty()) result = false; + if (!m_stream_zero.empty()) result = false; + pthread_mutex_unlock(&m_lock); + return result; } -bool stream_manager::empty() -{ - bool result = true; - if( !concurrent_streams_empty() ) - result = false; - if( !m_stream_zero.empty() ) - result = false; - return result; +bool stream_manager::empty() { + bool result = true; + if (!concurrent_streams_empty()) result = false; + if (!m_stream_zero.empty()) result = false; + return result; } - -void stream_manager::print( FILE *fp) -{ - pthread_mutex_lock(&m_lock); - print_impl(fp); - pthread_mutex_unlock(&m_lock); +void stream_manager::print(FILE *fp) { + pthread_mutex_lock(&m_lock); + print_impl(fp); + pthread_mutex_unlock(&m_lock); } -void stream_manager::print_impl( FILE *fp) -{ - fprintf(fp,"GPGPU-Sim API: Stream Manager State\n"); - std::list<struct CUstream_st *>::iterator s; - for( s=m_streams.begin(); s!=m_streams.end();++s ) { - struct CUstream_st *stream = *s; - if( !stream->empty() ) - stream->print(fp); - } - if( !m_stream_zero.empty() ) - m_stream_zero.print(fp); +void stream_manager::print_impl(FILE *fp) { + fprintf(fp, "GPGPU-Sim API: Stream Manager State\n"); + std::list<struct CUstream_st *>::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); ++s) { + struct CUstream_st *stream = *s; + if (!stream->empty()) stream->print(fp); + } + if (!m_stream_zero.empty()) m_stream_zero.print(fp); } -void stream_manager::push( stream_operation op ) -{ - struct CUstream_st *stream = op.get_stream(); - - // block if stream 0 (or concurrency disabled) and pending concurrent operations exist - bool block= !stream || m_cuda_launch_blocking; - while(block) { - pthread_mutex_lock(&m_lock); - block = !concurrent_streams_empty(); - pthread_mutex_unlock(&m_lock); - }; +void stream_manager::push(stream_operation op) { + struct CUstream_st *stream = op.get_stream(); + // block if stream 0 (or concurrency disabled) and pending concurrent + // operations exist + bool block = !stream || m_cuda_launch_blocking; + while (block) { pthread_mutex_lock(&m_lock); - if(!m_gpu->cycle_insn_cta_max_hit()) { - // Accept the stream operation if the maximum cycle/instruction/cta counts are not triggered - if( stream && !m_cuda_launch_blocking ) { - stream->push(op); - } else { - op.set_stream(&m_stream_zero); - m_stream_zero.push(op); - } - }else { - // Otherwise, ignore operation and continue - printf("GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. Skipping:"); - op.print(stdout); - printf("\n"); - } - if(g_debug_execution >= 3) - print_impl(stdout); + block = !concurrent_streams_empty(); pthread_mutex_unlock(&m_lock); - if( m_cuda_launch_blocking || stream == NULL ) { - unsigned int wait_amount = 100; - unsigned int wait_cap = 100000; // 100ms - while( !empty() ) { - // sleep to prevent CPU hog by empty spin - // sleep time increased exponentially ensure fast response when needed - usleep(wait_amount); - wait_amount *= 2; - if (wait_amount > wait_cap) - wait_amount = wait_cap; - } + }; + + pthread_mutex_lock(&m_lock); + if (!m_gpu->cycle_insn_cta_max_hit()) { + // Accept the stream operation if the maximum cycle/instruction/cta counts + // are not triggered + if (stream && !m_cuda_launch_blocking) { + stream->push(op); + } else { + op.set_stream(&m_stream_zero); + m_stream_zero.push(op); } + } else { + // Otherwise, ignore operation and continue + printf( + "GPGPU-Sim API: Maximum cycle, instruction, or CTA count hit. " + "Skipping:"); + op.print(stdout); + printf("\n"); + } + if (g_debug_execution >= 3) print_impl(stdout); + pthread_mutex_unlock(&m_lock); + if (m_cuda_launch_blocking || stream == NULL) { + unsigned int wait_amount = 100; + unsigned int wait_cap = 100000; // 100ms + while (!empty()) { + // sleep to prevent CPU hog by empty spin + // sleep time increased exponentially ensure fast response when needed + usleep(wait_amount); + wait_amount *= 2; + if (wait_amount > wait_cap) wait_amount = wait_cap; + } + } } -void stream_manager::pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ){ - std::list<CUstream_st *>::iterator s; - for( s=m_streams.begin(); s != m_streams.end(); s++ ) { - stream_operation op(*s,e,flags); - push(op); - } +void stream_manager::pushCudaStreamWaitEventToAllStreams(CUevent_st *e, + unsigned int flags) { + std::list<CUstream_st *>::iterator s; + for (s = m_streams.begin(); s != m_streams.end(); s++) { + stream_operation op(*s, e, flags); + push(op); + } } diff --git a/src/stream_manager.h b/src/stream_manager.h index 3fbdbaf..d543e68 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -7,269 +7,271 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef STREAM_MANAGER_H_INCLUDED #define STREAM_MANAGER_H_INCLUDED -#include "abstract_hardware_model.h" -#include <list> #include <pthread.h> #include <time.h> +#include <list> +#include "abstract_hardware_model.h" -//class stream_barrier { -//public: +// class stream_barrier { +// public: // stream_barrier() { m_pending_streams=0; } // void inc() { m_pending_streams++; } // void dec() { assert(m_pending_streams); m_pending_streams--; } // unsigned value() const { return m_pending_streams; } -//private: +// private: // unsigned m_pending_streams; //}; enum stream_operation_type { - stream_no_op, - stream_memcpy_host_to_device, - stream_memcpy_device_to_host, - stream_memcpy_device_to_device, - stream_memcpy_to_symbol, - stream_memcpy_from_symbol, - stream_kernel_launch, - stream_event, - stream_wait_event + stream_no_op, + stream_memcpy_host_to_device, + stream_memcpy_device_to_host, + stream_memcpy_device_to_device, + stream_memcpy_to_symbol, + stream_memcpy_from_symbol, + stream_kernel_launch, + stream_event, + stream_wait_event }; class stream_operation { -public: - stream_operation() - { - m_kernel=NULL; - m_type = stream_no_op; - m_stream = NULL; - m_done=true; - } - stream_operation( const void *src, const char *symbol, size_t count, size_t offset, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_stream = stream; - m_type=stream_memcpy_to_symbol; - m_host_address_src=src; - m_symbol=symbol; - m_cnt=count; - m_offset=offset; - m_done=false; - } - stream_operation( const char *symbol, void *dst, size_t count, size_t offset, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_stream = stream; - m_type=stream_memcpy_from_symbol; - m_host_address_dst=dst; - m_symbol=symbol; - m_cnt=count; - m_offset=offset; - m_done=false; - } - stream_operation( kernel_info_t *kernel, bool sim_mode, struct CUstream_st *stream ) - { - m_type=stream_kernel_launch; - m_kernel=kernel; - m_sim_mode=sim_mode; - m_stream=stream; - m_done=false; - } - stream_operation( struct CUevent_st *e, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_event; - m_event=e; - m_stream=stream; - m_done=false; - } - stream_operation( struct CUstream_st *stream, class CUevent_st *e, unsigned int flags ) - { - m_kernel=NULL; - m_type=stream_wait_event; - m_event=e; - m_stream=stream; - m_done=false; - } - stream_operation( const void *host_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_host_to_device; - m_host_address_src =host_address_src; - m_device_address_dst=device_address_dst; - m_host_address_dst=NULL; - m_device_address_src=0; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } - stream_operation( size_t device_address_src, void *host_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_device_to_host; - m_device_address_src=device_address_src; - m_host_address_dst=host_address_dst; - m_device_address_dst=0; - m_host_address_src=NULL; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } - stream_operation( size_t device_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) - { - m_kernel=NULL; - m_type=stream_memcpy_device_to_device; - m_device_address_src=device_address_src; - m_device_address_dst=device_address_dst; - m_host_address_src=NULL; - m_host_address_dst=NULL; - m_cnt=cnt; - m_stream=stream; - m_sim_mode=false; - m_done=false; - } + public: + stream_operation() { + m_kernel = NULL; + m_type = stream_no_op; + m_stream = NULL; + m_done = true; + } + stream_operation(const void *src, const char *symbol, size_t count, + size_t offset, struct CUstream_st *stream) { + m_kernel = NULL; + m_stream = stream; + m_type = stream_memcpy_to_symbol; + m_host_address_src = src; + m_symbol = symbol; + m_cnt = count; + m_offset = offset; + m_done = false; + } + stream_operation(const char *symbol, void *dst, size_t count, size_t offset, + struct CUstream_st *stream) { + m_kernel = NULL; + m_stream = stream; + m_type = stream_memcpy_from_symbol; + m_host_address_dst = dst; + m_symbol = symbol; + m_cnt = count; + m_offset = offset; + m_done = false; + } + stream_operation(kernel_info_t *kernel, bool sim_mode, + struct CUstream_st *stream) { + m_type = stream_kernel_launch; + m_kernel = kernel; + m_sim_mode = sim_mode; + m_stream = stream; + m_done = false; + } + stream_operation(struct CUevent_st *e, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_event; + m_event = e; + m_stream = stream; + m_done = false; + } + stream_operation(struct CUstream_st *stream, class CUevent_st *e, + unsigned int flags) { + m_kernel = NULL; + m_type = stream_wait_event; + m_event = e; + m_stream = stream; + m_done = false; + } + stream_operation(const void *host_address_src, size_t device_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_host_to_device; + m_host_address_src = host_address_src; + m_device_address_dst = device_address_dst; + m_host_address_dst = NULL; + m_device_address_src = 0; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } + stream_operation(size_t device_address_src, void *host_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_device_to_host; + m_device_address_src = device_address_src; + m_host_address_dst = host_address_dst; + m_device_address_dst = 0; + m_host_address_src = NULL; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } + stream_operation(size_t device_address_src, size_t device_address_dst, + size_t cnt, struct CUstream_st *stream) { + m_kernel = NULL; + m_type = stream_memcpy_device_to_device; + m_device_address_src = device_address_src; + m_device_address_dst = device_address_dst; + m_host_address_src = NULL; + m_host_address_dst = NULL; + m_cnt = cnt; + m_stream = stream; + m_sim_mode = false; + m_done = false; + } - bool is_kernel() const { return m_type == stream_kernel_launch; } - bool is_mem() const { - return m_type == stream_memcpy_host_to_device || - m_type == stream_memcpy_device_to_host || - m_type == stream_memcpy_host_to_device; - } - bool is_noop() const { return m_type == stream_no_op; } - bool is_done() const { return m_done; } - kernel_info_t *get_kernel() { return m_kernel; } - bool do_operation( gpgpu_sim *gpu ); - void print( FILE *fp ) const; - struct CUstream_st *get_stream() { return m_stream; } - void set_stream( CUstream_st *stream ) { m_stream = stream; } + bool is_kernel() const { return m_type == stream_kernel_launch; } + bool is_mem() const { + return m_type == stream_memcpy_host_to_device || + m_type == stream_memcpy_device_to_host || + m_type == stream_memcpy_host_to_device; + } + bool is_noop() const { return m_type == stream_no_op; } + bool is_done() const { return m_done; } + kernel_info_t *get_kernel() { return m_kernel; } + bool do_operation(gpgpu_sim *gpu); + void print(FILE *fp) const; + struct CUstream_st *get_stream() { + return m_stream; + } + void set_stream(CUstream_st *stream) { m_stream = stream; } -private: - struct CUstream_st *m_stream; + private: + struct CUstream_st *m_stream; - bool m_done; + bool m_done; - stream_operation_type m_type; - size_t m_device_address_dst; - size_t m_device_address_src; - void *m_host_address_dst; - const void *m_host_address_src; - size_t m_cnt; + stream_operation_type m_type; + size_t m_device_address_dst; + size_t m_device_address_src; + void *m_host_address_dst; + const void *m_host_address_src; + size_t m_cnt; - const char *m_symbol; - size_t m_offset; + const char *m_symbol; + size_t m_offset; - bool m_sim_mode; - kernel_info_t *m_kernel; - struct CUevent_st *m_event; + bool m_sim_mode; + kernel_info_t *m_kernel; + struct CUevent_st *m_event; }; struct CUevent_st { -public: - CUevent_st( bool blocking ) - { - m_uid = ++m_next_event_uid; - m_blocking = blocking; - m_updates = 0; - m_wallclock = 0; - m_gpu_tot_sim_cycle = 0; - m_done = false; - } - void update( double cycle, time_t clk ) - { - m_updates++; - m_wallclock=clk; - m_gpu_tot_sim_cycle=cycle; - m_done = true; - } - //void set_done() { assert(!m_done); m_done=true; } - int get_uid() const { return m_uid; } - unsigned num_updates() const { return m_updates; } - bool done() const { return m_done; } - time_t clock() const { return m_wallclock; } -private: - int m_uid; - bool m_blocking; - bool m_done; - int m_updates; - time_t m_wallclock; - double m_gpu_tot_sim_cycle; + public: + CUevent_st(bool blocking) { + m_uid = ++m_next_event_uid; + m_blocking = blocking; + m_updates = 0; + m_wallclock = 0; + m_gpu_tot_sim_cycle = 0; + m_done = false; + } + void update(double cycle, time_t clk) { + m_updates++; + m_wallclock = clk; + m_gpu_tot_sim_cycle = cycle; + m_done = true; + } + // void set_done() { assert(!m_done); m_done=true; } + int get_uid() const { return m_uid; } + unsigned num_updates() const { return m_updates; } + bool done() const { return m_done; } + time_t clock() const { return m_wallclock; } + + private: + int m_uid; + bool m_blocking; + bool m_done; + int m_updates; + time_t m_wallclock; + double m_gpu_tot_sim_cycle; - static int m_next_event_uid; + static int m_next_event_uid; }; struct CUstream_st { -public: - CUstream_st(); - bool empty(); - bool busy(); - void synchronize(); - void push( const stream_operation &op ); - void record_next_done(); - stream_operation next(); - void cancel_front(); //front operation fails, cancle the pending status - stream_operation &front() { return m_operations.front(); } - void print( FILE *fp ); - unsigned get_uid() const { return m_uid; } + public: + CUstream_st(); + bool empty(); + bool busy(); + void synchronize(); + void push(const stream_operation &op); + void record_next_done(); + stream_operation next(); + void cancel_front(); // front operation fails, cancle the pending status + stream_operation &front() { return m_operations.front(); } + void print(FILE *fp); + unsigned get_uid() const { return m_uid; } -private: - unsigned m_uid; - static unsigned sm_next_stream_uid; + private: + unsigned m_uid; + static unsigned sm_next_stream_uid; - std::list<stream_operation> m_operations; - bool m_pending; // front operation has started but not yet completed + std::list<stream_operation> m_operations; + bool m_pending; // front operation has started but not yet completed - pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream operation at one time + pthread_mutex_t m_lock; // ensure only one host or gpu manipulates stream + // operation at one time }; class stream_manager { -public: - stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ); - bool register_finished_kernel(unsigned grid_uid ); - bool check_finished_kernel( ); - stream_operation front(); - void add_stream( CUstream_st *stream ); - void destroy_stream( CUstream_st *stream ); - bool concurrent_streams_empty(); - bool empty_protected(); - bool empty(); - void print( FILE *fp); - void push( stream_operation op ); - void pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ); - bool operation(bool * sim); - void stop_all_running_kernels(); - unsigned size() {return m_streams.size(); }; - bool is_blocking() {return m_cuda_launch_blocking; }; -private: - void print_impl( FILE *fp); + public: + stream_manager(gpgpu_sim *gpu, bool cuda_launch_blocking); + bool register_finished_kernel(unsigned grid_uid); + bool check_finished_kernel(); + stream_operation front(); + void add_stream(CUstream_st *stream); + void destroy_stream(CUstream_st *stream); + bool concurrent_streams_empty(); + bool empty_protected(); + bool empty(); + void print(FILE *fp); + void push(stream_operation op); + void pushCudaStreamWaitEventToAllStreams(CUevent_st *e, unsigned int flags); + bool operation(bool *sim); + void stop_all_running_kernels(); + unsigned size() { return m_streams.size(); }; + bool is_blocking() { return m_cuda_launch_blocking; }; + + private: + void print_impl(FILE *fp); - bool m_cuda_launch_blocking; - gpgpu_sim *m_gpu; - std::list<CUstream_st *> m_streams; - std::map<unsigned,CUstream_st *> m_grid_id_to_stream; - CUstream_st m_stream_zero; - bool m_service_stream_zero; - pthread_mutex_t m_lock; + bool m_cuda_launch_blocking; + gpgpu_sim *m_gpu; + std::list<CUstream_st *> m_streams; + std::map<unsigned, CUstream_st *> m_grid_id_to_stream; + CUstream_st m_stream_zero; + bool m_service_stream_zero; + pthread_mutex_t m_lock; }; #endif diff --git a/src/tr1_hash_map.h b/src/tr1_hash_map.h index 8c7513e..c92ce15 100644 --- a/src/tr1_hash_map.h +++ b/src/tr1_hash_map.h @@ -7,44 +7,46 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once // detection and fallback for unordered_map in C++0x #ifdef __cplusplus - // detect GCC 4.3 or later and use unordered map (part of C++0x) - // unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its enabled. - #if defined( __GNUC__ ) and not defined( _GLIBCXX_DEBUG ) - #if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 - #include <unordered_map> - #define tr1_hash_map std::unordered_map - #define tr1_hash_map_ismap 0 - #else - #include <map> - #define tr1_hash_map std::map - #define tr1_hash_map_ismap 1 - #endif - #else - #include <map> - #define tr1_hash_map std::map - #define tr1_hash_map_ismap 1 - #endif +// detect GCC 4.3 or later and use unordered map (part of C++0x) +// unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its +// enabled. +#if defined(__GNUC__) and not defined(_GLIBCXX_DEBUG) +#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 +#include <unordered_map> +#define tr1_hash_map std::unordered_map +#define tr1_hash_map_ismap 0 +#else +#include <map> +#define tr1_hash_map std::map +#define tr1_hash_map_ismap 1 +#endif +#else +#include <map> +#define tr1_hash_map std::map +#define tr1_hash_map_ismap 1 +#endif #endif diff --git a/src/trace.cc b/src/trace.cc index 5171e46..1c435b2 100644 --- a/src/trace.cc +++ b/src/trace.cc @@ -7,50 +7,51 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "trace.h" #include "string.h" namespace Trace { - #define TS_TUP_BEGIN(X) const char* trace_streams_str[] = { #define TS_TUP(X) #X -#define TS_TUP_END(X) }; +#define TS_TUP_END(X) \ + } \ + ; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END - bool enabled = false; - int sampling_core = 0; - int sampling_memory_partition = -1; - bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; - const char* config_str; +bool enabled = false; +int sampling_core = 0; +int sampling_memory_partition = -1; +bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; +const char* config_str; - void init() - { - for ( unsigned i = 0; i < NUM_TRACE_STREAMS; ++i ) { - if ( strstr( config_str, trace_streams_str[i] ) != NULL ) { - trace_streams_enabled[ i ] = true; - } - } +void init() { + for (unsigned i = 0; i < NUM_TRACE_STREAMS; ++i) { + if (strstr(config_str, trace_streams_str[i]) != NULL) { + trace_streams_enabled[i] = true; } -} + } +} +} // namespace Trace diff --git a/src/trace.h b/src/trace.h index 0b96dcf..8d74151 100644 --- a/src/trace.h +++ b/src/trace.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. // This file is inspired by the trace system in gem5. // This is a highly simplified version adpated for gpgpusim @@ -35,52 +36,57 @@ namespace Trace { #define TS_TUP_BEGIN(X) enum X { #define TS_TUP(X) X -#define TS_TUP_END(X) }; +#define TS_TUP_END(X) \ + } \ + ; #include "trace_streams.tup" #undef TS_TUP_BEGIN #undef TS_TUP #undef TS_TUP_END - extern bool enabled; - extern int sampling_core; - extern int sampling_memory_partition; - extern const char* trace_streams_str[]; - extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; - extern const char* config_str; +extern bool enabled; +extern int sampling_core; +extern int sampling_memory_partition; +extern const char* trace_streams_str[]; +extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; +extern const char* config_str; - void init(); - -} // namespace Trace +void init(); +} // namespace Trace #if TRACING_ON #define SIM_PRINT_STR "GPGPU-Sim Cycle %llu: %s - " #define DTRACE(x) ((Trace::trace_streams_enabled[Trace::x]) && Trace::enabled) -#define DPRINTF(x, ...) do {\ - if (DTRACE(x)) {\ - printf( SIM_PRINT_STR,\ - m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::x] );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define DPRINTF(x, ...) \ + do { \ + if (DTRACE(x)) { \ + printf(SIM_PRINT_STR, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::x]); \ + printf(__VA_ARGS__); \ + } \ + } while (0) -#define DPRINTFG(x, ...) do {\ - if (DTRACE(x)) {\ - printf( SIM_PRINT_STR,\ - gpu_sim_cycle + gpu_tot_sim_cycle,\ - Trace::trace_streams_str[Trace::x] );\ - printf(__VA_ARGS__);\ - }\ -} while (0) +#define DPRINTFG(x, ...) \ + do { \ + if (DTRACE(x)) { \ + printf(SIM_PRINT_STR, gpu_sim_cycle + gpu_tot_sim_cycle, \ + Trace::trace_streams_str[Trace::x]); \ + printf(__VA_ARGS__); \ + } \ + } while (0) -#else +#else #define DTRACE(x) (false) -#define DPRINTF(x, ...) do {} while (0) -#define DPRINTFG(x, ...) do {} while (0) +#define DPRINTF(x, ...) \ + do { \ + } while (0) +#define DPRINTFG(x, ...) \ + do { \ + } while (0) -#endif +#endif -#endif +#endif |