From dcf7e6a8445a5d8e4383add44ca6f5d0d198ab02 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Mon, 17 Jul 2017 22:32:30 -0400 Subject: Improving GPU core model. This commits contains: 1- round robin inst issue for warp multiple schedulers 2- add sector mask in the memory request (to bused later for L2 sector cache) 3- Adding Fermi coalescer 4- Ensure different exen units are used in dual_issue mode 5- Report how many dual_issue happened 6- Adding oldest_first scheduler --- src/abstract_hardware_model.cc | 54 +++++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 14 deletions(-) (limited to 'src/abstract_hardware_model.cc') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index fe6f8ab..f75c525 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -314,12 +314,12 @@ void warp_inst_t::generate_mem_accesses() break; case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch == 13 ) { - if(isatomic()) - memory_coalescing_arch_13_atomic(is_write, access_type); - else - memory_coalescing_arch_13(is_write, access_type); - } else abort(); + if( m_config->gpgpu_coalesce_arch == 13 || m_config->gpgpu_coalesce_arch == 20) { + if(isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else abort(); break; @@ -343,7 +343,7 @@ void warp_inst_t::generate_mem_accesses() byte_mask.set(idx+i); } for( a=accesses.begin(); a != accesses.end(); ++a ) - m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second,byte_mask) ); + m_accessq.push_back( mem_access_t(access_type,a->first,cache_block_size,is_write,a->second, byte_mask, mem_access_sector_mask_t())); } if ( space.get_type() == global_space ) { @@ -352,11 +352,37 @@ void warp_inst_t::generate_mem_accesses() m_mem_accesses_created=true; } -void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type access_type ) +void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_type ) { // see the CUDA manual where it discusses coalescing rules before reading this unsigned segment_size = 0; - unsigned warp_parts = m_config->mem_warp_parts; + unsigned warp_parts; + + //TO DO: need to double check how double number are coalesced! + if(data_size == 1) + { + //If it is byte data, then coalesce on the whole 32 threads, regardless the arch version + warp_parts = 1; + } + else if(m_config->gpgpu_coalesce_arch == 13) + { + //mem_warp_parts should equal 2 for arch=13 + //we still use the parameter mem_warp_parts for arch=13 to ensure it is backward compatibility with older gpgpu config files + warp_parts = m_config->mem_warp_parts; + } + else if(m_config->gpgpu_coalesce_arch == 20) + { + //It is expected that L1_warp_parts_non_cached = 4 and L1_warp_parts_cached = 1 for arch=20 + //non cached, coalesce on 8 threads to generate 32 bytes accesses + //cached, coalesce on 32 threads to generate 128 bytes accesses + if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + warp_parts = m_config->L1_warp_parts_non_cached; + else + warp_parts = m_config->L1_warp_parts_cached; + } + else + abort(); + switch( data_size ) { case 1: segment_size = 32; break; case 2: segment_size = 64; break; @@ -410,13 +436,13 @@ void warp_inst_t::memory_coalescing_arch_13( bool is_write, mem_access_type acce new_addr_type addr = t->first; const transaction_info &info = t->second; - memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); } } } -void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_type access_type ) +void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type access_type ) { assert(space.get_type() == global_space); // Atomics allowed only for global memory @@ -485,13 +511,13 @@ void warp_inst_t::memory_coalescing_arch_13_atomic( bool is_write, mem_access_ty for(t=transaction_list.begin(); t!=transaction_list.end(); t++) { // For each transaction const transaction_info &info = *t; - memory_coalescing_arch_13_reduce_and_send(is_write, access_type, info, addr, segment_size); + memory_coalescing_arch_reduce_and_send(is_write, access_type, info, addr, segment_size); } } } } -void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) +void warp_inst_t::memory_coalescing_arch_reduce_and_send( bool is_write, mem_access_type access_type, const transaction_info &info, new_addr_type addr, unsigned segment_size ) { assert( (addr & (segment_size-1)) == 0 ); @@ -540,7 +566,7 @@ void warp_inst_t::memory_coalescing_arch_13_reduce_and_send( bool is_write, mem_ assert(lower_half_used && upper_half_used); } } - m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes) ); + m_accessq.push_back( mem_access_t(access_type,addr,size,is_write,info.active,info.bytes, info.chunks) ); } void warp_inst_t::completed( unsigned long long cycle ) const -- cgit v1.3 From b806d3a4021a12638e1e67fcd67012dca4b1689e Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Mon, 17 Jul 2017 23:20:42 -0400 Subject: Fixing some typos and adding comments --- src/abstract_hardware_model.cc | 14 +++++++------- src/abstract_hardware_model.h | 2 +- src/gpgpu-sim/gpu-cache.cc | 2 +- src/gpgpu-sim/gpu-sim.cc | 12 ++++++------ src/gpgpu-sim/gpu-sim.h | 2 -- src/gpgpu-sim/shader.cc | 43 +++++++++++++++++++++--------------------- src/gpgpu-sim/shader.h | 4 ++-- 7 files changed, 39 insertions(+), 40 deletions(-) (limited to 'src/abstract_hardware_model.cc') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index f75c525..64eb43c 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -315,11 +315,11 @@ void warp_inst_t::generate_mem_accesses() case global_space: case local_space: case param_space_local: if( m_config->gpgpu_coalesce_arch == 13 || m_config->gpgpu_coalesce_arch == 20) { - if(isatomic()) - memory_coalescing_arch_atomic(is_write, access_type); - else - memory_coalescing_arch(is_write, access_type); - } else abort(); + if(isatomic()) + memory_coalescing_arch_atomic(is_write, access_type); + else + memory_coalescing_arch(is_write, access_type); + } else abort(); break; @@ -358,7 +358,7 @@ void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_ unsigned segment_size = 0; unsigned warp_parts; - //TO DO: need to double check how double number are coalesced! + //TO DO: need to double check how doubles are coalesced! if(data_size == 1) { //If it is byte data, then coalesce on the whole 32 threads, regardless the arch version @@ -367,7 +367,7 @@ void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_ else if(m_config->gpgpu_coalesce_arch == 13) { //mem_warp_parts should equal 2 for arch=13 - //we still use the parameter mem_warp_parts for arch=13 to ensure it is backward compatibility with older gpgpu config files + //use the parameter mem_warp_parts for arch=13 to ensure it is backward compatibility with older gpgpu config files warp_parts = m_config->mem_warp_parts; } else if(m_config->gpgpu_coalesce_arch == 20) diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index cd185a1..910a9ed 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -685,7 +685,7 @@ public: bool wr, const active_mask_t &active_mask, const mem_access_byte_mask_t &byte_mask, - const mem_access_sector_mask_t §or_mask) + const mem_access_sector_mask_t §or_mask) : m_warp_mask(active_mask), m_byte_mask(byte_mask), m_sector_mask(sector_mask) { init(); diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index cfd0dc8..7af7db0 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -868,7 +868,7 @@ data_cache::wr_miss_wa( new_addr_type addr, false, // Now performing a read mf->get_access_warp_mask(), mf->get_access_byte_mask(), - mf->get_access_sector_mask()); + mf->get_access_sector_mask()); mem_fetch *n_mf = new mem_fetch( *ma, NULL, diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 26432c4..483a7b6 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -302,7 +302,7 @@ void shader_core_config::reg_options(class OptionParser * opp) "0"); option_parser_register(opp, "-gpgpu_local_mem_map", OPT_BOOL, &gpgpu_local_mem_map, "Mapping from local memory space address to simulated GPU physical address space (default = enabled)", - "1"); + "1"); option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, "Number of register banks (default = 8)", "8"); @@ -346,17 +346,17 @@ void shader_core_config::reg_options(class OptionParser * opp) "number of collector unit in ports (default = 0)", "0"); option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, - "Coalescing arch (GT200 = 13, Fermi = 20)", + "Coalescing arch (GT200 = 13, Fermi = 20)", "13"); option_parser_register(opp, "-gpgpu_num_sched_per_core", OPT_INT32, &gpgpu_num_sched_per_core, "Number of warp schedulers per core", "1"); option_parser_register(opp, "-gpgpu_max_insn_issue_per_warp", OPT_INT32, &gpgpu_max_insn_issue_per_warp, - "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)", - "2"); + "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)", + "2"); option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, &gpgpu_dual_issue_diff_exec_units, - "should dual issue use two different execution unit resources", - "1"); + "should dual issue use two different execution unit resources", + "1"); option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, &simt_core_sim_order, "Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)", "1"); diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 23e6144..7d92c66 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -314,8 +314,6 @@ public: unsigned num_shader() const { return m_shader_config.num_shader(); } unsigned num_cluster() const { return m_shader_config.n_simt_clusters; } unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; } - unsigned get_gpu_max_cycle_opt() const { return gpu_max_cycle_opt; } - unsigned get_gpu_max_insn_opt() const { return gpu_max_insn_opt; } private: void init_clock_domains(void ); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 015995e..d826b69 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -132,8 +132,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE : sched_config.find("gto") != std::string::npos ? CONCRETE_SCHEDULER_GTO : - sched_config.find("old") != std::string::npos ? - CONCRETE_SCHEDULER_OLDEST_FIRST : + sched_config.find("old") != std::string::npos ? + CONCRETE_SCHEDULER_OLDEST_FIRST : sched_config.find("warp_limiting") != std::string::npos ? CONCRETE_SCHEDULER_WARP_LIMITING: NUM_CONCRETE_SCHEDULERS; @@ -187,16 +187,16 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, break; case CONCRETE_SCHEDULER_OLDEST_FIRST: schedulers.push_back( - new oldest_scheduler( m_stats, - this, - m_scoreboard, - m_simt_stack, - &m_warp, - &m_pipeline_reg[ID_OC_SP], - &m_pipeline_reg[ID_OC_SFU], - &m_pipeline_reg[ID_OC_MEM], - i - ) + new oldest_scheduler( m_stats, + this, + m_scoreboard, + m_simt_stack, + &m_warp, + &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_MEM], + i + ) ); break; case CONCRETE_SCHEDULER_WARP_LIMITING: @@ -750,12 +750,13 @@ void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* void shader_core_ctx::issue(){ - unsigned j; - for (unsigned i = 0; i < schedulers.size(); i++) { - j = (Issue_Prio + i) % schedulers.size(); - schedulers[j]->cycle(); - } - Issue_Prio = (Issue_Prio+1)% schedulers.size(); + //Ensure fair round robin issu between schedulers + unsigned j; + for (unsigned i = 0; i < schedulers.size(); i++) { + j = (Issue_Prio + i) % schedulers.size(); + schedulers[j]->cycle(); + } + Issue_Prio = (Issue_Prio+1)% schedulers.size(); //really is issue; //for (unsigned i = 0; i < schedulers.size(); i++) { @@ -876,8 +877,8 @@ void scheduler_unit::cycle() unsigned checked=0; unsigned issued=0; exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; - unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; - bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; + unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); //Jin: handle cdp latency; @@ -1062,7 +1063,7 @@ void oldest_scheduler::order_warps() m_supervised_warps, m_last_supervised_issued, m_supervised_warps.size(), - ORDERED_PRIORITY_FUNC_ONLY, + ORDERED_PRIORITY_FUNC_ONLY, scheduler_unit::sort_warps_by_oldest_dynamic_id ); } diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 395c1ed..2fed420 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -316,7 +316,7 @@ enum concrete_scheduler CONCRETE_SCHEDULER_GTO, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE, CONCRETE_SCHEDULER_WARP_LIMITING, - CONCRETE_SCHEDULER_OLDEST_FIRST, + CONCRETE_SCHEDULER_OLDEST_FIRST, NUM_CONCRETE_SCHEDULERS }; @@ -456,7 +456,7 @@ public: : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id ){} virtual ~oldest_scheduler () {} virtual void order_warps (); - virtual void done_adding_supervised_warps() { + virtual void done_adding_supervised_warps() { m_last_supervised_issued = m_supervised_warps.begin(); } -- cgit v1.3 From f23021ad8663636e1103bd75a742480cb6238435 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Fri, 27 Oct 2017 22:35:52 -0400 Subject: add more statistics and chaging Pascal config --- configs/Pascal-P102-GDDR5X/gpgpusim.config | 12 +++++++----- src/abstract_hardware_model.cc | 2 +- src/gpgpu-sim/dram.cc | 3 ++- src/gpgpu-sim/dram_sched.cc | 2 ++ src/gpgpu-sim/gpu-cache.cc | 2 ++ src/gpgpu-sim/gpu-sim.cc | 2 +- src/gpgpu-sim/gpu-sim.h | 2 +- src/gpgpu-sim/l2cache.cc | 2 +- src/gpgpu-sim/mem_latency_stat.cc | 17 +++++++++++++---- src/gpgpu-sim/mem_latency_stat.h | 4 ++++ src/gpgpu-sim/shader.cc | 8 ++++---- 11 files changed, 38 insertions(+), 18 deletions(-) (limited to 'src/abstract_hardware_model.cc') diff --git a/configs/Pascal-P102-GDDR5X/gpgpusim.config b/configs/Pascal-P102-GDDR5X/gpgpusim.config index 5419c51..e830023 100644 --- a/configs/Pascal-P102-GDDR5X/gpgpusim.config +++ b/configs/Pascal-P102-GDDR5X/gpgpusim.config @@ -59,22 +59,24 @@ # ** Optional parameter - Required when mshr_type==Texture Fifo # Note: Hashing set index function (H) only applies to a set size of 32 or 64. # Pascal GP102 has 96KB Shared memory -# Pascal GP102 has 64KB L1 cache +# Pascal GP102 has 24KB L1 cache # The defulat is to disable the L1 cache, unless cache modifieres is used --gpgpu_cache:dl1 S:64:128:6,L:L:m:N:H,A:128:8,8 +-gpgpu_cache:dl1 N:32:128:6,L:L:m:N:H,S:128:8,16 -gpgpu_shmem_size 49152 -gmem_skip_L1D 1 -icnt_flit_size 40 -gpgpu_n_cluster_ejection_buffer_size 32 # 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache --gpgpu_cache:dl2 S:64:128:16,L:B:m:F:L,A:256:4,4:0,32 +-gpgpu_cache:dl2 S:64:128:16,L:B:m:F:L,A:128:4,16:0,32 -gpgpu_cache:dl2_texture_only 0 -gpgpu_dram_partition_queues 32:32:32:32 +#-gpgpu_flush_l2_cache 1 # 4 KB Inst. -gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 # 48 KB Tex +# this is unused -gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,F:128:4,128:2 # 12 KB Const -gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 @@ -100,7 +102,7 @@ -gpgpu_shmem_limited_broadcast 0 -gpgpu_shmem_warp_parts 1 # Use Fermi Coalsce arhitetecture which is the same as Pascal --gpgpu_coalesce_arch 20 +-gpgpu_coalesce_arch 61 ## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units -gpgpu_max_insn_issue_per_warp 2 @@ -143,7 +145,7 @@ -dram_bnkgrp_indexing_policy 1 #-Seperate_Write_Queue_Enable 1 -#-Write_Queue_Size 32:28:16 +#-Write_Queue_Size 64:56:32 # Pascal 102 has four schedulers per core -gpgpu_num_sched_per_core 2 diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 51265fd..d2a155c 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -314,7 +314,7 @@ void warp_inst_t::generate_mem_accesses() break; case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch == 13 || m_config->gpgpu_coalesce_arch == 20) { + if( m_config->gpgpu_coalesce_arch >= 13 && m_config->gpgpu_coalesce_arch <= 62) { if(isatomic()) memory_coalescing_arch_atomic(is_write, access_type); else diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index 7ed7b6f..de37f64 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -570,9 +570,10 @@ bool dram_t::issue_col_command(int j) bkgrp[grp]->RTPLc = m_config->tRTPL; issued = true; if(bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R) - n_rd_L2_A++; + n_rd_L2_A++; else n_rd++; + bwutil += m_config->BL/m_config->data_command_freq_ratio; bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; bk[j]->n_access++; diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index 7a140c5..ac4c827 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -220,6 +220,8 @@ void dram_t::scheduler_frfcfs() bk[b]->mrq = req; if (m_config->gpgpu_memlatency_stat) { mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - bk[b]->mrq->timestamp; + m_stats->tot_mrq_latency += mrq_latency; + m_stats->tot_mrq_num++; bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle; m_stats->mrq_lat_table[LOGB2(mrq_latency)]++; if (mrq_latency > m_stats->max_mrq_latency) { diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index d199cca..32c2bb1 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -333,8 +333,10 @@ void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf) void tag_array::flush() { for (unsigned i=0; i < m_config.get_num_lines(); i++) + if(m_lines[i]->is_modified_line()) { for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + } } float tag_array::windowed_miss_rate( ) const diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 6179d46..0e06c5c 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1412,7 +1412,7 @@ void gpgpu_sim::cycle() if (mf) { unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size(); if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) { - if (!mf->get_is_write()) + //if (!mf->get_is_write()) mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle); mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle); ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size ); diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 52c4643..043fcee 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -199,7 +199,7 @@ struct memory_config { assert(nbkgrp>0 && "Number of bank groups cannot be zero"); tRCDWR = tRCD-(WL+1); tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); - tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); tWTP = (WL+(BL/data_command_freq_ratio)+tWR); dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index cac59f1..8fbf448 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -628,6 +628,7 @@ std::vector memory_sub_partition::breakdown_request_to_sector_reques void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle ) { if (m_req) { + m_stats->memlatstat_icnt2mem_pop(m_req); std::vector reqs; if(m_config->m_L2_config.m_cache_type == SECTOR) reqs = breakdown_request_to_sector_requests(m_req); @@ -637,7 +638,6 @@ void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle ) for(unsigned i=0; imemlatstat_icnt2mem_pop(req); if( req->istexture() ) { m_icnt_L2_queue->push(req); req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index fde0eff..35d6d84 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -75,6 +75,10 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf max_mf_latency = 0; max_icnt2mem_latency = 0; max_icnt2sh_latency = 0; + tot_icnt2mem_latency = 0; + tot_icnt2sh_latency = 0; + tot_mrq_num = 0; + tot_mrq_latency = 0; memset(mrq_lat_table, 0, sizeof(unsigned)*32); memset(dq_lat_table, 0, sizeof(unsigned)*32); memset(mf_lat_table, 0, sizeof(unsigned)*32); @@ -158,6 +162,7 @@ void memory_stats_t::memlatstat_read_done(mem_fetch *mf) mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency; unsigned icnt2sh_latency; icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_return_timestamp(); + tot_icnt2sh_latency += icnt2sh_latency; icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; if (icnt2sh_latency > max_icnt2sh_latency) max_icnt2sh_latency = icnt2sh_latency; @@ -191,6 +196,7 @@ void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) if (m_memory_config->gpgpu_memlatency_stat) { unsigned icnt2mem_latency; icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_timestamp(); + tot_icnt2mem_latency += icnt2mem_latency; icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; if (icnt2mem_latency > max_icnt2mem_latency) max_icnt2mem_latency = icnt2mem_latency; @@ -216,14 +222,17 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk ) unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses; if (m_memory_config->gpgpu_memlatency_stat) { + printf("maxmflatency = %d \n", max_mf_latency); + printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); printf("maxmrqlatency = %d \n", max_mrq_latency); - printf("maxdqlatency = %d \n", max_dq_latency); - printf("maxmflatency = %d \n", max_mf_latency); + //printf("maxdqlatency = %d \n", max_dq_latency); + printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); if (num_mfs) { printf("averagemflatency = %lld \n", mf_total_lat/num_mfs); + printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs); + printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num); + printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs); } - printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); - printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); printf("mrq_lat_table:"); for (i=0; i< 32; i++) { printf("%d \t", mrq_lat_table[i]); diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h index 4968a3b..5b89202 100644 --- a/src/gpgpu-sim/mem_latency_stat.h +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -56,6 +56,10 @@ public: unsigned max_dq_latency; unsigned max_mf_latency; unsigned max_icnt2mem_latency; + unsigned long long int tot_icnt2mem_latency; + unsigned long long int tot_icnt2sh_latency; + unsigned long long int tot_mrq_latency; + unsigned long long int tot_mrq_num; unsigned max_icnt2sh_latency; unsigned mrq_lat_table[32]; unsigned dq_lat_table[32]; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index c14e19f..bf482fb 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -1458,7 +1458,7 @@ ldst_unit::process_cache_access( cache_t* cache, if( !write_sent ) delete mf; } else if ( status == RESERVATION_FAIL ) { - result = COAL_STALL; + result = BK_CONF; assert( !read_sent ); assert( !write_sent ); delete mf; @@ -1467,8 +1467,8 @@ ldst_unit::process_cache_access( cache_t* cache, //inst.clear_active( access.get_warp_mask() ); // threads in mf writeback when mf returns inst.accessq_pop_back(); } - if( !inst.accessq_empty() ) - result = BK_CONF; + if( !inst.accessq_empty() && result == NO_RC_FAIL) + result = COAL_STALL; return result; } @@ -1563,7 +1563,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea assert( CACHE_UNDEFINED != inst.cache_op ); stall_cond = process_memory_access_queue(m_L1D,inst); } - if( !inst.accessq_empty() ) + if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL) stall_cond = COAL_STALL; if (stall_cond != NO_RC_FAIL) { stall_reason = stall_cond; -- cgit v1.3 From 0e46a261dfeba9c19d1637f46277986d7eb1b9d8 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Tue, 9 Oct 2018 17:12:57 -0400 Subject: adding adaptive volta cache config --- configs/4.x-cfgs/SM7_TITANV/gpgpusim.config | 19 ++++++++++-------- src/abstract_hardware_model.cc | 2 ++ src/abstract_hardware_model.h | 2 ++ src/gpgpu-sim/gpu-cache.cc | 9 ++++----- src/gpgpu-sim/gpu-cache.h | 31 +++++++++++++++++++++++++---- src/gpgpu-sim/gpu-sim.cc | 3 +++ src/gpgpu-sim/shader.cc | 29 +++++++++++++++++++++++++++ src/gpgpu-sim/shader.h | 2 ++ 8 files changed, 80 insertions(+), 17 deletions(-) (limited to 'src/abstract_hardware_model.cc') diff --git a/configs/4.x-cfgs/SM7_TITANV/gpgpusim.config b/configs/4.x-cfgs/SM7_TITANV/gpgpusim.config index cc4c931..a7056db 100644 --- a/configs/4.x-cfgs/SM7_TITANV/gpgpusim.config +++ b/configs/4.x-cfgs/SM7_TITANV/gpgpusim.config @@ -65,19 +65,21 @@ # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo -# Defualt config is 64KB DL1 and 64KB shared memory --gpgpu_cache:dl1 S:4:128:128,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_cache:dl1PrefL1 S:4:128:192,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_cache:dl1PrefShared S:4:128:64,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_shmem_size 65536 --gpgpu_shmem_size_PrefL1 32768 --gpgpu_shmem_size_PrefShared 98304 +# Defualt config is 32KB DL1 and 96KB shared memory +# In Volta, we assign the remaining shared memory to L1 cache +# if the assigned shd mem = 0, then L1 cache = 128KB +# For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x +# disable this mode in case of multi kernels/apps execution +-adpative_volta_cache_config 1 +-gpgpu_cache:dl1 S:4:128:64,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 98304 -gmem_skip_L1D 0 -icnt_flit_size 40 -gpgpu_n_cluster_ejection_buffer_size 32 -l1_latency 28 -smem_latency 19 -gpgpu_flush_l1_cache 1 +-adpative_volta_cache_config 1 # 64 sets, each 128 bytes 24-way for each memory sub partition (192 KB per memory sub partition). This gives 4.5MB L2 cache -gpgpu_cache:dl2 S:64:128:24,L:B:m:L:L,A:384:4,32:0,32 @@ -87,7 +89,8 @@ # 128 KB Inst. -gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 -# 48 KB Tex +# 48 KB Tex +# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod -gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 # 64 KB Const -gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index d2a155c..39ed81c 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -591,6 +591,8 @@ kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info * //Jin: launch latency management m_launch_latency = g_kernel_launch_latency; + + volta_cache_config_set=false; } kernel_info_t::~kernel_info_t() diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index e708fa7..6fa2ba0 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -301,6 +301,8 @@ public: unsigned long long start_cycle; unsigned long long end_cycle; unsigned m_launch_latency; + + mutable bool volta_cache_config_set; }; struct core_config { diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index a11853a..8ae4702 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -29,7 +29,6 @@ #include "stat-tool.h" #include -#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 // used to allocate memory that is large enough to adapt the changes in cache size across kernels const char * cache_request_status_str(enum cache_request_status status) @@ -165,7 +164,7 @@ unsigned l2_cache_config::set_index(new_addr_type addr) const{ tag_array::~tag_array() { - unsigned cache_lines_num = MAX_DEFAULT_CACHE_SIZE_MULTIBLIER*m_config.get_num_lines(); + unsigned cache_lines_num = m_config.get_max_num_lines(); for(unsigned i=0; iis_modified_line()) { for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; @@ -396,7 +395,7 @@ void tag_array::flush() void tag_array::invalidate() { - for (unsigned i=0; i < m_config.get_num_lines(); i++) + for (unsigned i=0; i < m_config.get_max_num_lines(); i++) for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; } diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h index 4ed382c..9174d7f 100644 --- a/src/gpgpu-sim/gpu-cache.h +++ b/src/gpgpu-sim/gpu-cache.h @@ -38,6 +38,8 @@ #include "addrdec.h" #include +#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 + enum cache_block_state { INVALID=0, RESERVED, @@ -557,15 +559,15 @@ public: } if(m_alloc_policy == STREAMING) { //For streaming cache, we set the alloc policy to be on-fill to remove all line_alloc_fail stalls - //we set the MSHRs to be equal to the cache line. This is possible by moving TAG to be shared between cache line and MSHR enrty (i.e. for each cache line, there is an MSHR rntey associated with it) - // This is the easiest think we can think about to model (mimics) L1 streaming cache in Pascal and Volta + //we set the MSHRs to be equal to max allocated cache lines. This is possible by moving TAG to be shared between cache line and MSHR enrty (i.e. for each cache line, there is an MSHR rntey associated with it) + //This is the easiest think we can think about to model (mimic) L1 streaming cache in Pascal and Volta //Based on our microbenchmakrs, MSHRs entries have been increasing substantially in Pascal and Volta //For more information about streaming cache, see: // http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf // https://ieeexplore.ieee.org/document/8344474/ m_alloc_policy = ON_FILL; - m_mshr_entries = m_nset*m_assoc; + m_mshr_entries = m_nset*m_assoc*MAX_DEFAULT_CACHE_SIZE_MULTIBLIER; if(m_cache_type == SECTOR) m_mshr_entries *= SECTOR_CHUNCK_SIZE; m_mshr_max_merge = MAX_WARP_PER_SM; @@ -581,6 +583,7 @@ public: m_nset_log2 = LOGB2(m_nset); m_valid = true; m_atom_sz = (m_cache_type == SECTOR)? SECTOR_SIZE : m_line_sz; + original_m_assoc = m_assoc; //For more details about difference between FETCH_ON_WRITE and WRITE VALIDAE policies //Read: Jouppi, Norman P. "Cache write policies and performance". ISCA 93. @@ -646,7 +649,11 @@ public: assert( m_valid ); return m_nset * m_assoc; } - + unsigned get_max_num_lines() const + { + assert( m_valid ); + return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER * m_nset * original_m_assoc; + } void print( FILE *fp ) const { fprintf( fp, "Size = %d B (%d Set x %d-way x %d byte line)\n", @@ -687,6 +694,21 @@ public: { return m_mshr_type; } + void set_assoc(unsigned n) + { + //set new assoc. L1 cache dynamically resized in Volta + m_assoc = n; + } + unsigned get_nset() const + { + assert( m_valid ); + return m_nset; + } + unsigned get_total_size_inKB() const + { + assert( m_valid ); + return (m_assoc*m_nset*m_line_sz)/1024; + } FuncCache get_cache_status() {return cache_status;} char *m_config_string; char *m_config_stringPrefL1; @@ -708,6 +730,7 @@ protected: unsigned m_nset_log2; unsigned m_assoc; unsigned m_atom_sz; + unsigned original_m_assoc; enum replacement_policy_t m_replacement_policy; // 'L' = LRU, 'F' = FIFO enum write_policy_t m_write_policy; // 'T' = write through, 'B' = write back, 'R' = read only diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 08d4525..080cbac 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -307,6 +307,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register(opp, "-adpative_volta_cache_config", OPT_BOOL, &adpative_volta_cache_config, + "adpative_volta_cache_config", + "0"); option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_sizeDefault, "Size of shared memory per shader core (default 16kB)", "16384"); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 0e2e1c2..2cc8a2e 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -2744,6 +2744,35 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const abort(); } + if(adpative_volta_cache_config && !k.volta_cache_config_set) { + //For Volta, we assign the remaining shared memory to L1 cache + //For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x + unsigned total_shmed = kernel_info->smem * result; + assert(total_shmed >=0 && total_shmed <= gpgpu_shmem_size); + assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared + assert(m_L1D_config.get_nset() == 4); //Volta L1 has four sets + if(total_shmed < gpgpu_shmem_size){ + if(total_shmed == 0) + m_L1D_config.set_assoc(256); //L1 is 128KB ans shd=0 + else if(total_shmed > 0 && total_shmed <= 8192) + m_L1D_config.set_assoc(240); //L1 is 120KB ans shd=8KB + else if(total_shmed > 8192 && total_shmed <= 16384) + m_L1D_config.set_assoc(224); //L1 is 112KB ans shd=16KB + else if(total_shmed > 16384 && total_shmed <= 32768) + m_L1D_config.set_assoc(192); //L1 is 96KB ans shd=32KB + else if(total_shmed > 32768 && total_shmed <= 65536) + m_L1D_config.set_assoc(128); //L1 is 64KB ans shd=64KB + else if(total_shmed > 65536 && total_shmed <= gpgpu_shmem_size) + m_L1D_config.set_assoc(64); //L1 is 32KB and shd=96KB + else + assert(0); + + printf ("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", m_L1D_config.get_total_size_inKB()); + } + + k.volta_cache_config_set = true; + } + return result; } diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index e07096e..92b4159 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1409,6 +1409,8 @@ struct shader_core_config : public core_config //Jin: concurrent kernel on sm bool gpgpu_concurrent_kernel_sm; + + bool adpative_volta_cache_config; }; struct shader_core_stats_pod { -- cgit v1.3