From f678104dcc3e5c970b871244e18e38f97c0caaa5 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Tue, 12 Sep 2017 18:53:26 -0400 Subject: Adding HBM model --- src/gpgpu-sim/dram.h | 59 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 5 deletions(-) (limited to 'src/gpgpu-sim/dram.h') diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index a8bff14..0ec5166 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -31,9 +31,15 @@ #include "delayqueue.h" #include +#include +#include +#include +#include +#include #include #include #include +#include #define READ 'R' //define read and write states #define WRITE 'W' @@ -42,7 +48,7 @@ class dram_req_t { public: - dram_req_t( class mem_fetch *data ); + dram_req_t( class mem_fetch *data , unsigned banks, unsigned dram_bnk_indexing_policy); unsigned int row; unsigned int col; @@ -106,6 +112,7 @@ public: class mem_fetch* return_queue_pop(); class mem_fetch* return_queue_top(); + void push( class mem_fetch *data ); void cycle(); void dram_log (int task); @@ -123,17 +130,24 @@ public: unsigned &wr, unsigned &req) const; -private: - void scheduler_fifo(); - void scheduler_frfcfs(); + const struct memory_config *m_config; +private: bankgrp_t **bkgrp; bank_t **bk; unsigned int prio; + unsigned get_bankgrp_number(unsigned i); + + void scheduler_fifo(); + void scheduler_frfcfs(); + + bool issue_col_command(int j); + bool issue_row_command(int j); + unsigned int RRDc; unsigned int CCDc; unsigned int RTWc; //read to write penalty applies across banks @@ -146,7 +160,7 @@ private: fifo_pipeline *rwq; fifo_pipeline *mrqq; //buffer to hold packets when DRAM processing is over - //should be filled with dram clock and popped with l2or icnt clock + //should be filled with dram clock and popped with l2or icnt clock fifo_pipeline *returnq; unsigned int dram_util_bins[10]; @@ -158,11 +172,46 @@ private: unsigned int n_nop; unsigned int n_act; unsigned int n_pre; + unsigned int n_ref; unsigned int n_rd; + unsigned int n_rd_L2_A; unsigned int n_wr; + unsigned int n_wr_WB; unsigned int n_req; unsigned int max_mrqs_temp; + unsigned wasted_bw_row; + unsigned wasted_bw_col; + unsigned util_bw; + unsigned idle_bw; + unsigned RCDc_limit; + unsigned CCDLc_limit; + unsigned CCDLc_limit_alone; + unsigned CCDc_limit; + unsigned WTRc_limit; + unsigned WTRc_limit_alone; + unsigned RCDWRc_limit; + unsigned RTWc_limit; + unsigned RTWc_limit_alone; + unsigned rwq_limit; + + //row locality, BLP and other statistics + unsigned long access_num; + unsigned long long hits_num; + unsigned long long banks_1time; + unsigned long long banks_acess_total; + unsigned long long banks_acess_total_after; + unsigned long long banks_time_rw; + unsigned long long banks_access_rw_total; + unsigned long long banks_time_ready; + unsigned long long banks_access_ready_total; + unsigned long long issued_two; + unsigned long long issued_total; + unsigned long long issued_total_row; + unsigned long long issued_total_col; + double write_to_read_ratio_blp_rw_average; + unsigned long long bkgrp_parallsim_rw; + unsigned int bwutil; unsigned int max_mrqs; unsigned int ave_mrqs; -- cgit v1.3 From bd45b8f49f35827dfaa7b5273dda85f0c442fe13 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Wed, 13 Sep 2017 09:47:38 -0400 Subject: Fixing some typos --- src/gpgpu-sim/dram.cc | 39 ++++++++++++++++++++++----------------- src/gpgpu-sim/dram.h | 1 + 2 files changed, 23 insertions(+), 17 deletions(-) (limited to 'src/gpgpu-sim/dram.h') diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index 352bd58..dcc0a7d 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -304,29 +304,29 @@ void dram_t::cycle() //collect row buffer locality, BLP and other statistics ///////////////////////////////////////////////////////////////////////// - unsigned int memory_Pending=0; + unsigned int memory_pending=0; for (unsigned i=0;inbk;i++) { if (bk[i]->mrq) - memory_Pending++; + memory_pending++; } - banks_1time += memory_Pending; - if(memory_Pending >0) + banks_1time += memory_pending; + if(memory_pending >0) banks_acess_total++; unsigned int memory_pending_rw=0; - unsigned read_BLP_RW=0; - unsigned write_BLP_RW=0; - std::bitset<8> bnkgrp_RW_found; - //bool memory_pending_rw_found=false; - for (unsigned j=0;jnbk;j++) { + unsigned read_blp_rw=0; + unsigned write_blp_rw=0; + std::bitset<8> bnkgrp_rw_found; //assume max we have 8 bank groups + + for (unsigned j=0;jnbk;j++) { unsigned grp = get_bankgrp_number(j); if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && (bk[j]->mrq->rw == READ) && (bk[j]->state == BANK_ACTIVE)))) { memory_pending_rw++; - read_BLP_RW++; - bnkgrp_RW_found.set(grp); + read_blp_rw++; + bnkgrp_rw_found.set(grp); } else if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && @@ -334,15 +334,15 @@ void dram_t::cycle() (bk[j]->state == BANK_ACTIVE)))) { memory_pending_rw++; - write_BLP_RW++; - bnkgrp_RW_found.set(grp); + write_blp_rw++; + bnkgrp_rw_found.set(grp); } } banks_time_rw += memory_pending_rw; - bkgrp_parallsim_rw += bnkgrp_RW_found.count(); + bkgrp_parallsim_rw += bnkgrp_rw_found.count(); if(memory_pending_rw >0) { - write_to_read_ratio_blp_rw_average += (double)write_BLP_RW/(write_BLP_RW+read_BLP_RW); + write_to_read_ratio_blp_rw_average += (double)write_blp_rw/(write_blp_rw+read_blp_rw); banks_access_rw_total++; } @@ -441,6 +441,10 @@ void dram_t::cycle() if(issued_col_cmd) issued_total_col++; if(issued_row_cmd) issued_total_row++; + + //Collect some statistics + //check the limitation, see where BW is wasted? + ///////////////////////////////////////////////////////// unsigned int memory_pending_found=0; for (unsigned i=0;inbk;i++) { if (bk[i]->mrq) @@ -463,8 +467,7 @@ void dram_t::cycle() memory_pending_rw_found=true; } - //Collect some statistics - //check the limitation, why BW is wasted? + if(issued_col_cmd || CCDc) util_bw++; else if (memory_pending_rw_found) @@ -507,6 +510,8 @@ void dram_t::cycle() else assert(1); + ///////////////////////////////////////////////////////// + // decrements counters once for each time dram_issueCMD is called DEC2ZERO(RRDc); DEC2ZERO(CCDc); diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 0ec5166..331b4f1 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -180,6 +180,7 @@ private: unsigned int n_req; unsigned int max_mrqs_temp; + //some statistics to collect to see where BW is wasted? unsigned wasted_bw_row; unsigned wasted_bw_col; unsigned util_bw; -- cgit v1.3 From 50b7ac49a78948f61fa685d717de90feaa277b9b Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Wed, 25 Oct 2017 13:15:44 -0400 Subject: The commits includes: 1- REEAD/WERITE buffer for DRAM 2- Fixing FETCH_ON_WRITE cahce policy bug --- src/gpgpu-sim/dram.cc | 11 +++++- src/gpgpu-sim/dram.h | 2 +- src/gpgpu-sim/dram_sched.cc | 95 ++++++++++++++++++++++++++++++++++----------- src/gpgpu-sim/dram_sched.h | 12 ++++++ src/gpgpu-sim/gpu-cache.cc | 34 ++++++++++++---- src/gpgpu-sim/gpu-sim.cc | 7 +++- src/gpgpu-sim/gpu-sim.h | 9 +++++ src/gpgpu-sim/l2cache.cc | 29 ++++++++------ src/gpgpu-sim/mem_fetch.cc | 4 +- src/gpgpu-sim/mem_fetch.h | 4 +- 10 files changed, 160 insertions(+), 47 deletions(-) (limited to 'src/gpgpu-sim/dram.h') diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index 5c1ddab..7ed7b6f 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -149,11 +149,18 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m } -bool dram_t::full() const +bool dram_t::full(bool is_write) const { if(m_config->scheduler_type == DRAM_FRFCFS){ if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false; - return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + if(m_config->seperate_write_queue_enabled){ + if(is_write) + return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size; + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + } + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; } else return mrqq->full(); } diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 331b4f1..29731a7 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -101,7 +101,7 @@ public: dram_t( unsigned int parition_id, const struct memory_config *config, class memory_stats_t *stats, class memory_partition_unit *mp ); - bool full() const; + bool full(bool is_write) const; void print( FILE* simFile ) const; void visualize() const; void print_stat( FILE* simFile ); diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index 008b5bb..7a140c5 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -36,6 +36,7 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem m_config = config; m_stats = stats; m_num_pending = 0; + m_num_write_pending = 0; m_dram = dm; m_queue = new std::list[m_config->nbk]; m_bins = new std::map::iterator> >[ m_config->nbk ]; @@ -49,15 +50,36 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem curr_row_service_time[i] = 0; row_service_timestamp[i] = 0; } + if(m_config->seperate_write_queue_enabled) { + m_write_queue = new std::list[m_config->nbk]; + m_write_bins = new std::map::iterator> >[ m_config->nbk ]; + m_last_write_row = new std::list::iterator>*[ m_config->nbk ]; + + for ( unsigned i=0; i < m_config->nbk; i++ ) { + m_write_queue[i].clear(); + m_write_bins[i].clear(); + m_last_write_row[i] = NULL; + } + } + m_mode = READ_MODE; } void frfcfs_scheduler::add_req( dram_req_t *req ) { - m_num_pending++; - m_queue[req->bk].push_front(req); - std::list::iterator ptr = m_queue[req->bk].begin(); - m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size); + m_num_write_pending++; + m_write_queue[req->bk].push_front(req); + std::list::iterator ptr = m_write_queue[req->bk].begin(); + m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } else { + assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size); + m_num_pending++; + m_queue[req->bk].push_front(req); + std::list::iterator ptr = m_queue[req->bk].begin(); + m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } } void frfcfs_scheduler::data_collection(unsigned int bank) @@ -80,21 +102,43 @@ dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) { //row bool rowhit = true; + std::list *m_current_queue = m_queue; + std::map::iterator> > *m_current_bins = m_bins ; + std::list::iterator> **m_current_last_row = m_last_row; + + if(m_config->seperate_write_queue_enabled) { + if(m_mode == READ_MODE && + ((m_num_write_pending >= m_config->write_high_watermark ) + || (m_queue[bank].empty() && !m_write_queue[bank].empty()))) { + m_mode = WRITE_MODE; + } + else if(m_mode == WRITE_MODE && + (( m_num_write_pending < m_config->write_low_watermark ) + || (!m_queue[bank].empty() && m_write_queue[bank].empty()))){ + m_mode = READ_MODE; + } + } + + if(m_mode == WRITE_MODE) { + m_current_queue = m_write_queue; + m_current_bins = m_write_bins ; + m_current_last_row = m_last_write_row; + } - if ( m_last_row[bank] == NULL ) { - if ( m_queue[bank].empty() ) + if ( m_current_last_row[bank] == NULL ) { + if ( m_current_queue[bank].empty() ) return NULL; - std::map::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row ); - if ( bin_ptr == m_bins[bank].end()) { - dram_req_t *req = m_queue[bank].back(); - bin_ptr = m_bins[bank].find( req->row ); - assert( bin_ptr != m_bins[bank].end() ); // where did the request go??? - m_last_row[bank] = &(bin_ptr->second); + std::map::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row ); + if ( bin_ptr == m_current_bins[bank].end()) { + dram_req_t *req = m_current_queue[bank].back(); + bin_ptr = m_current_bins[bank].find( req->row ); + assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go??? + m_current_last_row[bank] = &(bin_ptr->second); data_collection(bank); rowhit = false; } else { - m_last_row[bank] = &(bin_ptr->second); + m_current_last_row[bank] = &(bin_ptr->second); rowhit = true; } } @@ -103,25 +147,32 @@ dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) if(rowhit) m_dram->hits_num++; - std::list::iterator next = m_last_row[bank]->back(); + std::list::iterator next = m_current_last_row[bank]->back(); dram_req_t *req = (*next); m_stats->concurrent_row_access[m_dram->id][bank]++; m_stats->row_access[m_dram->id][bank]++; - m_last_row[bank]->pop_back(); + m_current_last_row[bank]->pop_back(); - m_queue[bank].erase(next); - if ( m_last_row[bank]->empty() ) { - m_bins[bank].erase( req->row ); - m_last_row[bank] = NULL; + m_current_queue[bank].erase(next); + if ( m_current_last_row[bank]->empty() ) { + m_current_bins[bank].erase( req->row ); + m_current_last_row[bank] = NULL; } #ifdef DEBUG_FAST_IDEAL_SCHED if ( req ) printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row ); #endif - assert( req != NULL && m_num_pending != 0 ); - m_num_pending--; + + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert( req != NULL && m_num_write_pending != 0 ); + m_num_write_pending--; + } + else { + assert( req != NULL && m_num_pending != 0 ); + m_num_pending--; + } return req; } @@ -138,7 +189,7 @@ void dram_t::scheduler_frfcfs() { unsigned mrq_latency; frfcfs_scheduler *sched = m_frfcfs_scheduler; - while ( !mrqq->empty() && (!m_config->gpgpu_frfcfs_dram_sched_queue_size || sched->num_pending() < m_config->gpgpu_frfcfs_dram_sched_queue_size)) { + while ( !mrqq->empty() ) { dram_req_t *req = mrqq->pop(); // Power stats diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h index 3860f5b..63f5831 100644 --- a/src/gpgpu-sim/dram_sched.h +++ b/src/gpgpu-sim/dram_sched.h @@ -35,6 +35,11 @@ #include #include +enum memory_mode { + READ_MODE = 0, + WRITE_MODE +}; + class frfcfs_scheduler { public: frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats ); @@ -43,17 +48,24 @@ public: dram_req_t *schedule( unsigned bank, unsigned curr_row ); void print( FILE *fp ); unsigned num_pending() const { return m_num_pending;} + unsigned num_write_pending() const { return m_num_write_pending;} private: const memory_config *m_config; dram_t *m_dram; unsigned m_num_pending; + unsigned m_num_write_pending; std::list *m_queue; std::map::iterator> > *m_bins; std::list::iterator> **m_last_row; unsigned *curr_row_service_time; //one set of variables for each bank. unsigned *row_service_timestamp; //tracks when scheduler began servicing current row + std::list *m_write_queue; + std::map::iterator> > *m_write_bins; + std::list::iterator> **m_last_write_row; + + enum memory_mode m_mode; memory_stats_t *m_stats; }; diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index eadc094..d199cca 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -945,10 +945,9 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a mf->set_addr( block_addr ); m_miss_queue.push_back(mf); mf->set_status(m_miss_queue_status,time); - if(wa) - events.push_back(cache_event(WRITE_ALLOCATE_SENT)); - else + if(!wa) events.push_back(cache_event(READ_REQUEST_SENT)); + do_miss = true; } else if(mshr_hit && !mshr_avail) @@ -1087,6 +1086,8 @@ data_cache::wr_miss_wa_naive( new_addr_type addr, send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, evicted, events, false, true); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + if( do_miss ){ // If evicted block is modified and not a write-through // (already modified lower level) @@ -1111,7 +1112,7 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, { new_addr_type block_addr = m_config.block_addr(addr); - new_addr_type mshr_addr = m_config.block_addr(mf->get_addr()); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) { @@ -1147,10 +1148,23 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, } else { - if(miss_queue_full(1)) { - m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); - return RESERVATION_FAIL; - } + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if(miss_queue_full(1) + || (!(mshr_hit && mshr_avail) + && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + //check what is the exactly the failure reason + if(miss_queue_full(1) ) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); + + return RESERVATION_FAIL; + } //prevent Write - Read - Write in pending mshr @@ -1177,8 +1191,10 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, mf->get_sid(), mf->get_tpc(), mf->get_mem_config(), + NULL, mf); + new_addr_type block_addr = m_config.block_addr(addr); bool do_miss = false; bool wb = false; @@ -1191,6 +1207,8 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, cache_block_t* block = m_tag_array->get_block(cache_index); block->set_modified_on_fill(true, mf->get_access_sector_mask()); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + if( do_miss ){ // If evicted block is modified and not a write-through // (already modified lower level) diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 470fcf4..7838875 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -201,7 +201,12 @@ void memory_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy, "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)", "0"); - + option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled, + "Seperate_Write_Queue_Enable", + "0"); + option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt, + "Write_Queue_Size", + "32:28:16"); m_address_mapping.addrdec_setoption(opp); } diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index f379a17..197350b 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -214,6 +214,9 @@ struct memory_config { m_valid = true; icnt_flit_size = 32; // Default 32 + + sscanf(write_queue_size_opt,"%d:%d:%d", + &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark); } void reg_options(class OptionParser * opp); @@ -274,6 +277,12 @@ struct memory_config { unsigned dram_bnk_indexing_policy; unsigned dram_bnkgrp_indexing_policy; bool dual_bus_interface; + + bool seperate_write_queue_enabled; + char *write_queue_size_opt; + unsigned gpgpu_frfcfs_dram_write_queue_size; + unsigned write_high_watermark; + unsigned write_low_watermark; }; // global counters and flags (please try not to add to this list!!!) diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index f7323c5..cac59f1 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -93,7 +93,9 @@ memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct m m_private_credit_limit = 1; m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size + config->gpgpu_dram_return_queue_size - - (config->m_n_sub_partition_per_memory_channel - 1); + - (config->m_n_sub_partition_per_memory_channel - 1); + if(config->seperate_write_queue_enabled ) + m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size; if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 or config->gpgpu_dram_return_queue_size == 0) { @@ -220,7 +222,8 @@ void memory_partition_unit::dram_cycle() m_dram->cycle(); m_dram->dram_log(SAMPLELOG); - if( !m_dram->full() ) { + // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + //if( !m_dram->full(mf->is_write()) ) { // L2->DRAM queue to DRAM latency queue // Arbitrate among multiple L2 subpartitions int last_issued_partition = m_arbitration_metadata.last_borrower(); @@ -228,6 +231,9 @@ void memory_partition_unit::dram_cycle() int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + if(m_dram->full(mf->is_write()) ) + break; + m_sub_partition[spid]->L2_dram_queue_pop(); MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); dram_delay_t d; @@ -239,12 +245,13 @@ void memory_partition_unit::dram_cycle() break; // the DRAM should only accept one request per cycle } } - } + //} // DRAM latency queue - if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) { - mem_fetch* mf = m_dram_latency_queue.front().req; - m_dram_latency_queue.pop_front(); + + if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) { + mem_fetch* mf = m_dram_latency_queue.front().req; + m_dram_latency_queue.pop_front(); m_dram->push(mf); } } @@ -343,12 +350,12 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); m_L2_icnt_queue->push(mf); }else{ - if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE && mf->original_mf) + if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE) { - assert(mf->original_mf); - mf->original_mf->set_reply(); - mf->original_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - m_L2_icnt_queue->push(mf->original_mf); + assert(mf->original_wr_mf); + mf->original_wr_mf->set_reply(); + mf->original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf->original_wr_mf); } m_request_tracker.erase(mf); delete mf; diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index b8e918f..c05a693 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -40,7 +40,8 @@ mem_fetch::mem_fetch( const mem_access_t &access, unsigned sid, unsigned tpc, const class memory_config *config, - mem_fetch *m_original_mf) + mem_fetch *m_original_mf, + mem_fetch *m_original_wr_mf) { m_request_uid = sm_next_mf_request_uid++; m_access = access; @@ -63,6 +64,7 @@ mem_fetch::mem_fetch( const mem_access_t &access, m_mem_config = config; icnt_flit_size = config->icnt_flit_size; original_mf = m_original_mf; + original_wr_mf = m_original_wr_mf; } mem_fetch::~mem_fetch() diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index 76e7419..278cf32 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -56,7 +56,8 @@ public: unsigned sid, unsigned tpc, const class memory_config *config, - mem_fetch *original_mf = NULL); + mem_fetch *original_mf = NULL, + mem_fetch *original_wr_mf = NULL); ~mem_fetch(); void set_status( enum mem_fetch_status status, unsigned long long cycle ); @@ -115,6 +116,7 @@ public: unsigned get_num_flits(bool simt_to_mem); mem_fetch* original_mf; + mem_fetch* original_wr_mf; private: // request source information unsigned m_request_uid; -- cgit v1.3 From 0b1a646c0ddf5d4db64a2b6fe9dfa30789cf1cd4 Mon Sep 17 00:00:00 2001 From: Mahmoud Date: Mon, 30 Oct 2017 20:23:51 -0400 Subject: adding new stats and change the PascalP100-HBM config --- configs/Pascal-P100-HBM/config_fermi_islip.icnt | 9 +++++--- configs/Pascal-P100-HBM/gpgpusim.config | 25 ++++++++++++---------- configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt | 3 +++ src/gpgpu-sim/dram.cc | 13 +++++++---- src/gpgpu-sim/dram.h | 4 ++++ src/gpgpu-sim/dram_sched.cc | 21 +++++++++++++----- src/gpgpu-sim/gpu-sim.cc | 4 +++- src/gpgpu-sim/gpu-sim.h | 8 +++++++ 8 files changed, 63 insertions(+), 24 deletions(-) (limited to 'src/gpgpu-sim/dram.h') diff --git a/configs/Pascal-P100-HBM/config_fermi_islip.icnt b/configs/Pascal-P100-HBM/config_fermi_islip.icnt index a788090..0a73c81 100644 --- a/configs/Pascal-P100-HBM/config_fermi_islip.icnt +++ b/configs/Pascal-P100-HBM/config_fermi_islip.icnt @@ -1,13 +1,13 @@ //21*1 fly with 32 flits per packet under gpgpusim injection mode use_map = 0; -flit_size = 32; +flit_size = 40; // currently we do not use this, see subnets below network_count = 2; // Topology topology = fly; -k = 62; +k = 60; n = 1; // Routing @@ -17,7 +17,10 @@ routing_function = dest_tag; // Flow control num_vcs = 1; -vc_buf_size = 8; +vc_buf_size = 64; +input_buffer_size = 64; +ejection_buffer_size = 64; +boundary_buffer_size = 64; wait_for_tail_credit = 0; diff --git a/configs/Pascal-P100-HBM/gpgpusim.config b/configs/Pascal-P100-HBM/gpgpusim.config index 5b038de..1029194 100644 --- a/configs/Pascal-P100-HBM/gpgpusim.config +++ b/configs/Pascal-P100-HBM/gpgpusim.config @@ -54,13 +54,17 @@ # ** Optional parameter - Required when mshr_type==Texture Fifo # Note: Hashing set index function (H) only applies to a set size of 32 or 64. # Pascal GP100 has 64KB Shared memory --gpgpu_cache:dl1 N:64:128:6,L:L:m:N:H,S:128:8,8 +-gpgpu_cache:dl1 S:64:128:6,L:L:m:N:H,A:128:8,32:0,32 -gpgpu_shmem_size 65536 --gmem_skip_L1D 0 +-gmem_skip_L1D 1 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 -# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 4MB L2 cache --gpgpu_cache:dl2 S:64:128:16,L:B:m:W:L,A:128:4,4:0,32 +# 32 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 4MB L2 cache +-gpgpu_cache:dl2 S:64:128:16,L:B:m:W:L,A:256:4,32:0,32 -gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 +#-gpgpu_flush_l2_cache 1 # 4 KB Inst. -gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 @@ -89,8 +93,7 @@ -gpgpu_shmem_num_banks 32 -gpgpu_shmem_limited_broadcast 0 -gpgpu_shmem_warp_parts 1 -# Use Fermi Coalscer arhitetecture for now! Need to be canged to pascal Coalscer --gpgpu_coalesce_arch 20 +-gpgpu_coalesce_arch 60 ## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units -gpgpu_max_insn_issue_per_warp 2 @@ -123,10 +126,6 @@ -gpgpu_mem_address_mask 1 -gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBBCCC.CCCSSSSS -# GDDR5 timing -#-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=6:RCD=12:RAS=28:RP=12:RC=40: -# CL=12:WL=4:CDLR=5:WR=12:nbkgrp=4:CCDL=3:RTPL=2" - # HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) # Timing for 1 GHZ # tRRDl and tWTR are missing, need to be added @@ -140,8 +139,12 @@ # HBM has dual bus interface, in which it can issue two col and row commands at a time -dual_bus_interface 1 # select lower bits for bnkgrp to increase bnkgrp parallelism +-dram_bnk_indexing_policy 0 -dram_bnkgrp_indexing_policy 1 +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + # Pascal has two schedulers per core -gpgpu_num_sched_per_core 2 # Two Level Scheduler with active and pending pools @@ -158,7 +161,7 @@ -visualizer_enabled 0 # power model configs, disable it untill we create a real energy model for Pascal 100 --power_simulation_enabled 1 +-power_simulation_enabled 0 -gpuwattch_xml_file gpuwattch_gtx480.xml # tracing functionality diff --git a/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt b/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt index 58e596d..94b2378 100644 --- a/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt +++ b/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt @@ -18,6 +18,9 @@ routing_function = dest_tag; num_vcs = 1; vc_buf_size = 32; +input_buffer_size = 32; +ejection_buffer_size = 32; +boundary_buffer_size = 32; wait_for_tail_credit = 0; diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index de37f64..a57508c 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -52,6 +52,10 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m //rowblp access_num=0; hits_num=0; + read_num=0; + write_num=0; + hits_read_num=0; + hits_write_num=0; banks_1time=0; banks_acess_total=0; banks_acess_total_after=0; @@ -718,6 +722,8 @@ void dram_t::print( FILE* simFile) const fprintf(simFile, "\n------------------------------------------------------------------------\n"); printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num); + printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num); + printf("\nRow_Buffer_Locality_write = %.6f", (float)hits_write_num / write_num); printf("\nBank_Level_Parallism = %.6f", (float)banks_1time / banks_acess_total); printf("\nBank_Level_Parallism_Col = %.6f", (float)banks_time_rw / banks_access_rw_total); printf("\nBank_Level_Parallism_Ready = %.6f", (float)banks_time_ready /banks_access_ready_total); @@ -735,7 +741,7 @@ void dram_t::print( FILE* simFile) const printf("RCDWRc_limit = %d \n", RCDWRc_limit); printf("WTRc_limit = %d \n", WTRc_limit); printf("RTWc_limit = %d \n", RTWc_limit); - printf("CCDLc_limit %d \n", CCDLc_limit); + printf("CCDLc_limit = %d \n", CCDLc_limit); printf("rwq = %d \n", rwq_limit); printf("CCDLc_limit_alone = %d \n", CCDLc_limit_alone); printf("WTRc_limit_alone = %d \n", WTRc_limit_alone); @@ -751,15 +757,14 @@ void dram_t::print( FILE* simFile) const printf("n_pre = %d \n", n_pre); printf("n_ref = %d \n", n_ref); printf("n_req = %d \n", n_req ); - printf("n_req4 = %d \n", n_req*4 ); printf("total_req = %d \n\n", n_rd+n_wr+n_rd_L2_A+n_wr_WB); printf("issued_total_row = %lu \n", issued_total_row); printf("issued_total_col = %lu \n", issued_total_col); printf("Row_Bus_Util = %.6f \n", (float)issued_total_row / n_cmd); printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd); - printf("Either_Row_CoL_Bus_Util %.6f \n", (float)issued_total / n_cmd); - printf("Issued_on_Two_Bus_Simul_Util %.6f \n", (float)issued_two /n_cmd); + printf("Either_Row_CoL_Bus_Util = %.6f \n", (float)issued_total / n_cmd); + printf("Issued_on_Two_Bus_Simul_Util = %.6f \n", (float)issued_two /n_cmd); printf("issued_two_Eff = %.6f \n", (float)issued_two /issued_total); printf("queue_avg = %.6f \n\n", (float)ave_mrqs/n_cmd ); diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 29731a7..0d4c0e7 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -198,7 +198,11 @@ private: //row locality, BLP and other statistics unsigned long access_num; + unsigned long read_num; + unsigned long write_num; unsigned long long hits_num; + unsigned long long hits_read_num; + unsigned long long hits_write_num; unsigned long long banks_1time; unsigned long long banks_acess_total; unsigned long long banks_acess_total_after; diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index ac4c827..f754d36 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -142,14 +142,25 @@ dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) rowhit = true; } } - //rowblp - m_dram->access_num++; - if(rowhit) - m_dram->hits_num++; - std::list::iterator next = m_current_last_row[bank]->back(); dram_req_t *req = (*next); + //rowblp stats + m_dram->access_num++; + bool is_write = req->data->is_write(); + if(is_write) + m_dram->write_num++; + else + m_dram->read_num++; + + if(rowhit) { + m_dram->hits_num++; + if(is_write) + m_dram->hits_write_num++; + else + m_dram->hits_read_num++; + } + m_stats->concurrent_row_access[m_dram->id][bank]++; m_stats->row_access[m_dram->id][bank]++; m_current_last_row[bank]->pop_back(); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 0e06c5c..11ac5df 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -207,6 +207,9 @@ void memory_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt, "Write_Queue_Size", "32:28:16"); + option_parser_register(opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround, + "elimnate_rw_turnaround i.e set tWTR and tRTW = 0", + "0"); option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size, "icnt_flit_size", "32"); @@ -435,7 +438,6 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, &gpgpu_flush_l2_cache, "Flush L2 cache at the end of each kernel call", "0"); - option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, "Stop the simulation at deadlock (1=on (default), 0=off)", "1"); diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 043fcee..c04648c 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -198,8 +198,14 @@ struct memory_config { bk_tag_length = i-1; assert(nbkgrp>0 && "Number of bank groups cannot be zero"); tRCDWR = tRCD-(WL+1); + if(elimnate_rw_turnaround) + { + tRTW = 0; + tWTR = 0; + } else { tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + } tWTP = (WL+(BL/data_command_freq_ratio)+tWR); dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition @@ -266,6 +272,8 @@ struct memory_config { unsigned nbk; + bool elimnate_rw_turnaround; + unsigned data_command_freq_ratio; // frequency ratio between DRAM data bus and command bus (2 for GDDR3, 4 for GDDR5) unsigned dram_atom_size; // number of bytes transferred per read or write command -- cgit v1.3