From eae030c9d1d607d1c14e4ade99cb5caea6403efd Mon Sep 17 00:00:00 2001 From: aamir Date: Sat, 3 Nov 2018 17:03:58 -0700 Subject: merged with memory subsytem. Regression is passing but tensorcore kernel is stuck in deadlock --- src/gpgpu-sim/addrdec.cc | 71 ++++ src/gpgpu-sim/addrdec.h | 9 + src/gpgpu-sim/delayqueue.h | 1 + src/gpgpu-sim/dram.cc | 637 ++++++++++++++++++++++++------- src/gpgpu-sim/dram.h | 77 +++- src/gpgpu-sim/dram_sched.cc | 121 ++++-- src/gpgpu-sim/dram_sched.h | 12 + src/gpgpu-sim/gpu-cache.cc | 782 +++++++++++++++++++++++++++++++------- src/gpgpu-sim/gpu-cache.h | 644 +++++++++++++++++++++++++++---- src/gpgpu-sim/gpu-sim.cc | 125 +++++- src/gpgpu-sim/gpu-sim.h | 57 ++- src/gpgpu-sim/l2cache.cc | 166 ++++++-- src/gpgpu-sim/l2cache.h | 18 + src/gpgpu-sim/l2cache_trace.h | 16 + src/gpgpu-sim/mem_fetch.cc | 7 +- src/gpgpu-sim/mem_fetch.h | 13 +- src/gpgpu-sim/mem_latency_stat.cc | 19 +- src/gpgpu-sim/mem_latency_stat.h | 4 + src/gpgpu-sim/shader.cc | 431 ++++++++++++++++++--- src/gpgpu-sim/shader.h | 93 ++++- 20 files changed, 2810 insertions(+), 493 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc index 422576d..8651869 100644 --- a/src/gpgpu-sim/addrdec.cc +++ b/src/gpgpu-sim/addrdec.cc @@ -62,6 +62,9 @@ void linear_to_raw_address_translation::addrdec_setoption(option_parser_t opp) option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits", "0"); + option_parser_register(opp, "-memory_partition_indexing", OPT_UINT32, &memory_partition_indexing, + "0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing", + "0"); } new_addr_type linear_to_raw_address_translation::partition_address( new_addr_type addr ) const @@ -103,6 +106,74 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_ tlx->burst= addrdec_packbits(addrdec_mask[BURST], rest_of_addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); } + switch(memory_partition_indexing){ + case CONSECUTIVE: + //Do nothing + break; + case BITWISE_PERMUTATION: + { + assert(!gap); + tlx->chip = (tlx->chip) ^ (tlx->row & (m_n_channel-1)); + assert(tlx->chip < m_n_channel); + break; + } + case IPOLY: + { + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * equations are adopted from: + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." + * Khairy et al. + * IEEE TPDS 2017. + */ + if(m_n_channel == 32) { + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + chip[0] = a[13]^a[12]^a[11]^a[10]^a[9]^a[6]^a[5]^a[3]^a[0]^chip[0]; + chip[1] = a[14]^a[13]^a[12]^a[11]^a[10]^a[7]^a[6]^a[4]^a[1]^chip[1]; + chip[2] = a[14]^a[10]^a[9]^a[8]^a[7]^a[6]^a[3]^a[2]^a[0]^chip[2]; + chip[3] = a[11]^a[10]^a[9]^a[8]^a[7]^a[4]^a[3]^a[1]^chip[3]; + chip[4] = a[12]^a[11]^a[10]^a[9]^a[8]^a[5]^a[4]^a[2]^chip[4]; + tlx->chip = chip.to_ulong(); + + } + else{ /* Else incorrect number of channels for the hashing function */ + assert("\nGPGPU-Sim memory_partition_indexing error: The number of channels should be " + "32 for the hashing IPOLY index function.\n" && 0); + } + assert(tlx->chip < m_n_channel); + break; + } + case PAE: + { + //Page Address Entropy + //random selected bits from the page and bank bits + //similar to + //Liu, Yuxi, et al. "Get Out of the Valley: Power-Efficient Address Mapping for GPUs." ISCA 2018 + std::bitset<64> a(tlx->row); + std::bitset<5> chip(tlx->chip); + std::bitset<4> b(tlx->bk); + chip[0] = a[13]^a[10]^a[9]^a[5]^a[0]^b[3]^b[0]^chip[0]; + chip[1] = a[12]^a[11]^a[6]^a[1]^b[3]^b[2]^b[1]^chip[1]; + chip[2] = a[14]^a[9]^a[8]^a[7]^a[2]^b[1]^chip[2]; + chip[3] = a[11]^a[10]^a[8]^a[3]^b[2]^b[3]^chip[3]; + chip[4] = a[12]^a[9]^a[8]^a[5]^a[4]^b[1]^b[0]^chip[4]; + tlx->chip = chip.to_ulong(); + assert(tlx->chip < m_n_channel); + break; + } + case CUSTOM: + /* No custom set function implemented */ + //Do you custom index here + break; + default: + assert("\nUndefined set index function.\n" && 0); + break; + } + // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h index fd9af8d..bdc5fec 100644 --- a/src/gpgpu-sim/addrdec.h +++ b/src/gpgpu-sim/addrdec.h @@ -35,6 +35,14 @@ #include "../abstract_hardware_model.h" +enum partition_index_function{ + CONSECUTIVE = 0, + BITWISE_PERMUTATION, + IPOLY, + PAE, + CUSTOM +}; + struct addrdec_t { void print( FILE *fp ) const; @@ -72,6 +80,7 @@ private: const char *addrdec_option; int gpgpu_mem_address_mask; + partition_index_function memory_partition_indexing; bool run_test; int ADDR_CHIP_S; diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h index b25f143..0caa5d4 100644 --- a/src/gpgpu-sim/delayqueue.h +++ b/src/gpgpu-sim/delayqueue.h @@ -161,6 +161,7 @@ public: } bool full() const { return (m_max_len && m_length >= m_max_len); } + bool is_avilable_size(unsigned size) const { return (m_max_len && m_length+size-1 >= m_max_len); } bool empty() const { return m_head == NULL; } unsigned get_n_element() const { return m_n_element; } unsigned get_length() const { return m_length; } diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index a0e024b..6c11b43 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -49,11 +49,45 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m m_stats = stats; m_config = config; + //rowblp + access_num=0; + hits_num=0; + read_num=0; + write_num=0; + hits_read_num=0; + hits_write_num=0; + banks_1time=0; + banks_acess_total=0; + banks_acess_total_after=0; + banks_time_ready=0; + banks_access_ready_total=0; + issued_two=0; + issued_total=0; + issued_total_row=0; + issued_total_col=0; + CCDc = 0; RRDc = 0; RTWc = 0; WTRc = 0; + wasted_bw_row=0; + wasted_bw_col=0; + util_bw=0; + idle_bw=0; + RCDc_limit=0; + CCDLc_limit=0; + CCDLc_limit_alone=0; + CCDc_limit=0; + WTRc_limit=0; + WTRc_limit_alone=0; + RCDWRc_limit=0; + RTWc_limit=0; + RTWc_limit_alone=0; + rwq_limit=0; + write_to_read_ratio_blp_rw_average=0; + bkgrp_parallsim_rw=0; + rw = READ; //read mode is default bkgrp = (bankgrp_t**) calloc(sizeof(bankgrp_t*), m_config->nbkgrp); @@ -74,12 +108,13 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m bk[i]->state = BANK_IDLE; bk[i]->bkgrpindex = i/(m_config->nbk/m_config->nbkgrp); } - prio = 0; + prio = 0; + rwq = new fifo_pipeline("rwq",m_config->CL,m_config->CL+1); mrqq = new fifo_pipeline("mrqq",0,2); returnq = new fifo_pipeline("dramreturnq",0,m_config->gpgpu_dram_return_queue_size==0?1024:m_config->gpgpu_dram_return_queue_size); m_frfcfs_scheduler = NULL; - if ( m_config->scheduler_type == DRAM_FRFCFS ) + if ( m_config->scheduler_type == DRAM_FRFCFS) m_frfcfs_scheduler = new frfcfs_scheduler(m_config,this,stats); n_cmd = 0; n_activity = 0; @@ -88,6 +123,8 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m n_pre = 0; n_rd = 0; n_wr = 0; + n_wr_WB=0; + n_rd_L2_A=0; n_req = 0; max_mrqs_temp = 0; bwutil = 0; @@ -113,13 +150,21 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m mrqq_Dist = StatCreate("mrqq_length",1, queue_limit()); else //queue length is unlimited; mrqq_Dist = StatCreate("mrqq_length",1,64); //track up to 64 entries + } -bool dram_t::full() const +bool dram_t::full(bool is_write) const { - if(m_config->scheduler_type == DRAM_FRFCFS ){ + if(m_config->scheduler_type == DRAM_FRFCFS){ if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false; - return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + if(m_config->seperate_write_queue_enabled){ + if(is_write) + return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size; + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; + } + else + return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size; } else return mrqq->full(); } @@ -127,7 +172,7 @@ bool dram_t::full() const unsigned dram_t::que_length() const { unsigned nreqs = 0; - if (m_config->scheduler_type == DRAM_FRFCFS ) { + if (m_config->scheduler_type == DRAM_FRFCFS) { nreqs = m_frfcfs_scheduler->num_pending(); } else { nreqs = mrqq->get_length(); @@ -146,7 +191,7 @@ unsigned int dram_t::queue_limit() const } -dram_req_t::dram_req_t( class mem_fetch *mf ) +dram_req_t::dram_req_t( class mem_fetch *mf, unsigned banks, unsigned dram_bnk_indexing_policy) { txbytes = 0; dqbytes = 0; @@ -154,7 +199,29 @@ dram_req_t::dram_req_t( class mem_fetch *mf ) const addrdec_t &tlx = mf->get_tlx_addr(); - bk = tlx.bk; + switch(dram_bnk_indexing_policy){ + case LINEAR_BK_INDEX: + { + bk = tlx.bk; + break; + } + case BITWISE_XORING_BK_INDEX: + { + //xoring bank bits with lower bits of the page + int lbank = log2(banks); + bk = tlx.bk ^ (tlx.row & ((1<get_data_size(); @@ -169,14 +236,15 @@ void dram_t::push( class mem_fetch *data ) { assert(id == data->get_tlx_addr().chip); // Ensure request is in correct memory partition - dram_req_t *mrq = new dram_req_t(data); + dram_req_t *mrq = new dram_req_t(data,m_config->nbk,m_config->dram_bnk_indexing_policy); + data->set_status(IN_PARTITION_MC_INTERFACE_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - mrqq->push(mrq); + mrqq->push(mrq); // stats... n_req += 1; n_req_partial += 1; - if ( m_config->scheduler_type == DRAM_FRFCFS ) { + if ( m_config->scheduler_type == DRAM_FRFCFS) { unsigned nreqs = m_frfcfs_scheduler->num_pending(); if ( nreqs > max_mrqs_temp) max_mrqs_temp = nreqs; @@ -212,6 +280,7 @@ void dram_t::cycle() printf("\tDQ: BK%d Row:%03x Col:%03x", cmd->bk, cmd->row, cmd->col + cmd->dqbytes); #endif cmd->dqbytes += m_config->dram_atom_size; + if (cmd->dqbytes >= cmd->nbytes) { mem_fetch *data = cmd->data; data->set_status(IN_PARTITION_MC_RETURNQ,gpu_sim_cycle+gpu_tot_sim_cycle); @@ -240,7 +309,7 @@ void dram_t::cycle() printf("Error: Unknown DRAM scheduler type\n"); assert(0); } - if ( m_config->scheduler_type == DRAM_FRFCFS ) { + if ( m_config->scheduler_type == DRAM_FRFCFS) { unsigned nreqs = m_frfcfs_scheduler->num_pending(); if ( nreqs > max_mrqs) { max_mrqs = nreqs; @@ -258,130 +327,123 @@ void dram_t::cycle() unsigned k=m_config->nbk; bool issued = false; - // check if any bank is ready to issue a new read + //collect row buffer locality, BLP and other statistics + ///////////////////////////////////////////////////////////////////////// + unsigned int memory_pending=0; for (unsigned i=0;inbk;i++) { - unsigned j = (i + prio) % m_config->nbk; - unsigned grp = j>>m_config->bk_tag_length; - if (bk[j]->mrq) { //if currently servicing a memory request - bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); - // correct row activated for a READ - if ( !issued && !CCDc && !bk[j]->RCDc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==WRITE) { - rw=READ; - rwq->set_min_length(m_config->CL); - } - rwq->push(bk[j]->mrq); - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - RTWc = m_config->tRTW; - bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio; - bkgrp[grp]->RTPLc = m_config->tRTPL; - issued = true; - n_rd++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; - bk[j]->n_access++; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tRD Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); -#endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } else - // correct row activated for a WRITE - if ( !issued && !CCDc && !bk[j]->RCDWRc && - !(bkgrp[grp]->CCDLc) && - (bk[j]->curr_row == bk[j]->mrq->row) && - (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && - (bk[j]->state == BANK_ACTIVE) && - !rwq->full() ) { - if (rw==READ) { - rw=WRITE; - rwq->set_min_length(m_config->WL); - } - rwq->push(bk[j]->mrq); - - bk[j]->mrq->txbytes += m_config->dram_atom_size; - CCDc = m_config->tCCD; - bkgrp[grp]->CCDLc = m_config->tCCDL; - WTRc = m_config->tWTR; - bk[j]->WTPc = m_config->tWTP; - issued = true; - n_wr++; - bwutil += m_config->BL/m_config->data_command_freq_ratio; - bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tWR Bk:%d Row:%03x Col:%03x \n", - j, bk[j]->curr_row, - bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); -#endif - // transfer done - if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { - bk[j]->mrq = NULL; - } - } - - else - // bank is idle - if ( !issued && !RRDc && - (bk[j]->state == BANK_IDLE) && - !bk[j]->RPc && !bk[j]->RCc ) { -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tACT BK:%d NewRow:%03x From:%03x \n", - j,bk[j]->mrq->row,bk[j]->curr_row); -#endif - // activate the row with current memory request - bk[j]->curr_row = bk[j]->mrq->row; - bk[j]->state = BANK_ACTIVE; - RRDc = m_config->tRRD; - bk[j]->RCDc = m_config->tRCD; - bk[j]->RCDWRc = m_config->tRCDWR; - bk[j]->RASc = m_config->tRAS; - bk[j]->RCc = m_config->tRC; - prio = (j + 1) % m_config->nbk; - issued = true; - n_act_partial++; - n_act++; - } - - else - // different row activated - if ( (!issued) && - (bk[j]->curr_row != bk[j]->mrq->row) && - (bk[j]->state == BANK_ACTIVE) && - (!bk[j]->RASc && !bk[j]->WTPc && - !bk[j]->RTPc && - !bkgrp[grp]->RTPLc) ) { - // make the bank idle again - bk[j]->state = BANK_IDLE; - bk[j]->RPc = m_config->tRP; - prio = (j + 1) % m_config->nbk; - issued = true; - n_pre++; - n_pre_partial++; -#ifdef DRAM_VERIFY - PRINT_CYCLE=1; - printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row); -#endif - } - } else { - if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc - && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; - bk[j]->n_idle++; - } + if (bk[i]->mrq) + memory_pending++; } + banks_1time += memory_pending; + if(memory_pending >0) + banks_acess_total++; + + unsigned int memory_pending_rw=0; + unsigned read_blp_rw=0; + unsigned write_blp_rw=0; + std::bitset<8> bnkgrp_rw_found; //assume max we have 8 bank groups + + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) + { + memory_pending_rw++; + read_blp_rw++; + bnkgrp_rw_found.set(grp); + } + else if + (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE)))) + { + memory_pending_rw++; + write_blp_rw++; + bnkgrp_rw_found.set(grp); + } + } + banks_time_rw += memory_pending_rw; + bkgrp_parallsim_rw += bnkgrp_rw_found.count(); + if(memory_pending_rw >0) + { + write_to_read_ratio_blp_rw_average += (double)write_blp_rw/(write_blp_rw+read_blp_rw); + banks_access_rw_total++; + } + + unsigned int memory_Pending_ready=0; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && ((!CCDc && !bk[j]->RCDc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full()) + || + (!CCDc && !bk[j]->RCDWRc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full()))) + { + memory_Pending_ready++; + } + } + banks_time_ready += memory_Pending_ready; + if(memory_Pending_ready >0) + banks_access_ready_total++; + /////////////////////////////////////////////////////////////////////////////////// + + bool issued_col_cmd = false; + bool issued_row_cmd = false; + + if(m_config->dual_bus_interface) + { + //dual bus interface + //issue one row command and one column command + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_col_cmd = issue_col_command(j); + if(issued_col_cmd) break; + } + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + issued_row_cmd = issue_row_command(j); + if(issued_row_cmd) break; + } + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + if(!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc + && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; + bk[j]->n_idle++; + } + } + } + else + { + //single bus interface + //issue only one row/column command + for (unsigned i=0;inbk;i++) { + unsigned j = (i + prio) % m_config->nbk; + if(!issued_col_cmd) + issued_col_cmd = issue_col_command(j); + + if(!issued_col_cmd && !issued_row_cmd) + issued_row_cmd = issue_row_command(j); + + if(!bk[j]->mrq) { + if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc + && !bk[j]->RCc && !bk[j]->RPc && !bk[j]->RCDWRc) k--; + bk[j]->n_idle++; + } + + } + } + + issued = issued_row_cmd || issued_col_cmd; if (!issued) { n_nop++; n_nop_partial++; @@ -395,6 +457,85 @@ void dram_t::cycle() } n_cmd++; n_cmd_partial++; + if(issued) + { + issued_total++; + if(issued_col_cmd && issued_row_cmd) + issued_two++; + } + if(issued_col_cmd) issued_total_col++; + if(issued_row_cmd) issued_total_row++; + + + //Collect some statistics + //check the limitation, see where BW is wasted? + ///////////////////////////////////////////////////////// + unsigned int memory_pending_found=0; + for (unsigned i=0;inbk;i++) { + if (bk[i]->mrq) + memory_pending_found++; + } + if(memory_pending_found>0) + banks_acess_total_after++; + + bool memory_pending_rw_found=false; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)) + || + ( + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE)))) + memory_pending_rw_found=true; + } + + + if(issued_col_cmd || CCDc) + util_bw++; + else if (memory_pending_rw_found) + { + wasted_bw_col++; + for (unsigned j=0;jnbk;j++) { + unsigned grp = get_bankgrp_number(j); + //read + if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && + (bk[j]->state == BANK_ACTIVE)))) + { + if(bk[j]->RCDc) RCDc_limit++; + if(bkgrp[grp]->CCDLc) CCDLc_limit++; + if(WTRc) WTRc_limit++; + if(CCDc) CCDc_limit++; + if(rwq->full()) rwq_limit++; + if(bkgrp[grp]->CCDLc && !WTRc) CCDLc_limit_alone++; + if(!bkgrp[grp]->CCDLc && WTRc) WTRc_limit_alone++; + } + //write + else if (bk[j]->mrq && ((bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && + (bk[j]->state == BANK_ACTIVE))) + { + if(bk[j]->RCDWRc) RCDWRc_limit++; + if(bkgrp[grp]->CCDLc) CCDLc_limit++; + if(RTWc) RTWc_limit++; + if(CCDc) CCDc_limit++; + if(rwq->full()) rwq_limit++; + if(bkgrp[grp]->CCDLc && !RTWc) CCDLc_limit_alone++; + if(!bkgrp[grp]->CCDLc && RTWc) RTWc_limit_alone++; + } + } + } + else if (memory_pending_found) + wasted_bw_row++; + else if (!memory_pending_found) + idle_bw++; + else + assert(1); + + ///////////////////////////////////////////////////////// // decrements counters once for each time dram_issueCMD is called DEC2ZERO(RRDc); @@ -420,39 +561,237 @@ void dram_t::cycle() #endif } +bool dram_t::issue_col_command(int j) +{ + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { //if currently servicing a memory request + bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); + // correct row activated for a READ + if ( !issued && !CCDc && !bk[j]->RCDc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == READ) && (WTRc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full() ) { + if (rw==WRITE) { + rw=READ; + rwq->set_min_length(m_config->CL); + } + rwq->push(bk[j]->mrq); + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + RTWc = m_config->tRTW; + bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio; + bkgrp[grp]->RTPLc = m_config->tRTPL; + issued = true; + if(bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R) + n_rd_L2_A++; + else + n_rd++; + + bwutil += m_config->BL/m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; + bk[j]->n_access++; + +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tRD Bk:%d Row:%03x Col:%03x \n", + j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); +#endif + // transfer done + if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { + bk[j]->mrq = NULL; + } + } else + // correct row activated for a WRITE + if ( !issued && !CCDc && !bk[j]->RCDWRc && + !(bkgrp[grp]->CCDLc) && + (bk[j]->curr_row == bk[j]->mrq->row) && + (bk[j]->mrq->rw == WRITE) && (RTWc == 0 ) && + (bk[j]->state == BANK_ACTIVE) && + !rwq->full() ) { + if (rw==READ) { + rw=WRITE; + rwq->set_min_length(m_config->WL); + } + rwq->push(bk[j]->mrq); + + bk[j]->mrq->txbytes += m_config->dram_atom_size; + CCDc = m_config->tCCD; + bkgrp[grp]->CCDLc = m_config->tCCDL; + WTRc = m_config->tWTR; + bk[j]->WTPc = m_config->tWTP; + issued = true; + + if(bk[j]->mrq->data->get_access_type() == L2_WRBK_ACC) + n_wr_WB++; + else + n_wr++; + bwutil += m_config->BL/m_config->data_command_freq_ratio; + bwutil_partial += m_config->BL/m_config->data_command_freq_ratio; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tWR Bk:%d Row:%03x Col:%03x \n", + j, bk[j]->curr_row, + bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size); +#endif + // transfer done + if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) { + bk[j]->mrq = NULL; + } + } + + } + + return issued; +} + +bool dram_t::issue_row_command(int j) +{ + bool issued = false; + unsigned grp = get_bankgrp_number(j); + if (bk[j]->mrq) { //if currently servicing a memory request + bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle); + // bank is idle + //else + if ( !issued && !RRDc && + (bk[j]->state == BANK_IDLE) && + !bk[j]->RPc && !bk[j]->RCc) { // +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tACT BK:%d NewRow:%03x From:%03x \n", + j,bk[j]->mrq->row,bk[j]->curr_row); +#endif + // activate the row with current memory request + bk[j]->curr_row = bk[j]->mrq->row; + bk[j]->state = BANK_ACTIVE; + RRDc = m_config->tRRD; + bk[j]->RCDc = m_config->tRCD; + bk[j]->RCDWRc = m_config->tRCDWR; + bk[j]->RASc = m_config->tRAS; + bk[j]->RCc = m_config->tRC; + prio = (j + 1) % m_config->nbk; + issued = true; + n_act_partial++; + n_act++; + } + + else + // different row activated + if ( (!issued) && + (bk[j]->curr_row != bk[j]->mrq->row) && + (bk[j]->state == BANK_ACTIVE) && + (!bk[j]->RASc && !bk[j]->WTPc && + !bk[j]->RTPc && + !bkgrp[grp]->RTPLc) ) { + // make the bank idle again + bk[j]->state = BANK_IDLE; + bk[j]->RPc = m_config->tRP; + prio = (j + 1) % m_config->nbk; + issued = true; + n_pre++; + n_pre_partial++; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row); +#endif + } + } + return issued; +} + + //if mrq is being serviced by dram, gets popped after CL latency fulfilled -class mem_fetch* dram_t::return_queue_pop() +class mem_fetch* dram_t::return_queue_pop() { return returnq->pop(); } -class mem_fetch* dram_t::return_queue_top() +class mem_fetch* dram_t::return_queue_top() { return returnq->top(); } + void dram_t::print( FILE* simFile) const { unsigned i; fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", id, m_config->nbk, m_config->busW, m_config->BL, m_config->CL ); fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n", - m_config->tCCD, m_config->tRRD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC ); - fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g\n", - n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr, + m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC ); + fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref_event=%d n_req=%d n_rd=%d n_rd_L2_A=%d n_write=%d n_wr_bk=%d bw_util=%.4g\n", + n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB, (float)bwutil/n_cmd); fprintf(simFile,"n_activity=%d dram_eff=%.4g\n", n_activity, (float)bwutil/n_activity); for (i=0;inbk;i++) { fprintf(simFile, "bk%d: %da %di ",i,bk[i]->n_access,bk[i]->n_idle); } + fprintf(simFile, "\n"); + fprintf(simFile, "\n------------------------------------------------------------------------\n"); + + printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num); + printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num); + printf("\nRow_Buffer_Locality_write = %.6f", (float)hits_write_num / write_num); + printf("\nBank_Level_Parallism = %.6f", (float)banks_1time / banks_acess_total); + printf("\nBank_Level_Parallism_Col = %.6f", (float)banks_time_rw / banks_access_rw_total); + printf("\nBank_Level_Parallism_Ready = %.6f", (float)banks_time_ready /banks_access_ready_total); + printf("\nwrite_to_read_ratio_blp_rw_average = %.6f", write_to_read_ratio_blp_rw_average /banks_access_rw_total); + printf("\nGrpLevelPara = %.6f \n", (float)bkgrp_parallsim_rw /banks_access_rw_total); + + printf("\nBW Util details:\n"); + printf("bwutil = %.6f \n", (float)bwutil/n_cmd); + printf("total_CMD = %d \n", n_cmd); + printf("util_bw = %d \n", util_bw); + printf("Wasted_Col = %d \n", wasted_bw_col); + printf("Wasted_Row = %d \n", wasted_bw_row); + printf("Idle = %d \n", idle_bw); + + printf("\nBW Util Bottlenecks: \n"); + printf("RCDc_limit = %d \n", RCDc_limit); + printf("RCDWRc_limit = %d \n", RCDWRc_limit); + printf("WTRc_limit = %d \n", WTRc_limit); + printf("RTWc_limit = %d \n", RTWc_limit); + printf("CCDLc_limit = %d \n", CCDLc_limit); + printf("rwq = %d \n", rwq_limit); + printf("CCDLc_limit_alone = %d \n", CCDLc_limit_alone); + printf("WTRc_limit_alone = %d \n", WTRc_limit_alone); + printf("RTWc_limit_alone = %d \n", RTWc_limit_alone); + + printf("\nCommands details: \n"); + printf("total_CMD = %d \n", n_cmd); + printf("n_nop = %d \n", n_nop); + printf("Read = %d \n", n_rd); + printf("Write = %d \n",n_wr); + printf("L2_Alloc = %d \n", n_rd_L2_A); + printf("L2_WB = %d \n", n_wr_WB); + printf("n_act = %d \n", n_act); + printf("n_pre = %d \n", n_pre); + printf("n_ref = %d \n", n_ref); + printf("n_req = %d \n", n_req ); + printf("total_req = %d \n", n_rd+n_wr+n_rd_L2_A+n_wr_WB); + + printf("\nDual Bus Interface Util: \n"); + printf("issued_total_row = %lu \n", issued_total_row); + printf("issued_total_col = %lu \n", issued_total_col); + printf("Row_Bus_Util = %.6f \n", (float)issued_total_row / n_cmd); + printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd); + printf("Either_Row_CoL_Bus_Util = %.6f \n", (float)issued_total / n_cmd); + printf("Issued_on_Two_Bus_Simul_Util = %.6f \n", (float)issued_two /n_cmd); + printf("issued_two_Eff = %.6f \n", (float)issued_two /issued_total); + printf("queue_avg = %.6f \n\n", (float)ave_mrqs/n_cmd ); + fprintf(simFile, "\n"); fprintf(simFile, "dram_util_bins:"); for (i=0;i<10;i++) fprintf(simFile, " %d", dram_util_bins[i]); fprintf(simFile, "\ndram_eff_bins:"); for (i=0;i<10;i++) fprintf(simFile, " %d", dram_eff_bins[i]); fprintf(simFile, "\n"); - if(m_config->scheduler_type== DRAM_FRFCFS) + if(m_config->scheduler_type== DRAM_FRFCFS) fprintf(simFile, "mrqq: max=%d avg=%g\n", max_mrqs, (float)ave_mrqs/n_cmd); } @@ -476,8 +815,8 @@ void dram_t::visualize() const void dram_t::print_stat( FILE* simFile ) { - fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ", - id, n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr, + fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ", + id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr, (float)bwutil/n_cmd); fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", max_mrqs, (float)ave_mrqs/n_cmd, max_mrqs_temp); fprintf(simFile, "\n"); @@ -516,6 +855,7 @@ void dram_t::visualizer_print( gzFile visualizer_file ) n_pre_partial = 0; n_req_partial = 0; + // dram access type classification for (unsigned j = 0; j < m_config->nbk; j++) { gzprintf(visualizer_file,"dramglobal_acc_r: %u %u %u\n", id, j, @@ -553,3 +893,16 @@ void dram_t::set_dram_power_stats( unsigned &cmd, wr = n_wr; req = n_req; } + +unsigned dram_t::get_bankgrp_number(unsigned i) +{ + if(m_config->dram_bnkgrp_indexing_policy == HIGHER_BITS) { //higher bits + return i>>m_config->bk_tag_length; + } + else if (m_config->dram_bnkgrp_indexing_policy == LOWER_BITS) { //lower bits + return i&((m_config->nbkgrp-1)); + } + else { + assert(1); + } +} diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 15c63e7..bee5b7b 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -31,9 +31,15 @@ #include "delayqueue.h" #include +#include +#include +#include +#include +#include #include #include #include +#include #define READ 'R' //define read and write states #define WRITE 'W' @@ -42,7 +48,7 @@ class dram_req_t { public: - dram_req_t( class mem_fetch *data ); + dram_req_t( class mem_fetch *data , unsigned banks, unsigned dram_bnk_indexing_policy); unsigned int row; unsigned int col; @@ -87,6 +93,17 @@ struct bank_t unsigned int bkgrpindex; }; +enum bank_index_function{ + LINEAR_BK_INDEX = 0, + BITWISE_XORING_BK_INDEX, + CUSTOM_BK_INDEX +}; + +enum bank_grp_bits_position{ + HIGHER_BITS = 0, + LOWER_BITS +}; + class mem_fetch; class dram_t @@ -95,7 +112,7 @@ public: dram_t( unsigned int parition_id, const struct memory_config *config, class memory_stats_t *stats, class memory_partition_unit *mp ); - bool full() const; + bool full(bool is_write) const; void print( FILE* simFile ) const; void visualize() const; void print_stat( FILE* simFile ); @@ -106,6 +123,7 @@ public: class mem_fetch* return_queue_pop(); class mem_fetch* return_queue_top(); + void push( class mem_fetch *data ); void cycle(); void dram_log (int task); @@ -123,17 +141,24 @@ public: unsigned &wr, unsigned &req) const; -private: - void scheduler_fifo(); - void scheduler_frfcfs(); + const struct memory_config *m_config; +private: bankgrp_t **bkgrp; bank_t **bk; unsigned int prio; + unsigned get_bankgrp_number(unsigned i); + + void scheduler_fifo(); + void scheduler_frfcfs(); + + bool issue_col_command(int j); + bool issue_row_command(int j); + unsigned int RRDc; unsigned int CCDc; unsigned int RTWc; //read to write penalty applies across banks @@ -146,7 +171,7 @@ private: fifo_pipeline *rwq; fifo_pipeline *mrqq; //buffer to hold packets when DRAM processing is over - //should be filled with dram clock and popped with l2or icnt clock + //should be filled with dram clock and popped with l2or icnt clock fifo_pipeline *returnq; unsigned int dram_util_bins[10]; @@ -158,11 +183,51 @@ private: unsigned int n_nop; unsigned int n_act; unsigned int n_pre; + unsigned int n_ref; unsigned int n_rd; + unsigned int n_rd_L2_A; unsigned int n_wr; + unsigned int n_wr_WB; unsigned int n_req; unsigned int max_mrqs_temp; + //some statistics to collect to see where BW is wasted? + unsigned wasted_bw_row; + unsigned wasted_bw_col; + unsigned util_bw; + unsigned idle_bw; + unsigned RCDc_limit; + unsigned CCDLc_limit; + unsigned CCDLc_limit_alone; + unsigned CCDc_limit; + unsigned WTRc_limit; + unsigned WTRc_limit_alone; + unsigned RCDWRc_limit; + unsigned RTWc_limit; + unsigned RTWc_limit_alone; + unsigned rwq_limit; + + //row locality, BLP and other statistics + unsigned long access_num; + unsigned long read_num; + unsigned long write_num; + unsigned long long hits_num; + unsigned long long hits_read_num; + unsigned long long hits_write_num; + unsigned long long banks_1time; + unsigned long long banks_acess_total; + unsigned long long banks_acess_total_after; + unsigned long long banks_time_rw; + unsigned long long banks_access_rw_total; + unsigned long long banks_time_ready; + unsigned long long banks_access_ready_total; + unsigned long long issued_two; + unsigned long long issued_total; + unsigned long long issued_total_row; + unsigned long long issued_total_col; + double write_to_read_ratio_blp_rw_average; + unsigned long long bkgrp_parallsim_rw; + unsigned int bwutil; unsigned int max_mrqs; unsigned int ave_mrqs; diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc index 8303e86..ff50050 100644 --- a/src/gpgpu-sim/dram_sched.cc +++ b/src/gpgpu-sim/dram_sched.cc @@ -36,6 +36,7 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem m_config = config; m_stats = stats; m_num_pending = 0; + m_num_write_pending = 0; m_dram = dm; m_queue = new std::list[m_config->nbk]; m_bins = new std::map::iterator> >[ m_config->nbk ]; @@ -49,15 +50,36 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem curr_row_service_time[i] = 0; row_service_timestamp[i] = 0; } + if(m_config->seperate_write_queue_enabled) { + m_write_queue = new std::list[m_config->nbk]; + m_write_bins = new std::map::iterator> >[ m_config->nbk ]; + m_last_write_row = new std::list::iterator>*[ m_config->nbk ]; + + for ( unsigned i=0; i < m_config->nbk; i++ ) { + m_write_queue[i].clear(); + m_write_bins[i].clear(); + m_last_write_row[i] = NULL; + } + } + m_mode = READ_MODE; } void frfcfs_scheduler::add_req( dram_req_t *req ) { - m_num_pending++; - m_queue[req->bk].push_front(req); - std::list::iterator ptr = m_queue[req->bk].begin(); - m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size); + m_num_write_pending++; + m_write_queue[req->bk].push_front(req); + std::list::iterator ptr = m_write_queue[req->bk].begin(); + m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } else { + assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size); + m_num_pending++; + m_queue[req->bk].push_front(req); + std::list::iterator ptr = m_queue[req->bk].begin(); + m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + } } void frfcfs_scheduler::data_collection(unsigned int bank) @@ -78,41 +100,92 @@ void frfcfs_scheduler::data_collection(unsigned int bank) dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row ) { - if ( m_last_row[bank] == NULL ) { - if ( m_queue[bank].empty() ) + //row + bool rowhit = true; + std::list *m_current_queue = m_queue; + std::map::iterator> > *m_current_bins = m_bins ; + std::list::iterator> **m_current_last_row = m_last_row; + + if(m_config->seperate_write_queue_enabled) { + if(m_mode == READ_MODE && + ((m_num_write_pending >= m_config->write_high_watermark ) + // || (m_queue[bank].empty() && !m_write_queue[bank].empty()) + )) { + m_mode = WRITE_MODE; + } + else if(m_mode == WRITE_MODE && + (( m_num_write_pending < m_config->write_low_watermark ) + // || (!m_queue[bank].empty() && m_write_queue[bank].empty()) + )){ + m_mode = READ_MODE; + } + } + + if(m_mode == WRITE_MODE) { + m_current_queue = m_write_queue; + m_current_bins = m_write_bins ; + m_current_last_row = m_last_write_row; + } + + if ( m_current_last_row[bank] == NULL ) { + if ( m_current_queue[bank].empty() ) return NULL; - std::map::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row ); - if ( bin_ptr == m_bins[bank].end()) { - dram_req_t *req = m_queue[bank].back(); - bin_ptr = m_bins[bank].find( req->row ); - assert( bin_ptr != m_bins[bank].end() ); // where did the request go??? - m_last_row[bank] = &(bin_ptr->second); + std::map::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row ); + if ( bin_ptr == m_current_bins[bank].end()) { + dram_req_t *req = m_current_queue[bank].back(); + bin_ptr = m_current_bins[bank].find( req->row ); + assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go??? + m_current_last_row[bank] = &(bin_ptr->second); data_collection(bank); + rowhit = false; } else { - m_last_row[bank] = &(bin_ptr->second); - + m_current_last_row[bank] = &(bin_ptr->second); + rowhit = true; } } - std::list::iterator next = m_last_row[bank]->back(); + std::list::iterator next = m_current_last_row[bank]->back(); dram_req_t *req = (*next); + //rowblp stats + m_dram->access_num++; + bool is_write = req->data->is_write(); + if(is_write) + m_dram->write_num++; + else + m_dram->read_num++; + + if(rowhit) { + m_dram->hits_num++; + if(is_write) + m_dram->hits_write_num++; + else + m_dram->hits_read_num++; + } + m_stats->concurrent_row_access[m_dram->id][bank]++; m_stats->row_access[m_dram->id][bank]++; - m_last_row[bank]->pop_back(); + m_current_last_row[bank]->pop_back(); - m_queue[bank].erase(next); - if ( m_last_row[bank]->empty() ) { - m_bins[bank].erase( req->row ); - m_last_row[bank] = NULL; + m_current_queue[bank].erase(next); + if ( m_current_last_row[bank]->empty() ) { + m_current_bins[bank].erase( req->row ); + m_current_last_row[bank] = NULL; } #ifdef DEBUG_FAST_IDEAL_SCHED if ( req ) printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row ); #endif - assert( req != NULL && m_num_pending != 0 ); - m_num_pending--; + + if(m_config->seperate_write_queue_enabled && req->data->is_write()) { + assert( req != NULL && m_num_write_pending != 0 ); + m_num_write_pending--; + } + else { + assert( req != NULL && m_num_pending != 0 ); + m_num_pending--; + } return req; } @@ -129,7 +202,7 @@ void dram_t::scheduler_frfcfs() { unsigned mrq_latency; frfcfs_scheduler *sched = m_frfcfs_scheduler; - while ( !mrqq->empty() && (!m_config->gpgpu_frfcfs_dram_sched_queue_size || sched->num_pending() < m_config->gpgpu_frfcfs_dram_sched_queue_size)) { + while ( !mrqq->empty() ) { dram_req_t *req = mrqq->pop(); // Power stats @@ -160,6 +233,8 @@ void dram_t::scheduler_frfcfs() bk[b]->mrq = req; if (m_config->gpgpu_memlatency_stat) { mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - bk[b]->mrq->timestamp; + m_stats->tot_mrq_latency += mrq_latency; + m_stats->tot_mrq_num++; bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle; m_stats->mrq_lat_table[LOGB2(mrq_latency)]++; if (mrq_latency > m_stats->max_mrq_latency) { diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h index 3860f5b..63f5831 100644 --- a/src/gpgpu-sim/dram_sched.h +++ b/src/gpgpu-sim/dram_sched.h @@ -35,6 +35,11 @@ #include #include +enum memory_mode { + READ_MODE = 0, + WRITE_MODE +}; + class frfcfs_scheduler { public: frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats ); @@ -43,17 +48,24 @@ public: dram_req_t *schedule( unsigned bank, unsigned curr_row ); void print( FILE *fp ); unsigned num_pending() const { return m_num_pending;} + unsigned num_write_pending() const { return m_num_write_pending;} private: const memory_config *m_config; dram_t *m_dram; unsigned m_num_pending; + unsigned m_num_write_pending; std::list *m_queue; std::map::iterator> > *m_bins; std::list::iterator> **m_last_row; unsigned *curr_row_service_time; //one set of variables for each bank. unsigned *row_service_timestamp; //tracks when scheduler began servicing current row + std::list *m_write_queue; + std::map::iterator> > *m_write_bins; + std::list::iterator> **m_last_write_row; + + enum memory_mode m_mode; memory_stats_t *m_stats; }; diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc index 8886398..ba81440 100644 --- a/src/gpgpu-sim/gpu-cache.cc +++ b/src/gpgpu-sim/gpu-cache.cc @@ -29,7 +29,6 @@ #include "stat-tool.h" #include -#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 // used to allocate memory that is large enough to adapt the changes in cache size across kernels const char * cache_request_status_str(enum cache_request_status status) @@ -38,7 +37,8 @@ const char * cache_request_status_str(enum cache_request_status status) "HIT", "HIT_RESERVED", "MISS", - "RESERVATION_FAIL" + "RESERVATION_FAIL", + "SECTOR_MISS" }; assert(sizeof(static_cache_request_status_str) / sizeof(const char*) == NUM_CACHE_REQUEST_STATUS); @@ -47,6 +47,22 @@ const char * cache_request_status_str(enum cache_request_status status) return static_cache_request_status_str[status]; } +const char * cache_fail_status_str(enum cache_reservation_fail_reason status) +{ + static const char * static_cache_reservation_fail_reason_str[] = { + "LINE_ALLOC_FAIL", + "MISS_QUEUE_FULL", + "MSHR_ENRTY_FAIL", + "MSHR_MERGE_ENRTY_FAIL", + "MSHR_RW_PENDING" + }; + + assert(sizeof(static_cache_reservation_fail_reason_str) / sizeof(const char*) == NUM_CACHE_RESERVATION_FAIL_STATUS); + assert(status < NUM_CACHE_RESERVATION_FAIL_STATUS); + + return static_cache_reservation_fail_reason_str[status]; +} + unsigned l1d_cache_config::set_index(new_addr_type addr) const{ unsigned set_index = m_nset; // Default to linear set index function unsigned lower_xor = 0; @@ -54,10 +70,11 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ switch(m_set_index_function){ case FERMI_HASH_SET_FUNCTION: + case BITWISE_XORING_FUNCTION: /* * Set Indexing function from "A Detailed GPU Cache Model Based on Reuse Distance Theory" * Cedric Nugteren et al. - * ISCA 2014 + * HPCA 2014 */ if(m_nset == 32 || m_nset == 64){ // Lower xor value is bits 7-11 @@ -80,6 +97,36 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ } break; + case HASH_IPOLY_FUNCTION: + /* + * Set Indexing function from "Pseudo-randomly interleaved memory." + * Rau, B. R et al. + * ISCA 1991 + * + * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme." + * Khairy et al. + * IEEE TPDS 2017. + */ + if(m_nset == 32 || m_nset == 64){ + std::bitset<64> a(addr); + std::bitset<6> index; + index[0] = a[25]^a[24]^a[23]^a[22]^a[21]^a[18]^a[17]^a[15]^a[12]^a[7]; //10 + index[1] = a[26]^a[25]^a[24]^a[23]^a[22]^a[19]^a[18]^a[16]^a[13]^a[8]; //10 + index[2] = a[26]^a[22]^a[21]^a[20]^a[19]^a[18]^a[15]^a[14]^a[12]^a[9]; //10 + index[3] = a[23]^a[22]^a[21]^a[20]^a[19]^a[16]^a[15]^a[13]^a[10]; //9 + index[4] = a[24]^a[23]^a[22]^a[21]^a[20]^a[17]^a[16]^a[14]^a[11]; //9 + + if(m_nset == 64) + index[5] = a[12]; + + set_index = index.to_ulong(); + + }else{ /* Else incorrect number of sets for the hashing function */ + assert("\nGPGPU-Sim cache configuration error: The number of sets should be " + "32 or 64 for the hashing set index function.\n" && 0); + } + break; + case CUSTOM_SET_FUNCTION: /* No custom set function implemented */ break; @@ -87,6 +134,10 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{ case LINEAR_SET_FUNCTION: set_index = (addr >> m_line_sz_log2) & (m_nset-1); break; + + default: + assert("\nUndefined set index function.\n" && 0); + break; } // Linear function selected or custom set index function not implemented @@ -113,13 +164,16 @@ unsigned l2_cache_config::set_index(new_addr_type addr) const{ tag_array::~tag_array() { + unsigned cache_lines_num = m_config.get_max_num_lines(); + for(unsigned i=0; iget_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if ( i == pending_lines.end() ) { + pending_lines[addr] = mf->get_inst().get_uid(); + } +} + +void tag_array::remove_pending_line(mem_fetch *mf){ + assert(mf); + new_addr_type addr = m_config.block_addr(mf->get_addr()); + line_table::const_iterator i = pending_lines.find(addr); + if ( i != pending_lines.end() ) { + pending_lines.erase(addr); + } } -enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) const { +enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_fetch* mf, bool probe_mode) const { + mem_access_sector_mask_t mask = mf->get_access_sector_mask(); + return probe(addr, idx, mask, probe_mode, mf); +} + + +enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask, bool probe_mode, mem_fetch* mf) const { //assert( m_config.m_write_policy == READ_ONLY ); unsigned set_index = m_config.set_index(addr); new_addr_type tag = m_config.tag(addr); @@ -169,35 +263,45 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) // check for hit or pending hit for (unsigned way=0; waym_tag == tag) { - if ( line->m_status == RESERVED ) { + if ( line->get_status(mask) == RESERVED ) { idx = index; return HIT_RESERVED; - } else if ( line->m_status == VALID ) { + } else if ( line->get_status(mask) == VALID ) { idx = index; return HIT; - } else if ( line->m_status == MODIFIED ) { + } else if ( line->get_status(mask) == MODIFIED) { + if(line->is_readable(mask)) { + idx = index; + return HIT; + } + else { + idx = index; + return SECTOR_MISS; + } + + } else if ( line->is_valid_line() && line->get_status(mask) == INVALID ) { idx = index; - return HIT; - } else { - assert( line->m_status == INVALID ); + return SECTOR_MISS; + }else { + assert( line->get_status(mask) == INVALID ); } } - if (line->m_status != RESERVED) { + if (!line->is_reserved_line()) { all_reserved = false; - if (line->m_status == INVALID) { + if (line->is_invalid_line()) { invalid_line = index; } else { // valid line : keep track of most appropriate replacement candidate if ( m_config.m_replacement_policy == LRU ) { - if ( line->m_last_access_time < valid_timestamp ) { - valid_timestamp = line->m_last_access_time; + if ( line->get_last_access_time() < valid_timestamp ) { + valid_timestamp = line->get_last_access_time(); valid_line = index; } } else if ( m_config.m_replacement_policy == FIFO ) { - if ( line->m_alloc_time < valid_timestamp ) { - valid_timestamp = line->m_alloc_time; + if ( line->get_alloc_time() < valid_timestamp ) { + valid_timestamp = line->get_alloc_time(); valid_line = index; } } @@ -215,40 +319,59 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) idx = valid_line; } else abort(); // if an unreserved block exists, it is either invalid or replaceable + + if(probe_mode && m_config.is_streaming()){ + line_table::const_iterator i = pending_lines.find(m_config.block_addr(addr)); + assert(mf); + if ( !mf->is_write() && i != pending_lines.end() ) { + if(i->second != mf->get_inst().get_uid()) + return SECTOR_MISS; + } + } + return MISS; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx ) +enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf) { bool wb=false; - cache_block_t evicted; - enum cache_request_status result = access(addr,time,idx,wb,evicted); + evicted_block_info evicted; + enum cache_request_status result = access(addr,time,idx,wb,evicted,mf); assert(!wb); return result; } -enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, cache_block_t &evicted ) +enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf ) { m_access++; + is_used = true; shader_cache_access_log(m_core_id, m_type_id, 0); // log accesses to cache - enum cache_request_status status = probe(addr,idx); + enum cache_request_status status = probe(addr,idx,mf); switch (status) { case HIT_RESERVED: m_pending_hit++; case HIT: - m_lines[idx].m_last_access_time=time; + m_lines[idx]->set_last_access_time(time, mf->get_access_sector_mask()); break; case MISS: m_miss++; shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses if ( m_config.m_alloc_policy == ON_MISS ) { - if( m_lines[idx].m_status == MODIFIED ) { + if( m_lines[idx]->is_modified_line()) { wb = true; - evicted = m_lines[idx]; + evicted.set_info(m_lines[idx]->m_block_addr, m_lines[idx]->get_modified_size()); } - m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time ); + m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mf->get_access_sector_mask()); } break; + case SECTOR_MISS: + assert(m_config.m_cache_type == SECTOR); + m_sector_miss++; + shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses + if ( m_config.m_alloc_policy == ON_MISS ) { + ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mf->get_access_sector_mask() ); + } + break; case RESERVATION_FAIL: m_res_fail++; shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses @@ -261,37 +384,70 @@ enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, return status; } -void tag_array::fill( new_addr_type addr, unsigned time ) +void tag_array::fill( new_addr_type addr, unsigned time, mem_fetch* mf) +{ + fill(addr, time, mf->get_access_sector_mask()); +} + +void tag_array::fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) { - assert( m_config.m_alloc_policy == ON_FILL ); + //assert( m_config.m_alloc_policy == ON_FILL ); unsigned idx; - enum cache_request_status status = probe(addr,idx); - assert(status==MISS); // MSHR should have prevented redundant memory request - m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time ); - m_lines[idx].fill(time); + enum cache_request_status status = probe(addr,idx,mask); + //assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request + if(status==MISS) + m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mask ); + else if (status==SECTOR_MISS) { + assert(m_config.m_cache_type == SECTOR); + ((sector_cache_block*)m_lines[idx])->allocate_sector( time, mask ); + } + + m_lines[idx]->fill(time, mask); } -void tag_array::fill( unsigned index, unsigned time ) +void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf) { assert( m_config.m_alloc_policy == ON_MISS ); - m_lines[index].fill(time); + m_lines[index]->fill(time, mf->get_access_sector_mask()); } + +//TODO: we need write back the flushed data to the upper level void tag_array::flush() { + if(!is_used) + return; + for (unsigned i=0; i < m_config.get_num_lines(); i++) - m_lines[i].m_status = INVALID; + if(m_lines[i]->is_modified_line()) { + for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + } + + is_used = false; +} + +void tag_array::invalidate() +{ + if(!is_used) + return; + + for (unsigned i=0; i < m_config.get_num_lines(); i++) + for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++) + m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ; + + is_used = false; } float tag_array::windowed_miss_rate( ) const { unsigned n_access = m_access - m_prev_snapshot_access; - unsigned n_miss = m_miss - m_prev_snapshot_miss; + unsigned n_miss = (m_miss+m_sector_miss) - m_prev_snapshot_miss; // unsigned n_pending_hit = m_pending_hit - m_prev_snapshot_pending_hit; float missrate = 0.0f; if (n_access != 0) - missrate = (float) n_miss / n_access; + missrate = (float) (n_miss+m_sector_miss) / n_access; return missrate; } @@ -299,23 +455,24 @@ void tag_array::new_window() { m_prev_snapshot_access = m_access; m_prev_snapshot_miss = m_miss; + m_prev_snapshot_miss = m_miss + m_sector_miss; m_prev_snapshot_pending_hit = m_pending_hit; } void tag_array::print( FILE *stream, unsigned &total_access, unsigned &total_misses ) const { m_config.print(stream); - fprintf( stream, "\t\tAccess = %d, Miss = %d (%.3g), PendingHit = %d (%.3g)\n", - m_access, m_miss, (float) m_miss / m_access, + fprintf( stream, "\t\tAccess = %d, Miss = %d, Sector_Miss = %d, Total_Miss = %d (%.3g), PendingHit = %d (%.3g)\n", + m_access, m_miss, m_sector_miss, (m_miss+m_sector_miss), (float) (m_miss+m_sector_miss) / m_access, m_pending_hit, (float) m_pending_hit / m_access); - total_misses+=m_miss; + total_misses+=(m_miss+m_sector_miss); total_access+=m_access; } void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsigned &total_hit_res, unsigned &total_res_fail) const{ // Update statistics from the tag array total_access = m_access; - total_misses = m_miss; + total_misses = (m_miss+m_sector_miss); total_hit_res = m_pending_hit; total_res_fail = m_res_fail; } @@ -324,16 +481,17 @@ void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsign bool was_write_sent( const std::list &events ) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == WRITE_REQUEST_SENT ) + if( (*e).m_cache_event_type == WRITE_REQUEST_SENT ) return true; } return false; } -bool was_writeback_sent( const std::list &events ) +bool was_writeback_sent( const std::list &events, cache_event& wb_event) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == WRITE_BACK_REQUEST_SENT ) + if( (*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT ) + wb_event = *e; return true; } return false; @@ -342,7 +500,16 @@ bool was_writeback_sent( const std::list &events ) bool was_read_sent( const std::list &events ) { for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { - if( *e == READ_REQUEST_SENT ) + if( (*e).m_cache_event_type == READ_REQUEST_SENT ) + return true; + } + return false; +} + +bool was_writeallocate_sent( const std::list &events ) +{ + for( std::list::const_iterator e=events.begin(); e!=events.end(); e++ ) { + if( (*e).m_cache_event_type == WRITE_ALLOCATE_SENT ) return true; } return false; @@ -375,11 +542,27 @@ void mshr_table::add( new_addr_type block_addr, mem_fetch *mf ){ } } +/// check is_read_after_write_pending +bool mshr_table::is_read_after_write_pending( new_addr_type block_addr){ + std::list my_list = m_data[block_addr].m_list; + bool write_found = false; + for (std::list::iterator it=my_list.begin(); it != my_list.end(); ++it) + { + if((*it)->is_write()) //Pending Write Request + write_found = true; + else if(write_found) //Pending Read Request and we found previous Write + return true; + } + + return false; + +} + /// Accept a new cache fill response: mark entry ready for processing void mshr_table::mark_ready( new_addr_type block_addr, bool &has_atomic ){ assert( !busy() ); table::iterator a = m_data.find(block_addr); - assert( a != m_data.end() ); // don't remove same request twice + assert( a != m_data.end() ); m_current_response.push_back( block_addr ); has_atomic = a->second.m_has_atomic; assert( m_current_response.size() <= m_data.size() ); @@ -417,9 +600,11 @@ void mshr_table::display( FILE *fp ) const{ /***************************************************************** Caches *****************************************************************/ cache_stats::cache_stats(){ m_stats.resize(NUM_MEM_ACCESS_TYPE); + m_fail_stats.resize(NUM_MEM_ACCESS_TYPE); for(unsigned i=0; i 0) - fprintf(fout, "\t%s[%s][%s] = %u\n", - m_cache_name.c_str(), - mem_access_type_str((enum mem_access_type)type), - "TOTAL_ACCESS", - total_access[type]); + if(total_access[type] > 0) + fprintf(fout, "\t%s[%s][%s] = %u\n", + m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + "TOTAL_ACCESS", + total_access[type]); } } +void cache_stats::print_fail_stats(FILE *fout, const char *cache_name) const{ + std::string m_cache_name = cache_name; + for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { + for (unsigned fail = 0; fail < NUM_CACHE_RESERVATION_FAIL_STATUS; ++fail) { + if(m_fail_stats[type][fail] > 0){ + fprintf(fout, "\t%s[%s][%s] = %u\n", + m_cache_name.c_str(), + mem_access_type_str((enum mem_access_type)type), + cache_fail_status_str((enum cache_reservation_fail_reason)fail), + m_fail_stats[type][fail]); + } + } + } +} + void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const { float data_port_util = 0.0f; @@ -580,10 +813,10 @@ void cache_stats::get_sub_stats(struct cache_sub_stats &css) const{ for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) { for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) { - if(status == HIT || status == MISS || status == HIT_RESERVED) + if(status == HIT || status == MISS || status == SECTOR_MISS || status == HIT_RESERVED) t_css.accesses += m_stats[type][status]; - if(status == MISS) + if(status == MISS || status == SECTOR_MISS) t_css.misses += m_stats[type][status]; if(status == HIT_RESERVED) @@ -611,6 +844,16 @@ bool cache_stats::check_valid(int type, int status) const{ return false; } +bool cache_stats::check_fail_valid(int type, int fail) const{ + /// + /// Verify a valid access_type/access_status + /// + if((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (fail >= 0) && (fail < NUM_CACHE_RESERVATION_FAIL_STATUS)) + return true; + else + return false; +} + void cache_stats::sample_cache_port_utility(bool data_port_busy, bool fill_port_busy) { m_cache_port_available_cycles += 1; @@ -639,15 +882,18 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac unsigned data_cycles = data_size / port_width + ((data_size % port_width > 0)? 1 : 0); m_data_port_occupied_cycles += data_cycles; } break; - case HIT_RESERVED: + case HIT_RESERVED: case MISS: { // the data array is accessed to read out the entire line for write-back - if (was_writeback_sent(events)) { - unsigned data_cycles = m_config.m_line_sz / port_width; + // in case of sector cache we need to write bank only the modified sectors + cache_event ev(WRITE_BACK_REQUEST_SENT); + if (was_writeback_sent(events, ev)) { + unsigned data_cycles = ev.m_evicted_block.m_modified_size / port_width; m_data_port_occupied_cycles += data_cycles; } } break; - case RESERVATION_FAIL: + case SECTOR_MISS: + case RESERVATION_FAIL: // Does not consume any port bandwidth break; default: @@ -660,7 +906,7 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac void baseline_cache::bandwidth_management::use_fill_port(mem_fetch *mf) { // assume filling the entire line with the returned request - unsigned fill_cycles = m_config.m_line_sz / m_config.m_data_port_width; + unsigned fill_cycles = m_config.get_atom_sz() / m_config.m_data_port_width; m_fill_port_occupied_cycles += fill_cycles; } @@ -707,21 +953,43 @@ void baseline_cache::cycle(){ /// Interface for response from lower memory level (model bandwidth restictions in caller) void baseline_cache::fill(mem_fetch *mf, unsigned time){ + + if(m_config.m_mshr_type == SECTOR_ASSOC) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); + assert( e != m_extra_mf_fields.end() ); + e->second.pending_read--; + + if(e->second.pending_read > 0) { + //wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; + } + } + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); assert( e != m_extra_mf_fields.end() ); assert( e->second.m_valid ); mf->set_data_size( e->second.m_data_size ); + mf->set_addr( e->second.m_addr ); if ( m_config.m_alloc_policy == ON_MISS ) - m_tag_array->fill(e->second.m_cache_index,time); - else if ( m_config.m_alloc_policy == ON_FILL ) - m_tag_array->fill(e->second.m_block_addr,time); + m_tag_array->fill(e->second.m_cache_index,time,mf); + else if ( m_config.m_alloc_policy == ON_FILL ) { + m_tag_array->fill(e->second.m_block_addr,time,mf); + if(m_config.is_streaming()) + m_tag_array->remove_pending_line(mf); + } else abort(); bool has_atomic = false; m_mshrs.mark_ready(e->second.m_block_addr, has_atomic); if (has_atomic) { assert(m_config.m_alloc_policy == ON_MISS); - cache_block_t &block = m_tag_array->get_block(e->second.m_cache_index); - block.m_status = MODIFIED; // mark line as dirty for atomic operation + cache_block_t* block = m_tag_array->get_block(e->second.m_cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); // mark line as dirty for atomic operation } m_extra_mf_fields.erase(mf); m_bandwidth_management.use_fill_port(mf); @@ -749,45 +1017,59 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a unsigned time, bool &do_miss, std::list &events, bool read_only, bool wa){ bool wb=false; - cache_block_t e; + evicted_block_info e; send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, e, events, read_only, wa); } /// Read miss handler. Check MSHR hit or MSHR available void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list &events, bool read_only, bool wa){ + unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list &events, bool read_only, bool wa){ - bool mshr_hit = m_mshrs.probe(block_addr); - bool mshr_avail = !m_mshrs.full(block_addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); if ( mshr_hit && mshr_avail ) { if(read_only) - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); else - m_tag_array->access(block_addr,time,cache_index,wb,evicted); + m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); - m_mshrs.add(block_addr,mf); + m_mshrs.add(mshr_addr,mf); do_miss = true; + } else if ( !mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size) ) { if(read_only) - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); else - m_tag_array->access(block_addr,time,cache_index,wb,evicted); - - m_mshrs.add(block_addr,mf); - m_extra_mf_fields[mf] = extra_mf_fields(block_addr,cache_index, mf->get_data_size()); - mf->set_data_size( m_config.get_line_sz() ); + m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + + m_mshrs.add(mshr_addr,mf); + if(m_config.is_streaming() && m_config.m_cache_type == SECTOR){ + m_tag_array->add_pending_line(mf); + } + m_extra_mf_fields[mf] = extra_mf_fields(mshr_addr,mf->get_addr(),cache_index, mf->get_data_size(), m_config); + mf->set_data_size( m_config.get_atom_sz() ); + mf->set_addr( mshr_addr ); m_miss_queue.push_back(mf); mf->set_status(m_miss_queue_status,time); if(!wa) - events.push_back(READ_REQUEST_SENT); + events.push_back(cache_event(READ_REQUEST_SENT)); + do_miss = true; } + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); } /// Sends write request to lower level memory (write or writeback) void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned time, std::list &events){ - events.push_back(request); + + events.push_back(request); m_miss_queue.push_back(mf); mf->set_status(m_miss_queue_status,time); } @@ -798,40 +1080,44 @@ void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned /// Write-back hit: Mark block as modified cache_request_status data_cache::wr_hit_wb(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); // update LRU state - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; + m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); return HIT; } /// Write-through hit: Directly send request to lower level memory cache_request_status data_cache::wr_hit_wt(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(0)) + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; // cannot handle request this cycle + } new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); // update LRU state - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; + m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); // generate a write-through - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); return HIT; } /// Write-evict hit: Send request to lower level memory and invalidate corresponding block cache_request_status data_cache::wr_hit_we(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(0)) + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; // cannot handle request this cycle + } // generate a write-through/evict - cache_block_t &block = m_tag_array->get_block(cache_index); - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + cache_block_t* block = m_tag_array->get_block(cache_index); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); // Invalidate block - block.m_status = INVALID; + block->set_status(INVALID, mf->get_access_sector_mask()); return HIT; } @@ -850,34 +1136,46 @@ enum cache_request_status data_cache::wr_hit_global_we_local_wb(new_addr_type ad /// Write-allocate miss: Send write request to lower level memory // and send a read request for the same block enum cache_request_status -data_cache::wr_miss_wa( new_addr_type addr, +data_cache::wr_miss_wa_naive( new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list &events, enum cache_request_status status ) { new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); // Write allocate, maximum 3 requests (write miss, read request, write back request) // Conservatively ensure the worst-case request can be handled this cycle - bool mshr_hit = m_mshrs.probe(block_addr); - bool mshr_avail = !m_mshrs.full(block_addr); + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); if(miss_queue_full(2) || (!(mshr_hit && mshr_avail) - && !(!mshr_hit && mshr_avail - && (m_miss_queue.size() < m_config.m_miss_queue_size)))) + && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + //check what is the exactly the failure reason + if(miss_queue_full(2) ) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); + return RESERVATION_FAIL; + } - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); // Tries to send write allocate request, returns true on success and false on failure //if(!send_write_allocate(mf, addr, block_addr, cache_index, time, events)) // return RESERVATION_FAIL; const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, mf->get_addr(), - mf->get_data_size(), + m_config.get_atom_sz(), false, // Now performing a read mf->get_access_warp_mask(), - mf->get_access_byte_mask() ); + mf->get_access_byte_mask(), + mf->get_access_sector_mask()); mem_fetch *n_mf = new mem_fetch( *ma, NULL, @@ -889,20 +1187,22 @@ data_cache::wr_miss_wa( new_addr_type addr, bool do_miss = false; bool wb = false; - cache_block_t evicted; + evicted_block_info evicted; // Send read request resulting from write miss send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb, evicted, events, false, true); + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + if( do_miss ){ // If evicted block is modified and not a write-through // (already modified lower level) if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + assert(status == MISS); //SECTOR_MISS and HIT_RESERVED should not send write back mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,m_config.get_line_sz(),true); - m_miss_queue.push_back(wb); - wb->set_status(m_miss_queue_status,time); + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); } return MISS; } @@ -910,6 +1210,177 @@ data_cache::wr_miss_wa( new_addr_type addr, return RESERVATION_FAIL; } + +enum cache_request_status +data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, std::list &events, + enum cache_request_status status ) +{ + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + + if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) + { + //if the request writes to the whole cache line/sector, then, write and set cache line Modified. + //and no need to send read request to memory or reserve mshr + + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + + bool wb = false; + evicted_block_info evicted; + + cache_request_status status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + assert(status != HIT); + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if(status == HIT_RESERVED) + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + + if( status != RESERVATION_FAIL ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } + else + { + bool mshr_hit = m_mshrs.probe(mshr_addr); + bool mshr_avail = !m_mshrs.full(mshr_addr); + if(miss_queue_full(1) + || (!(mshr_hit && mshr_avail) + && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) { + //check what is the exactly the failure reason + if(miss_queue_full(1) ) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + else if(mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL); + else if (!mshr_hit && !mshr_avail) + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL); + else + assert(0); + + return RESERVATION_FAIL; + } + + + //prevent Write - Read - Write in pending mshr + //allowing another write will override the value of the first write, and the pending read request will read incorrect result from the second write + if(m_mshrs.probe(mshr_addr) && m_mshrs.is_read_after_write_pending(mshr_addr) && mf->is_write()) + { + //assert(0); + m_stats.inc_fail_stats(mf->get_access_type(), MSHR_RW_PENDING); + return RESERVATION_FAIL; + } + + const mem_access_t *ma = new mem_access_t( m_wr_alloc_type, + mf->get_addr(), + m_config.get_atom_sz(), + false, // Now performing a read + mf->get_access_warp_mask(), + mf->get_access_byte_mask(), + mf->get_access_sector_mask()); + + mem_fetch *n_mf = new mem_fetch( *ma, + NULL, + mf->get_ctrl_size(), + mf->get_wid(), + mf->get_sid(), + mf->get_tpc(), + mf->get_mem_config(), + NULL, + mf); + + + new_addr_type block_addr = m_config.block_addr(addr); + bool do_miss = false; + bool wb = false; + evicted_block_info evicted; + send_read_request( addr, + block_addr, + cache_index, + n_mf, time, do_miss, wb, evicted, events, false, true); + + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); + + events.push_back(cache_event(WRITE_ALLOCATE_SENT)); + + if( do_miss ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; + } +} + +enum cache_request_status +data_cache::wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, + unsigned cache_index, mem_fetch *mf, + unsigned time, std::list &events, + enum cache_request_status status ) +{ + + new_addr_type block_addr = m_config.block_addr(addr); + new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr()); + + + //if the request writes to the whole cache line/sector, then, write and set cache line Modified. + //and no need to send read request to memory or reserve mshr + + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + + bool wb = false; + evicted_block_info evicted; + + cache_request_status m_status = m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf); + assert(m_status != HIT); + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()); + if(m_status == HIT_RESERVED) { + block->set_ignore_on_fill(true, mf->get_access_sector_mask()); + block->set_modified_on_fill(true, mf->get_access_sector_mask()); + } + + if(mf->get_access_byte_mask().count() == m_config.get_atom_sz()) + { + block->set_m_readable(true, mf->get_access_sector_mask()); + } else + { + block->set_m_readable(false, mf->get_access_sector_mask()); + } + + if( m_status != RESERVATION_FAIL ){ + // If evicted block is modified and not a write-through + // (already modified lower level) + if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { + mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, + m_wrbk_type,evicted.m_modified_size,true); + send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events); + } + return MISS; + } + return RESERVATION_FAIL; +} + /// No write-allocate miss: Simply send write request to lower level memory enum cache_request_status data_cache::wr_miss_no_wa( new_addr_type addr, @@ -919,11 +1390,14 @@ data_cache::wr_miss_no_wa( new_addr_type addr, std::list &events, enum cache_request_status status ) { - if(miss_queue_full(0)) - return RESERVATION_FAIL; // cannot handle request this cycle + if(miss_queue_full(0)) { + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); + return RESERVATION_FAIL; // cannot handle request this cycle + } + // on miss, generate write through (no write buffering -- too many threads for that) - send_write_request(mf, WRITE_REQUEST_SENT, time, events); + send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events); return MISS; } @@ -941,13 +1415,13 @@ data_cache::rd_hit_base( new_addr_type addr, enum cache_request_status status ) { new_addr_type block_addr = m_config.block_addr(addr); - m_tag_array->access(block_addr,time,cache_index); + m_tag_array->access(block_addr,time,cache_index,mf); // Atomics treated as global read/write requests - Perform read, mark line as // MODIFIED if(mf->isatomic()){ assert(mf->get_access_type() == GLOBAL_ACC_R); - cache_block_t &block = m_tag_array->get_block(cache_index); - block.m_status = MODIFIED; // mark line as dirty + cache_block_t* block = m_tag_array->get_block(cache_index); + block->set_status(MODIFIED, mf->get_access_sector_mask()) ; // mark line as dirty } return HIT; } @@ -963,15 +1437,17 @@ data_cache::rd_miss_base( new_addr_type addr, unsigned time, std::list &events, enum cache_request_status status ){ - if(miss_queue_full(1)) + if(miss_queue_full(1)) { // cannot handle request this cycle // (might need to generate two requests) + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); return RESERVATION_FAIL; + } new_addr_type block_addr = m_config.block_addr(addr); bool do_miss = false; bool wb = false; - cache_block_t evicted; + evicted_block_info evicted; send_read_request( addr, block_addr, cache_index, @@ -982,12 +1458,12 @@ data_cache::rd_miss_base( new_addr_type addr, // (already modified lower level) if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr, - m_wrbk_type,m_config.get_line_sz(),true); + m_wrbk_type,evicted.m_modified_size,true); send_write_request(wb, WRITE_BACK_REQUEST_SENT, time, events); } return MISS; } - return RESERVATION_FAIL; + return RESERVATION_FAIL; } /// Access cache for read_only_cache: returns RESERVATION_FAIL if @@ -998,16 +1474,16 @@ read_only_cache::access( new_addr_type addr, unsigned time, std::list &events ) { - assert( mf->get_data_size() <= m_config.get_line_sz()); + assert( mf->get_data_size() <= m_config.get_atom_sz()); assert(m_config.m_write_policy == READ_ONLY); assert(!mf->get_is_write()); new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tag_array->probe(block_addr,cache_index); + enum cache_request_status status = m_tag_array->probe(block_addr,cache_index,mf); enum cache_request_status cache_status = RESERVATION_FAIL; if ( status == HIT ) { - cache_status = m_tag_array->access(block_addr,time,cache_index); // update LRU state + cache_status = m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state }else if ( status != RESERVATION_FAIL ) { if(!miss_queue_full(0)){ bool do_miss=false; @@ -1018,7 +1494,10 @@ read_only_cache::access( new_addr_type addr, cache_status = RESERVATION_FAIL; }else{ cache_status = RESERVATION_FAIL; + m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL); } + }else { + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } m_stats.inc_stats(mf->get_access_type(), m_stats.select_stats_status(status, cache_status)); @@ -1047,10 +1526,13 @@ data_cache::process_tag_probe( bool wr, access_status = (this->*m_wr_hit)( addr, cache_index, mf, time, events, probe_status ); - }else if ( probe_status != RESERVATION_FAIL ) { + }else if ( (probe_status != RESERVATION_FAIL) || (probe_status == RESERVATION_FAIL && m_config.m_write_alloc_policy == NO_WRITE_ALLOCATE) ) { access_status = (this->*m_wr_miss)( addr, cache_index, mf, time, events, probe_status ); + }else { + //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } }else{ // Read if(probe_status == HIT){ @@ -1061,6 +1543,9 @@ data_cache::process_tag_probe( bool wr, access_status = (this->*m_rd_miss)( addr, cache_index, mf, time, events, probe_status ); + }else { + //the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved) + m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL); } } @@ -1080,12 +1565,12 @@ data_cache::access( new_addr_type addr, std::list &events ) { - assert( mf->get_data_size() <= m_config.get_line_sz()); + assert( mf->get_data_size() <= m_config.get_atom_sz()); bool wr = mf->get_is_write(); new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; enum cache_request_status probe_status - = m_tag_array->probe( block_addr, cache_index ); + = m_tag_array->probe( block_addr, cache_index, mf, true); enum cache_request_status access_status = process_tag_probe( wr, probe_status, addr, cache_index, mf, time, events ); m_stats.inc_stats(mf->get_access_type(), @@ -1134,7 +1619,7 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf, // at this point, we will accept the request : access tags and immediately allocate line new_addr_type block_addr = m_config.block_addr(addr); unsigned cache_index = (unsigned)-1; - enum cache_request_status status = m_tags.access(block_addr,time,cache_index); + enum cache_request_status status = m_tags.access(block_addr,time,cache_index,mf); enum cache_request_status cache_status = RESERVATION_FAIL; assert( status != RESERVATION_FAIL ); assert( status != HIT_RESERVED ); // as far as tags are concerned: HIT or MISS @@ -1142,12 +1627,12 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf, if ( status == MISS ) { // we need to send a memory request... unsigned rob_index = m_rob.push( rob_entry(cache_index, mf, block_addr) ); - m_extra_mf_fields[mf] = extra_mf_fields(rob_index); + m_extra_mf_fields[mf] = extra_mf_fields(rob_index, m_config); mf->set_data_size(m_config.get_line_sz()); - m_tags.fill(cache_index,time); // mark block as valid + m_tags.fill(cache_index,time,mf); // mark block as valid m_request_fifo.push(mf); mf->set_status(m_request_queue_status,time); - events.push_back(READ_REQUEST_SENT); + events.push_back(cache_event(READ_REQUEST_SENT)); cache_status = MISS; } else { // the value *will* *be* in the cache already @@ -1174,7 +1659,7 @@ void tex_cache::cycle(){ unsigned rob_index = m_rob.next_pop_index(); const rob_entry &r = m_rob.peek(rob_index); assert( r.m_request == e.m_request ); - assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) ); + //assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) ); if ( r.m_ready ) { assert( r.m_index == e.m_cache_index ); m_cache[r.m_index].m_valid = true; @@ -1197,6 +1682,23 @@ void tex_cache::cycle(){ /// Place returning cache block into reorder buffer void tex_cache::fill( mem_fetch *mf, unsigned time ) { + if(m_config.m_mshr_type == SECTOR_TEX_FIFO) { + assert(mf->get_original_mf()); + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->get_original_mf()); + assert( e != m_extra_mf_fields.end() ); + e->second.pending_read--; + + if(e->second.pending_read > 0) { + //wait for the other requests to come back + delete mf; + return; + } else { + mem_fetch *temp = mf; + mf = mf->get_original_mf(); + delete temp; + } + } + extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf); assert( e != m_extra_mf_fields.end() ); assert( e->second.m_valid ); diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h index 7535a1d..e663cf6 100644 --- a/src/gpgpu-sim/gpu-cache.h +++ b/src/gpgpu-sim/gpu-cache.h @@ -36,9 +36,12 @@ #include "../tr1_hash_map.h" #include "addrdec.h" +#include + +#define MAX_DEFAULT_CACHE_SIZE_MULTIBLIER 4 enum cache_block_state { - INVALID, + INVALID=0, RESERVED, VALID, MODIFIED @@ -49,13 +52,51 @@ enum cache_request_status { HIT_RESERVED, MISS, RESERVATION_FAIL, + SECTOR_MISS, NUM_CACHE_REQUEST_STATUS }; -enum cache_event { +enum cache_reservation_fail_reason { + LINE_ALLOC_FAIL= 0,// all line are reserved + MISS_QUEUE_FULL, // MISS queue (i.e. interconnect or DRAM) is full + MSHR_ENRTY_FAIL, + MSHR_MERGE_ENRTY_FAIL, + MSHR_RW_PENDING, + NUM_CACHE_RESERVATION_FAIL_STATUS +}; + +enum cache_event_type { WRITE_BACK_REQUEST_SENT, READ_REQUEST_SENT, - WRITE_REQUEST_SENT + WRITE_REQUEST_SENT, + WRITE_ALLOCATE_SENT +}; + +struct evicted_block_info { + new_addr_type m_block_addr; + unsigned m_modified_size; + evicted_block_info() { + m_block_addr = 0; + m_modified_size = 0; + } + void set_info(new_addr_type block_addr, unsigned modified_size){ + m_block_addr = block_addr; + m_modified_size = modified_size; + } +}; + +struct cache_event { + enum cache_event_type m_cache_event_type; + evicted_block_info m_evicted_block; //if it was write_back event, fill the the evicted block info + + cache_event(enum cache_event_type m_cache_event){ + m_cache_event_type = m_cache_event; + } + + cache_event(enum cache_event_type cache_event, evicted_block_info evicted_block){ + m_cache_event_type = cache_event; + m_evicted_block = evicted_block; + } }; const char * cache_request_status_str(enum cache_request_status status); @@ -65,33 +106,340 @@ struct cache_block_t { { m_tag=0; m_block_addr=0; - m_alloc_time=0; - m_fill_time=0; - m_last_access_time=0; - m_status=INVALID; } - void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time ) + + virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) = 0; + virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) = 0; + + virtual bool is_invalid_line() = 0; + virtual bool is_valid_line() = 0; + virtual bool is_reserved_line() = 0; + virtual bool is_modified_line() = 0; + + virtual enum cache_block_state get_status( mem_access_sector_mask_t sector_mask) = 0; + virtual void set_status(enum cache_block_state m_status, mem_access_sector_mask_t sector_mask) = 0; + + virtual unsigned get_last_access_time() = 0; + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned get_alloc_time() = 0; + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) = 0; + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) = 0; + virtual unsigned get_modified_size() = 0; + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)=0; + virtual bool is_readable(mem_access_sector_mask_t sector_mask)=0; + virtual void print_status()=0; + virtual ~cache_block_t() {} + + + new_addr_type m_tag; + new_addr_type m_block_addr; + +}; + +struct line_cache_block: public cache_block_t { + line_cache_block() + { + m_alloc_time=0; + m_fill_time=0; + m_last_access_time=0; + m_status=INVALID; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + m_readable = true; + } + void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) + { + m_tag=tag; + m_block_addr=block_addr; + m_alloc_time=time; + m_last_access_time=time; + m_fill_time=0; + m_status=RESERVED; + m_ignore_on_fill_status = false; + m_set_modified_on_fill = false; + } + void fill( unsigned time, mem_access_sector_mask_t sector_mask ) + { + //if(!m_ignore_on_fill_status) + // assert( m_status == RESERVED ); + + m_status = m_set_modified_on_fill? MODIFIED : VALID; + + m_fill_time=time; + } + virtual bool is_invalid_line() + { + return m_status == INVALID; + } + virtual bool is_valid_line() + { + return m_status == VALID; + } + virtual bool is_reserved_line() + { + return m_status == RESERVED; + } + virtual bool is_modified_line() + { + return m_status == MODIFIED; + } + + virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) + { + return m_status; + } + virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) + { + m_status = status; + } + virtual unsigned get_last_access_time() + { + return m_last_access_time; + } + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) + { + m_last_access_time = time; + } + virtual unsigned get_alloc_time() + { + return m_alloc_time; + } + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) + { + m_ignore_on_fill_status = m_ignore; + } + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) + { + m_set_modified_on_fill = m_modified; + } + virtual unsigned get_modified_size() + { + return SECTOR_CHUNCK_SIZE * SECTOR_SIZE; //i.e. cache line size + } + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) + { + m_readable = readable; + } + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + return m_readable; + } + virtual void print_status() { + printf("m_block_addr is %llu, status = %u\n", m_block_addr, m_status); + } + + +private: + unsigned m_alloc_time; + unsigned m_last_access_time; + unsigned m_fill_time; + cache_block_state m_status; + bool m_ignore_on_fill_status; + bool m_set_modified_on_fill; + bool m_readable; +}; + +struct sector_cache_block : public cache_block_t { + sector_cache_block() { - m_tag=tag; - m_block_addr=block_addr; - m_alloc_time=time; - m_last_access_time=time; - m_fill_time=0; - m_status=RESERVED; + init(); } - void fill( unsigned time ) + + void init() { + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + m_sector_alloc_time[i]= 0; + m_sector_fill_time[i]= 0; + m_last_sector_access_time[i]= 0; + m_status[i]= INVALID; + m_ignore_on_fill_status[i] = false; + m_set_modified_on_fill[i] = false; + m_readable[i] = true; + } + m_line_alloc_time=0; + m_line_last_access_time=0; + m_line_fill_time=0; + } + + virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) { - assert( m_status == RESERVED ); - m_status=VALID; - m_fill_time=time; + allocate_line( tag, block_addr, time, sector_mask ); } - new_addr_type m_tag; - new_addr_type m_block_addr; - unsigned m_alloc_time; - unsigned m_last_access_time; - unsigned m_fill_time; - cache_block_state m_status; + void allocate_line( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask ) + { + //allocate a new line + //assert(m_block_addr != 0 && m_block_addr != block_addr); + init(); + m_tag=tag; + m_block_addr=block_addr; + + unsigned sidx = get_sector_index(sector_mask); + + //set sector stats + m_sector_alloc_time[sidx]=time; + m_last_sector_access_time[sidx]=time; + m_sector_fill_time[sidx]=0; + m_status[sidx]=RESERVED; + m_ignore_on_fill_status[sidx] = false; + m_set_modified_on_fill[sidx] = false; + + //set line stats + m_line_alloc_time=time; //only set this for the first allocated sector + m_line_last_access_time=time; + m_line_fill_time=0; + } + + void allocate_sector(unsigned time, mem_access_sector_mask_t sector_mask ) + { + //allocate invalid sector of this allocated valid line + assert(is_valid_line()); + unsigned sidx = get_sector_index(sector_mask); + + //set sector stats + m_sector_alloc_time[sidx]=time; + m_last_sector_access_time[sidx]=time; + m_sector_fill_time[sidx]=0; + if(m_status[sidx]==MODIFIED) //this should be the case only for fetch-on-write policy //TO DO + m_set_modified_on_fill[sidx] = true; + else + m_set_modified_on_fill[sidx] = false; + + m_status[sidx]=RESERVED; + m_ignore_on_fill_status[sidx] = false; + //m_set_modified_on_fill[sidx] = false; + m_readable[sidx] = true; + + //set line stats + m_line_last_access_time=time; + m_line_fill_time=0; + } + + virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + // if(!m_ignore_on_fill_status[sidx]) + // assert( m_status[sidx] == RESERVED ); + + m_status[sidx] = m_set_modified_on_fill[sidx]? MODIFIED : VALID; + + m_sector_fill_time[sidx]=time; + m_line_fill_time=time; + } + virtual bool is_invalid_line() { + //all the sectors should be invalid + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] != INVALID) + return false; + } + return true; + } + virtual bool is_valid_line() { return !(is_invalid_line()); } + virtual bool is_reserved_line() { + //if any of the sector is reserved, then the line is reserved + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == RESERVED) + return true; + } + return false; + } + virtual bool is_modified_line() { + //if any of the sector is modified, then the line is modified + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) + return true; + } + return false; + } + + virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + return m_status[sidx]; + } + + virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_status[sidx] = status; + } + + virtual unsigned get_last_access_time() + { + return m_line_last_access_time; + } + + virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + + m_last_sector_access_time[sidx] = time; + m_line_last_access_time = time; + } + + virtual unsigned get_alloc_time() + { + return m_line_alloc_time; + } + + virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_ignore_on_fill_status[sidx] = m_ignore; + } + + virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_set_modified_on_fill[sidx] = m_modified; + } + + virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask) + { + unsigned sidx = get_sector_index(sector_mask); + m_readable[sidx] = readable; + } + + virtual bool is_readable(mem_access_sector_mask_t sector_mask) { + unsigned sidx = get_sector_index(sector_mask); + return m_readable[sidx]; + } + + virtual unsigned get_modified_size() + { + unsigned modified=0; + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if (m_status[i] == MODIFIED) + modified++; + } + return modified * SECTOR_SIZE; + } + + virtual void print_status() { + printf("m_block_addr is %llu, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]); + } + + +private: + unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE]; + unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE]; + unsigned m_sector_fill_time[SECTOR_CHUNCK_SIZE]; + unsigned m_line_alloc_time; + unsigned m_line_last_access_time; + unsigned m_line_fill_time; + cache_block_state m_status[SECTOR_CHUNCK_SIZE]; + bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE]; + bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE]; + bool m_readable[SECTOR_CHUNCK_SIZE]; + + unsigned get_sector_index(mem_access_sector_mask_t sector_mask) + { + assert(sector_mask.count() == 1); + for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) { + if(sector_mask.to_ulong() & (1< line_table; + line_table pending_lines; }; class mshr_table { public: - mshr_table( unsigned num_entries, unsigned max_merged ) + mshr_table( unsigned num_entries, unsigned max_merged) : m_num_entries(num_entries), m_max_merged(max_merged) #if (tr1_hash_map_ismap == 0) @@ -414,6 +879,8 @@ public: /// Returns next ready access mem_fetch *next_access(); void display( FILE *fp ) const; + // Returns true if there is a pending read after write + bool is_read_after_write_pending(new_addr_type block_addr); void check_mshr_parameters( unsigned num_entries, unsigned max_merged ) { @@ -433,7 +900,9 @@ private: mshr_entry() : m_has_atomic(false) { } }; typedef tr1_hash_map table; + typedef tr1_hash_map line_table; table m_data; + line_table pending_lines; // it may take several cycles to process the merged requests bool m_current_response_ready; @@ -510,12 +979,14 @@ public: cache_stats(); void clear(); void inc_stats(int access_type, int access_outcome); + void inc_fail_stats(int access_type, int fail_outcome); enum cache_request_status select_stats_status(enum cache_request_status probe, enum cache_request_status access) const; - unsigned &operator()(int access_type, int access_outcome); - unsigned operator()(int access_type, int access_outcome) const; + unsigned &operator()(int access_type, int access_outcome, bool fail_outcome); + unsigned operator()(int access_type, int access_outcome, bool fail_outcome) const; cache_stats operator+(const cache_stats &cs); cache_stats &operator+=(const cache_stats &cs); void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const; + void print_fail_stats(FILE *fout, const char *cache_name = "Cache_fail_stats") const; unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const; void get_sub_stats(struct cache_sub_stats &css) const; @@ -523,8 +994,10 @@ public: void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); private: bool check_valid(int type, int status) const; + bool check_fail_valid(int type, int fail) const; std::vector< std::vector > m_stats; + std::vector< std::vector > m_fail_stats; unsigned long long m_cache_port_available_cycles; unsigned long long m_cache_data_port_busy_cycles; @@ -543,6 +1016,7 @@ public: bool was_write_sent( const std::list &events ); bool was_read_sent( const std::list &events ); +bool was_writeallocate_sent( const std::list &events ); /// Baseline cache /// Implements common functions for read_only_cache and data_cache @@ -552,7 +1026,7 @@ public: baseline_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport, enum mem_fetch_status status ) : m_config(config), m_tag_array(new tag_array(config,core_id,type_id)), - m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), + m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), m_bandwidth_management(config) { init( name, config, memport, status ); @@ -564,7 +1038,7 @@ public: enum mem_fetch_status status ) { m_name = name; - assert(config.m_mshr_type == ASSOC); + assert(config.m_mshr_type == ASSOC || config.m_mshr_type == SECTOR_ASSOC); m_memport=memport; m_miss_queue_status = status; } @@ -594,6 +1068,7 @@ public: mem_fetch *next_access(){return m_mshrs.next_access();} // flash invalidate all entries in cache void flush(){m_tag_array->flush();} + void invalidate(){m_tag_array->invalidate();} void print(FILE *fp, unsigned &accesses, unsigned &misses) const; void display_state( FILE *fp ) const; @@ -612,6 +1087,15 @@ public: bool data_port_free() const { return m_bandwidth_management.data_port_free(); } bool fill_port_free() const { return m_bandwidth_management.fill_port_free(); } + // This is a gapping hole we are poking in the system to quickly handle + // filling the cache on cudamemcopies. We don't care about anything other than + // L2 state after the memcopy - so just force the tag array to act as though + // something is read or written without doing anything else. + void force_tag_access( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask ) + { + m_tag_array->fill( addr, time, mask ); + } + protected: // Constructor that can be used by derived classes with custom tag arrays baseline_cache( const char *name, @@ -633,24 +1117,31 @@ protected: std::string m_name; cache_config &m_config; tag_array* m_tag_array; - mshr_table m_mshrs; + mshr_table m_mshrs; std::list m_miss_queue; enum mem_fetch_status m_miss_queue_status; mem_fetch_interface *m_memport; struct extra_mf_fields { extra_mf_fields() { m_valid = false;} - extra_mf_fields( new_addr_type a, unsigned i, unsigned d ) + extra_mf_fields( new_addr_type a, new_addr_type ad, unsigned i, unsigned d, const cache_config& m_config) { m_valid = true; m_block_addr = a; + m_addr = ad; m_cache_index = i; m_data_size = d; + pending_read = m_config.m_mshr_type == SECTOR_ASSOC? m_config.m_line_sz/SECTOR_SIZE : 0; + } bool m_valid; new_addr_type m_block_addr; + new_addr_type m_addr; unsigned m_cache_index; unsigned m_data_size; + //this variable is used when a load request generates multiple load transactions + //For example, a read request from non-sector L1 request sends a request to sector L2 + unsigned pending_read; }; typedef std::map extra_mf_fields_lookup; @@ -668,7 +1159,7 @@ protected: unsigned time, bool &do_miss, std::list &events, bool read_only, bool wa); /// Read miss handler. Check MSHR hit or MSHR available void send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf, - unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list &events, bool read_only, bool wa); + unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list &events, bool read_only, bool wa); /// Sub-class containing all metadata for port bandwidth management class bandwidth_management @@ -760,8 +1251,10 @@ public: // Set write miss function switch(m_config.m_write_alloc_policy){ - case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa; break; case NO_WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_no_wa; break; + case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa_naive; break; + case FETCH_ON_WRITE: m_wr_miss = &data_cache::wr_miss_wa_fetch_on_write; break; + case LAZY_FETCH_ON_READ: m_wr_miss = &data_cache::wr_miss_wa_lazy_fetch_on_read; break; default: assert(0 && "Error: Must set valid cache write miss policy\n"); break; // Need to set a write miss function @@ -870,12 +1363,33 @@ protected: /// Sends read request, and possible write-back request, // to lower level memory for a write miss with write-allocate enum cache_request_status - wr_miss_wa( new_addr_type addr, - unsigned cache_index, - mem_fetch *mf, - unsigned time, - std::list &events, - enum cache_request_status status ); // write-allocate + wr_miss_wa_naive( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate-send-write-and-read-request + enum cache_request_status + wr_miss_wa_fetch_on_write( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate with fetch-on-every-write + enum cache_request_status + wr_miss_wa_lazy_fetch_on_read( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate with read-fetch-only + enum cache_request_status + wr_miss_wa_write_validate( new_addr_type addr, + unsigned cache_index, + mem_fetch *mf, + unsigned time, + std::list &events, + enum cache_request_status status ); // write-allocate that writes with no read fetch enum cache_request_status wr_miss_no_wa( new_addr_type addr, unsigned cache_index, @@ -991,7 +1505,7 @@ public: m_result_fifo(config.m_result_fifo_entries) { m_name = name; - assert(config.m_mshr_type == TEX_FIFO); + assert(config.m_mshr_type == TEX_FIFO || config.m_mshr_type == SECTOR_TEX_FIFO ); assert(config.m_write_policy == READ_ONLY); assert(config.m_alloc_policy == ON_MISS); m_memport=memport; @@ -1144,13 +1658,15 @@ private: struct extra_mf_fields { extra_mf_fields() { m_valid = false;} - extra_mf_fields( unsigned i ) + extra_mf_fields( unsigned i, const cache_config &m_config ) { m_valid = true; m_rob_index = i; + pending_read = m_config.m_mshr_type == SECTOR_TEX_FIFO? m_config.m_line_sz/SECTOR_SIZE : 0; } bool m_valid; unsigned m_rob_index; + unsigned pending_read; }; cache_stats m_stats; diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 63ba759..a8be4d2 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -150,6 +150,8 @@ void power_config::reg_options(class OptionParser * opp) void memory_config::reg_options(class OptionParser * opp) { + option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, + "Fill the L2 cache on memcpy", "1"); option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, "0 = fifo, 1 = FR-FCFS (defaul)", "1"); option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, @@ -202,7 +204,27 @@ void memory_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-dram_latency", OPT_UINT32, &dram_latency, "DRAM latency (default 30)", "30"); - + option_parser_register(opp, "-dual_bus_interface", OPT_UINT32, &dual_bus_interface, + "dual_bus_interface (default = 0) ", + "0"); + option_parser_register(opp, "-dram_bnk_indexing_policy", OPT_UINT32, &dram_bnk_indexing_policy, + "dram_bnk_indexing_policy (0 = normal indexing, 1 = Xoring with the higher bits) (Default = 0)", + "0"); + option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy, + "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)", + "0"); + option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled, + "Seperate_Write_Queue_Enable", + "0"); + option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt, + "Write_Queue_Size", + "32:28:16"); + option_parser_register(opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround, + "elimnate_rw_turnaround i.e set tWTR and tRTW = 0", + "0"); + option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size, + "icnt_flit_size", + "32"); m_address_mapping.addrdec_setoption(opp); } @@ -229,6 +251,12 @@ void shader_core_config::reg_options(class OptionParser * opp) "per-shader L1 data cache config " " {::,:::,::, | none}", "none" ); + option_parser_register(opp, "-l1_latency", OPT_UINT32, &m_L1D_config.l1_latency, + "L1 Hit Latency", + "0"); + option_parser_register(opp, "-smem_latency", OPT_UINT32, &smem_latency, + "smem Latency", + "3"); option_parser_register(opp, "-gpgpu_cache:dl1PrefL1", OPT_CSTR, &m_L1D_config.m_config_stringPrefL1, "per-shader L1 data cache config " " {::,:::,::, | none}", @@ -256,6 +284,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", "8192"); + option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation, + "gpgpu_ignore_resources_limitation (default 0)", + "0"); option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, "Maximum number of concurrent CTAs in shader (default 8)", "8"); @@ -277,6 +308,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, "Size of shared memory per shader core (default 16kB)", "16384"); + option_parser_register(opp, "-adpative_volta_cache_config", OPT_BOOL, &adpative_volta_cache_config, + "adpative_volta_cache_config", + "0"); option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_sizeDefault, "Size of shared memory per shader core (default 16kB)", "16384"); @@ -295,6 +329,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, "Number of portions a warp is divided into for shared memory bank conflict check ", "2"); + option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, + "Number of portions a warp is divided into for shared memory bank conflict check ", + "2"); option_parser_register(opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, "Specify which shader core to collect the warp size distribution from", "-1"); @@ -303,7 +340,7 @@ void shader_core_config::reg_options(class OptionParser * opp) "0"); option_parser_register(opp, "-gpgpu_local_mem_map", OPT_BOOL, &gpgpu_local_mem_map, "Mapping from local memory space address to simulated GPU physical address space (default = enabled)", - "1"); + "1"); option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, "Number of register banks (default = 8)", "8"); @@ -313,6 +350,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", OPT_INT32, &gpgpu_operand_collector_num_units_sp, "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", OPT_INT32, &gpgpu_operand_collector_num_units_dp, + "number of collector units (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu, "number of collector units (default = 4)", "4"); @@ -328,6 +368,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sp, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu, "number of collector unit in ports (default = 1)", "1"); @@ -343,6 +386,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sp, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_dp, + "number of collector unit in ports (default = 0)", + "0"); option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu, "number of collector unit in ports (default = 1)", "1"); @@ -356,27 +402,33 @@ void shader_core_config::reg_options(class OptionParser * opp) "number of collector unit in ports (default = 0)", "0"); option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, - "Coalescing arch (default = 13, anything else is off for now)", + "Coalescing arch (GT200 = 13, Fermi = 20)", "13"); option_parser_register(opp, "-gpgpu_num_sched_per_core", OPT_INT32, &gpgpu_num_sched_per_core, "Number of warp schedulers per core", "1"); option_parser_register(opp, "-gpgpu_max_insn_issue_per_warp", OPT_INT32, &gpgpu_max_insn_issue_per_warp, - "Max number of instructions that can be issued per warp in one cycle by scheduler", - "2"); + "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)", + "2"); + option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, &gpgpu_dual_issue_diff_exec_units, + "should dual issue use two different execution unit resources (Default = 1)", + "1"); option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, &simt_core_sim_order, "Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)", "1"); option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, "Pipeline widths " - "ID_OC_SP,ID_OC_SFU,ID_OC_TENSOR_CORE,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_TENSOR_CORE,OC_EX_MEM,EX_WB", - "1,1,1,1,1,1,1,1,1,1" ); + "ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE", + "1,1,1,1,1,1,1,1,1,1,1" ); option_parser_register(opp, "-gpgpu_tensor_core_avail", OPT_INT32, &gpgpu_tensor_core_avail, "Tensor Core Available (default=0)", "0"); option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units, "Number of SP units (default=1)", "1"); + option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, &gpgpu_num_dp_units, + "Number of DP units (default=0)", + "0"); option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, &gpgpu_num_sfu_units, "Number of SF units (default=1)", "1"); @@ -425,7 +477,6 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, &gpgpu_flush_l2_cache, "Flush L2 cache at the end of each kernel call", "0"); - option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, "Stop the simulation at deadlock (1=on (default), 0=off)", "1"); @@ -669,7 +720,7 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config ) m_running_kernels.resize( config.max_concurrent_kernel, NULL ); m_last_issued_kernel = 0; - m_last_cluster_issue = 0; + m_last_cluster_issue = m_shader_config->n_simt_clusters-1; // this causes first launch to use simt cluster 0 *average_pipeline_duty_cycle=0; *active_sms=0; @@ -813,6 +864,7 @@ void gpgpu_sim::update_stats() { partiton_replys_in_parallel_total += partiton_replys_in_parallel; partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util; gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ; + gpu_tot_occupancy += gpu_occupancy; gpu_sim_cycle = 0; partiton_reqs_in_parallel = 0; @@ -821,6 +873,7 @@ void gpgpu_sim::update_stats() { gpu_sim_cycle_parition_util = 0; gpu_sim_insn = 0; m_total_cta_launched = 0; + gpu_occupancy = occupancy_stats(); } void gpgpu_sim::print_stats() @@ -994,6 +1047,9 @@ void gpgpu_sim::gpu_print_stat() printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn+gpu_sim_insn); printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn+gpu_sim_insn) / (gpu_tot_sim_cycle+gpu_sim_cycle)); printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta + m_total_cta_launched); + printf("gpu_occupancy = %.4f\% \n", gpu_occupancy.get_occ_fraction() * 100); + printf("gpu_tot_occupancy = %.4f\% \n", (gpu_occupancy + gpu_tot_occupancy).get_occ_fraction() * 100); + extern unsigned long long g_max_total_param_size; fprintf(statfout, "max_total_param_size = %llu\n", g_max_total_param_size); @@ -1032,6 +1088,8 @@ void gpgpu_sim::gpu_print_stat() } printf("\nTotal_core_cache_stats:\n"); core_cache_stats.print_stats(stdout, "Total_core_cache_stats_breakdown"); + printf("\nTotal_core_cache_fail_stats:\n"); + core_cache_stats.print_fail_stats(stdout, "Total_core_cache_fail_stats_breakdown"); shader_print_scheduler_stat( stdout, false ); m_shader_stats->print(stdout); @@ -1076,6 +1134,8 @@ void gpgpu_sim::gpu_print_stat() printf("L2_total_cache_reservation_fails = %u\n", total_l2_css.res_fails); printf("L2_total_cache_breakdown:\n"); l2_stats.print_stats(stdout, "L2_cache_stats_breakdown"); + printf("L2_total_cache_reservation_fail_breakdown:\n"); + l2_stats.print_fail_stats(stdout, "L2_cache_stats_fail_breakdown"); total_l2_css.print_port_stats(stdout, "L2_cache"); } } @@ -1429,7 +1489,7 @@ void gpgpu_sim::cycle() if (mf) { unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size(); if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) { - if (!mf->get_is_write()) + //if (!mf->get_is_write()) mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle); mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle); ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size ); @@ -1462,12 +1522,14 @@ void gpgpu_sim::cycle() for (unsigned i=0;im_n_mem_sub_partition;i++) { //move memory request from interconnect into memory partition (if not backed up) //Note:This needs to be called in DRAM clock domain if there is no L2 cache in the system - if ( m_memory_sub_partition[i]->full() ) { + //In the worst case, we may need to push SECTOR_CHUNCK_SIZE requests, so ensure you have enough buffer for them + if ( m_memory_sub_partition[i]->full(SECTOR_CHUNCK_SIZE) ) { gpu_stall_dramfull++; } else { mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) ); m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle ); - partiton_reqs_in_parallel_per_cycle++; + if(mf) + partiton_reqs_in_parallel_per_cycle++; } m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle); m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); @@ -1494,6 +1556,8 @@ void gpgpu_sim::cycle() // Update core icnt/cache stats for GPUWattch m_cluster[i]->get_icnt_stats(m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]); m_cluster[i]->get_cache_stats(m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX]); + m_cluster[i]->get_current_occupancy(gpu_occupancy.aggregate_warp_slot_filled, gpu_occupancy.aggregate_theoretical_warp_slots); + } float temp=0; for (unsigned i=0;inum_shader();i++){ @@ -1521,12 +1585,12 @@ void gpgpu_sim::cycle() issue_block2core(); - // Depending on configuration, flush the caches once all of threads are completed. + // Depending on configuration, invalidate the caches once all of threads are completed. int all_threads_complete = 1; if (m_config.gpgpu_flush_l1_cache) { for (unsigned i=0;in_simt_clusters;i++) { if (m_cluster[i]->get_not_completed() == 0) - m_cluster[i]->cache_flush(); + m_cluster[i]->cache_invalidate(); else all_threads_complete = 0 ; } @@ -1548,7 +1612,7 @@ void gpgpu_sim::cycle() int dlc = 0; for (unsigned i=0;im_n_mem;i++) { dlc = m_memory_sub_partition[i]->flushL2(); - assert (dlc == 0); // need to model actual writes to DRAM here + assert (dlc == 0); // TODO: need to model actual writes to DRAM here printf("Dirty lines flushed from L2 %d is %d\n", i, dlc ); } } @@ -1560,15 +1624,20 @@ void gpgpu_sim::cycle() time_t curr_time; time(&curr_time); unsigned long long elapsed_time = MAX(curr_time - g_simulation_starttime, 1); - if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq ) { + if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq && DTRACE(LIVENESS) ) { days = elapsed_time/(3600*24); hrs = elapsed_time/3600 - 24*days; minutes = elapsed_time/60 - 60*(hrs + 24*days); sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - DPRINTF(LIVENESS, "GPGPU-Sim uArch: cycles simulated: %lld inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", - gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, + + unsigned long long active = 0, total = 0; + for (unsigned i=0;in_simt_clusters;i++) { + m_cluster[i]->get_current_occupancy(active, total); + } + DPRINTF(LIVENESS, "uArch: inst.: %lld (ipc=%4.1f, occ=%0.4f\% [%llu / %llu]) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", + gpu_tot_sim_insn + gpu_sim_insn, (double)gpu_sim_insn/(double)gpu_sim_cycle, + float(active)/float(total) * 100, active, total, (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time), (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec, ctime(&curr_time)); @@ -1620,6 +1689,24 @@ void shader_core_ctx::dump_warp_state( FILE *fout ) const m_warp[w].print(fout); } + +void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ) +{ + if (m_memory_config->m_perf_sim_memcpy) { + assert (dst_start_addr % 32 == 0); + + for ( unsigned counter = 0; counter < count; counter += 32 ) { + const unsigned wr_addr = dst_start_addr + counter; + addrdec_t raw_addr; + mem_access_sector_mask_t mask; + mask.set(wr_addr % 128 / 32); + m_memory_config->m_address_mapping.addrdec_tlx( wr_addr, &raw_addr ); + const unsigned partition_id = raw_addr.sub_partition / m_memory_config->m_n_sub_partition_per_memory_channel; + m_memory_partition_unit[ partition_id ]->handle_memcpy_to_gpu( wr_addr, raw_addr.sub_partition, mask ); + } + } +} + void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const { /* diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 7d92c66..1bae1fa 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -198,8 +198,14 @@ struct memory_config { bk_tag_length = i-1; assert(nbkgrp>0 && "Number of bank groups cannot be zero"); tRCDWR = tRCD-(WL+1); + if(elimnate_rw_turnaround) + { + tRTW = 0; + tWTR = 0; + } else { tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); - tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); + } tWTP = (WL+(BL/data_command_freq_ratio)+tWR); dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition @@ -213,7 +219,9 @@ struct memory_config { m_L2_config.init(&m_address_mapping); m_valid = true; - icnt_flit_size = 32; // Default 32 + + sscanf(write_queue_size_opt,"%d:%d:%d", + &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark); } void reg_options(class OptionParser * opp); @@ -264,12 +272,25 @@ struct memory_config { unsigned nbk; + bool elimnate_rw_turnaround; + unsigned data_command_freq_ratio; // frequency ratio between DRAM data bus and command bus (2 for GDDR3, 4 for GDDR5) unsigned dram_atom_size; // number of bytes transferred per read or write command linear_to_raw_address_translation m_address_mapping; unsigned icnt_flit_size; + + unsigned dram_bnk_indexing_policy; + unsigned dram_bnkgrp_indexing_policy; + bool dual_bus_interface; + + bool seperate_write_queue_enabled; + char *write_queue_size_opt; + unsigned gpgpu_frfcfs_dram_write_queue_size; + unsigned write_high_watermark; + unsigned write_low_watermark; + bool m_perf_sim_memcpy; }; // global counters and flags (please try not to add to this list!!!) @@ -362,6 +383,32 @@ private: friend class gpgpu_sim; }; +struct occupancy_stats { + occupancy_stats() : aggregate_warp_slot_filled(0), aggregate_theoretical_warp_slots(0){} + occupancy_stats( unsigned long long wsf, unsigned long long tws ) + : aggregate_warp_slot_filled(wsf), aggregate_theoretical_warp_slots(tws){} + + unsigned long long aggregate_warp_slot_filled; + unsigned long long aggregate_theoretical_warp_slots; + + float get_occ_fraction() const { + return float(aggregate_warp_slot_filled) / float(aggregate_theoretical_warp_slots); + } + + occupancy_stats& operator+=(const occupancy_stats& rhs) { + aggregate_warp_slot_filled += rhs.aggregate_warp_slot_filled; + aggregate_theoretical_warp_slots += rhs.aggregate_theoretical_warp_slots; + return *this; + } + + occupancy_stats operator+(const occupancy_stats& rhs) const{ + return occupancy_stats( aggregate_warp_slot_filled + rhs.aggregate_warp_slot_filled, + aggregate_theoretical_warp_slots + rhs.aggregate_theoretical_warp_slots + ); + } +}; + + class gpgpu_sim : public gpgpu_t { public: gpgpu_sim( const gpgpu_sim_config &config ); @@ -405,6 +452,8 @@ public: void gpu_print_stat(); void dump_pipeline( int mask, int s, int m ) const; + void perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ); + //The next three functions added to be used by the functional simulation function //! Get shader core configuration @@ -498,6 +547,9 @@ public: unsigned long long gpu_tot_sim_insn; unsigned long long gpu_sim_insn_last_update; unsigned gpu_sim_insn_last_update_sid; + occupancy_stats gpu_occupancy; + occupancy_stats gpu_tot_occupancy; + FuncCache get_cache_config(std::string kernel_name); void set_cache_config(std::string kernel_name, FuncCache cacheConfig ); @@ -526,4 +578,5 @@ public: } }; + #endif diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index c5fc44e..25da107 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -74,6 +74,15 @@ memory_partition_unit::memory_partition_unit( unsigned partition_id, } } +void memory_partition_unit::handle_memcpy_to_gpu( size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask ) +{ + unsigned p = global_sub_partition_id_to_local_id(global_subpart_id); + std::string mystring = + mask.to_string(); + MEMPART_DPRINTF("Copy Engine Request Received For Address=%llx, local_subpart=%u, global_subpart=%u, sector_mask=%s \n", addr, p, global_subpart_id, mystring.c_str()); + m_sub_partition[p]->force_l2_tag_update(addr,gpu_sim_cycle+gpu_tot_sim_cycle, mask); +} + memory_partition_unit::~memory_partition_unit() { delete m_dram; @@ -93,7 +102,9 @@ memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct m m_private_credit_limit = 1; m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size + config->gpgpu_dram_return_queue_size - - (config->m_n_sub_partition_per_memory_channel - 1); + - (config->m_n_sub_partition_per_memory_channel - 1); + if(config->seperate_write_queue_enabled ) + m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size; if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 or config->gpgpu_dram_return_queue_size == 0) { @@ -220,7 +231,8 @@ void memory_partition_unit::dram_cycle() m_dram->cycle(); m_dram->dram_log(SAMPLELOG); - if( !m_dram->full() ) { + // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + //if( !m_dram->full(mf->is_write()) ) { // L2->DRAM queue to DRAM latency queue // Arbitrate among multiple L2 subpartitions int last_issued_partition = m_arbitration_metadata.last_borrower(); @@ -228,6 +240,9 @@ void memory_partition_unit::dram_cycle() int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + if(m_dram->full(mf->is_write()) ) + break; + m_sub_partition[spid]->L2_dram_queue_pop(); MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); dram_delay_t d; @@ -239,12 +254,13 @@ void memory_partition_unit::dram_cycle() break; // the DRAM should only accept one request per cycle } } - } + //} // DRAM latency queue - if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) { - mem_fetch* mf = m_dram_latency_queue.front().req; - m_dram_latency_queue.pop_front(); + + if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) { + mem_fetch* mf = m_dram_latency_queue.front().req; + m_dram_latency_queue.pop_front(); m_dram->push(mf); } } @@ -299,6 +315,7 @@ memory_sub_partition::memory_sub_partition( unsigned sub_partition_id, m_id = sub_partition_id; m_config=config; m_stats=stats; + m_memcpy_cycle_offset = 0; assert(m_id < m_config->m_n_mem_sub_partition); @@ -343,6 +360,14 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); m_L2_icnt_queue->push(mf); }else{ + if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE) + { + mem_fetch* original_wr_mf = mf->get_original_wr_mf(); + assert(original_wr_mf); + original_wr_mf->set_reply(); + original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2_icnt_queue->push(original_wr_mf); + } m_request_tracker.erase(mf); delete mf; } @@ -355,10 +380,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) if ( !m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf) ) { if (m_L2cache->fill_port_free()) { mf->set_status(IN_PARTITION_L2_FILL_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset); m_dram_L2_queue->pop(); } } else if ( !m_L2_icnt_queue->full() ) { + if(mf->is_write() && mf->get_type() == WRITE_ACK) mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); m_L2_icnt_queue->push(mf); m_dram_L2_queue->pop(); @@ -380,9 +406,10 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) bool port_free = m_L2cache->data_port_free(); if ( !output_full && port_free ) { std::list events; - enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events); + enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset,events); bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); + MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", mf->get_addr(), status); if ( status == HIT ) { if( !write_sent ) { @@ -402,6 +429,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle ) m_icnt_L2_queue->pop(); } } else if ( status != RESERVATION_FAIL ) { + if(mf->is_write() && (m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE || m_config->m_L2_config.m_write_alloc_policy == LAZY_FETCH_ON_READ) && !was_writeallocate_sent(events)) { + mf->set_reply(); + mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_L2_icnt_queue->push(mf); + } // L2 cache accepted request m_icnt_L2_queue->pop(); } else { @@ -432,6 +464,11 @@ bool memory_sub_partition::full() const return m_icnt_L2_queue->full(); } +bool memory_sub_partition::full(unsigned size) const +{ + return m_icnt_L2_queue->is_avilable_size(size); +} + bool memory_sub_partition::L2_dram_queue_empty() const { return m_L2_dram_queue->empty(); @@ -532,7 +569,15 @@ unsigned memory_sub_partition::flushL2() if (!m_config->m_L2_config.disabled()) { m_L2cache->flush(); } - return 0; // L2 is read only in this version + return 0; //TODO: write the flushed data to the main memory +} + +unsigned memory_sub_partition::invalidateL2() +{ + if (!m_config->m_L2_config.disabled()) { + m_L2cache->invalidate(); + } + return 0; } bool memory_sub_partition::busy() const @@ -540,21 +585,94 @@ bool memory_sub_partition::busy() const return !m_request_tracker.empty(); } -void memory_sub_partition::push( mem_fetch* req, unsigned long long cycle ) -{ - if (req) { - m_request_tracker.insert(req); - m_stats->memlatstat_icnt2mem_pop(req); - if( req->istexture() ) { - m_icnt_L2_queue->push(req); - req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - } else { - rop_delay_t r; - r.req = req; - r.ready_cycle = cycle + m_config->rop_latency; - m_rop.push(r); - req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle); - } +std::vector memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch* mf) +{ + std::vector result; + + if(mf->get_data_size() == SECTOR_SIZE && mf->get_access_sector_mask().count() == 1) { + result.push_back(mf); + } else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) { + //We only accept 32, 64 and 128 bytes reqs + unsigned start=0, end=0; + if(mf->get_data_size() == 128) { + start=0; end=3; + } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "1100") { + start=2; end=3; + } else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "0011") { + start=0; end=1; + } else if (mf->get_data_size() == 64 && (mf->get_access_sector_mask().to_string() == "1111" || mf->get_access_sector_mask().to_string() == "0000")) { + if(mf->get_addr() % 128 == 0) { + start=0; end=1; + } else { + start=2; end=3; + } + } else + { + printf("Invalid sector received, address = 0x%06x, sector mask = %s, data size = %d", + mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size()); + assert(0 && "Undefined sector mask is received"); + } + + std::bitset byte_sector_mask; + byte_sector_mask.reset(); + for(unsigned k=start*SECTOR_SIZE; k< SECTOR_SIZE; ++k) + byte_sector_mask.set(k); + + for(unsigned j=start, i=0; j<= end ; ++j, ++i){ + + const mem_access_t *ma = new mem_access_t( mf->get_access_type(), + mf->get_addr() + SECTOR_SIZE*i, + SECTOR_SIZE, + mf->is_write(), + mf->get_access_warp_mask(), + mf->get_access_byte_mask() & byte_sector_mask, + std::bitset().set(j)); + + mem_fetch *n_mf = new mem_fetch( *ma, + NULL, + mf->get_ctrl_size(), + mf->get_wid(), + mf->get_sid(), + mf->get_tpc(), + mf->get_mem_config(), + mf); + + result.push_back(n_mf); + byte_sector_mask <<= SECTOR_SIZE; + } + } else { + printf("Invalid sector received, address = 0x%06x, sector mask = %d, byte mask = , data size = %d", + mf->get_addr(), mf->get_access_sector_mask().count(), mf->get_data_size()); + assert(0 && "Undefined data size is received"); + } + + return result; +} + +void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle ) +{ + if (m_req) { + m_stats->memlatstat_icnt2mem_pop(m_req); + std::vector reqs; + if(m_config->m_L2_config.m_cache_type == SECTOR) + reqs = breakdown_request_to_sector_requests(m_req); + else + reqs.push_back(m_req); + + for(unsigned i=0; iistexture() ) { + m_icnt_L2_queue->push(req); + req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + } else { + rop_delay_t r; + r.req = req; + r.ready_cycle = cycle + m_config->rop_latency; + m_rop.push(r); + req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle); + } + } } } diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h index 3df54b1..18c0a8b 100644 --- a/src/gpgpu-sim/l2cache.h +++ b/src/gpgpu-sim/l2cache.h @@ -72,6 +72,7 @@ public: void print_stat( FILE *fp ) { m_dram->print_stat(fp); } void visualize() const { m_dram->visualize(); } void print( FILE *fp ) const; + void handle_memcpy_to_gpu( size_t dst_start_addr, unsigned subpart_id, mem_access_sector_mask_t mask ); class memory_sub_partition * get_sub_partition(int sub_partition_id) { @@ -154,12 +155,14 @@ public: void cache_cycle( unsigned cycle ); bool full() const; + bool full(unsigned size) const; void push( class mem_fetch* mf, unsigned long long clock_cycle ); class mem_fetch* pop(); class mem_fetch* top(); void set_done( mem_fetch *mf ); unsigned flushL2(); + unsigned invalidateL2(); // interface to L2_dram_queue bool L2_dram_queue_empty() const; @@ -177,6 +180,12 @@ public: void accumulate_L2cache_stats(class cache_stats &l2_stats) const; void get_L2cache_sub_stats(struct cache_sub_stats &css) const; + void force_l2_tag_update(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask) + { + m_L2cache->force_tag_access( addr, m_memcpy_cycle_offset + time, mask ); + m_memcpy_cycle_offset += 1; + } + private: // data unsigned m_id; //< the global sub partition ID @@ -207,6 +216,15 @@ private: std::set m_request_tracker; friend class L2interface; + + std::vector breakdown_request_to_sector_requests(mem_fetch* mf); + + // This is a cycle offset that has to be applied to the l2 accesses to account for + // the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for kernel execution + // but we want cudamemcpy to go through the L2. Everytime an access is made from cudamemcpy + // this counter is incremented, and when the l2 is accessed (in both cudamemcpyies and otherwise) + // this value is added to the gpgpu-sim cycle counters. + unsigned m_memcpy_cycle_offset; }; class L2interface : public mem_fetch_interface { diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h index 3dac87d..2235cdc 100644 --- a/src/gpgpu-sim/l2cache_trace.h +++ b/src/gpgpu-sim/l2cache_trace.h @@ -34,6 +34,9 @@ #define MEMPART_PRINT_STR SIM_PRINT_STR " %d - " #define MEMPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) ) +#define MEM_SUBPART_PRINT_STR SIM_PRINT_STR " %d - " +#define MEM_SUBPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)m_id) ) + // Intended to be called from inside components of a memory partition // Depends on a get_mpid() function #define MEMPART_DPRINTF(...) do {\ @@ -46,10 +49,23 @@ }\ } while (0) +#define MEM_SUBPART_DPRINTF(...) do {\ + if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) {\ + printf( MEM_SUBPART_PRINT_STR,\ + gpu_sim_cycle + gpu_tot_sim_cycle,\ + Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT],\ + m_id );\ + printf(__VA_ARGS__);\ + }\ +} while (0) + #else #define MEMPART_DTRACE(x) (false) #define MEMPART_DPRINTF(x, ...) do {} while (0) +#define MEM_SUBPART_DTRACE(x) (false) +#define MEM_SUBPART_DPRINTF(x, ...) do {} while (0) + #endif diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index 729636d..a260a35 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -39,7 +39,10 @@ mem_fetch::mem_fetch( const mem_access_t &access, unsigned wid, unsigned sid, unsigned tpc, - const struct memory_config *config ) + const struct memory_config *config, + mem_fetch *m_original_mf, + mem_fetch *m_original_wr_mf) + { m_request_uid = sm_next_mf_request_uid++; m_access = access; @@ -61,6 +64,8 @@ mem_fetch::mem_fetch( const mem_access_t &access, m_status_change = gpu_sim_cycle + gpu_tot_sim_cycle; m_mem_config = config; icnt_flit_size = config->icnt_flit_size; + original_mf = m_original_mf; + original_wr_mf = m_original_wr_mf; } mem_fetch::~mem_fetch() diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index de98748..e5efffd 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -55,7 +55,9 @@ public: unsigned wid, unsigned sid, unsigned tpc, - const struct memory_config *config ); + const struct memory_config *config, + mem_fetch *original_mf = NULL, + mem_fetch *original_wr_mf = NULL); ~mem_fetch(); void set_status( enum mem_fetch_status status, unsigned long long cycle ); @@ -104,6 +106,7 @@ public: enum mem_access_type get_access_type() const { return m_access.get_type(); } const active_mask_t& get_access_warp_mask() const { return m_access.get_warp_mask(); } mem_access_byte_mask_t get_access_byte_mask() const { return m_access.get_byte_mask(); } + mem_access_sector_mask_t get_access_sector_mask() const { return m_access.get_sector_mask(); } address_type get_pc() const { return m_inst.empty()?-1:m_inst.pc; } const warp_inst_t &get_inst() { return m_inst; } @@ -112,6 +115,10 @@ public: const memory_config *get_mem_config(){return m_mem_config;} unsigned get_num_flits(bool simt_to_mem); + + mem_fetch* get_original_mf() { return original_mf; } + mem_fetch* get_original_wr_mf() { return original_wr_mf; } + private: // request source information unsigned m_request_uid; @@ -143,6 +150,10 @@ private: const struct memory_config *m_mem_config; unsigned icnt_flit_size; + + mem_fetch* original_mf; //this pointer is set up when a request is divided into sector requests at L2 cache (if the req size > L2 sector size), so the pointer refers to the original request + mem_fetch* original_wr_mf; //this pointer refers to the original write req, when fetch-on-write policy is used + }; #endif diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc index fde0eff..c5452b9 100644 --- a/src/gpgpu-sim/mem_latency_stat.cc +++ b/src/gpgpu-sim/mem_latency_stat.cc @@ -75,6 +75,10 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf max_mf_latency = 0; max_icnt2mem_latency = 0; max_icnt2sh_latency = 0; + tot_icnt2mem_latency = 0; + tot_icnt2sh_latency = 0; + tot_mrq_num = 0; + tot_mrq_latency = 0; memset(mrq_lat_table, 0, sizeof(unsigned)*32); memset(dq_lat_table, 0, sizeof(unsigned)*32); memset(mf_lat_table, 0, sizeof(unsigned)*32); @@ -158,6 +162,7 @@ void memory_stats_t::memlatstat_read_done(mem_fetch *mf) mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency; unsigned icnt2sh_latency; icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_return_timestamp(); + tot_icnt2sh_latency += icnt2sh_latency; icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; if (icnt2sh_latency > max_icnt2sh_latency) max_icnt2sh_latency = icnt2sh_latency; @@ -191,6 +196,7 @@ void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf) if (m_memory_config->gpgpu_memlatency_stat) { unsigned icnt2mem_latency; icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_timestamp(); + tot_icnt2mem_latency += icnt2mem_latency; icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; if (icnt2mem_latency > max_icnt2mem_latency) max_icnt2mem_latency = icnt2mem_latency; @@ -216,14 +222,19 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk ) unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses; if (m_memory_config->gpgpu_memlatency_stat) { + printf("maxmflatency = %d \n", max_mf_latency); + printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); printf("maxmrqlatency = %d \n", max_mrq_latency); - printf("maxdqlatency = %d \n", max_dq_latency); - printf("maxmflatency = %d \n", max_mf_latency); + //printf("maxdqlatency = %d \n", max_dq_latency); + printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); if (num_mfs) { printf("averagemflatency = %lld \n", mf_total_lat/num_mfs); + printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs); + if(tot_mrq_num) + printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num); + + printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs); } - printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); - printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); printf("mrq_lat_table:"); for (i=0; i< 32; i++) { printf("%d \t", mrq_lat_table[i]); diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h index 4968a3b..5b89202 100644 --- a/src/gpgpu-sim/mem_latency_stat.h +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -56,6 +56,10 @@ public: unsigned max_dq_latency; unsigned max_mf_latency; unsigned max_icnt2mem_latency; + unsigned long long int tot_icnt2mem_latency; + unsigned long long int tot_icnt2sh_latency; + unsigned long long int tot_mrq_latency; + unsigned long long int tot_mrq_num; unsigned max_icnt2sh_latency; unsigned mrq_lat_table[32]; unsigned dq_lat_table[32]; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index f9cfa58..82f9181 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -74,13 +74,14 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, shader_core_stats *stats ) : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ), m_barriers( this, config->max_warps_per_shader, config->max_cta_per_core, config->max_barriers_per_cta, config->warp_size ), - m_dynamic_warp_id(0) + m_dynamic_warp_id(0), m_active_warps(0) { m_cluster = cluster; m_config = config; m_memory_config = mem_config; m_stats = stats; unsigned warp_size=config->warp_size; + Issue_Prio = 0; m_sid = shader_id; m_tpc = tpc_id; @@ -131,6 +132,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE : sched_config.find("gto") != std::string::npos ? CONCRETE_SCHEDULER_GTO : + sched_config.find("old") != std::string::npos ? + CONCRETE_SCHEDULER_OLDEST_FIRST : sched_config.find("warp_limiting") != std::string::npos ? CONCRETE_SCHEDULER_WARP_LIMITING: NUM_CONCRETE_SCHEDULERS; @@ -147,6 +150,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -162,6 +166,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -178,6 +183,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -185,6 +191,22 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, ) ); break; + case CONCRETE_SCHEDULER_OLDEST_FIRST: + schedulers.push_back( + new oldest_scheduler( m_stats, + this, + m_scoreboard, + m_simt_stack, + &m_warp, + &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], + &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_TENSOR_CORE], + &m_pipeline_reg[ID_OC_MEM], + i + ) + ); + break; case CONCRETE_SCHEDULER_WARP_LIMITING: schedulers.push_back( new swl_scheduler( m_stats, @@ -193,6 +215,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_simt_stack, &m_warp, &m_pipeline_reg[ID_OC_SP], + &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], @@ -215,8 +238,9 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } //op collector configuration - enum { SP_CUS, SFU_CUS, TENSOR_CORE_CUS, MEM_CUS, GEN_CUS }; + enum { SP_CUS, DP_CUS, SFU_CUS, MEM_CUS, TENSOR_CORE_CUS, GEN_CUS }; m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); + m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp); m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core); m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); @@ -234,6 +258,15 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, in_ports.clear(),out_ports.clear(),cu_sets.clear(); } + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); + cu_sets.push_back((unsigned)DP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) { in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); @@ -279,7 +312,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_operand_collector.init( m_config->gpgpu_num_reg_banks, this ); // execute - m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_sfu_units + config->gpgpu_num_tensor_core_units + 1; // sp_unit, sfu, ldst_unit + m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + 1; // sp_unit, sfu, ldst_unit //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; @@ -291,6 +324,12 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_issue_port.push_back(OC_EX_SP); } + for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) { + m_fu.push_back(new dp_unit( &m_pipeline_reg[EX_WB], m_config, this )); + m_dispatch_port.push_back(ID_OC_DP); + m_issue_port.push_back(OC_EX_DP); + } + for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this )); m_dispatch_port.push_back(ID_OC_SFU); @@ -307,7 +346,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_fu.push_back(m_ldst_unit); m_dispatch_port.push_back(ID_OC_MEM); m_issue_port.push_back(OC_EX_MEM); - + assert(m_num_function_units == m_fu.size() and m_fu.size() == m_dispatch_port.size() and m_fu.size() == m_issue_port.size()); //there are as many result buses as the width of the EX_WB stage @@ -341,6 +380,7 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re m_occupied_ctas = 0; m_occupied_hwtid.reset(); m_occupied_cta_to_hwtid.clear(); + m_active_warps = 0; } for (unsigned i = start_thread; iget_pdom_stack_top_info(pc,rpc); } +float shader_core_ctx::get_current_occupancy( unsigned long long & active, unsigned long long & total ) const +{ + // To match the achieved_occupancy in nvprof, only SMs that are active are counted toward the occupancy. + if ( m_active_warps > 0 ) { + total += m_warp.size(); + active += m_active_warps; + return float(active) / float(total); + } else { + return 0; + } +} + void shader_core_stats::print( FILE* fout ) const { unsigned long long thread_icount_uarch=0; @@ -436,15 +489,15 @@ void shader_core_stats::print( FILE* fout ) const fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge); fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]); fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]); - fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][bk_conf] = %d\n", + fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] + gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] + gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] + @@ -462,22 +515,22 @@ void shader_core_stats::print( FILE* fout ) const gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] + gpu_stall_shd_mem_breakdown[L_MEM_ST][DATA_PORT_STALL] ); // data port stall at data cache - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); - fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); + //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls); @@ -488,6 +541,14 @@ void shader_core_stats::print( FILE* fout ) const for (unsigned i = 3; i < m_config->warp_size + 3; i++) fprintf(fout, "\tW%d:%d", i-2, shader_cycle_distro[i]); fprintf(fout, "\n"); + fprintf(fout, "single_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, single_issue_nums[i]); + fprintf(fout, "\n"); + fprintf(fout, "dual_issue_nums: "); + for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) + fprintf(fout, "WS%d:%d\t", i, dual_issue_nums[i]); + fprintf(fout, "\n"); m_outgoing_traffic_stats->print(fout); m_incoming_traffic_stats->print(fout); @@ -669,13 +730,15 @@ void shader_core_ctx::fetch() } if( did_exit ) m_warp[warp_id].set_done_exit(); + --m_active_warps; + assert(m_active_warps >= 0); } // this code fetches instructions from the i-cache or generates memory requests if( !m_warp[warp_id].functional_done() && !m_warp[warp_id].imiss_pending() && m_warp[warp_id].ibuffer_empty() ) { address_type pc = m_warp[warp_id].get_pc(); address_type ppc = pc + PROGRAM_MEM_START; - unsigned nbytes=16; + unsigned nbytes=16; unsigned offset_in_block = pc & (m_config->m_L1I_config.get_line_sz()-1); if( (offset_in_block+nbytes) > m_config->m_L1I_config.get_line_sz() ) nbytes = (m_config->m_L1I_config.get_line_sz()-offset_in_block); @@ -750,10 +813,19 @@ void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* } void shader_core_ctx::issue(){ + + //Ensure fair round robin issu between schedulers + unsigned j; + for (unsigned i = 0; i < schedulers.size(); i++) { + j = (Issue_Prio + i) % schedulers.size(); + schedulers[j]->cycle(); + } + Issue_Prio = (Issue_Prio+1)% schedulers.size(); + //really is issue; - for (unsigned i = 0; i < schedulers.size(); i++) { - schedulers[i]->cycle(); - } + //for (unsigned i = 0; i < schedulers.size(); i++) { + // schedulers[i]->cycle(); + //} } shd_warp_t& scheduler_unit::warp(int i){ @@ -868,7 +940,10 @@ void scheduler_unit::cycle() unsigned warp_id = (*iter)->get_warp_id(); unsigned checked=0; unsigned issued=0; - unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; + unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; + while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); //Jin: handle cdp latency; @@ -901,18 +976,21 @@ void scheduler_unit::cycle() ready_inst = true; const active_mask_t &active_mask = m_simt_stack[warp_id]->get_active_mask(); assert( warp(warp_id).inst_in_pipeline() ); - if ( (pI->op == LOAD_OP)||(pI->op ==TENSOR_CORE_LOAD_OP)|| (pI->op == STORE_OP)|| (pI->op==TENSOR_CORE_STORE_OP) || (pI->op == MEMORY_BARRIER_OP) ) { - if( m_mem_out->has_free() ) { - m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - } + if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP)||(pI->op==TENSOR_CORE_LOAD_OP)||(pI->op==TENSOR_CORE_STORE_OP) ) { + if( m_mem_out->has_free() && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { + m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::MEM; + } } else { + bool sp_pipe_avail = m_sp_out->has_free(); bool sfu_pipe_avail = m_sfu_out->has_free(); bool tensor_core_pipe_avail = m_tensor_core_out->has_free(); - if( sp_pipe_avail && (pI->op != SFU_OP) && (pI->op != TENSOR_CORE_OP) ) { + bool dp_pipe_avail = m_dp_out->has_free(); + if( sp_pipe_avail && (pI->op != TENSOR_CORE_OP) && (pI->op != SFU_OP && pI->op != DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) { //Jin: special for CDP api if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { @@ -937,12 +1015,23 @@ void scheduler_unit::cycle() issued++; issued_inst=true; warp_inst_issued = true; - } else if ( (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP) ) { + previous_issued_inst_exec_type = exec_unit_type_t::SP; + } else if ( (m_shader->m_config->gpgpu_num_dp_units != 0) && (pI->op == DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::DP)) { + if( dp_pipe_avail ) { + m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::DP; + } + } //If the DP units = 0 (like in Fermi archi), then change DP inst to SFU inst + else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) { if( sfu_pipe_avail ) { m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id); issued++; issued_inst=true; warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::SFU; } } else if ( (pI->op == TENSOR_CORE_OP) ) { @@ -952,12 +1041,12 @@ void scheduler_unit::cycle() issued_inst=true; warp_inst_issued = true; } - } - } - } else { + } + }//end of else + } else { SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - } + } } } else if( valid ) { // this case can happen after a return instruction in diverged warp @@ -988,6 +1077,14 @@ void scheduler_unit::cycle() m_last_supervised_issued = supervised_iter; } } + + if(issued == 1) + m_stats->single_issue_nums[m_id]++; + else if(issued > 1) + m_stats->dual_issue_nums[m_id]++; + else + abort(); //issued should be > 0 + break; } } @@ -1045,6 +1142,16 @@ void gto_scheduler::order_warps() scheduler_unit::sort_warps_by_oldest_dynamic_id ); } +void oldest_scheduler::order_warps() +{ + order_by_priority( m_next_cycle_prioritized_warps, + m_supervised_warps, + m_last_supervised_issued, + m_supervised_warps.size(), + ORDERED_PRIORITY_FUNC_ONLY, + scheduler_unit::sort_warps_by_oldest_dynamic_id ); +} + void two_level_active_scheduler::do_on_warp_issued( unsigned warp_id, unsigned num_issued, @@ -1117,12 +1224,13 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id, char* config_string ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ) { unsigned m_prioritization_readin; int ret = sscanf( config_string, @@ -1386,8 +1494,14 @@ ldst_unit::process_cache_access( cache_t* cache, mem_stage_stall_type result = NO_RC_FAIL; bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); - if( write_sent ) - m_core->inc_store_req( inst.warp_id() ); + if( write_sent ) { + unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf->get_data_size()/SECTOR_SIZE) : 1; + + for(unsigned i=0; i< inc_ack; ++i) + m_core->inc_store_req( inst.warp_id() ); + + } if ( status == HIT ) { assert( !read_sent ); inst.accessq_pop_back(); @@ -1399,7 +1513,7 @@ ldst_unit::process_cache_access( cache_t* cache, if( !write_sent ) delete mf; } else if ( status == RESERVATION_FAIL ) { - result = COAL_STALL; + result = BK_CONF; assert( !read_sent ); assert( !write_sent ); delete mf; @@ -1408,8 +1522,8 @@ ldst_unit::process_cache_access( cache_t* cache, //inst.clear_active( access.get_warp_mask() ); // threads in mf writeback when mf returns inst.accessq_pop_back(); } - if( !inst.accessq_empty() ) - result = BK_CONF; + if( !inst.accessq_empty() && result == NO_RC_FAIL) + result = COAL_STALL; return result; } @@ -1429,6 +1543,111 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue( cache_t *cache, war return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); } +mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ) +{ + mem_stage_stall_type result = NO_RC_FAIL; + if( inst.accessq_empty() ) + return result; + + mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back()); + + if(m_config->m_L1D_config.l1_latency > 0) + { + if((l1_latency_queue[m_config->m_L1D_config.l1_latency-1]) == NULL) + { + l1_latency_queue[m_config->m_L1D_config.l1_latency-1] = mf; + + if( mf->get_inst().is_store() ) { + unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf->get_data_size()/SECTOR_SIZE) : 1; + + for(unsigned i=0; i< inc_ack; ++i) + m_core->inc_store_req( inst.warp_id() ); + } + + inst.accessq_pop_back(); + } + else + { + result = BK_CONF; + delete mf; + } + if( !inst.accessq_empty() && result !=BK_CONF) + result = COAL_STALL; + return result; + } + else + { + std::list events; + enum cache_request_status status = cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events); + return process_cache_access( cache, mf->get_addr(), inst, events, mf, status ); + } +} + +void ldst_unit::L1_latency_queue_cycle() +{ + //std::deque< std::pair >::iterator it = m_latency_queue.begin(); + if((l1_latency_queue[0]) != NULL) + { + mem_fetch* mf_next = l1_latency_queue[0]; + std::list events; + enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,gpu_sim_cycle+gpu_tot_sim_cycle,events); + + bool write_sent = was_write_sent(events); + bool read_sent = was_read_sent(events); + + if ( status == HIT ) { + assert( !read_sent ); + l1_latency_queue[0] = NULL; + if ( mf_next->get_inst().is_load() ) { + for ( unsigned r=0; r < 4; r++) + if (mf_next->get_inst().out[r] > 0) + { + assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0); + unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]; + if(!still_pending) + { + m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]); + m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]); + m_core->warp_inst_complete(mf_next->get_inst()); + } + } + } + + //For write hit in WB policy + if(mf_next->get_inst().is_store() && !write_sent) + { + unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)? + (mf_next->get_data_size()/SECTOR_SIZE) : 1; + + mf_next->set_reply(); + + for(unsigned i=0; i< dec_ack; ++i) + m_core->store_ack(mf_next); + } + + if( !write_sent ) + delete mf_next; + + } else if ( status == RESERVATION_FAIL ) { + assert( !read_sent ); + assert( !write_sent ); + } else { + assert( status == MISS || status == HIT_RESERVED ); + l1_latency_queue[0] = NULL; + } + } + + for( unsigned stage = 0; stagem_L1D_config.l1_latency-1; ++stage) + if( l1_latency_queue[stage] == NULL) { + l1_latency_queue[stage] = l1_latency_queue[stage+1] ; + l1_latency_queue[stage+1] = NULL; + } + +} + + + bool ldst_unit::constant_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) { if( inst.empty() || ((inst.space.get_type() != const_space) && (inst.space.get_type() != param_space_kernel)) ) @@ -1478,7 +1697,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea bypassL1D = true; } else if (inst.space.is_global()) { // global memory access // skip L1 cache if the option is enabled - if (m_core->get_config()->gmem_skip_L1D) + if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op)) bypassL1D = true; } if( bypassL1D ) { @@ -1502,9 +1721,9 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea } } else { assert( CACHE_UNDEFINED != inst.cache_op ); - stall_cond = process_memory_access_queue(m_L1D,inst); + stall_cond = process_memory_access_queue_l1cache(m_L1D,inst); } - if( !inst.accessq_empty() ) + if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL) stall_cond = COAL_STALL; if (stall_cond != NO_RC_FAIL) { stall_reason = stall_cond; @@ -1534,6 +1753,11 @@ void ldst_unit::flush(){ m_L1D->flush(); } +void ldst_unit::invalidate(){ + // Flush L1D cache + m_L1D->invalidate(); +} + simd_function_unit::simd_function_unit( const shader_core_config *config ) { m_config=config; @@ -1586,6 +1810,13 @@ void sp_unit::active_lanes_in_pipeline(){ m_core->incfuactivelanes_stat(active_count); m_core->incfumemactivelanes_stat(active_count); } +void dp_unit::active_lanes_in_pipeline(){ + unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count<=m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); +} void sfu::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); @@ -1610,6 +1841,12 @@ sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,sh m_name = "SP "; } +dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) + : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core) +{ + m_name = "DP "; +} + void sp_unit :: issue(register_set& source_reg) { warp_inst_t** ready_reg = source_reg.get_ready(); @@ -1619,6 +1856,14 @@ void sp_unit :: issue(register_set& source_reg) pipelined_simd_unit::issue(source_reg); } +void dp_unit :: issue(register_set& source_reg) +{ + warp_inst_t** ready_reg = source_reg.get_ready(); + //m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe=DP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); +} pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core ) : simd_function_unit(config) @@ -1712,8 +1957,9 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt, const memory_config *mem_config, shader_core_stats *stats, unsigned sid, - unsigned tpc ) : pipelined_simd_unit(NULL,config,3,core), m_next_wb(config) + unsigned tpc ) : pipelined_simd_unit(NULL,config,config->smem_latency,core), m_next_wb(config) { + assert(config->smem_latency > 1); init( icnt, mf_allocator, core, @@ -1734,6 +1980,12 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt, m_icnt, m_mf_allocator, IN_L1D_MISS_QUEUE ); + + if(m_config->m_L1D_config.l1_latency > 0) + { + for(int i=0; im_L1D_config.l1_latency; i++ ) + l1_latency_queue.push_back((mem_fetch*)NULL); + } } } @@ -1916,12 +2168,12 @@ void ldst_unit::cycle() if( !m_response_fifo.empty() ) { mem_fetch *mf = m_response_fifo.front(); - if (mf->istexture()) { + if (mf->get_access_type() == TEXTURE_ACC_R) { if (m_L1T->fill_port_free()) { m_L1T->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); m_response_fifo.pop_front(); } - } else if (mf->isconst()) { + } else if (mf->get_access_type() == CONST_ACC_R) { if (m_L1C->fill_port_free()) { mf->set_status(IN_SHADER_FETCHED,gpu_sim_cycle+gpu_tot_sim_cycle); m_L1C->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle); @@ -1960,7 +2212,11 @@ void ldst_unit::cycle() m_L1T->cycle(); m_L1C->cycle(); - if( m_L1D ) m_L1D->cycle(); + if( m_L1D ) { + m_L1D->cycle(); + if(m_config->m_L1D_config.l1_latency > 0) + L1_latency_queue_cycle(); + } warp_inst_t &pipe_reg = *m_dispatch_reg; enum mem_stage_stall_type rc_fail = NO_RC_FAIL; @@ -1983,9 +2239,9 @@ void ldst_unit::cycle() unsigned warp_id = pipe_reg.warp_id(); if( pipe_reg.is_load() ) { if( pipe_reg.space.get_type() == shared_space ) { - if( m_pipeline_reg[2]->empty() ) { + if( m_pipeline_reg[m_config->smem_latency-1]->empty() ) { // new shared memory request - move_warp(m_pipeline_reg[2],m_dispatch_reg); + move_warp(m_pipeline_reg[m_config->smem_latency-1],m_dispatch_reg); m_dispatch_reg->clear(); } } else { @@ -2562,9 +2818,42 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const assert( result <= MAX_CTA_PER_SHADER ); if (result < 1) { printf ("GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader has.\n"); + if(gpgpu_ignore_resources_limitation) { + printf ("GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore the ERROR!\n"); + return 1; + } abort(); } + if(adpative_volta_cache_config && !k.volta_cache_config_set) { + //For Volta, we assign the remaining shared memory to L1 cache + //For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x + unsigned total_shmed = kernel_info->smem * result; + assert(total_shmed >=0 && total_shmed <= gpgpu_shmem_size); + assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared + assert(m_L1D_config.get_nset() == 4); //Volta L1 has four sets + if(total_shmed < gpgpu_shmem_size){ + if(total_shmed == 0) + m_L1D_config.set_assoc(256); //L1 is 128KB ans shd=0 + else if(total_shmed > 0 && total_shmed <= 8192) + m_L1D_config.set_assoc(240); //L1 is 120KB ans shd=8KB + else if(total_shmed > 8192 && total_shmed <= 16384) + m_L1D_config.set_assoc(224); //L1 is 112KB ans shd=16KB + else if(total_shmed > 16384 && total_shmed <= 32768) + m_L1D_config.set_assoc(192); //L1 is 96KB ans shd=32KB + else if(total_shmed > 32768 && total_shmed <= 65536) + m_L1D_config.set_assoc(128); //L1 is 64KB ans shd=64KB + else if(total_shmed > 65536 && total_shmed <= gpgpu_shmem_size) + m_L1D_config.set_assoc(64); //L1 is 32KB and shd=96KB + else + assert(0); + + printf ("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", m_L1D_config.get_total_size_inKB()); + } + + k.volta_cache_config_set = true; + } + return result; } @@ -2586,6 +2875,11 @@ void shader_core_ctx::cache_flush() m_ldst_unit->flush(); } +void shader_core_ctx::cache_invalidate() +{ + m_ldst_unit->invalidate(); +} + // modifiers std::list opndcoll_rfu_t::arbiter_t::allocate_reads() { @@ -3340,6 +3634,15 @@ void simt_core_cluster::print_not_completed( FILE *fp ) const } } + +float simt_core_cluster::get_current_occupancy( unsigned long long& active, unsigned long long& total ) const { + float aggregate = 0.f; + for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) { + aggregate+=m_core[i]->get_current_occupancy( active, total ); + } + return aggregate / m_config->n_simt_cores_per_cluster; +} + unsigned simt_core_cluster::get_n_active_cta() const { unsigned n=0; @@ -3403,6 +3706,12 @@ void simt_core_cluster::cache_flush() m_core[i]->cache_flush(); } +void simt_core_cluster::cache_invalidate() +{ + for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) + m_core[i]->cache_invalidate(); +} + bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write) { unsigned request_size = size; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 97e438f..437506c 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -70,6 +70,14 @@ #define WRITE_MASK_SIZE 8 +enum exec_unit_type_t +{ + NONE = 0, + SP = 1, + SFU = 2, + MEM = 3, + DP = 4 +}; class thread_ctx_t { public: @@ -308,6 +316,7 @@ enum concrete_scheduler CONCRETE_SCHEDULER_GTO, CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE, CONCRETE_SCHEDULER_WARP_LIMITING, + CONCRETE_SCHEDULER_OLDEST_FIRST, NUM_CONCRETE_SCHEDULERS }; @@ -317,13 +326,14 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id) : m_supervised_warps(), m_stats(stats), m_shader(shader), m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), - m_sp_out(sp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} + m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} virtual ~scheduler_unit(){} virtual void add_supervised_warp_id(int i) { m_supervised_warps.push_back(&warp(i)); @@ -395,6 +405,7 @@ protected: //warp_inst_t** m_pipeline_reg; std::vector* m_warp; register_set* m_sp_out; + register_set* m_dp_out; register_set* m_sfu_out; register_set* m_tensor_core_out; register_set* m_mem_out; @@ -408,11 +419,12 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} virtual ~lrr_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -426,11 +438,12 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} virtual ~gto_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -439,6 +452,25 @@ public: }; +class oldest_scheduler : public scheduler_unit { +public: + oldest_scheduler ( shader_core_stats* stats, shader_core_ctx* shader, + Scoreboard* scoreboard, simt_stack** simt, + std::vector* warp, + register_set* sp_out, + register_set* dp_out, + register_set* sfu_out, + register_set* tensor_core_out, + register_set* mem_out, + int id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ){} + virtual ~oldest_scheduler () {} + virtual void order_warps (); + virtual void done_adding_supervised_warps() { + m_last_supervised_issued = m_supervised_warps.begin(); + } + +}; class two_level_active_scheduler : public scheduler_unit { public: @@ -446,12 +478,13 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, int id, char* config_str ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, tensor_core_out, mem_out, id ), + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out,tensor_core_out, mem_out, id ), m_pending_warps() { unsigned inner_level_readin; @@ -497,6 +530,7 @@ public: Scoreboard* scoreboard, simt_stack** simt, std::vector* warp, register_set* sp_out, + register_set* dp_out, register_set* sfu_out, register_set* tensor_core_out, register_set* mem_out, @@ -1060,6 +1094,23 @@ public: switch(inst.op) { case SFU_OP: break; case ALU_SFU_OP: break; + case DP_OP: break; //for compute <= 29 (i..e Fermi and GT200) + default: return false; + } + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue( register_set& source_reg ); +}; + +class dp_unit : public pipelined_simd_unit +{ +public: + dp_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); + virtual bool can_issue( const warp_inst_t &inst ) const + { + switch(inst.op) { + case DP_OP: break; default: return false; } return pipelined_simd_unit::can_issue(inst); @@ -1098,6 +1149,7 @@ public: case STORE_OP: return false; case TENSOR_CORE_STORE_OP: return false; case MEMORY_BARRIER_OP: return false; + case DP_OP: return false; default: break; } return pipelined_simd_unit::can_issue(inst); @@ -1129,6 +1181,7 @@ public: void fill( mem_fetch *mf ); void flush(); + void invalidate(); void writeback(); // accessors @@ -1195,6 +1248,7 @@ protected: mem_fetch *mf, enum cache_request_status status ); mem_stage_stall_type process_memory_access_queue( cache_t *cache, warp_inst_t &inst ); + mem_stage_stall_type process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst ); const memory_config *m_memory_config; class mem_fetch_interface *m_icnt; @@ -1223,13 +1277,18 @@ protected: // for debugging unsigned long long m_last_inst_gpu_sim_cycle; unsigned long long m_last_inst_gpu_tot_sim_cycle; + + std::deque l1_latency_queue; + void L1_latency_queue_cycle(); }; enum pipeline_stage_name_t { ID_OC_SP=0, + ID_OC_DP, ID_OC_SFU, ID_OC_MEM, OC_EX_SP, + OC_EX_DP, OC_EX_SFU, OC_EX_MEM, EX_WB, @@ -1240,9 +1299,11 @@ enum pipeline_stage_name_t { const char* const pipeline_stage_name_decode[] = { "ID_OC_SP", + "ID_OC_DP", "ID_OC_SFU", "ID_OC_MEM", "OC_EX_SP", + "OC_EX_DP", "OC_EX_SFU", "OC_EX_MEM", "EX_WB", @@ -1340,27 +1401,29 @@ struct shader_core_config : public core_config mutable cache_config m_L1C_config; mutable l1d_cache_config m_L1D_config; - bool gmem_skip_L1D; // on = global memory access always skip the L1 cache - bool gpgpu_dwf_reg_bankconflict; int gpgpu_num_sched_per_core; int gpgpu_max_insn_issue_per_warp; + bool gpgpu_dual_issue_diff_exec_units; //op collector int gpgpu_operand_collector_num_units_sp; + int gpgpu_operand_collector_num_units_dp; int gpgpu_operand_collector_num_units_sfu; int gpgpu_operand_collector_num_units_tensor_core; int gpgpu_operand_collector_num_units_mem; int gpgpu_operand_collector_num_units_gen; unsigned int gpgpu_operand_collector_num_in_ports_sp; + unsigned int gpgpu_operand_collector_num_in_ports_dp; unsigned int gpgpu_operand_collector_num_in_ports_sfu; unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; unsigned int gpgpu_operand_collector_num_in_ports_mem; unsigned int gpgpu_operand_collector_num_in_ports_gen; unsigned int gpgpu_operand_collector_num_out_ports_sp; + unsigned int gpgpu_operand_collector_num_out_ports_dp; unsigned int gpgpu_operand_collector_num_out_ports_sfu; unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; unsigned int gpgpu_operand_collector_num_out_ports_mem; @@ -1368,6 +1431,7 @@ struct shader_core_config : public core_config int gpgpu_num_sp_units; int gpgpu_tensor_core_avail; + int gpgpu_num_dp_units; int gpgpu_num_sfu_units; int gpgpu_num_tensor_core_units; int gpgpu_num_mem_units; @@ -1379,6 +1443,7 @@ struct shader_core_config : public core_config unsigned gpgpu_num_reg_banks; bool gpgpu_reg_bank_use_warp_id; bool gpgpu_local_mem_map; + bool gpgpu_ignore_resources_limitation; unsigned max_sp_latency; unsigned max_sfu_latency; @@ -1391,10 +1456,14 @@ struct shader_core_config : public core_config int simt_core_sim_order; + unsigned smem_latency; + unsigned mem2device(unsigned memid) const { return memid + n_simt_clusters; } //Jin: concurrent kernel on sm bool gpgpu_concurrent_kernel_sm; + + bool adpative_volta_cache_config; }; struct shader_core_stats_pod { @@ -1457,6 +1526,8 @@ struct shader_core_stats_pod { unsigned *last_shader_cycle_distro; unsigned *num_warps_issuable; unsigned gpgpu_n_stall_shd_mem; + unsigned* single_issue_nums; + unsigned* dual_issue_nums; //memory access classification int gpgpu_n_mem_read_local; @@ -1528,6 +1599,8 @@ public: m_n_diverge = (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); shader_cycle_distro = (unsigned*) calloc(config->warp_size+3, sizeof(unsigned)); last_shader_cycle_distro = (unsigned*) calloc(m_config->warp_size+3, sizeof(unsigned)); + single_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core,sizeof(unsigned)); + dual_issue_nums = (unsigned*) calloc(config->gpgpu_num_sched_per_core, sizeof(unsigned)); n_simt_to_mem = (long *)calloc(config->num_shader(), sizeof(long)); n_mem_to_simt = (long *)calloc(config->num_shader(), sizeof(long)); @@ -1651,6 +1724,7 @@ public: void issue_block2core( class kernel_info_t &kernel ); void cache_flush(); + void cache_invalidate(); void accept_fetch_response( mem_fetch *mf ); void accept_ldst_unit_response( class mem_fetch * mf ); void broadcast_barrier_reduction(unsigned cta_id, unsigned bar_id,warp_set_t warps); @@ -1679,6 +1753,7 @@ public: // accessors virtual bool warp_waiting_at_barrier( unsigned warp_id ) const; void get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsigned *rpc ) const; + float get_current_occupancy( unsigned long long & active, unsigned long long & total ) const; // used by pipeline timing model components: // modifiers @@ -1885,10 +1960,14 @@ public: std::vector m_pipeline_reg; Scoreboard *m_scoreboard; opndcoll_rfu_t m_operand_collector; + int m_active_warps; //schedule std::vector schedulers; + //issue + unsigned int Issue_Prio; + // execute unsigned m_num_function_units; std::vector m_dispatch_port; @@ -1940,6 +2019,7 @@ public: void reinit(); unsigned issue_block2core(); void cache_flush(); + void cache_invalidate(); bool icnt_injection_buffer_full(unsigned size, bool write); void icnt_inject_request_packet(class mem_fetch *mf); @@ -1970,6 +2050,7 @@ public: void get_L1T_sub_stats(struct cache_sub_stats &css) const; void get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const; + float get_current_occupancy( unsigned long long& active, unsigned long long & total ) const; private: unsigned m_cluster_id; -- cgit v1.3