The commits includes:

1- REEAD/WERITE buffer for DRAM
2- Fixing FETCH_ON_WRITE cahce policy  bug

10 files changed, 160 insertions, 47 deletions

diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index 5c1ddab..7ed7b6f 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -149,11 +149,18 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m
 
 }
 
-bool dram_t::full() const 
+bool dram_t::full(bool is_write) const
 {
     if(m_config->scheduler_type == DRAM_FRFCFS){
         if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false;
-        return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
+        if(m_config->seperate_write_queue_enabled){
+        	if(is_write)
+        		return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size;
+        	else
+        		return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
+        }
+        else
+        	return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
     }
    else return mrqq->full();
 }
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
index 331b4f1..29731a7 100644
--- a/src/gpgpu-sim/dram.h
+++ b/src/gpgpu-sim/dram.h
@@ -101,7 +101,7 @@ public:
    dram_t( unsigned int parition_id, const struct memory_config *config, class memory_stats_t *stats, 
            class memory_partition_unit *mp );
 
-   bool full() const;
+   bool full(bool is_write) const;
    void print( FILE* simFile ) const;
    void visualize() const;
    void print_stat( FILE* simFile );
diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc
index 008b5bb..7a140c5 100644
--- a/src/gpgpu-sim/dram_sched.cc
+++ b/src/gpgpu-sim/dram_sched.cc
@@ -36,6 +36,7 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem
    m_config = config;
    m_stats = stats;
    m_num_pending = 0;
+   m_num_write_pending = 0;
    m_dram = dm;
    m_queue = new std::list<dram_req_t*>[m_config->nbk];
    m_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ];
@@ -49,15 +50,36 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem
       curr_row_service_time[i] = 0;
       row_service_timestamp[i] = 0;
    }
+   if(m_config->seperate_write_queue_enabled) {
+	   m_write_queue = new std::list<dram_req_t*>[m_config->nbk];
+	   m_write_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ];
+	   m_last_write_row = new std::list<std::list<dram_req_t*>::iterator>*[ m_config->nbk ];
+
+	   for ( unsigned i=0; i < m_config->nbk; i++ ) {
+	         m_write_queue[i].clear();
+	         m_write_bins[i].clear();
+	         m_last_write_row[i] = NULL;
+	      }
+   }
+   m_mode = READ_MODE;
 
 }
 
 void frfcfs_scheduler::add_req( dram_req_t *req )
 {
-   m_num_pending++;
-   m_queue[req->bk].push_front(req);
-   std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin();
-   m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  if(m_config->seperate_write_queue_enabled && req->data->is_write()) {
+	  assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size);
+	  m_num_write_pending++;
+	  m_write_queue[req->bk].push_front(req);
+	  std::list<dram_req_t*>::iterator ptr = m_write_queue[req->bk].begin();
+	  m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  } else {
+	   assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size);
+	   m_num_pending++;
+	   m_queue[req->bk].push_front(req);
+	   std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin();
+	   m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  }
 }
 
 void frfcfs_scheduler::data_collection(unsigned int bank)
@@ -80,21 +102,43 @@ dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row )
 {
    //row
    bool rowhit = true;
+   std::list<dram_req_t*> *m_current_queue = m_queue;
+   std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> > *m_current_bins = m_bins ;
+   std::list<std::list<dram_req_t*>::iterator> **m_current_last_row = m_last_row;
 
-   if ( m_last_row[bank] == NULL ) {
-      if ( m_queue[bank].empty() )
+   if(m_config->seperate_write_queue_enabled) {
+	   if(m_mode == READ_MODE &&
+			  ((m_num_write_pending >= m_config->write_high_watermark )
+			   || (m_queue[bank].empty() && !m_write_queue[bank].empty()))) {
+		   m_mode = WRITE_MODE;
+	   }
+	   else if(m_mode == WRITE_MODE &&
+				  (( m_num_write_pending < m_config->write_low_watermark )
+				   || (!m_queue[bank].empty() && m_write_queue[bank].empty()))){
+		   m_mode = READ_MODE;
+	   }
+   }
+
+   if(m_mode == WRITE_MODE) {
+	   m_current_queue = m_write_queue;
+	   m_current_bins = m_write_bins ;
+	   m_current_last_row = m_last_write_row;
+   }
+
+   if ( m_current_last_row[bank] == NULL ) {
+      if ( m_current_queue[bank].empty() )
          return NULL;
 
-      std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row );
-      if ( bin_ptr == m_bins[bank].end()) {
-         dram_req_t *req = m_queue[bank].back();
-         bin_ptr = m_bins[bank].find( req->row );
-         assert( bin_ptr != m_bins[bank].end() ); // where did the request go???
-         m_last_row[bank] = &(bin_ptr->second);
+      std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row );
+      if ( bin_ptr == m_current_bins[bank].end()) {
+         dram_req_t *req = m_current_queue[bank].back();
+         bin_ptr = m_current_bins[bank].find( req->row );
+         assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go???
+         m_current_last_row[bank] = &(bin_ptr->second);
          data_collection(bank);
          rowhit = false;
       } else {
-         m_last_row[bank] = &(bin_ptr->second);
+    	  m_current_last_row[bank] = &(bin_ptr->second);
          rowhit = true;
       }
    }
@@ -103,25 +147,32 @@ dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row )
   if(rowhit)
    m_dram->hits_num++;
 
-   std::list<dram_req_t*>::iterator next = m_last_row[bank]->back();
+   std::list<dram_req_t*>::iterator next = m_current_last_row[bank]->back();
    dram_req_t *req = (*next);
 
    m_stats->concurrent_row_access[m_dram->id][bank]++;
    m_stats->row_access[m_dram->id][bank]++;
-   m_last_row[bank]->pop_back();
+   m_current_last_row[bank]->pop_back();
 
-   m_queue[bank].erase(next);
-   if ( m_last_row[bank]->empty() ) {
-      m_bins[bank].erase( req->row );
-      m_last_row[bank] = NULL;
+   m_current_queue[bank].erase(next);
+   if ( m_current_last_row[bank]->empty() ) {
+	   m_current_bins[bank].erase( req->row );
+	   m_current_last_row[bank] = NULL;
    }
 #ifdef DEBUG_FAST_IDEAL_SCHED
    if ( req )
       printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", 
              (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row );
 #endif
-   assert( req != NULL && m_num_pending != 0 ); 
-   m_num_pending--;
+
+   if(m_config->seperate_write_queue_enabled && req->data->is_write()) {
+	   assert( req != NULL && m_num_write_pending != 0 );
+	   m_num_write_pending--;
+   }
+   else {
+	   assert( req != NULL && m_num_pending != 0 );
+	   m_num_pending--;
+   }
 
    return req;
 }
@@ -138,7 +189,7 @@ void dram_t::scheduler_frfcfs()
 {
    unsigned mrq_latency;
    frfcfs_scheduler *sched = m_frfcfs_scheduler;
-   while ( !mrqq->empty() && (!m_config->gpgpu_frfcfs_dram_sched_queue_size || sched->num_pending() < m_config->gpgpu_frfcfs_dram_sched_queue_size)) {
+   while ( !mrqq->empty() ) {
       dram_req_t *req = mrqq->pop();
 
       // Power stats
diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h
index 3860f5b..63f5831 100644
--- a/src/gpgpu-sim/dram_sched.h
+++ b/src/gpgpu-sim/dram_sched.h
@@ -35,6 +35,11 @@
 #include <list>
 #include <map>
 
+enum memory_mode {
+   READ_MODE = 0,
+   WRITE_MODE
+};
+
 class frfcfs_scheduler {
 public:
    frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats );
@@ -43,17 +48,24 @@ public:
    dram_req_t *schedule( unsigned bank, unsigned curr_row );
    void print( FILE *fp );
    unsigned num_pending() const { return m_num_pending;}
+   unsigned num_write_pending() const { return m_num_write_pending;}
 
 private:
    const memory_config *m_config;
    dram_t *m_dram;
    unsigned m_num_pending;
+   unsigned m_num_write_pending;
    std::list<dram_req_t*>                                    *m_queue;
    std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >    *m_bins;
    std::list<std::list<dram_req_t*>::iterator>                 **m_last_row;
    unsigned *curr_row_service_time; //one set of variables for each bank.
    unsigned *row_service_timestamp; //tracks when scheduler began servicing current row
 
+   std::list<dram_req_t*>                                    *m_write_queue;
+   std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >    *m_write_bins;
+   std::list<std::list<dram_req_t*>::iterator>                 **m_last_write_row;
+
+   enum memory_mode m_mode;
    memory_stats_t *m_stats;
 };
 
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index eadc094..d199cca 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -945,10 +945,9 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a
         mf->set_addr( block_addr );
         m_miss_queue.push_back(mf);
         mf->set_status(m_miss_queue_status,time);
-        if(wa)
-        	events.push_back(cache_event(WRITE_ALLOCATE_SENT));
-        else
+        if(!wa)
         	events.push_back(cache_event(READ_REQUEST_SENT));
+
         do_miss = true;
     }
     else if(mshr_hit && !mshr_avail)
@@ -1087,6 +1086,8 @@ data_cache::wr_miss_wa_naive( new_addr_type addr,
     send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb,
         evicted, events, false, true);
 
+    events.push_back(cache_event(WRITE_ALLOCATE_SENT));
+
     if( do_miss ){
         // If evicted block is modified and not a write-through
         // (already modified lower level)
@@ -1111,7 +1112,7 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr,
 {
 
 	new_addr_type block_addr = m_config.block_addr(addr);
-	new_addr_type mshr_addr = m_config.block_addr(mf->get_addr());
+    new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr());
 
 	if(mf->get_access_byte_mask().count() == m_config.get_atom_sz())
 	{
@@ -1147,10 +1148,23 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr,
 	}
 	else
 	{
-		 if(miss_queue_full(1)) {
-			 m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
-			 return RESERVATION_FAIL;
-		 }
+		bool mshr_hit = m_mshrs.probe(mshr_addr);
+		bool mshr_avail = !m_mshrs.full(mshr_addr);
+		if(miss_queue_full(1)
+			|| (!(mshr_hit && mshr_avail)
+			&& !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) {
+			//check what is the exactly the failure reason
+			 if(miss_queue_full(1) )
+				 m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+			 else if(mshr_hit && !mshr_avail)
+				  m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL);
+			 else if (!mshr_hit && !mshr_avail)
+				m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL);
+			 else
+				 assert(0);
+
+			return RESERVATION_FAIL;
+		}
 
 
 		  //prevent Write - Read - Write in pending mshr
@@ -1177,8 +1191,10 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr,
 								mf->get_sid(),
 								mf->get_tpc(),
 								mf->get_mem_config(),
+								NULL,
 								mf);
 
+
 			new_addr_type block_addr = m_config.block_addr(addr);
 			bool do_miss = false;
 			bool wb = false;
@@ -1191,6 +1207,8 @@ data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr,
 			cache_block_t* block = m_tag_array->get_block(cache_index);
 			block->set_modified_on_fill(true, mf->get_access_sector_mask());
 
+			events.push_back(cache_event(WRITE_ALLOCATE_SENT));
+
 			if( do_miss ){
 				// If evicted block is modified and not a write-through
 				// (already modified lower level)
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 470fcf4..7838875 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -201,7 +201,12 @@ void memory_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy,
                                             "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)",
                                             "0");
-
+    option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled,
+                           "Seperate_Write_Queue_Enable",
+                           "0");
+    option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt,
+                                  "Write_Queue_Size",
+                                  "32:28:16");
     m_address_mapping.addrdec_setoption(opp);
 }
 
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index f379a17..197350b 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -214,6 +214,9 @@ struct memory_config {
 
       m_valid = true;
       icnt_flit_size = 32; // Default 32
+
+      sscanf(write_queue_size_opt,"%d:%d:%d",
+                     &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark);
    }
    void reg_options(class OptionParser * opp);
 
@@ -274,6 +277,12 @@ struct memory_config {
    unsigned dram_bnk_indexing_policy;
    unsigned dram_bnkgrp_indexing_policy;
    bool dual_bus_interface;
+
+   bool seperate_write_queue_enabled;
+   char *write_queue_size_opt;
+   unsigned gpgpu_frfcfs_dram_write_queue_size;
+   unsigned write_high_watermark;
+   unsigned write_low_watermark;
 };
 
 // global counters and flags (please try not to add to this list!!!)
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index f7323c5..cac59f1 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -93,7 +93,9 @@ memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct m
     m_private_credit_limit = 1; 
     m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size 
                             + config->gpgpu_dram_return_queue_size 
-                            - (config->m_n_sub_partition_per_memory_channel - 1); 
+                            - (config->m_n_sub_partition_per_memory_channel - 1);
+    if(config->seperate_write_queue_enabled )
+    	m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size;
     if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 
         or config->gpgpu_dram_return_queue_size == 0) 
     {
@@ -220,7 +222,8 @@ void memory_partition_unit::dram_cycle()
     m_dram->cycle(); 
     m_dram->dram_log(SAMPLELOG);   
 
-    if( !m_dram->full() ) {
+   // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+    //if( !m_dram->full(mf->is_write()) ) {
         // L2->DRAM queue to DRAM latency queue
         // Arbitrate among multiple L2 subpartitions 
         int last_issued_partition = m_arbitration_metadata.last_borrower(); 
@@ -228,6 +231,9 @@ void memory_partition_unit::dram_cycle()
             int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; 
             if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) {
                 mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+                if(m_dram->full(mf->is_write()) )
+                	break;
+
                 m_sub_partition[spid]->L2_dram_queue_pop();
                 MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); 
                 dram_delay_t d;
@@ -239,12 +245,13 @@ void memory_partition_unit::dram_cycle()
                 break;  // the DRAM should only accept one request per cycle 
             }
         }
-    }
+    //}
 
     // DRAM latency queue
-    if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) {
-        mem_fetch* mf = m_dram_latency_queue.front().req;
-        m_dram_latency_queue.pop_front();
+
+    if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) {
+    	mem_fetch* mf = m_dram_latency_queue.front().req;
+    	m_dram_latency_queue.pop_front();
         m_dram->push(mf);
     }
 }
@@ -343,12 +350,12 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
 				mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
 				m_L2_icnt_queue->push(mf);
            }else{
-        	    if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE && mf->original_mf)
+        	    if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE)
         	    {
-			 		assert(mf->original_mf);
-			 		mf->original_mf->set_reply();
-			 		mf->original_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
-			 		m_L2_icnt_queue->push(mf->original_mf);
+			 		assert(mf->original_wr_mf);
+			 		mf->original_wr_mf->set_reply();
+			 		mf->original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
+			 		m_L2_icnt_queue->push(mf->original_wr_mf);
         	    }
 				m_request_tracker.erase(mf);
 				delete mf;
diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc
index b8e918f..c05a693 100644
--- a/src/gpgpu-sim/mem_fetch.cc
+++ b/src/gpgpu-sim/mem_fetch.cc
@@ -40,7 +40,8 @@ mem_fetch::mem_fetch( const mem_access_t &access,
                       unsigned sid, 
                       unsigned tpc, 
                       const class memory_config *config,
-					  mem_fetch *m_original_mf)
+					  mem_fetch *m_original_mf,
+					  mem_fetch *m_original_wr_mf)
 {
    m_request_uid = sm_next_mf_request_uid++;
    m_access = access;
@@ -63,6 +64,7 @@ mem_fetch::mem_fetch( const mem_access_t &access,
    m_mem_config = config;
    icnt_flit_size = config->icnt_flit_size;
    original_mf = m_original_mf;
+   original_wr_mf = m_original_wr_mf;
 }
 
 mem_fetch::~mem_fetch()
diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h
index 76e7419..278cf32 100644
--- a/src/gpgpu-sim/mem_fetch.h
+++ b/src/gpgpu-sim/mem_fetch.h
@@ -56,7 +56,8 @@ public:
                unsigned sid, 
                unsigned tpc, 
                const class memory_config *config,
-			   mem_fetch *original_mf = NULL);
+			   mem_fetch *original_mf = NULL,
+			   mem_fetch *original_wr_mf = NULL);
    ~mem_fetch();
 
    void set_status( enum mem_fetch_status status, unsigned long long cycle );
@@ -115,6 +116,7 @@ public:
 
    unsigned get_num_flits(bool simt_to_mem);
    mem_fetch* original_mf;
+   mem_fetch* original_wr_mf;
 private:
    // request source information
    unsigned m_request_uid;