Lengthened the DRAM return queue size to have enough credits in order to keep the DRAM utilized. Also extended the state printing function to print out mem_fetch entries inside the DRAM delay queue.

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 16639]

7 files changed, 47 insertions, 10 deletions

diff --git a/CHANGES b/CHANGES
index c743ce1..0be0d4c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -35,7 +35,10 @@ Version 3.2.1+edits (development branch) versus 3.2.1
 - Redesigned the memory partition microarchitecture model to allow multiple L2
   cache banks (sub partitions) in each memory partition.  Each memory partition
   contains a single DRAM scheduler, and one or more L2 cache banks.  Each L2
-  cache bank has an independent port to the interconnection network.  
+  cache bank has an independent port to the interconnection network.  The
+  configuration files are changes to have a larger DRAM return queue to allow
+  the credit-based arbiter between the sub partitions and the DRAM scheduler to
+  tolerate the minimum DRAM latency.  
 - Bug Fixes:
     - Fixed the flit count sent to GPUWattch for atomic operations. 
     - Fix for Bug 51 - Updated the function declaration of
diff --git a/configs/GTX480/gpgpusim.config b/configs/GTX480/gpgpusim.config
index 62dd078..a1d6517 100644
--- a/configs/GTX480/gpgpusim.config
+++ b/configs/GTX480/gpgpusim.config
@@ -85,8 +85,13 @@
 
 # dram model config
 -gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (100 core cycles).  I.e. 
+#   Total buffer space required = 100 x 924MHz / 700MHz = 132
 -gpgpu_frfcfs_dram_sched_queue_size 16
--gpgpu_dram_return_queue_size 16
+-gpgpu_dram_return_queue_size 116
 
 # for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition
 -gpgpu_n_mem_per_ctrlr 2
diff --git a/configs/QuadroFX5600/gpgpusim.config b/configs/QuadroFX5600/gpgpusim.config
index 256610f..8a9cd5b 100644
--- a/configs/QuadroFX5600/gpgpusim.config
+++ b/configs/QuadroFX5600/gpgpusim.config
@@ -49,10 +49,17 @@
 -network_mode 1 
 -inter_config_file icnt_config_islip.txt 
 
-# dram model config
+# dram scheduler config
 -gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (30 core cycles).  I.e. 
+#   Total buffer space required = 30 x 800MHz / 337.5MHz = 71
 -gpgpu_frfcfs_dram_sched_queue_size 16
--gpgpu_dram_return_queue_size 16
+-gpgpu_dram_return_queue_size 55
+
+# dram model config
 -gpgpu_n_mem_per_ctrlr 2
 -gpgpu_dram_buswidth 4 
 -gpgpu_dram_burst_length 4
diff --git a/configs/QuadroFX5800/gpgpusim.config b/configs/QuadroFX5800/gpgpusim.config
index 81a5f1f..0df4b64 100644
--- a/configs/QuadroFX5800/gpgpusim.config
+++ b/configs/QuadroFX5800/gpgpusim.config
@@ -45,10 +45,17 @@
 -network_mode 1 
 -inter_config_file config_quadro_islip.icnt
 
-# dram model config
+# dram scheduler config
 -gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (30 core cycles).  I.e. 
+#   Total buffer space required = 30 x 800MHz / 325MHz = 74
 -gpgpu_frfcfs_dram_sched_queue_size 16
--gpgpu_dram_return_queue_size 16
+-gpgpu_dram_return_queue_size 58
+
+# dram model config
 -gpgpu_n_mem_per_ctrlr 2
 -gpgpu_dram_buswidth 4 
 -gpgpu_dram_burst_length 4
diff --git a/configs/TeslaC2050/gpgpusim.config b/configs/TeslaC2050/gpgpusim.config
index 3100cbc..70e3420 100644
--- a/configs/TeslaC2050/gpgpusim.config
+++ b/configs/TeslaC2050/gpgpusim.config
@@ -88,8 +88,13 @@
 
 # dram model config
 -gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (100 core cycles).  I.e. 
+#   Total buffer space required = 100 x 750MHz / 575MHz = 130
 -gpgpu_frfcfs_dram_sched_queue_size 16
--gpgpu_dram_return_queue_size 16
+-gpgpu_dram_return_queue_size 114  
 
 # for Fermi, bus width is 384bits, this is 8 bytes (4 bytes at each DRAM chip) per memory partition
 -gpgpu_n_mem_per_ctrlr 2
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index d667fd7..c589e6c 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -233,7 +233,7 @@ void memory_partition_unit::dram_cycle()
                 dram_delay_t d;
                 d.req = mf;
                 d.ready_cycle = gpu_sim_cycle+gpu_tot_sim_cycle + m_config->dram_latency;
-                m_dram_latency_queue.push(d);
+                m_dram_latency_queue.push_back(d);
                 mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
                 m_arbitration_metadata.borrow_credit(spid); 
                 break;  // the DRAM should only accept one request per cycle 
@@ -244,7 +244,7 @@ void memory_partition_unit::dram_cycle()
     // DRAM latency queue
     if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) {
         mem_fetch* mf = m_dram_latency_queue.front().req;
-        m_dram_latency_queue.pop();
+        m_dram_latency_queue.pop_front();
         m_dram->push(mf);
     }
 }
@@ -279,6 +279,16 @@ void memory_partition_unit::print( FILE *fp ) const
     for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) {
         m_sub_partition[p]->print(fp); 
     }
+    fprintf(fp, "In Dram Latency Queue (total = %zd): \n", m_dram_latency_queue.size()); 
+    for (std::list<dram_delay_t>::const_iterator mf_dlq = m_dram_latency_queue.begin(); 
+         mf_dlq != m_dram_latency_queue.end(); ++mf_dlq) {
+        mem_fetch *mf = mf_dlq->req; 
+        fprintf(fp, "Ready @ %llu - ", mf_dlq->ready_cycle); 
+        if (mf) 
+            mf->print(fp); 
+        else 
+            fprintf(fp, " <NULL mem_fetch?>\n"); 
+    }
     m_dram->print(fp); 
 }
 
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
index 07e37ca..3df54b1 100644
--- a/src/gpgpu-sim/l2cache.h
+++ b/src/gpgpu-sim/l2cache.h
@@ -138,7 +138,7 @@ private:
       unsigned long long ready_cycle;
       class mem_fetch* req;
    };
-   std::queue<dram_delay_t> m_dram_latency_queue;
+   std::list<dram_delay_t> m_dram_latency_queue;
 };
 
 class memory_sub_partition