Merge branch 'dev' into fix_warnings

12 files changed, 247 insertions, 100 deletions

diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc
index 09bbc3c..e3713f3 100644
--- a/src/gpgpu-sim/addrdec.cc
+++ b/src/gpgpu-sim/addrdec.cc
@@ -167,6 +167,8 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_
 		}
 		case RANDOM:
 		{
+			//This is an unrealistic hashing using software hashtable
+			//we generate a random set for each memory address and save the value in a big hashtable for future reuse
 			new_addr_type chip_address = (addr>>ADDR_CHIP_S);
 			tr1_hash_map<new_addr_type,unsigned>::const_iterator got = address_random_interleaving.find (chip_address);
 			  if ( got == address_random_interleaving.end() ) {
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index dec61db..1e99fec 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -64,6 +64,14 @@ const char * cache_fail_status_str(enum cache_reservation_fail_reason status)
    return static_cache_reservation_fail_reason_str[status];
 }
 
+unsigned l1d_cache_config::set_bank(new_addr_type addr) const{
+
+	if(m_cache_type == SECTOR)
+		return (addr >> m_sector_sz_log2) & (l1_banks-1);
+	else
+		return (addr >> m_line_sz_log2) & (l1_banks-1);
+}
+
 unsigned l1d_cache_config::set_index(new_addr_type addr) const{
     unsigned set_index = m_nset; // Default to linear set index function
     unsigned lower_xor = 0;
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index be33d96..6f39221 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -584,6 +584,7 @@ public:
         m_nset_log2 = LOGB2(m_nset);
         m_valid = true;
         m_atom_sz = (m_cache_type == SECTOR)? SECTOR_SIZE : m_line_sz;
+        m_sector_sz_log2 = LOGB2(SECTOR_SIZE);
         original_m_assoc = m_assoc;
 
         //For more details about difference between FETCH_ON_WRITE and WRITE VALIDAE policies
@@ -734,6 +735,7 @@ protected:
     unsigned m_nset_log2;
     unsigned m_assoc;
     unsigned m_atom_sz;
+    unsigned m_sector_sz_log2;
     unsigned original_m_assoc;
     bool m_is_streaming;
 
@@ -775,7 +777,9 @@ class l1d_cache_config : public cache_config{
 public:
 	l1d_cache_config() : cache_config(){}
 	virtual unsigned set_index(new_addr_type addr) const;
+    unsigned set_bank(new_addr_type addr) const;
 	unsigned l1_latency;
+	unsigned l1_banks;
 };
 
 class l2_cache_config : public cache_config {
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 1f9a422..1720836 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -83,6 +83,7 @@ class  gpgpu_sim_wrapper {};
 
 
 bool g_interactive_debugger_enabled=false;
+
 tr1_hash_map<new_addr_type,unsigned> address_random_interleaving;
 
 /* Clock Domains */
@@ -137,6 +138,8 @@ void memory_config::reg_options(class OptionParser * opp)
 {
     option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, 
                                 "Fill the L2 cache on memcpy", "1");
+    option_parser_register(opp, "-simple_dram_model", OPT_BOOL, &simple_dram_model,
+                                "simple_dram_model with fixed latency and BW", "0");
     option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, 
                                 "0 = fifo, 1 = FR-FCFS (defaul)", "1");
     option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, 
@@ -236,9 +239,12 @@ void shader_core_config::reg_options(class OptionParser * opp)
                    "per-shader L1 data cache config "
                    " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                    "none" );
+    option_parser_register(opp, "-l1_banks", OPT_UINT32, &m_L1D_config.l1_banks,
+                 "The number of L1 cache banks",
+                 "1");
     option_parser_register(opp, "-l1_latency", OPT_UINT32, &m_L1D_config.l1_latency,
                  "L1 Hit Latency",
-                 "0");
+                 "1");
     option_parser_register(opp, "-smem_latency", OPT_UINT32, &smem_latency,
                  "smem Latency",
                  "3");
@@ -299,10 +305,10 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size,
                  "Size of shared memory per shader core (default 16kB)",
                  "16384");
-    option_parser_register(opp, "-adaptive_volta_cache_config", OPT_BOOL, &adaptive_volta_cache_config,
-                 "adaptive_volta_cache_config",
+    option_parser_register(opp, "-adaptive_cache_config", OPT_BOOL, &adaptive_volta_cache_config,
+                 "adaptive_cache_config",
                  "0");
-    option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_sizeDefault,
+    option_parser_register(opp, "-gpgpu_shmem_sizeDefault", OPT_UINT32, &gpgpu_shmem_sizeDefault,
                  "Size of shared memory per shader core (default 16kB)",
                  "16384");
     option_parser_register(opp, "-gpgpu_shmem_size_PrefL1", OPT_UINT32, &gpgpu_shmem_sizePrefL1,
@@ -795,6 +801,16 @@ int gpgpu_sim::shader_clock() const
    return m_config.core_freq/1000;
 }
 
+int gpgpu_sim::max_cta_per_core() const
+{
+   return m_shader_config->max_cta_per_core;
+}
+
+int gpgpu_sim::get_max_cta( const kernel_info_t &k ) const
+{
+   return m_shader_config->max_cta(k);
+}
+
 void gpgpu_sim::set_prop( cudaDeviceProp *prop )
 {
    m_cuda_properties = prop;
@@ -1044,7 +1060,7 @@ void gpgpu_sim::change_cache_config(FuncCache cache_config)
 	if(cache_config != m_shader_config->m_L1D_config.get_cache_status()){
 		printf("FLUSH L1 Cache at configuration change between kernels\n");
 		for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) {
-			m_cluster[i]->cache_flush();
+			m_cluster[i]->cache_invalidate();
 	    }
 	}
 
@@ -1582,7 +1598,10 @@ void gpgpu_sim::cycle()
 
    if (clock_mask & DRAM) {
       for (unsigned i=0;i<m_memory_config->m_n_mem;i++){
-         m_memory_partition_unit[i]->dram_cycle(); // Issue the dram command (scheduler + delay model)
+    	  if(m_memory_config->simple_dram_model)
+    		  m_memory_partition_unit[i]->simple_dram_model_cycle();
+    	  else
+    		  m_memory_partition_unit[i]->dram_cycle(); // Issue the dram command (scheduler + delay model)
          // Update performance counters for DRAM
          m_memory_partition_unit[i]->set_dram_power_stats(m_power_stats->pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i],
                         m_power_stats->pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_act[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i],
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 19e1eb3..76c7a06 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -293,9 +293,12 @@ class memory_config {
    unsigned write_high_watermark;
    unsigned write_low_watermark;
    bool m_perf_sim_memcpy;
+   bool simple_dram_model;
+
    gpgpu_context* gpgpu_ctx;
 };
 
+
 extern bool g_interactive_debugger_enabled;
 
 class gpgpu_sim_config : public power_config, public gpgpu_functional_sim_config {
@@ -482,6 +485,8 @@ public:
    int num_registers_per_block() const;
    int wrp_size() const;
    int shader_clock() const;
+   int max_cta_per_core() const;
+   int get_max_cta( const kernel_info_t &k ) const;
    const struct cudaDeviceProp *get_prop() const;
    enum divergence_support_t simd_model() const; 
 
diff --git a/src/gpgpu-sim/icnt_wrapper.cc b/src/gpgpu-sim/icnt_wrapper.cc
index 6e0950c..67724d0 100644
--- a/src/gpgpu-sim/icnt_wrapper.cc
+++ b/src/gpgpu-sim/icnt_wrapper.cc
@@ -182,6 +182,8 @@ void icnt_reg_options( class OptionParser * opp )
    option_parser_register(opp, "-inct_in_buffer_limit", OPT_UINT32, &g_inct_config.in_buffer_limit, "in_buffer_limit", "64");
    option_parser_register(opp, "-inct_out_buffer_limit", OPT_UINT32, &g_inct_config.out_buffer_limit, "out_buffer_limit", "64");
    option_parser_register(opp, "-inct_subnets", OPT_UINT32, &g_inct_config.subnets, "subnets", "2");
+   option_parser_register(opp, "-arbiter_algo", OPT_UINT32, &g_inct_config.arbiter_algo, "arbiter_algo", "1");
+
 
 }
 
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index 862461f..fb4ce32 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -206,7 +206,67 @@ int memory_partition_unit::global_sub_partition_id_to_local_id(int global_sub_pa
     return (global_sub_partition_id - m_id * m_config->m_n_sub_partition_per_memory_channel); 
 }
 
-void memory_partition_unit::dram_cycle() 
+void memory_partition_unit::simple_dram_model_cycle()
+{
+
+	// pop completed memory request from dram and push it to dram-to-L2 queue
+	// of the original sub partition
+	if (!m_dram_latency_queue.empty() && ( (m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle )) {
+		 mem_fetch* mf_return = m_dram_latency_queue.front().req;
+		 if( mf_return->get_access_type() != L1_WRBK_ACC && mf_return->get_access_type() != L2_WRBK_ACC ) {
+			 mf_return->set_reply();
+
+		    unsigned dest_global_spid = mf_return->get_sub_partition_id();
+			int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid);
+			assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid);
+			if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) {
+				if( mf_return->get_access_type() == L1_WRBK_ACC ) {
+					m_sub_partition[dest_spid]->set_done(mf_return);
+					delete mf_return;
+				} else {
+					m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return);
+					mf_return->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle);
+					m_arbitration_metadata.return_credit(dest_spid);
+					MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf_return, dest_spid);
+				}
+				m_dram_latency_queue.pop_front();
+			}
+
+		  } else {
+			 this->set_done(mf_return);
+			 delete mf_return;
+			 m_dram_latency_queue.pop_front();
+		  }
+	}
+
+   // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+	//if( !m_dram->full(mf->is_write()) ) {
+		// L2->DRAM queue to DRAM latency queue
+		// Arbitrate among multiple L2 subpartitions
+		int last_issued_partition = m_arbitration_metadata.last_borrower();
+		for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) {
+			int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel;
+			if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) {
+				mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+				if(m_dram->full(mf->is_write()) )
+					break;
+
+				m_sub_partition[spid]->L2_dram_queue_pop();
+				MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid);
+				dram_delay_t d;
+				d.req = mf;
+				d.ready_cycle = m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle + m_config->dram_latency;
+				m_dram_latency_queue.push_back(d);
+				mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle);
+				m_arbitration_metadata.borrow_credit(spid);
+				break;  // the DRAM should only accept one request per cycle
+			}
+		}
+	//}
+
+}
+
+void memory_partition_unit::dram_cycle()
 { 
     // pop completed memory request from dram and push it to dram-to-L2 queue 
     // of the original sub partition 
@@ -231,8 +291,8 @@ void memory_partition_unit::dram_cycle()
         m_dram->return_queue_pop(); 
     }
     
-    m_dram->cycle(); 
-    m_dram->dram_log(SAMPLELOG);   
+    m_dram->cycle();
+    m_dram->dram_log(SAMPLELOG);
 
    // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
     //if( !m_dram->full(mf->is_write()) ) {
@@ -260,7 +320,6 @@ void memory_partition_unit::dram_cycle()
     //}
 
     // DRAM latency queue
-
     if( !m_dram_latency_queue.empty() && ( (m_gpu->gpu_sim_cycle+m_gpu->gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) {
     	mem_fetch* mf = m_dram_latency_queue.front().req;
     	m_dram_latency_queue.pop_front();
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
index 1f74c47..0f6fe32 100644
--- a/src/gpgpu-sim/l2cache.h
+++ b/src/gpgpu-sim/l2cache.h
@@ -65,6 +65,7 @@ public:
 
    void cache_cycle( unsigned cycle );
    void dram_cycle();
+   void simple_dram_model_cycle();
 
    void set_done( mem_fetch *mf );
 
diff --git a/src/gpgpu-sim/local_interconnect.cc b/src/gpgpu-sim/local_interconnect.cc
index bb09d44..c70477c 100644
--- a/src/gpgpu-sim/local_interconnect.cc
+++ b/src/gpgpu-sim/local_interconnect.cc
@@ -36,7 +36,7 @@
 #include "local_interconnect.h"
 #include "mem_fetch.h"
 
-xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit)
+xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit, enum Arbiteration_type m_arbit_type)
 {
 	m_id=router_id;
 	router_type=m_type;
@@ -46,9 +46,10 @@ xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, unsi
 	in_buffers.resize(total_nodes);
 	out_buffers.resize(total_nodes);
 	next_node.resize(total_nodes,0);
-//	next_node = 0;
 	in_buffer_limit = m_in_buffer_limit;
 	out_buffer_limit = m_out_buffer_limit;
+	arbit_type = m_arbit_type;
+	next_node_id=0;
 	if(m_type == REQ_NET) {
 		active_in_buffers=n_shader;
 		active_out_buffers=n_mem;
@@ -109,14 +110,25 @@ bool xbar_router::Has_Buffer_In(unsigned input_deviceID, unsigned size, bool upd
 bool xbar_router::Has_Buffer_Out(unsigned output_deviceID, unsigned size){
 	return (out_buffers[output_deviceID].size() + size <= out_buffer_limit);
 }
-/*
+
 void xbar_router::Advance() {
+
+	if(arbit_type == NAIVE_RR)
+		RR_Advance();
+	else if(arbit_type == iSLIP)
+		iSLIP_Advance();
+	else
+		assert(0);
+
+}
+
+void xbar_router::RR_Advance() {
 	cycles++;
 
 	vector<bool> issued(total_nodes, false);
 
 	for(unsigned i=0; i<total_nodes; ++i){
-		unsigned node_id = (i+next_node)%total_nodes;
+		unsigned node_id = (i+next_node_id)%total_nodes;
 
 		if(!in_buffers[node_id].empty()) {
 			Packet _packet = in_buffers[node_id].front();
@@ -130,12 +142,16 @@ void xbar_router::Advance() {
 				else
 					conflicts++;
 			}
-			else
+			else {
 				out_buffer_full++;
+
+				if(issued[_packet.output_deviceID])
+					conflicts++;
+			}
 		}
 	}
 
-	next_node = (++next_node % total_nodes);
+	next_node_id = (++next_node_id % total_nodes);
 
 	//collect some stats about buffer util
 	for(unsigned i=0; i<total_nodes; ++i){
@@ -143,13 +159,17 @@ void xbar_router::Advance() {
 		out_buffer_util+=out_buffers[i].size();
 	}
 }
-*/
 
-void xbar_router::Advance() {
+//iSLIP algorithm
+//McKeown, Nick. "The iSLIP scheduling algorithm for input-queued switches." IEEE/ACM transactions on networking 2 (1999): 188-201.
+//https://www.cs.rutgers.edu/~sn624/552-F18/papers/islip.pdf
+void xbar_router::iSLIP_Advance() {
 	cycles++;
 
 	vector<unsigned> node_tmp;
 
+
+	//calcaulte how many conflicts are there for stats
 	for (unsigned i=0; i<total_nodes; ++i){
 		
 		if(!in_buffers[i].empty()){
@@ -167,6 +187,8 @@ void xbar_router::Advance() {
 		}
 	}
 
+
+	//do iSLIP
 	for(unsigned i=0; i<total_nodes; ++i){
 
 		if(Has_Buffer_Out(i, 1)) {
@@ -242,7 +264,7 @@ void LocalInterconnect::CreateInterconnect(unsigned m_n_shader, unsigned m_n_mem
 
 	net.resize(n_subnets);
 	for (unsigned i = 0; i < n_subnets; ++i) {
-		net[i] = new xbar_router( i, static_cast<Interconnect_type>(i), m_n_shader, m_n_mem, m_inct_config.in_buffer_limit, m_inct_config.out_buffer_limit );
+		net[i] = new xbar_router( i, static_cast<Interconnect_type>(i), m_n_shader, m_n_mem, m_inct_config.in_buffer_limit, m_inct_config.out_buffer_limit,m_inct_config.arbiter_algo);
 	}
 
 }
diff --git a/src/gpgpu-sim/local_interconnect.h b/src/gpgpu-sim/local_interconnect.h
index f4a2af1..a784da8 100644
--- a/src/gpgpu-sim/local_interconnect.h
+++ b/src/gpgpu-sim/local_interconnect.h
@@ -35,27 +35,35 @@
 using namespace std;
 
 
+enum Interconnect_type {
+	REQ_NET=0,
+	REPLY_NET=1
+};
+
+enum Arbiteration_type {
+	NAIVE_RR=0,
+	iSLIP=1
+};
+
 struct inct_config
 {
-
 	//config for local interconnect
 	unsigned   in_buffer_limit;
 	unsigned   out_buffer_limit;
 	unsigned   subnets;
+	Arbiteration_type   arbiter_algo;
 };
 
-enum Interconnect_type {
-	REQ_NET=0,
-	REPLY_NET=1
-};
 class xbar_router {
 
 public:
-	xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit);
+	xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit, enum Arbiteration_type m_arbit_type);
 	~xbar_router();
 	void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size);
 	void* Pop(unsigned ouput_deviceID);
-	void Advance();
+	void Advance( );
+
+
 	bool Busy() const;
 	bool Has_Buffer_In(unsigned input_deviceID, unsigned size, bool update_counter=false);
 	bool Has_Buffer_Out(unsigned output_deviceID, unsigned size);
@@ -70,6 +78,9 @@ public:
 	unsigned long long packets_num;
 
 private:
+	void iSLIP_Advance();
+	void RR_Advance();
+
 	struct Packet{
 		Packet(void* m_data, unsigned m_output_deviceID) {
 			data = m_data;
@@ -82,11 +93,12 @@ private:
 	vector<queue<Packet> > out_buffers;
 	unsigned _n_shader, _n_mem, total_nodes;
 	unsigned in_buffer_limit, out_buffer_limit;
-	vector<unsigned> next_node;
-//	unsigned next_node;
+	vector<unsigned> next_node;  //used for iSLIP arbit
+	unsigned next_node_id;   //used for RR arbit
 	unsigned m_id;
 	enum Interconnect_type router_type;
 	unsigned active_in_buffers,active_out_buffers;
+	Arbiteration_type arbit_type;
 
 	friend class LocalInterconnect;
 
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index c365ebb..6a0e3d6 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1659,35 +1659,46 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *c
     if( inst.accessq_empty() )
         return result;
 
-    mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle);
-
     if(m_config->m_L1D_config.l1_latency > 0)
 	{
-    	if((l1_latency_queue[m_config->m_L1D_config.l1_latency-1]) == NULL)
-    	{
-    		l1_latency_queue[m_config->m_L1D_config.l1_latency-1] = mf;
+    	for(int j=0; j<m_config->m_L1D_config.l1_banks; j++) {  //We can handle at max l1_banks reqs per cycle
 
-    		if( mf->get_inst().is_store() ) {
-				unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
-						(mf->get_data_size()/SECTOR_SIZE) : 1;
+    		if( inst.accessq_empty() )
+    		        return result;
 
-				for(unsigned i=0; i< inc_ack; ++i)
-					m_core->inc_store_req( inst.warp_id() );
-			}
+    	    mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle);
+    	    unsigned bank_id = m_config->m_L1D_config.set_bank(mf->get_addr());
+    	    assert(bank_id < m_config->m_L1D_config.l1_banks);
+
+			if((l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency-1]) == NULL)
+			{
+				l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency-1] = mf;
+
+				if( mf->get_inst().is_store() ) {
+					unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
+							(mf->get_data_size()/SECTOR_SIZE) : 1;
+
+					for(unsigned i=0; i< inc_ack; ++i)
+						m_core->inc_store_req( inst.warp_id() );
+				}
 
-    		inst.accessq_pop_back();
+				inst.accessq_pop_back();
+			}
+			else
+			{
+				result = BK_CONF;
+				delete mf;
+				break;   //do not try again, just break from the loop and try the next cycle
+			}
     	}
-    	else
-        {
-        	result = BK_CONF;
-        	delete mf;
-        }
-        if( !inst.accessq_empty() &&  result !=BK_CONF)
+		if( !inst.accessq_empty() &&  result !=BK_CONF)
 		   result = COAL_STALL;
-	   return result;
+
+	     return result;
 	}
     else
     {
+	    mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back(),m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle);
 		std::list<cache_event> events;
 		enum cache_request_status status = cache->access(mf->get_addr(),mf,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events);
 		return process_cache_access( cache, mf->get_addr(), inst, events, mf, status );
@@ -1696,63 +1707,64 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *c
 
 void ldst_unit::L1_latency_queue_cycle()
 {
-	//std::deque< std::pair<mem_fetch*,bool> >::iterator it = m_latency_queue.begin();
-	if((l1_latency_queue[0]) != NULL)
-    {
-		    mem_fetch* mf_next = l1_latency_queue[0];
-			std::list<cache_event> events;
-			enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events);
+	for(int j=0; j<m_config->m_L1D_config.l1_banks; j++) {
+		if((l1_latency_queue[j][0]) != NULL)
+		{
+				mem_fetch* mf_next = l1_latency_queue[j][0];
+				std::list<cache_event> events;
+				enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,m_core->get_gpu()->gpu_sim_cycle+m_core->get_gpu()->gpu_tot_sim_cycle,events);
 
-		   bool write_sent = was_write_sent(events);
-		   bool read_sent = was_read_sent(events);
+			   bool write_sent = was_write_sent(events);
+			   bool read_sent = was_read_sent(events);
 
-		   if ( status == HIT ) {
-			   assert( !read_sent );
-			   l1_latency_queue[0] = NULL;
-			   if ( mf_next->get_inst().is_load() ) {
-				   for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++)
-					   if (mf_next->get_inst().out[r] > 0)
-					   {
-						   assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0);
-						   unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]];
-						   if(!still_pending)
+			   if ( status == HIT ) {
+				   assert( !read_sent );
+				   l1_latency_queue[j][0] = NULL;
+				   if ( mf_next->get_inst().is_load() ) {
+					   for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++)
+						   if (mf_next->get_inst().out[r] > 0)
 						   {
-							m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]);
-							m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]);
-							m_core->warp_inst_complete(mf_next->get_inst());
+							   assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0);
+							   unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]];
+							   if(!still_pending)
+							   {
+								m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]);
+								m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]);
+								m_core->warp_inst_complete(mf_next->get_inst());
+							   }
 						   }
-					   }
-			   }
+				   }
 
-			   //For write hit in WB policy
-			   if(mf_next->get_inst().is_store() && !write_sent)
-			   {
-				   unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
-				   						(mf_next->get_data_size()/SECTOR_SIZE) : 1;
+				   //For write hit in WB policy
+				   if(mf_next->get_inst().is_store() && !write_sent)
+				   {
+					   unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
+											(mf_next->get_data_size()/SECTOR_SIZE) : 1;
 
-				   mf_next->set_reply();
+					   mf_next->set_reply();
 
-				   for(unsigned i=0; i< dec_ack; ++i)
-				      m_core->store_ack(mf_next);
-			   }
+					   for(unsigned i=0; i< dec_ack; ++i)
+						  m_core->store_ack(mf_next);
+				   }
 
-			   if( !write_sent )
-				   delete mf_next;
+				   if( !write_sent )
+					   delete mf_next;
 
-		   } else if ( status == RESERVATION_FAIL ) {
-			   assert( !read_sent );
-			   assert( !write_sent );
-		   } else {
-			   assert( status == MISS || status == HIT_RESERVED );
-			   l1_latency_queue[0] = NULL;
-	   }
-    }
+			   } else if ( status == RESERVATION_FAIL ) {
+				   assert( !read_sent );
+				   assert( !write_sent );
+			   } else {
+				   assert( status == MISS || status == HIT_RESERVED );
+				   l1_latency_queue[j][0] = NULL;
+		   }
+		}
 
-	 for( unsigned stage = 0; stage<m_config->m_L1D_config.l1_latency-1; ++stage)
-	  if( l1_latency_queue[stage] == NULL) {
-		   l1_latency_queue[stage] = l1_latency_queue[stage+1] ;
-		   l1_latency_queue[stage+1] = NULL;
-	   }
+		 for( unsigned stage = 0; stage<m_config->m_L1D_config.l1_latency-1; ++stage)
+		  if( l1_latency_queue[j][stage] == NULL) {
+			   l1_latency_queue[j][stage] = l1_latency_queue[j][stage+1] ;
+			   l1_latency_queue[j][stage+1] = NULL;
+		   }
+	}
 
 }
 
@@ -2132,11 +2144,12 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt,
                               IN_L1D_MISS_QUEUE,
 							  core->get_gpu());
 
-        if(m_config->m_L1D_config.l1_latency > 0)
-	    {
-        	for(unsigned i = 0; i < m_config->m_L1D_config.l1_latency; i++ )
-        		l1_latency_queue.push_back((mem_fetch*)NULL);
-	    }
+        l1_latency_queue.resize(m_config->m_L1D_config.l1_banks);
+        assert(m_config->m_L1D_config.l1_latency > 0);
+
+		for(unsigned j = 0; j < m_config->m_L1D_config.l1_banks; j++ )
+			l1_latency_queue[j].resize(m_config->m_L1D_config.l1_latency,(mem_fetch*)NULL);
+
     }
     m_name = "MEM ";
 }
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index dbe2285..667cb2d 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1326,7 +1326,7 @@ protected:
    unsigned long long m_last_inst_gpu_sim_cycle;
    unsigned long long m_last_inst_gpu_tot_sim_cycle;
 
-   std::deque<mem_fetch* > l1_latency_queue;
+   std::vector<std::deque<mem_fetch* >> l1_latency_queue;
    void L1_latency_queue_cycle();
 };