Merge branch 'dev' of https://github.com/mkhairy/gpgpu-sim-private into dev-purdue-integration-trace

1 files changed, 635 insertions, 175 deletions

diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index d2f40a1..e38eefd 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -74,7 +74,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                   shader_core_stats *stats )
    : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ),
      m_barriers( this, config->max_warps_per_shader, config->max_cta_per_core, config->max_barriers_per_cta, config->warp_size ),
-     m_dynamic_warp_id(0)
+     m_dynamic_warp_id(0), m_active_warps(0)
 {
     m_cluster = cluster;
     m_config = config;
@@ -90,6 +90,18 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
     for (int j = 0; j<N_PIPELINE_STAGES; j++) {
         m_pipeline_reg.push_back(register_set(m_config->pipe_widths[j],pipeline_stage_name_decode[j]));
     }
+    if(m_config->sub_core_model) {
+    	//in subcore model, each scheduler should has its own issue register, so num scheduler = reg width
+    	assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() );
+    	assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SFU].get_size() );
+    	assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_MEM].get_size() );
+    	if(m_config->gpgpu_tensor_core_avail)
+    		 assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_TENSOR_CORE].get_size() );
+    	if(m_config->gpgpu_num_dp_units > 0)
+    		assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_DP].get_size() );
+    	if(m_config->gpgpu_num_int_units > 0)
+    	    assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_INT].get_size() );
+    }
     
     m_threadState = (thread_ctx_t*) calloc(sizeof(thread_ctx_t), config->n_thread_per_shader);
     
@@ -152,6 +164,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        &m_pipeline_reg[ID_OC_SP],
 									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
+									   &m_pipeline_reg[ID_OC_INT],
+                                       &m_pipeline_reg[ID_OC_TENSOR_CORE],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i
                                      )
@@ -167,6 +181,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                                     &m_pipeline_reg[ID_OC_SP],
 													&m_pipeline_reg[ID_OC_DP],
                                                     &m_pipeline_reg[ID_OC_SFU],
+													&m_pipeline_reg[ID_OC_INT],
+                                                    &m_pipeline_reg[ID_OC_TENSOR_CORE],
                                                     &m_pipeline_reg[ID_OC_MEM],
                                                     i,
                                                     config->gpgpu_scheduler_string
@@ -183,6 +199,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        &m_pipeline_reg[ID_OC_SP],
 									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
+									   &m_pipeline_reg[ID_OC_INT],
+                                       &m_pipeline_reg[ID_OC_TENSOR_CORE],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i
                                      )
@@ -198,6 +216,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
 					  &m_pipeline_reg[ID_OC_SP],
 					  &m_pipeline_reg[ID_OC_DP],
 					  &m_pipeline_reg[ID_OC_SFU],
+					  &m_pipeline_reg[ID_OC_INT],
+                      &m_pipeline_reg[ID_OC_TENSOR_CORE],
 					  &m_pipeline_reg[ID_OC_MEM],
 					  i
 					 )
@@ -213,6 +233,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        &m_pipeline_reg[ID_OC_SP],
 									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
+									   &m_pipeline_reg[ID_OC_INT],
+                                       &m_pipeline_reg[ID_OC_TENSOR_CORE],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i,
                                        config->gpgpu_scheduler_string
@@ -233,53 +255,16 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
     }
     
     //op collector configuration
-    enum { SP_CUS, DP_CUS, SFU_CUS, MEM_CUS, GEN_CUS };
-    m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp);
-    m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp);
-    m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu);
-    m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem);
-    m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen);
-    
+
+	enum { SP_CUS, DP_CUS, SFU_CUS, TENSOR_CORE_CUS, INT_CUS, MEM_CUS,  GEN_CUS };
+
     opndcoll_rfu_t::port_vector_t in_ports;
     opndcoll_rfu_t::port_vector_t out_ports;
     opndcoll_rfu_t::uint_vector_t cu_sets;
-    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; i++) {
-        in_ports.push_back(&m_pipeline_reg[ID_OC_SP]);
-        out_ports.push_back(&m_pipeline_reg[OC_EX_SP]);
-        cu_sets.push_back((unsigned)SP_CUS);
-        cu_sets.push_back((unsigned)GEN_CUS);
-        m_operand_collector.add_port(in_ports,out_ports,cu_sets);
-        in_ports.clear(),out_ports.clear(),cu_sets.clear();
-    }
-    
-    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) {
-            in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
-            out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
-            cu_sets.push_back((unsigned)DP_CUS);
-            cu_sets.push_back((unsigned)GEN_CUS);
-            m_operand_collector.add_port(in_ports,out_ports,cu_sets);
-            in_ports.clear(),out_ports.clear(),cu_sets.clear();
-        }
 
-    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) {
-        in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
-        out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
-        cu_sets.push_back((unsigned)SFU_CUS);
-        cu_sets.push_back((unsigned)GEN_CUS);
-        m_operand_collector.add_port(in_ports,out_ports,cu_sets);
-        in_ports.clear(),out_ports.clear(),cu_sets.clear();
-    }
-    
-    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) {
-        in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]);
-        out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]);
-        cu_sets.push_back((unsigned)MEM_CUS);
-        cu_sets.push_back((unsigned)GEN_CUS);                       
-        m_operand_collector.add_port(in_ports,out_ports,cu_sets);
-        in_ports.clear(),out_ports.clear(),cu_sets.clear();
-    }   
-    
-    
+    //configure generic collectors
+    m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen);
+
     for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_gen; i++) {
         in_ports.push_back(&m_pipeline_reg[ID_OC_SP]);
         in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
@@ -287,15 +272,89 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
         out_ports.push_back(&m_pipeline_reg[OC_EX_SP]);
         out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
         out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]);
-        cu_sets.push_back((unsigned)GEN_CUS);   
+        if(m_config->gpgpu_tensor_core_avail) {
+        	in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]);
+        	out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]);
+        }
+        if(m_config->gpgpu_num_dp_units > 0) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
+        }
+        if(m_config->gpgpu_num_int_units > 0) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_INT]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_INT]);
+        }
+        cu_sets.push_back((unsigned)GEN_CUS);
         m_operand_collector.add_port(in_ports,out_ports,cu_sets);
         in_ports.clear(),out_ports.clear(),cu_sets.clear();
     }
+
+    if(m_config->enable_specialized_operand_collector) {
+		m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp);
+		m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp);
+	    m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core);
+	    m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu);
+		m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem);
+		m_operand_collector.add_cu_set(INT_CUS, m_config->gpgpu_operand_collector_num_units_int, m_config->gpgpu_operand_collector_num_out_ports_int);
+
+		for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; i++) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_SP]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_SP]);
+			cu_sets.push_back((unsigned)SP_CUS);
+			cu_sets.push_back((unsigned)GEN_CUS);
+			m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+			in_ports.clear(),out_ports.clear(),cu_sets.clear();
+		}
+
+		for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) {
+				in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
+				out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
+				cu_sets.push_back((unsigned)DP_CUS);
+				cu_sets.push_back((unsigned)GEN_CUS);
+				m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+				in_ports.clear(),out_ports.clear(),cu_sets.clear();
+			}
+
+		for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
+			cu_sets.push_back((unsigned)SFU_CUS);
+			cu_sets.push_back((unsigned)GEN_CUS);
+			m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+			in_ports.clear(),out_ports.clear(),cu_sets.clear();
+		}
+
+	    for (unsigned i = 0; i < config->gpgpu_operand_collector_num_in_ports_tensor_core; i++) {
+	        in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]);
+	        out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]);
+	        cu_sets.push_back((unsigned)TENSOR_CORE_CUS);
+	        cu_sets.push_back((unsigned)GEN_CUS);
+	        m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+	        in_ports.clear(),out_ports.clear(),cu_sets.clear();
+	    }
+
+		for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]);
+			cu_sets.push_back((unsigned)MEM_CUS);
+			cu_sets.push_back((unsigned)GEN_CUS);
+			m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+			in_ports.clear(),out_ports.clear(),cu_sets.clear();
+		}
+
+		for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_int; i++) {
+			in_ports.push_back(&m_pipeline_reg[ID_OC_INT]);
+			out_ports.push_back(&m_pipeline_reg[OC_EX_INT]);
+			cu_sets.push_back((unsigned)INT_CUS);
+			cu_sets.push_back((unsigned)GEN_CUS);
+			m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+			in_ports.clear(),out_ports.clear(),cu_sets.clear();
+		}
+    }
     
     m_operand_collector.init( m_config->gpgpu_num_reg_banks, this );
     
-    // execute
-    m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + 1; // sp_unit, sfu, ldst_unit
+    m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + m_config->gpgpu_num_int_units + 1; // sp_unit, sfu, dp, tensor, int, ldst_unit
     //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ];
     //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ];
     
@@ -312,18 +371,29 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
             m_dispatch_port.push_back(ID_OC_DP);
             m_issue_port.push_back(OC_EX_DP);
         }
+    for (int k = 0; k < m_config->gpgpu_num_int_units; k++) {
+            m_fu.push_back(new int_unit( &m_pipeline_reg[EX_WB], m_config, this ));
+            m_dispatch_port.push_back(ID_OC_INT);
+            m_issue_port.push_back(OC_EX_INT);
+        }
 
     for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) {
         m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this ));
         m_dispatch_port.push_back(ID_OC_SFU);
         m_issue_port.push_back(OC_EX_SFU);
     }
+       
+    for (int k = 0; k < config->gpgpu_num_tensor_core_units; k++) {
+        m_fu.push_back(new tensor_core( &m_pipeline_reg[EX_WB], m_config, this ));
+        m_dispatch_port.push_back(ID_OC_TENSOR_CORE);
+        m_issue_port.push_back(OC_EX_TENSOR_CORE);
+    }
 
     m_ldst_unit = new ldst_unit( m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, m_scoreboard, config, mem_config, stats, shader_id, tpc_id );
     m_fu.push_back(m_ldst_unit);
     m_dispatch_port.push_back(ID_OC_MEM);
     m_issue_port.push_back(OC_EX_MEM);
-    
+   
     assert(m_num_function_units == m_fu.size() and m_fu.size() == m_dispatch_port.size() and m_fu.size() == m_issue_port.size());
     
     //there are as many result buses as the width of the EX_WB stage
@@ -357,6 +427,7 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
        m_occupied_ctas = 0;
        m_occupied_hwtid.reset();
        m_occupied_cta_to_hwtid.clear();
+       m_active_warps = 0;
 
    }
    for (unsigned i = start_thread; i<end_thread; i++) {
@@ -369,11 +440,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
    }
 }
 
-void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread )
+void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id )
 {
     address_type start_pc = next_pc(start_thread);
     if (m_config->model == POST_DOMINATOR) {
         unsigned start_warp = start_thread / m_config->warp_size;
+        unsigned warp_per_cta =  cta_size / m_config->warp_size;
         unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0);
         for (unsigned i = start_warp; i < end_warp; ++i) {
             unsigned n_active=0;
@@ -388,9 +460,25 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign
                 }
             }
             m_simt_stack[i]->launch(start_pc,active_threads);
+
+              if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+               { 
+                char fname[2048];
+                snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid );
+                unsigned pc,rpc;
+                m_simt_stack[i]->resume(fname);
+                m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc);
+                for (unsigned t = 0; t < m_config->warp_size; t++) {
+                  m_thread[i * m_config->warp_size + t]->set_npc(pc);
+                  m_thread[i * m_config->warp_size + t]->update_pc();
+                }   
+                start_pc=pc;
+              }
+               
             m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id);
             ++m_dynamic_warp_id;
             m_not_completed += n_active;
+            ++m_active_warps;
       }
    }
 }
@@ -418,6 +506,18 @@ void shader_core_ctx::get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsig
     m_simt_stack[warp_id]->get_pdom_stack_top_info(pc,rpc);
 }
 
+float shader_core_ctx::get_current_occupancy( unsigned long long & active, unsigned long long & total ) const
+{
+    // To match the achieved_occupancy in nvprof, only SMs that are active are counted toward the occupancy.
+    if ( m_active_warps > 0 ) {
+        total += m_warp.size();
+        active += m_active_warps;
+        return float(active) / float(total);
+    } else {
+        return 0;
+    }
+}
+
 void shader_core_stats::print( FILE* fout ) const
 {
 	unsigned long long  thread_icount_uarch=0;
@@ -441,6 +541,7 @@ void shader_core_stats::print( FILE* fout ) const
    fprintf(fout, "gpgpu_n_load_insn  = %d\n", gpgpu_n_load_insn);
    fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn);
    fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn);
+   fprintf(fout, "gpgpu_n_sstarr_insn = %d\n", gpgpu_n_sstarr_insn);
    fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn);
    fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn);
    fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn);
@@ -451,15 +552,15 @@ void shader_core_stats::print( FILE* fout ) const
    fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge);
    fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict);
 
-   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]);
-   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]);
    fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]);
-   fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][bk_conf] = %d\n", 
+   fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n",
            gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] + 
            gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] + 
            gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] + 
@@ -477,22 +578,22 @@ void shader_core_stats::print( FILE* fout ) const
            gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] + 
            gpu_stall_shd_mem_breakdown[L_MEM_ST][DATA_PORT_STALL]    
            ); // data port stall at data cache 
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]);
-   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]);
+   //fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]);
 
    fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls);
 
@@ -692,6 +793,8 @@ void shader_core_ctx::fetch()
                     }
                     if( did_exit ) 
                         m_warp[warp_id].set_done_exit();
+                        --m_active_warps;
+                        assert(m_active_warps >= 0);
                 }
 
                 // this code fetches instructions from the i-cache or generates memory requests
@@ -742,18 +845,21 @@ void shader_core_ctx::func_exec_inst( warp_inst_t &inst )
 {
     execute_warp_inst_t(inst);
     if( inst.is_load() || inst.is_store() )
-        inst.generate_mem_accesses();
+    {
+	inst.generate_mem_accesses();
+        //inst.print_m_accessq();	
+    }	
 }
 
-void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* next_inst, const active_mask_t &active_mask, unsigned warp_id )
+void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* next_inst, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id )
 {
-    warp_inst_t** pipe_reg = pipe_reg_set.get_free();
+	warp_inst_t** pipe_reg = pipe_reg = pipe_reg_set.get_free(m_config->sub_core_model, sch_id);
     assert(pipe_reg);
 
     m_warp[warp_id].ibuffer_free();
     assert(next_inst->valid());
     **pipe_reg = *next_inst; // static instruction information
-    (*pipe_reg)->issue( active_mask, warp_id, gpu_tot_sim_cycle + gpu_sim_cycle, m_warp[warp_id].get_dynamic_warp_id() ); // dynamic instruction information
+    (*pipe_reg)->issue( active_mask, warp_id, gpu_tot_sim_cycle + gpu_sim_cycle, m_warp[warp_id].get_dynamic_warp_id(), sch_id ); // dynamic instruction information
     m_stats->shader_cycle_distro[2+(*pipe_reg)->active_count()]++;
     func_exec_inst( **pipe_reg );
     if( next_inst->op == BARRIER_OP ){
@@ -898,8 +1004,8 @@ void scheduler_unit::cycle()
         unsigned checked=0;
         unsigned issued=0;
         exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE;
-	unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp;
-	bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units;
+        unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp;
+        bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units;  //In tis mode, we only allow dual issue to diff execution units (as in Maxwell and Pascal)
 
         while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) {
             const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst();
@@ -933,9 +1039,10 @@ void scheduler_unit::cycle()
                         ready_inst = true;
                         const active_mask_t &active_mask = m_simt_stack[warp_id]->get_active_mask();
                         assert( warp(warp_id).inst_in_pipeline() );
-                        if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP) ) {
-                        	if( m_mem_out->has_free() && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) {
-                                m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id);
+
+                        if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP)||(pI->op==TENSOR_CORE_LOAD_OP)||(pI->op==TENSOR_CORE_STORE_OP) ) {
+                        	if( m_mem_out->has_free(m_shader->m_config->sub_core_model, m_id) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) {
+                                m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id,m_id);
                                 issued++;
                                 issued_inst=true;
                                 warp_inst_issued = true;
@@ -943,58 +1050,98 @@ void scheduler_unit::cycle()
                             }
                         } else {
 
-                            bool sp_pipe_avail = m_sp_out->has_free();
-                            bool sfu_pipe_avail = m_sfu_out->has_free();
-                            bool dp_pipe_avail = m_dp_out->has_free();
-                            if( sp_pipe_avail && (pI->op != SFU_OP && pI->op != DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) {
+                            bool sp_pipe_avail = m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id);
+                            bool sfu_pipe_avail = m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id);
+                            bool tensor_core_pipe_avail = m_tensor_core_out->has_free(m_shader->m_config->sub_core_model, m_id);
+                            bool dp_pipe_avail = m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id);
+                            bool int_pipe_avail = m_int_out->has_free(m_shader->m_config->sub_core_model, m_id);
+
+                            //This code need to be refactored
+                            if(pI->op != TENSOR_CORE_OP && pI->op != SFU_OP && pI->op != DP_OP) {
                                 
-                                //Jin: special for CDP api
-                                if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) {
-                                    assert(warp(warp_id).m_cdp_latency == 0);
-                                    
-                                    extern unsigned cdp_latency[5];
-                                    if(pI->m_is_cdp == 1)
-                                        warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1];
-                                    else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2
-                                        warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1]
-                                            + cdp_latency[pI->m_is_cdp] * active_mask.count();
-                                    warp(warp_id).m_cdp_dummy = true;
-                                    break;
-                                }
-                                else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) {
-                                    assert(warp(warp_id).m_cdp_latency == 0);
-                                    warp(warp_id).m_cdp_dummy = false;
-                                }
+									bool execute_on_SP = false;
+									bool execute_on_INT = false;
 
-                                // always prefer SP pipe for operations that can use both SP and SFU pipelines
-                                m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id);
-                                issued++;
-                                issued_inst=true;
-                                warp_inst_issued = true;
-                                previous_issued_inst_exec_type = exec_unit_type_t::SP;
-                            } else if ( (m_shader->m_config->gpgpu_num_dp_units != 0) && (pI->op == DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::DP)) {
+									//if INT unit pipline exist, then execute ALU and INT operations on INT unit and SP-FPU on SP unit (like in Volta)
+									//if INT unit pipline does not exist, then execute all ALU, INT and SP operations on SP unit (as in Fermi, Pascal GPUs)
+									if(m_shader->m_config->gpgpu_num_int_units > 0 &&
+											int_pipe_avail &&
+											pI->op != SP_OP &&
+											!(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::INT))
+										execute_on_INT = true;
+									else if (sp_pipe_avail &&
+											(m_shader->m_config->gpgpu_num_int_units == 0 ||
+											(m_shader->m_config->gpgpu_num_int_units > 0 && pI->op == SP_OP)) &&
+											!(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) )
+										execute_on_SP = true;
+
+
+									if(execute_on_INT || execute_on_SP) {
+										//Jin: special for CDP api
+										if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) {
+											assert(warp(warp_id).m_cdp_latency == 0);
+
+											extern unsigned cdp_latency[5];
+											if(pI->m_is_cdp == 1)
+												warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1];
+											else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2
+												warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1]
+													+ cdp_latency[pI->m_is_cdp] * active_mask.count();
+											warp(warp_id).m_cdp_dummy = true;
+											break;
+										}
+										else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) {
+											assert(warp(warp_id).m_cdp_latency == 0);
+											warp(warp_id).m_cdp_dummy = false;
+										}
+									}
+
+									if(execute_on_SP) {
+										m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id,m_id);
+										issued++;
+										issued_inst=true;
+										warp_inst_issued = true;
+										previous_issued_inst_exec_type = exec_unit_type_t::SP;
+									} else if (execute_on_INT) {
+										m_shader->issue_warp(*m_int_out,pI,active_mask,warp_id,m_id);
+										issued++;
+										issued_inst=true;
+										warp_inst_issued = true;
+										previous_issued_inst_exec_type = exec_unit_type_t::INT;
+                                   }
+                            } else if ( (m_shader->m_config->gpgpu_num_dp_units > 0) && (pI->op == DP_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::DP)) {
                                 if( dp_pipe_avail ) {
-                                    m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id);
+                                    m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id,m_id);
                                     issued++;
                                     issued_inst=true;
                                     warp_inst_issued = true;
                                     previous_issued_inst_exec_type = exec_unit_type_t::DP;
                                 }
-                            }  //If the DP units = 0 (like in Fermi archi), then change DP inst to SFU inst
-                            else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) {
+                            }  //If the DP units = 0 (like in Fermi archi), then execute DP inst on SFU unit
+                            else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SFU)) {
                                 if( sfu_pipe_avail ) {
-                                    m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id);
+                                    m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id,m_id);
                                     issued++;
                                     issued_inst=true;
                                     warp_inst_issued = true;
                                     previous_issued_inst_exec_type = exec_unit_type_t::SFU;
                                 }
-                            }
-                        }
-                    } else {
+                            }                         
+                             else if ( (pI->op == TENSOR_CORE_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) {
+                                if( tensor_core_pipe_avail ) {
+                                    m_shader->issue_warp(*m_tensor_core_out,pI,active_mask,warp_id,m_id);
+                                    issued++;
+                                    issued_inst=true;
+                                    warp_inst_issued = true;
+                                    previous_issued_inst_exec_type = exec_unit_type_t::TENSOR;
+                                }
+			    }
+                         }//end of else
+                   } else {
+
                         SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n",
                                        (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() );
-                    }
+                   }
                 }
             } else if( valid ) {
                // this case can happen after a return instruction in diverged warp
@@ -1128,7 +1275,7 @@ void two_level_active_scheduler::order_warps()
     for (   std::vector< shd_warp_t* >::iterator iter = m_next_cycle_prioritized_warps.begin();
             iter != m_next_cycle_prioritized_warps.end(); ) {
         bool waiting = (*iter)->waiting();
-        for (int i=0; i<4; i++){
+        for (int i=0; i<MAX_INPUT_VALUES; i++){
             const warp_inst_t* inst = (*iter)->ibuffer_next_inst();
             //Is the instruction waiting on a long operation?
             if ( inst && inst->in[i] > 0 && this->m_scoreboard->islongop((*iter)->get_warp_id(), inst->in[i])){
@@ -1174,10 +1321,12 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader
                                register_set* sp_out,
 							   register_set* dp_out,
                                register_set* sfu_out,
+							   register_set* int_out,
+                               register_set* tensor_core_out,
                                register_set* mem_out,
                                int id,
                                char* config_string )
-    : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id )
+    : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id )
 {
     unsigned m_prioritization_readin;
     int ret = sscanf( config_string,
@@ -1345,10 +1494,12 @@ void ldst_unit::get_L1T_sub_stats(struct cache_sub_stats &css) const{
 
 void shader_core_ctx::warp_inst_complete(const warp_inst_t &inst)
 {
-   #if 0
-      printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu issued@%llu\n", 
-             inst.get_uid(), m_sid, inst.warp_id(), inst.pc, gpu_tot_sim_cycle + gpu_sim_cycle, inst.get_issue_cycle()); 
-   #endif
+
+  #if 0
+      printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu issued@%llu\n",
+             inst.get_uid(), m_sid, inst.warp_id(), inst.pc, gpu_tot_sim_cycle + gpu_sim_cycle, inst.get_issue_cycle());
+  #endif
+
   if(inst.op_pipe==SP__OP)
 	  m_stats->m_num_sp_committed[m_sid]++;
   else if(inst.op_pipe==SFU__OP)
@@ -1451,7 +1602,7 @@ ldst_unit::process_cache_access( cache_t* cache,
         assert( !read_sent );
         inst.accessq_pop_back();
         if ( inst.is_load() ) {
-            for ( unsigned r=0; r < 4; r++)
+            for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++)
                 if (inst.out[r] > 0)
                     m_pending_writes[inst.warp_id()][inst.out[r]]--; 
         }
@@ -1488,6 +1639,111 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue( cache_t *cache, war
     return process_cache_access( cache, mf->get_addr(), inst, events, mf, status );
 }
 
+mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( l1_cache *cache, warp_inst_t &inst )
+{
+    mem_stage_stall_type result = NO_RC_FAIL;
+    if( inst.accessq_empty() )
+        return result;
+
+    mem_fetch *mf = m_mf_allocator->alloc(inst,inst.accessq_back());
+
+    if(m_config->m_L1D_config.l1_latency > 0)
+	{
+    	if((l1_latency_queue[m_config->m_L1D_config.l1_latency-1]) == NULL)
+    	{
+    		l1_latency_queue[m_config->m_L1D_config.l1_latency-1] = mf;
+
+    		if( mf->get_inst().is_store() ) {
+				unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
+						(mf->get_data_size()/SECTOR_SIZE) : 1;
+
+				for(unsigned i=0; i< inc_ack; ++i)
+					m_core->inc_store_req( inst.warp_id() );
+			}
+
+    		inst.accessq_pop_back();
+    	}
+    	else
+        {
+        	result = BK_CONF;
+        	delete mf;
+        }
+        if( !inst.accessq_empty() &&  result !=BK_CONF)
+		   result = COAL_STALL;
+	   return result;
+	}
+    else
+    {
+		std::list<cache_event> events;
+		enum cache_request_status status = cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events);
+		return process_cache_access( cache, mf->get_addr(), inst, events, mf, status );
+    }
+}
+
+void ldst_unit::L1_latency_queue_cycle()
+{
+	//std::deque< std::pair<mem_fetch*,bool> >::iterator it = m_latency_queue.begin();
+	if((l1_latency_queue[0]) != NULL)
+    {
+		    mem_fetch* mf_next = l1_latency_queue[0];
+			std::list<cache_event> events;
+			enum cache_request_status status = m_L1D->access(mf_next->get_addr(),mf_next,gpu_sim_cycle+gpu_tot_sim_cycle,events);
+
+		   bool write_sent = was_write_sent(events);
+		   bool read_sent = was_read_sent(events);
+
+		   if ( status == HIT ) {
+			   assert( !read_sent );
+			   l1_latency_queue[0] = NULL;
+			   if ( mf_next->get_inst().is_load() ) {
+				   for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++)
+					   if (mf_next->get_inst().out[r] > 0)
+					   {
+						   assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0);
+						   unsigned still_pending = --m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]];
+						   if(!still_pending)
+						   {
+							m_pending_writes[mf_next->get_inst().warp_id()].erase(mf_next->get_inst().out[r]);
+							m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),mf_next->get_inst().out[r]);
+							m_core->warp_inst_complete(mf_next->get_inst());
+						   }
+					   }
+			   }
+
+			   //For write hit in WB policy
+			   if(mf_next->get_inst().is_store() && !write_sent)
+			   {
+				   unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
+				   						(mf_next->get_data_size()/SECTOR_SIZE) : 1;
+
+				   mf_next->set_reply();
+
+				   for(unsigned i=0; i< dec_ack; ++i)
+				      m_core->store_ack(mf_next);
+			   }
+
+			   if( !write_sent )
+				   delete mf_next;
+
+		   } else if ( status == RESERVATION_FAIL ) {
+			   assert( !read_sent );
+			   assert( !write_sent );
+		   } else {
+			   assert( status == MISS || status == HIT_RESERVED );
+			   l1_latency_queue[0] = NULL;
+	   }
+    }
+
+	 for( unsigned stage = 0; stage<m_config->m_L1D_config.l1_latency-1; ++stage)
+	  if( l1_latency_queue[stage] == NULL) {
+		   l1_latency_queue[stage] = l1_latency_queue[stage+1] ;
+		   l1_latency_queue[stage+1] = NULL;
+	   }
+
+}
+
+
+
 bool ldst_unit::constant_cycle( warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type)
 {
    if( inst.empty() || ((inst.space.get_type() != const_space) && (inst.space.get_type() != param_space_kernel)) )
@@ -1537,14 +1793,14 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea
        bypassL1D = true; 
    } else if (inst.space.is_global()) { // global memory access 
        // skip L1 cache if the option is enabled
-       if (m_core->get_config()->gmem_skip_L1D) 
+       if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op))
            bypassL1D = true; 
    }
-
    if( bypassL1D ) {
        // bypass L1 cache
        unsigned control_size = inst.is_store() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE;
        unsigned size = access.get_size() + control_size;
+       //printf("Interconnect:Addr: %x, size=%d\n",access.get_addr(),size);
        if( m_icnt->full(size, inst.is_store() || inst.isatomic()) ) {
            stall_cond = ICNT_RC_FAIL;
        } else {
@@ -1553,7 +1809,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea
            inst.accessq_pop_back();
            //inst.clear_active( access.get_warp_mask() );
            if( inst.is_load() ) { 
-              for( unsigned r=0; r < 4; r++) 
+              for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) 
                   if(inst.out[r] > 0) 
                       assert( m_pending_writes[inst.warp_id()][inst.out[r]] > 0 );
            } else if( inst.is_store() ) 
@@ -1561,7 +1817,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea
        }
    } else {
        assert( CACHE_UNDEFINED != inst.cache_op );
-       stall_cond = process_memory_access_queue(m_L1D,inst);
+       stall_cond = process_memory_access_queue_l1cache(m_L1D,inst);
    }
    if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL)
        stall_cond = COAL_STALL;
@@ -1593,6 +1849,11 @@ void ldst_unit::flush(){
 	m_L1D->flush();
 }
 
+void ldst_unit::invalidate(){
+	// Flush L1D cache
+	m_L1D->invalidate();
+}
+
 simd_function_unit::simd_function_unit( const shader_core_config *config )
 { 
     m_config=config;
@@ -1606,6 +1867,12 @@ sfu:: sfu(  register_set* result_port, const shader_core_config *config,shader_c
     m_name = "SFU"; 
 }
 
+tensor_core:: tensor_core(  register_set* result_port, const shader_core_config *config,shader_core_ctx *core  )
+    : pipelined_simd_unit(result_port,config,config->max_tensor_core_latency,core)
+{ 
+    m_name = "TENSOR_CORE"; 
+}
+
 void sfu::issue( register_set& source_reg )
 {
     warp_inst_t** ready_reg = source_reg.get_ready();
@@ -1616,11 +1883,34 @@ void sfu::issue( register_set& source_reg )
 	pipelined_simd_unit::issue(source_reg);
 }
 
+void tensor_core::issue( register_set& source_reg )
+{
+    warp_inst_t** ready_reg = source_reg.get_ready();
+	//m_core->incexecstat((*ready_reg));
+
+	(*ready_reg)->op_pipe= TENSOR_CORE__OP;
+	m_core->incsfu_stat(m_core->get_config()->warp_size,(*ready_reg)->latency);
+	pipelined_simd_unit::issue(source_reg);
+}
+
+unsigned pipelined_simd_unit::get_active_lanes_in_pipeline(){
+	active_mask_t active_lanes;
+	active_lanes.reset();
+	 if(m_core->get_gpu()->get_config().g_power_simulation_enabled){
+		for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ){
+			if( !m_pipeline_reg[stage]->empty() )
+				active_lanes|=m_pipeline_reg[stage]->get_active_mask();
+		}
+	 }
+	return active_lanes.count();
+}
+
 void ldst_unit::active_lanes_in_pipeline(){
 	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
 	assert(active_count<=m_core->get_config()->warp_size);
 	m_core->incfumemactivelanes_stat(active_count);
 }
+
 void sp_unit::active_lanes_in_pipeline(){
 	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
 	assert(active_count<=m_core->get_config()->warp_size);
@@ -1636,6 +1926,13 @@ void dp_unit::active_lanes_in_pipeline(){
 	m_core->incfumemactivelanes_stat(active_count);
 }
 
+void int_unit::active_lanes_in_pipeline(){
+	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
+	assert(active_count<=m_core->get_config()->warp_size);
+	m_core->incspactivelanes_stat(active_count);
+	m_core->incfuactivelanes_stat(active_count);
+	m_core->incfumemactivelanes_stat(active_count);
+}
 void sfu::active_lanes_in_pipeline(){
 	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
 	assert(active_count<=m_core->get_config()->warp_size);
@@ -1644,6 +1941,15 @@ void sfu::active_lanes_in_pipeline(){
 	m_core->incfumemactivelanes_stat(active_count);
 }
 
+void tensor_core::active_lanes_in_pipeline(){
+	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
+	assert(active_count<=m_core->get_config()->warp_size);
+	m_core->incsfuactivelanes_stat(active_count);
+	m_core->incfuactivelanes_stat(active_count);
+	m_core->incfumemactivelanes_stat(active_count);
+}
+
+
 sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
     : pipelined_simd_unit(result_port,config,config->max_sp_latency,core)
 { 
@@ -1651,11 +1957,17 @@ sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,sh
 }
 
 dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
-    : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core)
+    : pipelined_simd_unit(result_port,config,config->max_dp_latency,core)
 {
     m_name = "DP ";
 }
 
+int_unit::int_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
+    : pipelined_simd_unit(result_port,config,config->max_int_latency,core)
+{
+    m_name = "INT ";
+}
+
 void sp_unit :: issue(register_set& source_reg)
 {
     warp_inst_t** ready_reg = source_reg.get_ready();
@@ -1674,6 +1986,15 @@ void dp_unit :: issue(register_set& source_reg)
 	pipelined_simd_unit::issue(source_reg);
 }
 
+void int_unit :: issue(register_set& source_reg)
+{
+    warp_inst_t** ready_reg = source_reg.get_ready();
+	//m_core->incexecstat((*ready_reg));
+	(*ready_reg)->op_pipe=INTP__OP;
+	m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency);
+	pipelined_simd_unit::issue(source_reg);
+}
+
 pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core )
     : simd_function_unit(config) 
 {
@@ -1683,19 +2004,25 @@ pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shade
     for( unsigned i=0; i < m_pipeline_depth; i++ ) 
 	m_pipeline_reg[i] = new warp_inst_t( config );
     m_core=core;
+    active_insts_in_pipeline=0;
 }
 
 void pipelined_simd_unit::cycle()
 {
     if( !m_pipeline_reg[0]->empty() ){
         m_result_port->move_in(m_pipeline_reg[0]);
+        assert(active_insts_in_pipeline > 0);
+        active_insts_in_pipeline--;
+    }
+    if(active_insts_in_pipeline){
+		for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ )
+			move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]);
     }
-    for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ )
-        move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]);
     if( !m_dispatch_reg->empty() ) {
         if( !m_dispatch_reg->dispatch_delay()){
             int start_stage = m_dispatch_reg->latency - m_dispatch_reg->initiation_interval;
             move_warp(m_pipeline_reg[start_stage],m_dispatch_reg);
+            active_insts_in_pipeline++;
         }
     }
     occupied >>=1;
@@ -1766,8 +2093,9 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt,
                       const memory_config *mem_config,  
                       shader_core_stats *stats,
                       unsigned sid,
-                      unsigned tpc ) : pipelined_simd_unit(NULL,config,3,core), m_next_wb(config)
+                      unsigned tpc ) : pipelined_simd_unit(NULL,config,config->smem_latency,core), m_next_wb(config)
 {
+	assert(config->smem_latency > 1);
     init( icnt,
           mf_allocator,
           core, 
@@ -1788,7 +2116,14 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt,
                               m_icnt,
                               m_mf_allocator,
                               IN_L1D_MISS_QUEUE );
+
+        if(m_config->m_L1D_config.l1_latency > 0)
+	    {
+        	for(int i=0; i<m_config->m_L1D_config.l1_latency; i++ )
+        		l1_latency_queue.push_back((mem_fetch*)NULL);
+	    }
     }
+    m_name = "MEM ";
 }
 
 ldst_unit::ldst_unit( mem_fetch_interface *icnt,
@@ -1825,7 +2160,7 @@ void ldst_unit:: issue( register_set &reg_set )
    if (inst->is_load() and inst->space.get_type() != shared_space) {
       unsigned warp_id = inst->warp_id();
       unsigned n_accesses = inst->accessq_count();
-      for (unsigned r = 0; r < 4; r++) {
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = inst->out[r];
          if (reg_id > 0) {
             m_pending_writes[warp_id][reg_id] += n_accesses;
@@ -1847,7 +2182,7 @@ void ldst_unit::writeback()
     if( !m_next_wb.empty() ) {
         if( m_operand_collector->writeback(m_next_wb) ) {
             bool insn_completed = false; 
-            for( unsigned r=0; r < 4; r++ ) {
+            for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++ ) {
                 if( m_next_wb.out[r] > 0 ) {
                     if( m_next_wb.space.get_type() != shared_space ) {
                         assert( m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0 );
@@ -1935,7 +2270,11 @@ void ldst_unit::writeback()
 
 unsigned ldst_unit::clock_multiplier() const
 { 
-    return m_config->mem_warp_parts; 
+	//to model multiple read port, we give multiple cycles for the memory units
+	if(m_config->mem_unit_ports)
+		return m_config->mem_unit_ports;
+	else
+		return m_config->mem_warp_parts;
 }
 /*
 void ldst_unit::issue( register_set &reg_set )
@@ -1949,7 +2288,7 @@ void ldst_unit::issue( register_set &reg_set )
    if (inst->is_load() and inst->space.get_type() != shared_space) {
       unsigned warp_id = inst->warp_id(); 
       unsigned n_accesses = inst->accessq_count(); 
-      for (unsigned r = 0; r < 4; r++) {
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = inst->out[r]; 
          if (reg_id > 0) {
             m_pending_writes[warp_id][reg_id] += n_accesses; 
@@ -2014,7 +2353,11 @@ void ldst_unit::cycle()
 
    m_L1T->cycle();
    m_L1C->cycle();
-   if( m_L1D ) m_L1D->cycle();
+   if( m_L1D ) {
+	   m_L1D->cycle();
+	   if(m_config->m_L1D_config.l1_latency > 0)
+	   		L1_latency_queue_cycle();
+   }
 
    warp_inst_t &pipe_reg = *m_dispatch_reg;
    enum mem_stage_stall_type rc_fail = NO_RC_FAIL;
@@ -2037,9 +2380,9 @@ void ldst_unit::cycle()
        unsigned warp_id = pipe_reg.warp_id();
        if( pipe_reg.is_load() ) {
            if( pipe_reg.space.get_type() == shared_space ) {
-               if( m_pipeline_reg[2]->empty() ) {
+               if( m_pipeline_reg[m_config->smem_latency-1]->empty() ) {
                    // new shared memory request
-                   move_warp(m_pipeline_reg[2],m_dispatch_reg);
+                   move_warp(m_pipeline_reg[m_config->smem_latency-1],m_dispatch_reg);
                    m_dispatch_reg->clear();
                }
            } else {
@@ -2049,7 +2392,7 @@ void ldst_unit::cycle()
                //} 
 
                bool pending_requests=false;
-               for( unsigned r=0; r<4; r++ ) {
+               for( unsigned r=0; r<MAX_OUTPUT_VALUES; r++ ) {
                    unsigned reg_id = pipe_reg.out[r];
                    if( reg_id > 0 ) {
                        if( m_pending_writes[warp_id].find(reg_id) != m_pending_writes[warp_id].end() ) {
@@ -2199,13 +2542,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1I_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1I_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1I_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1I_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1I_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1I_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1I_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1I_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1I_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1I_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 
     // L1D
@@ -2216,18 +2559,18 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
         for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++){
             m_cluster[i]->get_L1D_sub_stats(css);
 
-            fprintf( stdout, "\tL1D_cache_core[%d]: Access = %d, Miss = %d, Miss_rate = %.3lf, Pending_hits = %u, Reservation_fails = %u\n",
+            fprintf( stdout, "\tL1D_cache_core[%d]: Access = %llu, Miss = %llu, Miss_rate = %.3lf, Pending_hits = %llu, Reservation_fails = %llu\n",
                      i, css.accesses, css.misses, (double)css.misses / (double)css.accesses, css.pending_hits, css.res_fails);
 
             total_css += css;
         }
-        fprintf(fout, "\tL1D_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1D_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1D_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1D_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1D_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1D_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1D_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1D_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1D_total_cache_reservation_fails = %llu\n", total_css.res_fails);
         total_css.print_port_stats(fout, "\tL1D_cache"); 
     }
 
@@ -2240,13 +2583,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1C_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1C_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1C_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1C_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1C_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1C_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1C_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1C_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1C_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1C_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 
     // L1T
@@ -2258,13 +2601,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1T_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1T_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1T_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1T_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1T_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1T_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1T_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1T_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1T_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1T_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 }
 
@@ -2357,7 +2700,7 @@ void shader_core_ctx::incexecstat(warp_inst_t *&inst)
 
 	switch(inst->sp_op){
 	case INT__OP:
-		incialu_stat(inst->active_count(),25);
+		incialu_stat(inst->active_count(),32);
 		break;
 	case INT_MUL_OP:
 		incimul_stat(inst->active_count(),7.2);
@@ -2623,11 +2966,86 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const
        abort();
     }
 
+    if(adaptive_volta_cache_config && !k.volta_cache_config_set) {
+    	//For Volta, we assign the remaining shared memory to L1 cache
+    	//For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
+    	unsigned total_shmed = kernel_info->smem * result;
+    	assert(total_shmed >=0 && total_shmed <= gpgpu_shmem_size);
+    	assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared
+    	assert(m_L1D_config.get_nset() == 4);  //Volta L1 has four sets
+    	if(total_shmed < gpgpu_shmem_size){
+    		if(total_shmed == 0)
+    			m_L1D_config.set_assoc(256);  //L1 is 128KB ans shd=0
+    		else if(total_shmed > 0 && total_shmed <= 8192)
+    			m_L1D_config.set_assoc(240);  //L1 is 120KB ans shd=8KB
+    		else if(total_shmed > 8192 && total_shmed <= 16384)
+    		    m_L1D_config.set_assoc(224);  //L1 is 112KB ans shd=16KB
+    		else if(total_shmed > 16384 && total_shmed <= 32768)
+    		    m_L1D_config.set_assoc(192);  //L1 is 96KB ans shd=32KB
+    		else if(total_shmed > 32768 && total_shmed <= 65536)
+    		    m_L1D_config.set_assoc(128);	//L1 is 64KB ans shd=64KB
+    		else if(total_shmed > 65536 && total_shmed <= gpgpu_shmem_size)
+    		     m_L1D_config.set_assoc(64); //L1 is 32KB and shd=96KB
+    		else
+    			assert(0);
+
+    		 printf ("GPGPU-Sim: Reconfigure L1 cache in Volta Archi to %uKB\n", m_L1D_config.get_total_size_inKB());
+    	}
+
+    	k.volta_cache_config_set = true;
+    }
+
     return result;
 }
 
+void shader_core_config::set_pipeline_latency() {
+
+		//calculate the max latency  based on the input
+
+		unsigned int_latency[5];
+		unsigned fp_latency[5];
+		unsigned dp_latency[5];
+		unsigned sfu_latency;
+		unsigned tensor_latency;
+
+			/*
+			 * [0] ADD,SUB
+			 * [1] MAX,Min
+			 * [2] MUL
+			 * [3] MAD
+			 * [4] DIV
+			 */
+			sscanf(opcode_latency_int, "%u,%u,%u,%u,%u",
+					&int_latency[0],&int_latency[1],&int_latency[2],
+					&int_latency[3],&int_latency[4]);
+			sscanf(opcode_latency_fp, "%u,%u,%u,%u,%u",
+					&fp_latency[0],&fp_latency[1],&fp_latency[2],
+					&fp_latency[3],&fp_latency[4]);
+			sscanf(opcode_latency_dp, "%u,%u,%u,%u,%u",
+					&dp_latency[0],&dp_latency[1],&dp_latency[2],
+					&dp_latency[3],&dp_latency[4]);
+			sscanf(opcode_latency_sfu, "%u",
+					&sfu_latency);
+			sscanf(opcode_latency_tensor, "%u",
+					&tensor_latency);
+
+		//all div operation are executed on sfu
+		//assume that the max latency are dp div or normal sfu_latency
+		max_sfu_latency = std::max(dp_latency[4],sfu_latency);
+		//assume that the max operation has the max latency
+		max_sp_latency = fp_latency[1];
+		max_int_latency = int_latency[1];
+		max_dp_latency = dp_latency[1];
+		max_tensor_core_latency = tensor_latency;
+
+}
+
 void shader_core_ctx::cycle()
 {
+	if(!isactive() && get_not_completed() == 0)
+		return;
+
+	elapsed_cycles_sm_tot++;
 	m_stats->shader_cycles[m_sid]++;
     writeback();
     execute();
@@ -2644,6 +3062,11 @@ void shader_core_ctx::cache_flush()
    m_ldst_unit->flush();
 }
 
+void shader_core_ctx::cache_invalidate()
+{
+   m_ldst_unit->invalidate();
+}
+
 // modifiers
 std::list<opndcoll_rfu_t::op_t> opndcoll_rfu_t::arbiter_t::allocate_reads() 
 {
@@ -3134,33 +3557,50 @@ void opndcoll_rfu_t::init( unsigned num_banks, shader_core_ctx *shader )
    m_bank_warp_shift = (unsigned)(int) (log(m_warp_size+0.5) / log(2.0));
    assert( (m_bank_warp_shift == 5) || (m_warp_size != 32) );
 
+   sub_core_model = shader->get_config()->sub_core_model;
+   m_num_warp_sceds = shader->get_config()->gpgpu_num_sched_per_core;
+   if(sub_core_model)
+	   assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0);
+   m_num_banks_per_sched = num_banks / shader->get_config()->gpgpu_num_sched_per_core;
+
    for( unsigned j=0; j<m_cu.size(); j++) {
-       m_cu[j]->init(j,num_banks,m_bank_warp_shift,shader->get_config(),this);
+       m_cu[j]->init(j,num_banks,m_bank_warp_shift,shader->get_config(),this, sub_core_model, m_num_banks_per_sched );
    }
    m_initialized=true;
+
+
+
+
 }
 
-int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift)
+int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id)
 {
    int bank = regnum;
    if (bank_warp_shift)
       bank += wid;
-   return bank % num_banks;
+   if(sub_core_model) {
+	   unsigned bank_num  = (bank % banks_per_sched) + (sched_id * banks_per_sched);
+	   assert(bank_num < num_banks);
+	   return bank_num;
+   }
+   else
+	   return bank % num_banks;
 }
 
-bool opndcoll_rfu_t::writeback( const warp_inst_t &inst )
+bool opndcoll_rfu_t::writeback( warp_inst_t &inst )
 {
    assert( !inst.empty() );
    std::list<unsigned> regs = m_shader->get_regs_written(inst);
-   std::list<unsigned>::iterator r;
-   unsigned n=0;
-   for( r=regs.begin(); r!=regs.end();r++,n++ ) {
-      unsigned reg = *r;
-      unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift);
-      if( m_arbiter.bank_idle(bank) ) {
-          m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg,m_num_banks,m_bank_warp_shift));
-      } else {
-          return false;
+   for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) {
+      int reg_num = inst.arch_reg.dst[op]; // this math needs to match that used in function_info::ptx_decode_inst
+      if( reg_num >= 0 ){ // valid register
+         unsigned bank = register_bank(reg_num,inst.warp_id(),m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id());
+         if( m_arbiter.bank_idle(bank) ) {
+             m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg_num,m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id()));
+             inst.arch_reg.dst[op] = -1;
+         } else {
+             return false;
+         }
       }
    }
    for(unsigned i=0;i<(unsigned)regs.size();i++){
@@ -3242,7 +3682,7 @@ void opndcoll_rfu_t::allocate_reads()
       const op_t &rr = *r;
       unsigned reg = rr.get_reg();
       unsigned wid = rr.get_wid();
-      unsigned bank = register_bank(reg,wid,m_num_banks,m_bank_warp_shift);
+      unsigned bank = register_bank(reg,wid,m_num_banks,m_bank_warp_shift,sub_core_model, m_num_banks_per_sched, rr.get_sid());
       m_arbiter.allocate_for_read(bank,rr);
       read_ops[bank] = rr;
    }
@@ -3293,7 +3733,9 @@ void opndcoll_rfu_t::collector_unit_t::init( unsigned n,
                                              unsigned num_banks, 
                                              unsigned log2_warp_size,
                                              const core_config *config,
-                                             opndcoll_rfu_t *rfu ) 
+                                             opndcoll_rfu_t *rfu,
+											 bool sub_core_model,
+											 unsigned banks_per_sched)
 { 
    m_rfu=rfu;
    m_cuid=n; 
@@ -3301,6 +3743,8 @@ void opndcoll_rfu_t::collector_unit_t::init( unsigned n,
    assert(m_warp==NULL); 
    m_warp = new warp_inst_t(config);
    m_bank_warp_shift=log2_warp_size;
+   m_sub_core_model = sub_core_model;
+   m_num_banks_per_sched = banks_per_sched;
 }
 
 bool opndcoll_rfu_t::collector_unit_t::allocate( register_set* pipeline_reg_set, register_set* output_reg_set ) 
@@ -3315,7 +3759,7 @@ bool opndcoll_rfu_t::collector_unit_t::allocate( register_set* pipeline_reg_set,
       for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) {
          int reg_num = (*pipeline_reg)->arch_reg.src[op]; // this math needs to match that used in function_info::ptx_decode_inst
          if( reg_num >= 0 ) { // valid register
-            m_src_op[op] = op_t( this, op, reg_num, m_num_banks, m_bank_warp_shift );
+            m_src_op[op] = op_t( this, op, reg_num, m_num_banks, m_bank_warp_shift, m_sub_core_model, m_num_banks_per_sched, (*pipeline_reg)->get_schd_id() );
             m_not_ready.set(op);
          } else 
             m_src_op[op] = op_t();
@@ -3398,6 +3842,15 @@ void simt_core_cluster::print_not_completed( FILE *fp ) const
     }
 }
 
+
+float simt_core_cluster::get_current_occupancy( unsigned long long& active, unsigned long long& total ) const {
+    float aggregate = 0.f;
+    for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) {
+        aggregate+=m_core[i]->get_current_occupancy( active, total );
+    }
+    return aggregate / m_config->n_simt_cores_per_cluster;
+}
+
 unsigned simt_core_cluster::get_n_active_cta() const
 {
     unsigned n=0;
@@ -3461,6 +3914,12 @@ void simt_core_cluster::cache_flush()
         m_core[i]->cache_flush();
 }
 
+void simt_core_cluster::cache_invalidate()
+{
+    for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ )
+        m_core[i]->cache_invalidate();
+}
+
 bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write)
 {
     unsigned request_size = size;
@@ -3478,6 +3937,7 @@ void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf)
     case CONST_ACC_R: m_stats->gpgpu_n_mem_const++; break;
     case TEXTURE_ACC_R: m_stats->gpgpu_n_mem_texture++; break;
     case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; break;
+    //case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; printf("read_global%d\n",m_stats->gpgpu_n_mem_read_global); break;
     case GLOBAL_ACC_W: m_stats->gpgpu_n_mem_write_global++; break;
     case LOCAL_ACC_R: m_stats->gpgpu_n_mem_read_local++; break;
     case LOCAL_ACC_W: m_stats->gpgpu_n_mem_write_local++; break;