Merge pull request #118 from mkhairy/dev

Integrating Mahmoud's changes that improve simulation speed by 2-3x.
Also includes a change that mitigates memory channel conflicts by using a random hashing function.

18 files changed, 754 insertions, 141 deletions

diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index e8716ab..22ef509 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -386,6 +386,8 @@ struct core_config {
 
 	unsigned gpgpu_max_insn_issue_per_warp;
 	bool gmem_skip_L1D; // on = global memory access always skip the L1 cache
+
+	bool adaptive_volta_cache_config;
 };
 
 // bounded stack that implements simt reconvergence using pdom mechanism from MICRO'07 paper
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index abd32f9..e690356 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -47,6 +47,8 @@ extern int g_debug_thread_uid;
 extern void ** g_inst_classification_stat;
 extern void ** g_inst_op_classification_stat;
 extern int g_ptx_kernel_count; // used for classification stat collection purposes 
+extern char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor;
+
 
 void ptx_opcocde_latency_options (option_parser_t opp);
 extern class kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc
index 8651869..ca88ec9 100644
--- a/src/gpgpu-sim/addrdec.cc
+++ b/src/gpgpu-sim/addrdec.cc
@@ -165,6 +165,27 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_
 			assert(tlx->chip < m_n_channel);
 			break;
 		}
+		case RANDOM:
+		{
+			new_addr_type chip_address = (addr>>ADDR_CHIP_S);
+			tr1_hash_map<new_addr_type,unsigned>::const_iterator got = address_random_interleaving.find (chip_address);
+			  if ( got == address_random_interleaving.end() ) {
+				  unsigned new_chip_id = rand() % (m_n_channel*m_n_sub_partition_in_channel);
+				  address_random_interleaving[chip_address] = new_chip_id;
+				  tlx->chip = new_chip_id/m_n_sub_partition_in_channel;
+				  tlx->sub_partition = new_chip_id;
+			  }
+			  else {
+				  unsigned new_chip_id = got->second;
+				  tlx->chip = new_chip_id/m_n_sub_partition_in_channel;
+				  tlx->sub_partition = new_chip_id;
+			  }
+
+			  assert(tlx->chip < m_n_channel);
+			  assert(tlx->sub_partition < m_n_channel*m_n_sub_partition_in_channel);
+			  return;
+			break;
+		}
 		case CUSTOM:
 			/* No custom set function implemented */
 			//Do you custom index here
@@ -175,9 +196,9 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_
 	}
 
    // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID
-   unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; 
+   unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1;
    tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel
-                        + (tlx->bk & sub_partition_addr_mask); 
+                        + (tlx->bk & sub_partition_addr_mask);
 }
 
 void linear_to_raw_address_translation::addrdec_parseoption(const char *option)
@@ -396,6 +417,10 @@ void linear_to_raw_address_translation::init(unsigned int n_channel, unsigned in
    if (run_test) {
       sweep_test(); 
    }
+
+   if(memory_partition_indexing == RANDOM)
+     srand (1);
+
 }
 
 #include "../tr1_hash_map.h" 
diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h
index bdc5fec..a5333fb 100644
--- a/src/gpgpu-sim/addrdec.h
+++ b/src/gpgpu-sim/addrdec.h
@@ -40,6 +40,7 @@ enum partition_index_function{
 	BITWISE_PERMUTATION,
 	IPOLY,
 	PAE,
+	RANDOM,
     CUSTOM
 };
 
@@ -55,6 +56,7 @@ struct addrdec_t {
    unsigned sub_partition; 
 };
 
+
 class linear_to_raw_address_translation {
 public:
    linear_to_raw_address_translation();
@@ -62,7 +64,7 @@ public:
    void init(unsigned int n_channel, unsigned int n_sub_partition_in_channel); 
 
    // accessors
-   void addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const; 
+   void addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const;
    new_addr_type partition_address( new_addr_type addr ) const;
 
 private:
@@ -92,6 +94,7 @@ private:
    unsigned int gap;
    int m_n_channel;
    int m_n_sub_partition_in_channel; 
+
 };
 
 #endif
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index 6c11b43..192cb65 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -723,10 +723,10 @@ void dram_t::print( FILE* simFile) const
            id, m_config->nbk, m_config->busW, m_config->BL, m_config->CL );
    fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n",
            m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC );
-   fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref_event=%d n_req=%d n_rd=%d n_rd_L2_A=%d n_write=%d n_wr_bk=%d bw_util=%.4g\n",
+   fprintf(simFile,"n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref_event=%llu n_req=%llu n_rd=%llu n_rd_L2_A=%llu n_write=%llu n_wr_bk=%llu bw_util=%.4g\n",
            n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB,
            (float)bwutil/n_cmd);
-   fprintf(simFile,"n_activity=%d dram_eff=%.4g\n",
+   fprintf(simFile,"n_activity=%llu dram_eff=%.4g\n",
            n_activity, (float)bwutil/n_activity);
    for (i=0;i<m_config->nbk;i++) {
       fprintf(simFile, "bk%d: %da %di ",i,bk[i]->n_access,bk[i]->n_idle);
@@ -745,39 +745,39 @@ void dram_t::print( FILE* simFile) const
 
    printf("\nBW Util details:\n");
    printf("bwutil = %.6f \n", (float)bwutil/n_cmd);
-   printf("total_CMD = %d \n", n_cmd);
-   printf("util_bw = %d \n", util_bw);
-   printf("Wasted_Col = %d \n", wasted_bw_col);
-   printf("Wasted_Row = %d \n", wasted_bw_row);
-   printf("Idle = %d \n", idle_bw);
+   printf("total_CMD = %llu \n", n_cmd);
+   printf("util_bw = %llu \n", util_bw);
+   printf("Wasted_Col = %llu \n", wasted_bw_col);
+   printf("Wasted_Row = %llu \n", wasted_bw_row);
+   printf("Idle = %llu \n", idle_bw);
 
    printf("\nBW Util Bottlenecks: \n");
-   printf("RCDc_limit = %d \n", RCDc_limit);
-   printf("RCDWRc_limit = %d \n", RCDWRc_limit);
-   printf("WTRc_limit = %d \n", WTRc_limit);
-   printf("RTWc_limit = %d \n", RTWc_limit);
-   printf("CCDLc_limit = %d \n", CCDLc_limit);
-   printf("rwq = %d \n", rwq_limit);
-   printf("CCDLc_limit_alone = %d \n", CCDLc_limit_alone);
-   printf("WTRc_limit_alone = %d \n", WTRc_limit_alone);
-   printf("RTWc_limit_alone = %d \n", RTWc_limit_alone);
+   printf("RCDc_limit = %llu \n", RCDc_limit);
+   printf("RCDWRc_limit = %llu \n", RCDWRc_limit);
+   printf("WTRc_limit = %llu \n", WTRc_limit);
+   printf("RTWc_limit = %llu \n", RTWc_limit);
+   printf("CCDLc_limit = %llu \n", CCDLc_limit);
+   printf("rwq = %llu \n", rwq_limit);
+   printf("CCDLc_limit_alone = %llu \n", CCDLc_limit_alone);
+   printf("WTRc_limit_alone = %llu \n", WTRc_limit_alone);
+   printf("RTWc_limit_alone = %llu \n", RTWc_limit_alone);
 
    printf("\nCommands details: \n");
-   printf("total_CMD = %d \n", n_cmd);
-   printf("n_nop = %d \n", n_nop);
-   printf("Read = %d \n", n_rd);
-   printf("Write = %d \n",n_wr);
-   printf("L2_Alloc = %d \n", n_rd_L2_A);
-   printf("L2_WB = %d \n", n_wr_WB);
-   printf("n_act = %d \n", n_act);
-   printf("n_pre = %d \n", n_pre);
-   printf("n_ref = %d \n", n_ref);
-   printf("n_req = %d \n", n_req );
-   printf("total_req = %d \n", n_rd+n_wr+n_rd_L2_A+n_wr_WB);
+   printf("total_CMD = %llu \n", n_cmd);
+   printf("n_nop = %llu \n", n_nop);
+   printf("Read = %llu \n", n_rd);
+   printf("Write = %llu \n",n_wr);
+   printf("L2_Alloc = %llu \n", n_rd_L2_A);
+   printf("L2_WB = %llu \n", n_wr_WB);
+   printf("n_act = %llu \n", n_act);
+   printf("n_pre = %llu \n", n_pre);
+   printf("n_ref = %llu \n", n_ref);
+   printf("n_req = %llu \n", n_req );
+   printf("total_req = %llu \n", n_rd+n_wr+n_rd_L2_A+n_wr_WB);
 
    printf("\nDual Bus Interface Util: \n");
-   printf("issued_total_row = %lu \n", issued_total_row);
-   printf("issued_total_col = %lu \n", issued_total_col);
+   printf("issued_total_row = %llu \n", issued_total_row);
+   printf("issued_total_col = %llu \n", issued_total_col);
    printf("Row_Bus_Util =  %.6f \n", (float)issued_total_row / n_cmd);
    printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd);
    printf("Either_Row_CoL_Bus_Util = %.6f \n",  (float)issued_total / n_cmd);
@@ -815,7 +815,7 @@ void dram_t::visualize() const
 
 void dram_t::print_stat( FILE* simFile ) 
 {
-   fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ",
+   fprintf(simFile,"DRAM (%llu): n_cmd=%llu n_nop=%llu n_act=%llu n_pre=%llu n_ref=%llu n_req=%llu n_rd=%llu n_write=%llu bw_util=%.4g ",
            id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr,
            (float)bwutil/n_cmd);
    fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", max_mrqs, (float)ave_mrqs/n_cmd, max_mrqs_temp);
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
index bee5b7b..1ab0153 100644
--- a/src/gpgpu-sim/dram.h
+++ b/src/gpgpu-sim/dram.h
@@ -178,39 +178,39 @@ private:
    unsigned int dram_eff_bins[10];
    unsigned int last_n_cmd, last_n_activity, last_bwutil;
 
-   unsigned int n_cmd;
-   unsigned int n_activity;
-   unsigned int n_nop;
-   unsigned int n_act;
-   unsigned int n_pre;
-   unsigned int n_ref;
-   unsigned int n_rd;
-   unsigned int n_rd_L2_A;
-   unsigned int n_wr;
-   unsigned int n_wr_WB;
-   unsigned int n_req;
-   unsigned int max_mrqs_temp;
+   unsigned long long n_cmd;
+   unsigned long long n_activity;
+   unsigned long long n_nop;
+   unsigned long long n_act;
+   unsigned long long n_pre;
+   unsigned long long n_ref;
+   unsigned long long n_rd;
+   unsigned long long n_rd_L2_A;
+   unsigned long long n_wr;
+   unsigned long long n_wr_WB;
+   unsigned long long n_req;
+   unsigned long long max_mrqs_temp;
 
-   //some statistics to collect to see where BW is wasted?
-   unsigned wasted_bw_row;
-   unsigned wasted_bw_col;
-   unsigned util_bw;
-   unsigned idle_bw;
-   unsigned RCDc_limit;
-   unsigned CCDLc_limit;
-   unsigned CCDLc_limit_alone;
-   unsigned CCDc_limit;
-   unsigned WTRc_limit;
-   unsigned WTRc_limit_alone;
-   unsigned RCDWRc_limit;
-   unsigned RTWc_limit;
-   unsigned RTWc_limit_alone;
-   unsigned rwq_limit;
+   //some statistics to see where BW is wasted?
+   unsigned long long wasted_bw_row;
+   unsigned long long wasted_bw_col;
+   unsigned long long util_bw;
+   unsigned long long idle_bw;
+   unsigned long long RCDc_limit;
+   unsigned long long CCDLc_limit;
+   unsigned long long CCDLc_limit_alone;
+   unsigned long long CCDc_limit;
+   unsigned long long WTRc_limit;
+   unsigned long long WTRc_limit_alone;
+   unsigned long long RCDWRc_limit;
+   unsigned long long RTWc_limit;
+   unsigned long long RTWc_limit_alone;
+   unsigned long long rwq_limit;
 
    //row locality, BLP and other statistics
-   unsigned long  access_num;
-   unsigned long  read_num;
-   unsigned long  write_num;
+   unsigned long long access_num;
+   unsigned long long read_num;
+   unsigned long long write_num;
    unsigned long long hits_num;
    unsigned long long hits_read_num;
    unsigned long long hits_write_num;
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index ba81440..565fae1 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -256,7 +256,7 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, m
 
     unsigned invalid_line = (unsigned)-1;
     unsigned valid_line = (unsigned)-1;
-    unsigned valid_timestamp = (unsigned)-1;
+    unsigned long long valid_timestamp = (unsigned)-1;
 
     bool all_reserved = true;
 
@@ -654,7 +654,7 @@ enum cache_request_status cache_stats::select_stats_status(enum cache_request_st
 		return access;
 }
 
-unsigned &cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome){
+unsigned long long &cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome){
     ///
     /// Simple method to read/modify the stat corresponding to (access_type, access_outcome)
     /// Used overloaded () to avoid the need for separate read/write member functions
@@ -673,7 +673,7 @@ unsigned &cache_stats::operator()(int access_type, int access_outcome, bool fail
 	}
 }
 
-unsigned cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome) const{
+unsigned long long cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome) const{
     ///
     /// Const accessor into m_stats.
     ///
@@ -740,7 +740,7 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
     std::string m_cache_name = cache_name;
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
         for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) {
-            fprintf(fout, "\t%s[%s][%s] = %u\n",
+            fprintf(fout, "\t%s[%s][%s] = %llu\n",
                 m_cache_name.c_str(),
                 mem_access_type_str((enum mem_access_type)type),
                 cache_request_status_str((enum cache_request_status)status),
@@ -751,7 +751,7 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
     }
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
     	 if(total_access[type] > 0)
-    	  fprintf(fout, "\t%s[%s][%s] = %u\n",
+    	  fprintf(fout, "\t%s[%s][%s] = %llu\n",
 				m_cache_name.c_str(),
 				mem_access_type_str((enum mem_access_type)type),
 				"TOTAL_ACCESS",
@@ -788,13 +788,13 @@ void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const
     fprintf(fout, "%s_fill_port_util = %.3f\n", cache_name, fill_port_util); 
 }
 
-unsigned cache_stats::get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const{
+unsigned long long cache_stats::get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status) const{
     ///
     /// Returns a sum of the stats corresponding to each "access_type" and "access_status" pair.
     /// "access_type" is an array of "num_access_type" mem_access_types.
     /// "access_status" is an array of "num_access_status" cache_request_statuses.
     ///
-    unsigned total=0;
+    unsigned long long total=0;
     for(unsigned type =0; type < num_access_type; ++type){
         for(unsigned status=0; status < num_access_status; ++status){
             if(!check_valid((int)access_type[type], (int)access_status[status]))
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index e663cf6..673e128 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -119,9 +119,9 @@ struct cache_block_t {
     virtual enum cache_block_state get_status( mem_access_sector_mask_t sector_mask) = 0;
     virtual void set_status(enum cache_block_state m_status, mem_access_sector_mask_t sector_mask) = 0;
 
-    virtual unsigned get_last_access_time() = 0;
-    virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) = 0;
-    virtual unsigned get_alloc_time() = 0;
+    virtual unsigned long long get_last_access_time() = 0;
+    virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask) = 0;
+    virtual unsigned long long get_alloc_time() = 0;
     virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) = 0;
     virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) = 0;
     virtual unsigned get_modified_size() = 0;
@@ -192,15 +192,15 @@ struct line_cache_block: public cache_block_t  {
 	    {
 	    	m_status = status;
 	    }
-		virtual unsigned get_last_access_time()
+		virtual unsigned long long get_last_access_time()
 		{
 			return m_last_access_time;
 		}
-		virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask)
+		virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask)
 	    {
 	    	m_last_access_time = time;
 	    }
-		virtual unsigned get_alloc_time()
+		virtual unsigned long long get_alloc_time()
 	    {
 	    	return m_alloc_time;
 	    }
@@ -229,9 +229,9 @@ struct line_cache_block: public cache_block_t  {
 
 
 private:
-	    unsigned         m_alloc_time;
-	    unsigned         m_last_access_time;
-	    unsigned         m_fill_time;
+	    unsigned long long     m_alloc_time;
+	    unsigned long long     m_last_access_time;
+	    unsigned long long     m_fill_time;
 	    cache_block_state    m_status;
 	    bool m_ignore_on_fill_status;
 	    bool m_set_modified_on_fill;
@@ -364,12 +364,12 @@ struct sector_cache_block : public cache_block_t {
 		m_status[sidx] = status;
 	}
 
-    virtual unsigned get_last_access_time()
+    virtual unsigned long long get_last_access_time()
 	{
 		return m_line_last_access_time;
 	}
 
-    virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask)
+    virtual void set_last_access_time(unsigned long long time, mem_access_sector_mask_t sector_mask)
 	{
 		unsigned sidx = get_sector_index(sector_mask);
 
@@ -377,7 +377,7 @@ struct sector_cache_block : public cache_block_t {
 		m_line_last_access_time = time;
 	}
 
-    virtual unsigned get_alloc_time()
+    virtual unsigned long long get_alloc_time()
 	{
 		return m_line_alloc_time;
 	}
@@ -915,10 +915,10 @@ private:
 /// Simple struct to maintain cache accesses, misses, pending hits, and reservation fails.
 ///
 struct cache_sub_stats{
-    unsigned accesses;
-    unsigned misses;
-    unsigned pending_hits;
-    unsigned res_fails;
+    unsigned long long accesses;
+    unsigned long long misses;
+    unsigned long long pending_hits;
+    unsigned long long res_fails;
 
     unsigned long long port_available_cycles; 
     unsigned long long data_port_busy_cycles; 
@@ -981,14 +981,14 @@ public:
     void inc_stats(int access_type, int access_outcome);
     void inc_fail_stats(int access_type, int fail_outcome);
     enum cache_request_status select_stats_status(enum cache_request_status probe, enum cache_request_status access) const;
-    unsigned &operator()(int access_type, int access_outcome, bool fail_outcome);
-    unsigned operator()(int access_type, int access_outcome, bool fail_outcome) const;
+    unsigned long long &operator()(int access_type, int access_outcome, bool fail_outcome);
+    unsigned long long operator()(int access_type, int access_outcome, bool fail_outcome) const;
     cache_stats operator+(const cache_stats &cs);
     cache_stats &operator+=(const cache_stats &cs);
     void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const;
     void print_fail_stats(FILE *fout, const char *cache_name = "Cache_fail_stats") const;
 
-    unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status)  const;
+    unsigned long long get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status)  const;
     void get_sub_stats(struct cache_sub_stats &css) const;
 
     void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); 
@@ -996,8 +996,8 @@ private:
     bool check_valid(int type, int status) const;
     bool check_fail_valid(int type, int fail) const;
 
-    std::vector< std::vector<unsigned> > m_stats;
-    std::vector< std::vector<unsigned> > m_fail_stats;
+    std::vector< std::vector<unsigned long long> > m_stats;
+    std::vector< std::vector<unsigned long long> > m_fail_stats;
 
     unsigned long long m_cache_port_available_cycles; 
     unsigned long long m_cache_data_port_busy_cycles; 
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 4b68e2e..c1ba934 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -99,6 +99,8 @@ unsigned long long  gpu_tot_sim_cycle_parition_util = 0;
 unsigned long long partiton_replys_in_parallel = 0;
 unsigned long long partiton_replys_in_parallel_total = 0;
 
+tr1_hash_map<new_addr_type,unsigned> address_random_interleaving;
+
 /* Clock Domains */
 
 #define  CORE  0x01
@@ -476,6 +478,7 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_concurrent_kernel_sm", OPT_BOOL, &gpgpu_concurrent_kernel_sm, 
                 "Support concurrent kernels on a SM (default = disabled)", 
                 "0");
+
 }
 
 void gpgpu_sim_config::reg_options(option_parser_t opp)
@@ -1181,19 +1184,19 @@ void gpgpu_sim::gpu_print_stat()
            m_memory_sub_partition[i]->accumulate_L2cache_stats(l2_stats);
            m_memory_sub_partition[i]->get_L2cache_sub_stats(l2_css);
 
-           fprintf( stdout, "L2_cache_bank[%d]: Access = %u, Miss = %u, Miss_rate = %.3lf, Pending_hits = %u, Reservation_fails = %u\n",
+           fprintf( stdout, "L2_cache_bank[%d]: Access = %llu, Miss = %llu, Miss_rate = %.3lf, Pending_hits = %llu, Reservation_fails = %llu\n",
                     i, l2_css.accesses, l2_css.misses, (double)l2_css.misses / (double)l2_css.accesses, l2_css.pending_hits, l2_css.res_fails);
 
            total_l2_css += l2_css;
        }
        if (!m_memory_config->m_L2_config.disabled() && m_memory_config->m_L2_config.get_num_lines()) {
           //L2c_print_cache_stat();
-          printf("L2_total_cache_accesses = %u\n", total_l2_css.accesses);
-          printf("L2_total_cache_misses = %u\n", total_l2_css.misses);
+          printf("L2_total_cache_accesses = %llu\n", total_l2_css.accesses);
+          printf("L2_total_cache_misses = %llu\n", total_l2_css.misses);
           if(total_l2_css.accesses > 0)
               printf("L2_total_cache_miss_rate = %.4lf\n", (double)total_l2_css.misses/(double)total_l2_css.accesses);
-          printf("L2_total_cache_pending_hits = %u\n", total_l2_css.pending_hits);
-          printf("L2_total_cache_reservation_fails = %u\n", total_l2_css.res_fails);
+          printf("L2_total_cache_pending_hits = %llu\n", total_l2_css.pending_hits);
+          printf("L2_total_cache_reservation_fails = %llu\n", total_l2_css.res_fails);
           printf("L2_total_cache_breakdown:\n");
           l2_stats.print_stats(stdout, "L2_cache_stats_breakdown");
           printf("L2_total_cache_reservation_fail_breakdown:\n");
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index d4cc18f..c8dad89 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -62,8 +62,7 @@
 #define SAMPLELOG 222
 #define DUMPLOG 333
 
-
-
+extern tr1_hash_map<new_addr_type,unsigned> address_random_interleaving;
 
 
 enum dram_ctrl_t {
diff --git a/src/gpgpu-sim/icnt_wrapper.cc b/src/gpgpu-sim/icnt_wrapper.cc
index ee58ece..6e0950c 100644
--- a/src/gpgpu-sim/icnt_wrapper.cc
+++ b/src/gpgpu-sim/icnt_wrapper.cc
@@ -29,6 +29,8 @@
 #include <assert.h>
 #include "../intersim2/globals.hpp"
 #include "../intersim2/interconnect_interface.hpp"
+#include "local_interconnect.h"
+
 
 icnt_create_p                icnt_create;
 icnt_init_p                  icnt_init;
@@ -42,9 +44,13 @@ icnt_display_overall_stats_p icnt_display_overall_stats;
 icnt_display_state_p         icnt_display_state;
 icnt_get_flit_size_p         icnt_get_flit_size;
 
-int   g_network_mode;
+unsigned   g_network_mode;
 char* g_network_config_filename;
 
+
+struct inct_config g_inct_config;
+LocalInterconnect *g_localicnt_interface;
+
 #include "../option_parser.h"
 
 // Wrapper to intersim2 to accompany old icnt_wrapper
@@ -105,10 +111,78 @@ static unsigned intersim2_get_flit_size()
    return g_icnt_interface->GetFlitSize();
 }
 
+
+//////////////////////////////////////////////////////
+
+static void LocalInterconnect_create(unsigned int n_shader, unsigned int n_mem)
+{
+   g_localicnt_interface->CreateInterconnect(n_shader, n_mem);
+}
+
+static void LocalInterconnect_init()
+{
+   g_localicnt_interface->Init();
+}
+
+static bool LocalInterconnect_has_buffer(unsigned input, unsigned int size)
+{
+   return g_localicnt_interface->HasBuffer(input, size);
+}
+
+static void LocalInterconnect_push(unsigned input, unsigned output, void* data, unsigned int size)
+{
+   g_localicnt_interface->Push(input, output, data, size);
+}
+
+static void* LocalInterconnect_pop(unsigned output)
+{
+   return g_localicnt_interface->Pop(output);
+}
+
+static void LocalInterconnect_transfer()
+{
+   g_localicnt_interface->Advance();
+}
+
+static bool LocalInterconnect_busy()
+{
+   return g_localicnt_interface->Busy();
+}
+
+static void LocalInterconnect_display_stats()
+{
+   g_localicnt_interface->DisplayStats();
+}
+
+static void LocalInterconnect_display_overall_stats()
+{
+   g_localicnt_interface->DisplayOverallStats();
+}
+
+static void LocalInterconnect_display_state(FILE *fp)
+{
+   g_localicnt_interface->DisplayState(fp);
+}
+
+static unsigned LocalInterconnect_get_flit_size()
+{
+   return g_localicnt_interface->GetFlitSize();
+}
+
+
+///////////////////////////
+
 void icnt_reg_options( class OptionParser * opp )
 {
    option_parser_register(opp, "-network_mode", OPT_INT32, &g_network_mode, "Interconnection network mode", "1");
    option_parser_register(opp, "-inter_config_file", OPT_CSTR, &g_network_config_filename, "Interconnection network config file", "mesh");
+
+
+   //parameters for local xbar
+   option_parser_register(opp, "-inct_in_buffer_limit", OPT_UINT32, &g_inct_config.in_buffer_limit, "in_buffer_limit", "64");
+   option_parser_register(opp, "-inct_out_buffer_limit", OPT_UINT32, &g_inct_config.out_buffer_limit, "out_buffer_limit", "64");
+   option_parser_register(opp, "-inct_subnets", OPT_UINT32, &g_inct_config.subnets, "subnets", "2");
+
 }
 
 void icnt_wrapper_init()
@@ -129,6 +203,20 @@ void icnt_wrapper_init()
          icnt_display_state = intersim2_display_state;
          icnt_get_flit_size = intersim2_get_flit_size;
          break;
+      case LOCAL_XBAR:
+         g_localicnt_interface = LocalInterconnect::New(g_inct_config);
+         icnt_create     = LocalInterconnect_create;
+         icnt_init       = LocalInterconnect_init;
+         icnt_has_buffer = LocalInterconnect_has_buffer;
+         icnt_push       = LocalInterconnect_push;
+         icnt_pop        = LocalInterconnect_pop;
+         icnt_transfer   = LocalInterconnect_transfer;
+         icnt_busy       = LocalInterconnect_busy;
+         icnt_display_stats = LocalInterconnect_display_stats;
+         icnt_display_overall_stats = LocalInterconnect_display_overall_stats;
+         icnt_display_state = LocalInterconnect_display_state;
+         icnt_get_flit_size = LocalInterconnect_get_flit_size;
+         break;
       default:
          assert(0);
          break;
diff --git a/src/gpgpu-sim/icnt_wrapper.h b/src/gpgpu-sim/icnt_wrapper.h
index a4d123e..e1086f9 100644
--- a/src/gpgpu-sim/icnt_wrapper.h
+++ b/src/gpgpu-sim/icnt_wrapper.h
@@ -57,13 +57,16 @@ extern icnt_display_stats_p icnt_display_stats;
 extern icnt_display_overall_stats_p icnt_display_overall_stats;
 extern icnt_display_state_p icnt_display_state;
 extern icnt_get_flit_size_p icnt_get_flit_size;
-extern int g_network_mode;
+extern unsigned g_network_mode;
 
 enum network_mode {
    INTERSIM = 1,
+   LOCAL_XBAR = 2,
    N_NETWORK_MODE
 };
 
+
+
 void icnt_wrapper_init();
 void icnt_reg_options( class OptionParser * opp );
 
diff --git a/src/gpgpu-sim/local_interconnect.cc b/src/gpgpu-sim/local_interconnect.cc
new file mode 100644
index 0000000..66d6648
--- /dev/null
+++ b/src/gpgpu-sim/local_interconnect.cc
@@ -0,0 +1,301 @@
+// Copyright (c) 2019, Mahmoud Khairy
+// Purdue University
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// Redistributions in binary form must reproduce the above copyright notice, this
+// list of conditions and the following disclaimer in the documentation and/or
+// other materials provided with the distribution.
+// Neither the name of The University of British Columbia nor the names of its
+// contributors may be used to endorse or promote products derived from this
+// software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+#include <cmath>
+#include <utility>
+#include <algorithm>
+
+#include "local_interconnect.h"
+#include "mem_fetch.h"
+
+xbar_router::xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit)
+{
+	m_id=router_id;
+	router_type=m_type;
+	_n_mem = n_mem;
+	_n_shader = n_shader;
+	total_nodes = n_shader+n_mem;
+	in_buffers.resize(total_nodes);
+	out_buffers.resize(total_nodes);
+	next_node=0;
+	in_buffer_limit = m_in_buffer_limit;
+	out_buffer_limit = m_out_buffer_limit;
+	if(m_type == REQ_NET) {
+		active_in_buffers=n_shader;
+		active_out_buffers=n_mem;
+	}
+	else if(m_type == REPLY_NET) {
+		active_in_buffers=n_mem;
+		active_out_buffers=n_shader;
+	}
+
+	cycles = 0;
+	conflicts= 0;
+	out_buffer_full=0;
+	in_buffer_full=0;
+	out_buffer_util=0;
+	in_buffer_util=0;
+	packets_num=0;
+}
+
+
+xbar_router::~xbar_router()
+{
+
+}
+
+void xbar_router::Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size)
+{
+	assert(input_deviceID < total_nodes);
+	in_buffers[input_deviceID].push(Packet(data, output_deviceID));
+	packets_num++;
+}
+
+void* xbar_router::Pop(unsigned ouput_deviceID)
+{
+	assert(ouput_deviceID < total_nodes);
+	void* data = NULL;
+
+	if(!out_buffers[ouput_deviceID].empty()) {
+		data = out_buffers[ouput_deviceID].front().data;
+		out_buffers[ouput_deviceID].pop();
+	}
+
+	return data;
+}
+
+
+bool xbar_router::Has_Buffer_In(unsigned input_deviceID, unsigned size, bool update_counter){
+
+	assert(input_deviceID < total_nodes);
+
+	bool has_buffer = (in_buffers[input_deviceID].size() + size <= in_buffer_limit);
+	if(update_counter && !has_buffer)
+		in_buffer_full++;
+
+	return has_buffer;
+
+}
+
+bool xbar_router::Has_Buffer_Out(unsigned output_deviceID, unsigned size){
+	return (out_buffers[output_deviceID].size() + size <= out_buffer_limit);
+}
+
+void xbar_router::Advance() {
+	cycles++;
+
+	vector<bool> issued(total_nodes, false);
+
+	for(unsigned i=0; i<total_nodes; ++i){
+		unsigned node_id = (i+next_node)%total_nodes;
+
+		if(!in_buffers[node_id].empty()) {
+			Packet _packet = in_buffers[node_id].front();
+			//ensure that the outbuffer has space and not issued before in this cycle
+			if(Has_Buffer_Out(_packet.output_deviceID, 1)){
+				if(!issued[_packet.output_deviceID]) {
+					out_buffers[_packet.output_deviceID].push(_packet);
+					in_buffers[node_id].pop();
+					issued[_packet.output_deviceID]=true;
+				}
+				else
+					conflicts++;
+			}
+			else
+				out_buffer_full++;
+		}
+	}
+
+	next_node = (++next_node % total_nodes);
+
+	//collect some stats about buffer util
+	for(unsigned i=0; i<total_nodes; ++i){
+		in_buffer_util+=in_buffers[i].size();
+		out_buffer_util+=out_buffers[i].size();
+	}
+}
+
+bool xbar_router::Busy() const {
+
+	for(unsigned i=0; i<total_nodes; ++i){
+		if(!in_buffers[i].empty())
+			return true;
+
+		if(!out_buffers[i].empty())
+			return true;
+	}
+	return false;
+}
+
+
+////////////////////////////////////////////////////
+/////////////LocalInterconnect/////////////////////
+
+//assume all the packets are one flit
+#define LOCAL_INCT_FLIT_SIZE 40
+
+LocalInterconnect* LocalInterconnect::New(const struct inct_config& m_localinct_config)
+{
+
+	LocalInterconnect* icnt_interface = new LocalInterconnect(m_localinct_config);
+
+	return icnt_interface;
+}
+
+LocalInterconnect::LocalInterconnect(const struct inct_config& m_localinct_config): m_inct_config(m_localinct_config){
+	n_shader=0;
+	n_mem=0;
+	n_subnets = m_localinct_config.subnets;
+}
+
+LocalInterconnect::~LocalInterconnect(){
+	for (int i=0; i<m_inct_config.subnets; ++i) {
+		delete net[i];
+	}
+}
+
+void LocalInterconnect::CreateInterconnect(unsigned m_n_shader, unsigned m_n_mem){
+	n_shader = m_n_shader;
+	n_mem = m_n_mem;
+
+	net.resize(n_subnets);
+	for (unsigned i = 0; i < n_subnets; ++i) {
+		net[i] = new xbar_router( i, static_cast<Interconnect_type>(i), m_n_shader, m_n_mem, m_inct_config.in_buffer_limit, m_inct_config.out_buffer_limit );
+	}
+
+}
+
+
+void LocalInterconnect::Init() {
+	//empty
+	//there is nothing to do
+
+}
+
+void LocalInterconnect::Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size){
+
+	unsigned subnet;
+	if (n_subnets == 1) {
+		subnet = 0;
+	} else {
+		if (input_deviceID < n_shader ) {
+			subnet = 0;
+		} else {
+			subnet = 1;
+		}
+	}
+
+	// it should have free buffer
+	//assume all the packets have size of one
+	//no flits are implemented
+	assert(net[subnet]->Has_Buffer_In(input_deviceID, 1));
+
+	net[subnet]->Push(input_deviceID, output_deviceID, data, size);
+
+}
+
+void* LocalInterconnect::Pop(unsigned ouput_deviceID){
+
+	// 0-_n_shader-1 indicates reply(network 1), otherwise request(network 0)
+	int subnet = 0;
+	if (ouput_deviceID < n_shader)
+		subnet = 1;
+
+	return net[subnet]->Pop(ouput_deviceID);
+
+}
+
+void LocalInterconnect::Advance(){
+
+	for (unsigned i = 0; i < n_subnets; ++i) {
+		net[i]->Advance();
+	}
+
+}
+
+bool  LocalInterconnect::Busy() const{
+
+	for (unsigned i = 0; i < n_subnets; ++i) {
+		if(net[i]->Busy())
+			return true;
+	}
+	return false;
+}
+
+bool  LocalInterconnect::HasBuffer(unsigned deviceID, unsigned int size) const{
+
+	bool has_buffer = false;
+
+	if ((n_subnets>1) && deviceID >= n_shader) // deviceID is memory node
+		has_buffer = net[REPLY_NET]->Has_Buffer_In(deviceID, 1, true);
+	else
+		has_buffer = net[REQ_NET]->Has_Buffer_In(deviceID, 1, true);
+
+	return has_buffer;
+
+}
+
+void  LocalInterconnect::DisplayStats() const{
+
+	cout<<"Req_Network_injected_packets_num = "<<net[REQ_NET]->packets_num<<endl;
+	cout<<"Req_Network_cycles = "<<net[REQ_NET]->cycles<<endl;
+	cout<<"Req_Network_injected_packets_per_cycle = "<<(float)(net[REQ_NET]->packets_num) / (net[REQ_NET]->cycles)<<endl;
+	cout<<"Req_Network_conflicts_per_cycle = "<<(float)(net[REQ_NET]->conflicts) / (net[REQ_NET]->cycles)<<endl;
+	cout<<"Req_Network_in_buffer_full_per_cycle = "<<(float)(net[REQ_NET]->in_buffer_full) / (net[REQ_NET]->cycles)<<endl;
+	cout<<"Req_Network_in_buffer_avg_util = "<<((float)(net[REQ_NET]->in_buffer_util) / (net[REQ_NET]->cycles) / net[REQ_NET]->active_in_buffers)<<endl;
+	cout<<"Req_Network_out_buffer_full_per_cycle = "<<(float)(net[REQ_NET]->out_buffer_full) / (net[REQ_NET]->cycles)<<endl;
+	cout<<"Req_Network_out_buffer_avg_util = "<<((float)(net[REQ_NET]->out_buffer_util) / (net[REQ_NET]->cycles) / net[REQ_NET]->active_out_buffers)<<endl;
+
+	cout<<endl;
+	cout<<"Reply_Network_injected_packets_num = "<<net[REPLY_NET]->packets_num<<endl;
+	cout<<"Reply_Network_cycles = "<<net[REPLY_NET]->cycles<<endl;
+	cout<<"Reply_Network_injected_packets_per_cycle = "<<(float)(net[REPLY_NET]->packets_num) / (net[REPLY_NET]->cycles)<<endl;
+	cout<<"Reply_Network_conflicts_per_cycle = "<<(float)(net[REPLY_NET]->conflicts) / (net[REPLY_NET]->cycles)<<endl;
+	cout<<"Reply_Network_in_buffer_full_per_cycle = "<<(float)(net[REPLY_NET]->in_buffer_full) / (net[REPLY_NET]->cycles)<<endl;
+	cout<<"Reply_Network_in_buffer_avg_util = "<<((float)(net[REPLY_NET]->in_buffer_util) / (net[REPLY_NET]->cycles) / net[REPLY_NET]->active_in_buffers)<<endl;
+	cout<<"Reply_Network_out_buffer_full_per_cycle = "<<(float)(net[REPLY_NET]->out_buffer_full) / (net[REPLY_NET]->cycles)<<endl;
+	cout<<"Reply_Network_out_buffer_avg_util= "<<((float)(net[REPLY_NET]->out_buffer_util) / (net[REPLY_NET]->cycles) / net[REPLY_NET]->active_out_buffers)<<endl;
+
+}
+
+void  LocalInterconnect::DisplayOverallStats() const{
+
+}
+
+unsigned  LocalInterconnect::GetFlitSize() const{
+	return LOCAL_INCT_FLIT_SIZE;
+}
+
+void  LocalInterconnect::DisplayState(FILE* fp) const{
+
+	fprintf(fp, "GPGPU-Sim uArch: ICNT:Display State: Under implementation\n");
+}
+
diff --git a/src/gpgpu-sim/local_interconnect.h b/src/gpgpu-sim/local_interconnect.h
new file mode 100644
index 0000000..502c80d
--- /dev/null
+++ b/src/gpgpu-sim/local_interconnect.h
@@ -0,0 +1,127 @@
+// Copyright (c) 2019, Mahmoud Khairy
+// Purdue University
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+// Redistributions in binary form must reproduce the above copyright notice, this
+// list of conditions and the following disclaimer in the documentation and/or
+// other materials provided with the distribution.
+// Neither the name of The University of British Columbia nor the names of its
+// contributors may be used to endorse or promote products derived from this
+// software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef _LOCAL_INTERCONNECT_HPP_
+#define _LOCAL_INTERCONNECT_HPP_
+
+#include <vector>
+#include <queue>
+#include <iostream>
+#include <map>
+using namespace std;
+
+
+struct inct_config
+{
+
+	//config for local interconnect
+	unsigned   in_buffer_limit;
+	unsigned   out_buffer_limit;
+	unsigned   subnets;
+};
+
+enum Interconnect_type {
+	REQ_NET=0,
+	REPLY_NET=1
+};
+class xbar_router {
+
+public:
+	xbar_router(unsigned router_id, enum Interconnect_type m_type, unsigned n_shader, unsigned n_mem, unsigned m_in_buffer_limit, unsigned m_out_buffer_limit);
+	~xbar_router();
+	void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size);
+	void* Pop(unsigned ouput_deviceID);
+	void Advance();
+	bool Busy() const;
+	bool Has_Buffer_In(unsigned input_deviceID, unsigned size, bool update_counter=false);
+	bool Has_Buffer_Out(unsigned output_deviceID, unsigned size);
+
+	//some stats
+	unsigned long long cycles;
+	unsigned long long conflicts;
+	unsigned long long out_buffer_full;
+	unsigned long long out_buffer_util;
+	unsigned long long in_buffer_full;
+	unsigned long long in_buffer_util;
+	unsigned long long packets_num;
+
+private:
+	struct Packet{
+		Packet(void* m_data, unsigned m_output_deviceID) {
+			data = m_data;
+			output_deviceID = m_output_deviceID;
+		}
+		void* data;
+		unsigned output_deviceID;
+	};
+	vector<queue<Packet> > in_buffers;
+	vector<queue<Packet> > out_buffers;
+	unsigned _n_shader, _n_mem, total_nodes;
+	unsigned in_buffer_limit, out_buffer_limit;
+	unsigned next_node;
+	unsigned m_id;
+	enum Interconnect_type router_type;
+	unsigned active_in_buffers,active_out_buffers;
+
+	friend class LocalInterconnect;
+
+};
+
+class LocalInterconnect {
+public:
+	LocalInterconnect(const struct inct_config& m_localinct_config);
+	~LocalInterconnect();
+	static LocalInterconnect* New(const struct inct_config& m_inct_config);
+	void CreateInterconnect(unsigned n_shader, unsigned n_mem);
+
+	//node side functions
+	void Init();
+	void Push(unsigned input_deviceID, unsigned output_deviceID, void* data, unsigned int size);
+	void* Pop(unsigned ouput_deviceID);
+	void Advance();
+	bool Busy() const;
+	bool HasBuffer(unsigned deviceID, unsigned int size) const;
+	void DisplayStats() const;
+	void DisplayOverallStats() const;
+	unsigned GetFlitSize() const;
+
+	void DisplayState(FILE* fp) const;
+
+
+protected:
+
+	const inct_config& m_inct_config;
+
+	unsigned n_shader, n_mem;
+	unsigned n_subnets;
+	vector<xbar_router *> net;
+
+};
+
+#endif
+
+
diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc
index c5452b9..7f6cde9 100644
--- a/src/gpgpu-sim/mem_latency_stat.cc
+++ b/src/gpgpu-sim/mem_latency_stat.cc
@@ -366,7 +366,7 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk )
          m = 0;
          printf("\n");
       }
-      printf("total reads: %d\n", k);
+      printf("total dram reads = %d\n", k);
       if (min_bank_accesses)
          printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses);
       else
@@ -404,7 +404,7 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk )
          m = 0;
          printf("\n");
       }
-      printf("total reads: %d\n", k);
+      printf("total dram writes = %d\n", k);
       if (min_bank_accesses)
          printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses);
       else
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 2cecd0c..007ad42 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1893,6 +1893,17 @@ void tensor_core::issue( register_set& source_reg )
 	pipelined_simd_unit::issue(source_reg);
 }
 
+unsigned pipelined_simd_unit::get_active_lanes_in_pipeline(){
+	active_mask_t active_lanes;
+	active_lanes.reset();
+	 if(m_core->get_gpu()->get_config().g_power_simulation_enabled){
+		for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ){
+			if( !m_pipeline_reg[stage]->empty() )
+				active_lanes|=m_pipeline_reg[stage]->get_active_mask();
+		}
+	 }
+	return active_lanes.count();
+}
 
 void ldst_unit::active_lanes_in_pipeline(){
 	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
@@ -1946,13 +1957,13 @@ sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,sh
 }
 
 dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
-    : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core)
+    : pipelined_simd_unit(result_port,config,config->max_dp_latency,core)
 {
     m_name = "DP ";
 }
 
 int_unit::int_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
-    : pipelined_simd_unit(result_port,config,config->max_sp_latency,core)
+    : pipelined_simd_unit(result_port,config,config->max_int_latency,core)
 {
     m_name = "INT ";
 }
@@ -1993,19 +2004,25 @@ pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shade
     for( unsigned i=0; i < m_pipeline_depth; i++ ) 
 	m_pipeline_reg[i] = new warp_inst_t( config );
     m_core=core;
+    active_insts_in_pipeline=0;
 }
 
 void pipelined_simd_unit::cycle()
 {
     if( !m_pipeline_reg[0]->empty() ){
         m_result_port->move_in(m_pipeline_reg[0]);
+        assert(active_insts_in_pipeline > 0);
+        active_insts_in_pipeline--;
+    }
+    if(active_insts_in_pipeline){
+		for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ )
+			move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]);
     }
-    for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ )
-        move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]);
     if( !m_dispatch_reg->empty() ) {
         if( !m_dispatch_reg->dispatch_delay()){
             int start_stage = m_dispatch_reg->latency - m_dispatch_reg->initiation_interval;
             move_warp(m_pipeline_reg[start_stage],m_dispatch_reg);
+            active_insts_in_pipeline++;
         }
     }
     occupied >>=1;
@@ -2525,13 +2542,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1I_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1I_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1I_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1I_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1I_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1I_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1I_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1I_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1I_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1I_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 
     // L1D
@@ -2542,18 +2559,18 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
         for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++){
             m_cluster[i]->get_L1D_sub_stats(css);
 
-            fprintf( stdout, "\tL1D_cache_core[%d]: Access = %d, Miss = %d, Miss_rate = %.3lf, Pending_hits = %u, Reservation_fails = %u\n",
+            fprintf( stdout, "\tL1D_cache_core[%d]: Access = %llu, Miss = %llu, Miss_rate = %.3lf, Pending_hits = %llu, Reservation_fails = %llu\n",
                      i, css.accesses, css.misses, (double)css.misses / (double)css.accesses, css.pending_hits, css.res_fails);
 
             total_css += css;
         }
-        fprintf(fout, "\tL1D_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1D_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1D_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1D_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1D_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1D_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1D_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1D_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1D_total_cache_reservation_fails = %llu\n", total_css.res_fails);
         total_css.print_port_stats(fout, "\tL1D_cache"); 
     }
 
@@ -2566,13 +2583,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1C_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1C_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1C_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1C_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1C_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1C_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1C_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1C_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1C_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1C_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 
     // L1T
@@ -2584,13 +2601,13 @@ void gpgpu_sim::shader_print_cache_stats( FILE *fout ) const{
             m_cluster[i]->get_L1T_sub_stats(css);
             total_css += css;
         }
-        fprintf(fout, "\tL1T_total_cache_accesses = %u\n", total_css.accesses);
-        fprintf(fout, "\tL1T_total_cache_misses = %u\n", total_css.misses);
+        fprintf(fout, "\tL1T_total_cache_accesses = %llu\n", total_css.accesses);
+        fprintf(fout, "\tL1T_total_cache_misses = %llu\n", total_css.misses);
         if(total_css.accesses > 0){
             fprintf(fout, "\tL1T_total_cache_miss_rate = %.4lf\n", (double)total_css.misses / (double)total_css.accesses);
         }
-        fprintf(fout, "\tL1T_total_cache_pending_hits = %u\n", total_css.pending_hits);
-        fprintf(fout, "\tL1T_total_cache_reservation_fails = %u\n", total_css.res_fails);
+        fprintf(fout, "\tL1T_total_cache_pending_hits = %llu\n", total_css.pending_hits);
+        fprintf(fout, "\tL1T_total_cache_reservation_fails = %llu\n", total_css.res_fails);
     }
 }
 
@@ -2981,8 +2998,53 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const
     return result;
 }
 
+void shader_core_config::set_pipeline_latency() {
+
+		//calculate the max latency  based on the input
+
+		unsigned int_latency[5];
+		unsigned fp_latency[5];
+		unsigned dp_latency[5];
+		unsigned sfu_latency;
+		unsigned tensor_latency;
+
+			/*
+			 * [0] ADD,SUB
+			 * [1] MAX,Min
+			 * [2] MUL
+			 * [3] MAD
+			 * [4] DIV
+			 */
+			sscanf(opcode_latency_int, "%u,%u,%u,%u,%u",
+					&int_latency[0],&int_latency[1],&int_latency[2],
+					&int_latency[3],&int_latency[4]);
+			sscanf(opcode_latency_fp, "%u,%u,%u,%u,%u",
+					&fp_latency[0],&fp_latency[1],&fp_latency[2],
+					&fp_latency[3],&fp_latency[4]);
+			sscanf(opcode_latency_dp, "%u,%u,%u,%u,%u",
+					&dp_latency[0],&dp_latency[1],&dp_latency[2],
+					&dp_latency[3],&dp_latency[4]);
+			sscanf(opcode_latency_sfu, "%u",
+					&sfu_latency);
+			sscanf(opcode_latency_tensor, "%u",
+					&tensor_latency);
+
+		//all div operation are executed on sfu
+		//assume that the max latency are dp div or normal sfu_latency
+		max_sfu_latency = std::max(dp_latency[4],sfu_latency);
+		//assume that the max operation has the max latency
+		max_sp_latency = fp_latency[1];
+		max_int_latency = int_latency[1];
+		max_dp_latency = dp_latency[1];
+		max_tensor_core_latency = tensor_latency;
+
+}
+
 void shader_core_ctx::cycle()
 {
+	if(!isactive() && get_not_completed() == 0)
+		return;
+
 	m_stats->shader_cycles[m_sid]++;
     writeback();
     execute();
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 473d6b0..a0c2b63 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -55,7 +55,6 @@
 #include "traffic_breakdown.h"
 
 
-
 #define NO_OP_FLAG            0xFF
 
 /* READ_PACKET_SIZE:
@@ -1073,16 +1072,8 @@ public:
     //modifiers
     virtual void cycle();
     virtual void issue( register_set& source_reg );
-    virtual unsigned get_active_lanes_in_pipeline()
-    {
-    	active_mask_t active_lanes;
-    	active_lanes.reset();
-        for( unsigned stage=0; (stage+1)<m_pipeline_depth; stage++ ){
-        	if( !m_pipeline_reg[stage]->empty() )
-        		active_lanes|=m_pipeline_reg[stage]->get_active_mask();
-        }
-        return active_lanes.count();
-    }
+    virtual unsigned get_active_lanes_in_pipeline();
+
     virtual void active_lanes_in_pipeline() = 0;
 /*
     virtual void issue( register_set& source_reg )
@@ -1113,6 +1104,9 @@ protected:
     warp_inst_t **m_pipeline_reg;
     register_set *m_result_port;
     class shader_core_ctx *m_core;
+
+    unsigned active_insts_in_pipeline;
+
 };
 
 class sfu : public pipelined_simd_unit
@@ -1413,10 +1407,8 @@ struct shader_core_config : public core_config
         }
         max_warps_per_shader =  n_thread_per_shader/warp_size;
         assert( !(n_thread_per_shader % warp_size) );
-        max_sfu_latency = 512;
-        max_sp_latency = 32;
-        
-	max_tensor_core_latency = 64;
+
+        set_pipeline_latency();
         
 	m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone);
         m_L1T_config.init(m_L1T_config.m_config_string,FuncCachePreferNone);
@@ -1432,6 +1424,7 @@ struct shader_core_config : public core_config
     unsigned sid_to_cluster( unsigned sid ) const { return sid / n_simt_cores_per_cluster; }
     unsigned sid_to_cid( unsigned sid )     const { return sid % n_simt_cores_per_cluster; }
     unsigned cid_to_sid( unsigned cid, unsigned cluster_id ) const { return cluster_id*n_simt_cores_per_cluster + cid; }
+    void set_pipeline_latency();
 
 // data
     char *gpgpu_shader_core_pipeline_opt;
@@ -1506,7 +1499,9 @@ struct shader_core_config : public core_config
     bool sub_core_model;
     
     unsigned max_sp_latency;
+    unsigned max_int_latency;
     unsigned max_sfu_latency;
+    unsigned max_dp_latency;
     unsigned max_tensor_core_latency;
     
     unsigned n_simt_cores_per_cluster;
@@ -1523,7 +1518,8 @@ struct shader_core_config : public core_config
     //Jin: concurrent kernel on sm
     bool gpgpu_concurrent_kernel_sm;
 
-    bool adaptive_volta_cache_config;
+    bool adpative_volta_cache_config;
+
 };
 
 struct shader_core_stats_pod {
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 270bace..9e2bfa2 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -222,10 +222,12 @@ gpgpu_sim *gpgpu_ptx_sim_init_perf()
    read_parser_environment_variables();
    option_parser_t opp = option_parser_create();
 
-   icnt_reg_options(opp);
-   g_the_gpu_config.reg_options(opp); // register GPU microrachitecture options
    ptx_reg_options(opp);
    ptx_opcocde_latency_options(opp);
+
+   icnt_reg_options(opp);
+   g_the_gpu_config.reg_options(opp); // register GPU microrachitecture options
+
    option_parser_cmdline(opp, sg_argc, sg_argv); // parse configuration options
    fprintf(stdout, "GPGPU-Sim: Configuration options:\n\n");
    option_parser_print(opp, stdout);