Merge branch 'dev-purdue-integration' of https://github.rcac.purdue.edu/abdallm/gpgpu-sim_distribution into dev-purdue-integration-trace

21 files changed, 240 insertions, 48 deletions

diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index cec75f9..1b764e2 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -517,7 +517,14 @@ public:
     const struct textureReference* get_texref(const std::string &texname) const
     {
         std::map<std::string, const struct textureReference*>::const_iterator t=m_NameToTextureRef.find(texname);
-        assert( t != m_NameToTextureRef.end() );
+        if( t == m_NameToTextureRef.end() ) {
+	  // search for :: prefixed names
+	  std::string temp("::" + texname);
+	  t=m_NameToTextureRef.find(temp);
+	}
+
+	assert(t != m_NameToTextureRef.end());
+
         return t->second;
     }
     const struct cudaArray* get_texarray( const struct textureReference *texref ) const
@@ -568,6 +575,7 @@ struct gpgpu_ptx_sim_info
    int cmem;
    int gmem;
    int regs;
+   unsigned maxthreads;
    unsigned ptx_version;
    unsigned sm_target;
 };
diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h
index 4721e8a..f88c526 100644
--- a/src/cuda-sim/cuda-math.h
+++ b/src/cuda-sim/cuda-math.h
@@ -67,6 +67,8 @@
 #ifndef CUDA_MATH
 #define CUDA_MATH
 
+#include <cmath>
+
 // cuda math implementations
 #undef max
 #undef min
@@ -321,7 +323,7 @@ float __internal_accurate_fdividef(float a, float b)
 float __saturatef(float a)
 {
    float b; 
-   if (isnan(a)) b = 0.0f; 
+   if (std::isnan(a)) b = 0.0f; 
    else if (a >= 1.0f) b = 1.0f;
    else if (a <= 0.0f) b = 0.0f; 
    else b = a; 
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 2f166aa..9246613 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -1159,13 +1159,13 @@ void function_info::finalize( memory_space *param_mem )
       // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over
       //Jin: copy parameter using aligned rules
       const size_t word_size = 4; 
-      param_address = (param_address + size - 1) / size * size; //aligned with size 
+      //param_address = (param_address + size - 1) / size * size; //aligned with size TODO: align not correct
       for (size_t idx = 0; idx < size; idx += word_size) {
          const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic
          param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); 
       }
       unsigned offset = p.get_offset();
-      assert(offset == param_address);
+      //assert(offset == param_address);
       param->set_address(param_address);
       param_address += size;
    }
@@ -1432,7 +1432,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
    if ( (g_ptx_sim_num_insn % 100000) == 0 ) {
       dim3 ctaid = get_ctaid();
       dim3 tid = get_tid();
-      printf("GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n",
+      DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n",
              g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z );
       fflush(stdout);
    }
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 011c285..e3b8970 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -33,6 +33,7 @@
 #include "ptx.tab.h"
 #include <stdlib.h>
 #include <math.h>
+#include <cmath>
 #include <fenv.h>
 #include "cuda-math.h"
 #include "../abstract_hardware_model.h"
@@ -1961,7 +1962,7 @@ ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign,
       y.f64 = x.f64;
       break; 
    }
-   if (isnan(y.f64)) {
+   if (std::isnan(y.f64)) {
       y.u64 = 0xfff8000000000000ull;
    } else if (saturation_mode) {
       y.f64 = cuda_math::__saturatef(y.f64); 
@@ -2086,7 +2087,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
       }
    }
    if ((type == F64_TYPE)||(type == FF64_TYPE)) {
-      if (isnan(data.f64)) {
+      if (std::isnan(data.f64)) {
          data.u64 = 0xfff8000000000000ull;
       }
    }
@@ -2648,12 +2649,12 @@ void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry
 
 bool isNaN(float x)
 {
-   return isnan(x);
+   return std::isnan(x);
 }
 
 bool isNaN(double x)
 {
-   return isnan(x);
+   return std::isnan(x);
 }
 
 void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) 
diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y
index e00aa4b..c0c58a6 100644
--- a/src/cuda-sim/ptx.y
+++ b/src/cuda-sim/ptx.y
@@ -228,7 +228,8 @@ function_defn: function_decl { set_symtab($1); func_header(".skip"); } statement
 
 block_spec: MAXNTID_DIRECTIVE INT_OPERAND COMMA INT_OPERAND COMMA INT_OPERAND {func_header_info_int(".maxntid", $2);
 										func_header_info_int(",", $4);
-										func_header_info_int(",", $6); }
+										func_header_info_int(",", $6); 
+                                                                                maxnt_id($2, $4, $6);}
 	| MINNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".minnctapersm", $2); printf("GPGPU-Sim: Warning: .minnctapersm ignored. \n"); }
 	| MAXNCTAPERSM_DIRECTIVE INT_OPERAND { func_header_info_int(".maxnctapersm", $2); printf("GPGPU-Sim: Warning: .maxnctapersm ignored. \n"); }
 	;
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index 8ebdcf8..ee36957 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -222,6 +222,7 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio
    } else {
       *func_info = new function_info(entry_point);
       (*func_info)->set_name(name);
+      (*func_info)->set_maxnt_id(0);
       m_function_info_lookup[key] = *func_info;
    }
 
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 9ad1571..36ef3d5 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -1245,6 +1245,7 @@ public:
 
    const struct gpgpu_ptx_sim_info* get_kernel_info () const
    {
+      assert (m_kernel_info.maxthreads == maxnt_id);
       return &m_kernel_info;
    }
 
@@ -1252,6 +1253,8 @@ public:
       m_kernel_info = info;
       m_kernel_info.ptx_version = 10*get_ptx_version().ver();
       m_kernel_info.sm_target = get_ptx_version().target();
+      // THIS DEPENDS ON ptxas being called after the PTX is parsed.
+      m_kernel_info.maxthreads = maxnt_id;
    }
    symbol_table *get_symtab()
    {
@@ -1275,7 +1278,11 @@ public:
    }
    bool is_entry_point() const { return m_entry_point; }
 
+   void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;}
+   unsigned get_maxnt_id() { return maxnt_id;}
+
 private:
+   unsigned maxnt_id;
    unsigned m_uid;
    unsigned m_local_mem_framesize;
    bool m_entry_point;
diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc
index baa3bcd..e5731a8 100644
--- a/src/cuda-sim/ptx_parser.cc
+++ b/src/cuda-sim/ptx_parser.cc
@@ -969,6 +969,10 @@ void target_header3(char* a, char* b, char* c)
    g_global_symbol_table->set_sm_target(a,b,c);
 }
 
+void maxnt_id(int x, int y, int z) {
+  g_func_info->set_maxnt_id(x * y * z);
+}
+
 void func_header(const char* a) {} //intentional dummy function
 void func_header_info(const char* a) {} //intentional dummy function
 void func_header_info_int(const char* a, int b) {} //intentional dummy function
diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h
index 32f3903..13042e1 100644
--- a/src/cuda-sim/ptx_parser.h
+++ b/src/cuda-sim/ptx_parser.h
@@ -93,6 +93,7 @@ void change_double_operand_type( int addr_type );
 void change_operand_neg( );
 void set_immediate_operand_type( );
 void version_header(double a);
+void maxnt_id(int x, int y, int z);
 
 //Jin: handle instructino group for cdp
 void start_inst_group();
diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc
index 422576d..cfd90ec 100644
--- a/src/gpgpu-sim/addrdec.cc
+++ b/src/gpgpu-sim/addrdec.cc
@@ -62,6 +62,9 @@ void linear_to_raw_address_translation::addrdec_setoption(option_parser_t opp)
    option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, 
                "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits",
                "0");
+   option_parser_register(opp, "-memory_partition_indexing", OPT_UINT32, &memory_partition_indexing,
+                  "0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing",
+                  "0");
 }
 
 new_addr_type linear_to_raw_address_translation::partition_address( new_addr_type addr ) const 
@@ -103,6 +106,51 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_
       tlx->burst= addrdec_packbits(addrdec_mask[BURST], rest_of_addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]);
    }
 
+    switch(memory_partition_indexing){
+		case CONSECUTIVE:
+			//Do nothing
+			break;
+		case BITWISE_PERMUTATION:
+			assert(!gap);
+			tlx->chip = (tlx->chip) ^ (tlx->row & (m_n_channel-1));
+			assert(tlx->chip < m_n_channel);
+			break;
+		case IPOLY:
+		    /*
+			* Set Indexing function from "Pseudo-randomly interleaved memory."
+			* Rau, B. R et al.
+			* ISCA 1991
+			*
+			* equations are adopted from:
+			* "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme."
+			* Khairy et al.
+			* IEEE TPDS 2017.
+			*/
+			if(m_n_channel == 32) {
+				std::bitset<64> a(tlx->row);
+				std::bitset<5> chip(tlx->chip);
+				chip[0] = a[13]^a[12]^a[11]^a[10]^a[9]^a[6]^a[5]^a[3]^a[0]^chip[0];
+				chip[1] = a[14]^a[13]^a[12]^a[11]^a[10]^a[7]^a[6]^a[4]^a[1]^chip[1];
+				chip[2] = a[14]^a[10]^a[9]^a[8]^a[7]^a[6]^a[3]^a[2]^a[0]^chip[2];
+				chip[3] = a[11]^a[10]^a[9]^a[8]^a[7]^a[4]^a[3]^a[1]^chip[3]; 
+				chip[4] = a[12]^a[11]^a[10]^a[9]^a[8]^a[5]^a[4]^a[2]^chip[4];
+				tlx->chip = chip.to_ulong();
+				
+			}
+			else{ /* Else incorrect number of channels for the hashing function */
+            assert("\nGPGPU-Sim memory_partition_indexing error: The number of channels should be "
+                    "32 for the hashing IPOLY index function.\n" && 0);
+			}
+			assert(tlx->chip < m_n_channel);
+			break;
+		case CUSTOM:
+			/* No custom set function implemented */
+			break;
+		default:
+			 assert("\nUndefined set index function.\n" && 0);
+			 break;
+	}
+
    // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID
    unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; 
    tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel
diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h
index fd9af8d..a18ff63 100644
--- a/src/gpgpu-sim/addrdec.h
+++ b/src/gpgpu-sim/addrdec.h
@@ -35,6 +35,13 @@
 
 #include "../abstract_hardware_model.h"
 
+enum partition_index_function{
+	CONSECUTIVE = 0,
+	BITWISE_PERMUTATION,
+	IPOLY,
+    CUSTOM
+};
+
 struct addrdec_t {
    void print( FILE *fp ) const;
     
@@ -72,6 +79,7 @@ private:
 
    const char *addrdec_option;
    int gpgpu_mem_address_mask;
+   partition_index_function memory_partition_indexing;
    bool run_test; 
 
    int ADDR_CHIP_S;
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index a57508c..92aa819 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -204,7 +204,7 @@ dram_req_t::dram_req_t( class mem_fetch *mf, unsigned banks, unsigned dram_bnk_i
    }
     else if(dram_bnk_indexing_policy == 1) {
     	 int lbank = log2(banks);
-    	 bk  = tlx.bk ^ (((1<<lbank)-1) & tlx.row);
+    	 bk  = tlx.bk ^ (tlx.row & ((1<<lbank)-1));
     }
     else
     	assert(1);
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index 0602e20..75ec00a 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -74,7 +74,7 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{
         /*
         * Set Indexing function from "A Detailed GPU Cache Model Based on Reuse Distance Theory"
         * Cedric Nugteren et al.
-        * ISCA 2014
+        * HPCA 2014
         */
         if(m_nset == 32 || m_nset == 64){
             // Lower xor value is bits 7-11
@@ -97,6 +97,36 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{
         }
         break;
 
+    case HASH_IPOLY_FUNCTION:
+    	/*
+		* Set Indexing function from "Pseudo-randomly interleaved memory."
+		* Rau, B. R et al.
+		* ISCA 1991
+		*
+		* "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu cache management scheme."
+		* Khairy et al.
+		* IEEE TPDS 2017.
+    	*/
+    	if(m_nset == 32 || m_nset == 64){
+		std::bitset<64> a(addr);
+		std::bitset<6> index;
+		index[0] = a[25]^a[24]^a[23]^a[22]^a[21]^a[18]^a[17]^a[15]^a[12]^a[7]; //10
+		index[1] = a[26]^a[25]^a[24]^a[23]^a[22]^a[19]^a[18]^a[16]^a[13]^a[8]; //10
+		index[2] = a[26]^a[22]^a[21]^a[20]^a[19]^a[18]^a[15]^a[14]^a[12]^a[9]; //10
+		index[3] = a[23]^a[22]^a[21]^a[20]^a[19]^a[16]^a[15]^a[13]^a[10]; //9
+		index[4] = a[24]^a[23]^a[22]^a[21]^a[20]^a[17]^a[16]^a[14]^a[11]; //9
+
+		 if(m_nset == 64)
+			 index[5] = a[12];
+
+		set_index = index.to_ulong();
+
+    	}else{ /* Else incorrect number of sets for the hashing function */
+    	            assert("\nGPGPU-Sim cache configuration error: The number of sets should be "
+    	                    "32 or 64 for the hashing set index function.\n" && 0);
+    	 }
+        break;
+
     case CUSTOM_SET_FUNCTION:
         /* No custom set function implemented */
         break;
@@ -104,6 +134,10 @@ unsigned l1d_cache_config::set_index(new_addr_type addr) const{
     case LINEAR_SET_FUNCTION:
         set_index = (addr >> m_line_sz_log2) & (m_nset-1);
         break;
+
+    default:
+    	 assert("\nUndefined set index function.\n" && 0);
+    	 break;
     }
 
     // Linear function selected or custom set index function not implemented
@@ -658,15 +692,13 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
     std::string m_cache_name = cache_name;
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
         for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) {
-            if(m_stats[type][status] > 0){
-                fprintf(fout, "\t%s[%s][%s] = %u\n",
-                    m_cache_name.c_str(),
-                    mem_access_type_str((enum mem_access_type)type),
-                    cache_request_status_str((enum cache_request_status)status),
-                    m_stats[type][status]);
-                if(status != RESERVATION_FAIL)
-                	 total_access[type]+= m_stats[type][status];
-            }
+            fprintf(fout, "\t%s[%s][%s] = %u\n",
+                m_cache_name.c_str(),
+                mem_access_type_str((enum mem_access_type)type),
+                cache_request_status_str((enum cache_request_status)status),
+                m_stats[type][status]);
+            if(status != RESERVATION_FAIL)
+            	 total_access[type]+= m_stats[type][status];
         }
     }
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
@@ -962,7 +994,7 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a
         m_mshrs.add(mshr_addr,mf);
         m_extra_mf_fields[mf] = extra_mf_fields(mshr_addr,mf->get_addr(),cache_index, mf->get_data_size(), m_config);
         mf->set_data_size( m_config.get_atom_sz() );
-        mf->set_addr( block_addr );
+        mf->set_addr( mshr_addr );
         m_miss_queue.push_back(mf);
         mf->set_status(m_miss_queue_status,time);
         if(!wa)
@@ -1432,7 +1464,7 @@ data_cache::process_tag_probe( bool wr,
             access_status = (this->*m_wr_hit)( addr,
                                       cache_index,
                                       mf, time, events, probe_status );
-        }else if ( probe_status != RESERVATION_FAIL ) {
+        }else if ( (probe_status != RESERVATION_FAIL) || (probe_status == RESERVATION_FAIL && m_config.m_write_alloc_policy == NO_WRITE_ALLOCATE) ) {
             access_status = (this->*m_wr_miss)( addr,
                                        cache_index,
                                        mf, time, events, probe_status );
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index 0d07878..d2b7757 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -39,7 +39,7 @@
 #include <iostream>
 
 enum cache_block_state {
-    INVALID,
+    INVALID=0,
     RESERVED,
     VALID,
     MODIFIED
@@ -125,6 +125,7 @@ struct cache_block_t {
     virtual unsigned get_modified_size() = 0;
     virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)=0;
     virtual bool is_readable(mem_access_sector_mask_t sector_mask)=0;
+    virtual void print_status()=0;
     virtual ~cache_block_t() {}
 
 
@@ -144,7 +145,7 @@ struct line_cache_block: public cache_block_t  {
 	        m_set_modified_on_fill = false;
 	        m_readable = true;
 	    }
-	    void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask )
+	    void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask)
 	    {
 	        m_tag=tag;
 	        m_block_addr=block_addr;
@@ -220,6 +221,9 @@ struct line_cache_block: public cache_block_t  {
 		virtual bool is_readable(mem_access_sector_mask_t sector_mask) {
 			return m_readable;
 		}
+		virtual void print_status() {
+			 printf("m_block_addr is %u, status = %u\n", m_block_addr, m_status);
+		}
 
 
 private:
@@ -401,6 +405,11 @@ struct sector_cache_block : public cache_block_t {
 		return modified * SECTOR_SIZE;
 	}
 
+    virtual void print_status() {
+    	 printf("m_block_addr is %u, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]);
+    }
+
+
 private:
     unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE];
     unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE];
@@ -455,8 +464,10 @@ enum mshr_config_t {
 };
 
 enum set_index_function{
-    FERMI_HASH_SET_FUNCTION = 0,
-    LINEAR_SET_FUNCTION,
+	LINEAR_SET_FUNCTION = 0,
+	BITWISE_XORING_FUNCTION,
+	HASH_IPOLY_FUNCTION,
+	FERMI_HASH_SET_FUNCTION,
     CUSTOM_SET_FUNCTION
 };
 
@@ -578,6 +589,7 @@ public:
 
         switch(sif){
         case 'H': m_set_index_function = FERMI_HASH_SET_FUNCTION; break;
+        case 'P': m_set_index_function = HASH_IPOLY_FUNCTION; break;
         case 'C': m_set_index_function = CUSTOM_SET_FUNCTION; break;
         case 'L': m_set_index_function = LINEAR_SET_FUNCTION; break;
         default: exit_parse_error();
@@ -1030,7 +1042,7 @@ protected:
     std::string m_name;
     cache_config &m_config;
     tag_array*  m_tag_array;
-    mshr_table m_mshrs;
+	mshr_table m_mshrs;
     std::list<mem_fetch*> m_miss_queue;
     enum mem_fetch_status m_miss_queue_status;
     mem_fetch_interface *m_memport;
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 17f1714..c5d4464 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -36,6 +36,7 @@
 
 
 #include "shader.h"
+#include "shader_trace.h"
 #include "dram.h"
 #include "mem_fetch.h"
 
@@ -87,6 +88,14 @@ unsigned long long  gpu_tot_sim_cycle = 0;
 // performance counter for stalls due to congestion.
 unsigned int gpu_stall_dramfull = 0; 
 unsigned int gpu_stall_icnt2sh = 0;
+unsigned long long partiton_reqs_in_parallel = 0;
+unsigned long long partiton_reqs_in_parallel_total = 0;
+unsigned long long partiton_reqs_in_parallel_util = 0;
+unsigned long long partiton_reqs_in_parallel_util_total = 0;
+unsigned long long  gpu_sim_cycle_parition_util = 0;
+unsigned long long  gpu_tot_sim_cycle_parition_util = 0;
+unsigned long long partiton_replys_in_parallel = 0;
+unsigned long long partiton_replys_in_parallel_total = 0;
 
 /* Clock Domains */
 
@@ -245,7 +254,7 @@ void shader_core_config::reg_options(class OptionParser * opp)
                    "per-shader L1 data cache config "
                    " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                    "none" );
-    option_parser_register(opp, "-gpgpu_cache:dl1PreShared", OPT_CSTR, &m_L1D_config.m_config_stringPrefShared,
+    option_parser_register(opp, "-gpgpu_cache:dl1PrefShared", OPT_CSTR, &m_L1D_config.m_config_stringPrefShared,
                    "per-shader L1 data cache config "
                    " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                    "none" );
@@ -268,6 +277,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, 
                  "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)",
                  "8192");
+    option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation,
+                 "gpgpu_ignore_resources_limitation (default 0)",
+                 "0");
     option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, 
                  "Maximum number of concurrent CTAs in shader (default 8)",
                  "8");
@@ -783,6 +795,10 @@ void gpgpu_sim::init()
     gpu_sim_insn = 0;
     last_gpu_sim_insn = 0;
     m_total_cta_launched=0;
+    partiton_reqs_in_parallel = 0;
+    partiton_replys_in_parallel = 0;
+    partiton_reqs_in_parallel_util = 0;
+    gpu_sim_cycle_parition_util = 0;
 
     reinit_clock_domains();
     set_param_gpgpu_num_shaders(m_config.num_shader());
@@ -819,8 +835,16 @@ void gpgpu_sim::update_stats() {
     gpu_tot_sim_cycle += gpu_sim_cycle;
     gpu_tot_sim_insn += gpu_sim_insn;
     gpu_tot_issued_cta += m_total_cta_launched;
+    partiton_reqs_in_parallel_total += partiton_reqs_in_parallel;
+    partiton_replys_in_parallel_total += partiton_replys_in_parallel;
+    partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util;
+    gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ;
 
     gpu_sim_cycle = 0;
+    partiton_reqs_in_parallel = 0;
+    partiton_replys_in_parallel = 0;
+    partiton_reqs_in_parallel_util = 0;
+    gpu_sim_cycle_parition_util = 0;
     gpu_sim_insn = 0;
     m_total_cta_launched = 0;
 }
@@ -1004,6 +1028,21 @@ void gpgpu_sim::gpu_print_stat()
    printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull);
    printf("gpu_stall_icnt2sh    = %d\n", gpu_stall_icnt2sh );
 
+   printf("partiton_reqs_in_parallel = %lld\n", partiton_reqs_in_parallel);
+   printf("partiton_reqs_in_parallel_total    = %lld\n", partiton_reqs_in_parallel_total );
+   printf("partiton_level_parallism = %12.4f\n", (float)partiton_reqs_in_parallel / gpu_sim_cycle);
+   printf("partiton_level_parallism_total  = %12.4f\n", (float)(partiton_reqs_in_parallel+partiton_reqs_in_parallel_total) / (gpu_tot_sim_cycle+gpu_sim_cycle) );
+   printf("partiton_reqs_in_parallel_util = %lld\n", partiton_reqs_in_parallel_util);
+   printf("partiton_reqs_in_parallel_util_total    = %lld\n", partiton_reqs_in_parallel_util_total );
+   printf("gpu_sim_cycle_parition_util = %lld\n", gpu_sim_cycle_parition_util);
+   printf("gpu_tot_sim_cycle_parition_util    = %lld\n", gpu_tot_sim_cycle_parition_util );
+   printf("partiton_level_parallism_util = %12.4f\n", (float)partiton_reqs_in_parallel_util / gpu_sim_cycle_parition_util);
+   printf("partiton_level_parallism_util_total  = %12.4f\n", (float)(partiton_reqs_in_parallel_util+partiton_reqs_in_parallel_util_total) / (gpu_sim_cycle_parition_util+gpu_tot_sim_cycle_parition_util) );
+   printf("partiton_replys_in_parallel = %lld\n", partiton_replys_in_parallel);
+   printf("partiton_replys_in_parallel_total    = %lld\n", partiton_replys_in_parallel_total );
+   printf("L2_BW  = %12.4f GB/Sec\n", ((float)(partiton_replys_in_parallel * 32) / (gpu_sim_cycle * m_config.icnt_period)) / 1000000000);
+   printf("L2_BW_total  = %12.4f GB/Sec\n", ((float)((partiton_replys_in_parallel+partiton_replys_in_parallel_total) * 32) / ((gpu_tot_sim_cycle+gpu_sim_cycle) * m_config.icnt_period)) / 1000000000 );
+
    time_t curr_time;
    time(&curr_time);
    unsigned long long elapsed_time = MAX( curr_time - g_simulation_starttime, 1 );
@@ -1218,8 +1257,8 @@ bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t & k, bool occu
        m_occupied_regs += (padded_cta_size * ((kernel_info->regs+3)&~3));
        m_occupied_ctas++;
 
-       printf("GPGPU-Sim uArch: Shader %d occupied %d threads, %d shared mem, %d registers, %d ctas\n",
-            m_sid, m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas);  
+      SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Occupied %d threads, %d shared mem, %d registers, %d ctas\n",
+            m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas);  
    }
 
    return true;
@@ -1344,8 +1383,8 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     m_n_active_cta++;
 
     shader_CTA_count_log(m_sid, 1);
-    printf("GPGPU-Sim uArch: core:%3d, cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", 
-        m_sid, free_cta_hw_id, start_thread, end_thread, gpu_sim_cycle, gpu_tot_sim_cycle );
+    SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", 
+        free_cta_hw_id, start_thread, end_thread, gpu_sim_cycle, gpu_tot_sim_cycle );
 
 }
 
@@ -1409,6 +1448,7 @@ void gpgpu_sim::cycle()
       for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) 
          m_cluster[i]->icnt_cycle(); 
    }
+    unsigned partiton_replys_in_parallel_per_cycle = 0;
     if (clock_mask & ICNT) {
         // pop from memory controller to interconnect
         for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) {
@@ -1421,6 +1461,7 @@ void gpgpu_sim::cycle()
                     mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle);
                     ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size );
                     m_memory_sub_partition[i]->pop();
+                    partiton_replys_in_parallel_per_cycle++;
                 } else {
                     gpu_stall_icnt2sh++;
                 }
@@ -1429,6 +1470,7 @@ void gpgpu_sim::cycle()
             }
         }
     }
+    partiton_replys_in_parallel += partiton_replys_in_parallel_per_cycle;
 
    if (clock_mask & DRAM) {
       for (unsigned i=0;i<m_memory_config->m_n_mem;i++){
@@ -1441,6 +1483,7 @@ void gpgpu_sim::cycle()
    }
 
    // L2 operations follow L2 clock domain
+   unsigned partiton_reqs_in_parallel_per_cycle = 0;
    if (clock_mask & L2) {
        m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].clear();
       for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) {
@@ -1452,11 +1495,17 @@ void gpgpu_sim::cycle()
           } else {
               mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) );
               m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle );
+              partiton_reqs_in_parallel_per_cycle++;
           }
           m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle);
           m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]);
        }
    }
+   partiton_reqs_in_parallel += partiton_reqs_in_parallel_per_cycle;
+   if(partiton_reqs_in_parallel_per_cycle > 0){
+	   partiton_reqs_in_parallel_util += partiton_reqs_in_parallel_per_cycle;
+	   gpu_sim_cycle_parition_util++;
+   }
 
    if (clock_mask & ICNT) {
       icnt_transfer();
@@ -1543,7 +1592,8 @@ void gpgpu_sim::cycle()
             hrs     = elapsed_time/3600 - 24*days;
             minutes = elapsed_time/60 - 60*(hrs + 24*days);
             sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days));
-            printf("GPGPU-Sim uArch: cycles simulated: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
+
+            DPRINTF(LIVENESS, "GPGPU-Sim uArch: cycles simulated: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
                    gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, 
                    (double)gpu_sim_insn/(double)gpu_sim_cycle,
                    (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time),
diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc
index 35d6d84..c5452b9 100644
--- a/src/gpgpu-sim/mem_latency_stat.cc
+++ b/src/gpgpu-sim/mem_latency_stat.cc
@@ -230,7 +230,9 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk )
       if (num_mfs) {
          printf("averagemflatency = %lld \n", mf_total_lat/num_mfs);
          printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs);
-         printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num);
+         if(tot_mrq_num)
+        	 printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num);
+
          printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs);
       }
       printf("mrq_lat_table:");
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index bf482fb..d2f40a1 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -2088,12 +2088,12 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t
       m_barriers.deallocate_barrier(cta_num);
       shader_CTA_count_unlog(m_sid, 1);
 
-      printf("GPGPU-Sim uArch: Shader %d finished CTA #%d (%lld,%lld), %u CTAs running\n", m_sid, cta_num, gpu_sim_cycle, gpu_tot_sim_cycle,
-             m_n_active_cta );
+     SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Finished CTA #%d (%lld,%lld), %u CTAs running\n",
+        cta_num, gpu_sim_cycle, gpu_tot_sim_cycle, m_n_active_cta);
 
       if( m_n_active_cta == 0 ) {
-          printf("GPGPU-Sim uArch: Shader %u empty (last released kernel %u \'%s\').\n", m_sid, kernel->get_uid(),
-                 kernel->name().c_str() );
+        SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Empty (last released kernel %u \'%s\').\n",
+            kernel->get_uid(), kernel->name().c_str());
           fflush(stdout);
 
           //Shader can only be empty when no more cta are dispatched
@@ -2108,8 +2108,10 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t
       kernel->dec_running();
       if( !m_gpu->kernel_more_cta_left(kernel) ) {
           if( !kernel->running() ) {
-              printf("GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(),
-                kernel->name().c_str(), m_sid );
+              SHADER_DPRINTF(LIVENESS,
+                "GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(),
+                kernel->name().c_str(), m_sid);
+
               if(m_kernel == kernel)
                 m_kernel = NULL;
               m_gpu->set_kernel_done( kernel );
@@ -2614,6 +2616,10 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const
     assert( result <= MAX_CTA_PER_SHADER );
     if (result < 1) {
        printf ("GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader has.\n");
+       if(gpgpu_ignore_resources_limitation) {
+    	   printf ("GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore the ERROR!\n");
+    	   return 1;
+       }
        abort();
     }
 
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 5b41c06..ae22eaa 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1386,6 +1386,7 @@ struct shader_core_config : public core_config
     unsigned gpgpu_num_reg_banks;
     bool gpgpu_reg_bank_use_warp_id;
     bool gpgpu_local_mem_map;
+    bool gpgpu_ignore_resources_limitation;
     
     unsigned max_sp_latency;
     unsigned max_sfu_latency;
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index ad4587a..52e2f5e 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -93,8 +93,15 @@ bool g_sim_active = false;
 bool g_sim_done = true;
 bool break_limit = false;
 
+static void termination_callback()
+{
+    printf("GPGPU-Sim: *** exit detected ***\n");
+    fflush(stdout);
+}
+
 void *gpgpu_sim_thread_concurrent(void*)
 {
+    atexit(termination_callback);
     // concurrent kernel execution simulation thread
     do {
         if(g_debug_execution >= 3) {
@@ -156,8 +163,8 @@ void *gpgpu_sim_thread_concurrent(void*)
            printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n");
            fflush(stdout);
         }
-        g_the_gpu->print_stats();
         if(sim_cycles) {
+            g_the_gpu->print_stats();
             g_the_gpu->update_stats();
             print_simulation_time();
         }
@@ -165,10 +172,10 @@ void *gpgpu_sim_thread_concurrent(void*)
         g_sim_active = false;
         pthread_mutex_unlock(&g_sim_lock);
     } while( !g_sim_done );
-    if(g_debug_execution >= 3) {
-       printf("GPGPU-Sim: *** simulation thread exiting ***\n");
-       fflush(stdout);
-    }
+
+    printf("GPGPU-Sim: *** simulation thread exiting ***\n");
+    fflush(stdout);
+
     if(break_limit) {
     	printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n");
     	exit(1);
diff --git a/src/gpuwattch/makefile b/src/gpuwattch/makefile
index ab718cc..354c9ec 100644
--- a/src/gpuwattch/makefile
+++ b/src/gpuwattch/makefile
@@ -11,10 +11,10 @@ opt: $(TAR).mk obj_opt
 	@$(MAKE) TAG=opt -C . -f $(TAR).mk
 
 obj_dbg:
-	mkdir $@
+	mkdir -p $@
 
 obj_opt:
-	mkdir $@
+	mkdir -p $@
 
 depend:
 	@$(MAKE) TAG=opt -C . -f $(TAR).mk depend
diff --git a/src/trace_streams.tup b/src/trace_streams.tup
index 3455a00..074c7c8 100644
--- a/src/trace_streams.tup
+++ b/src/trace_streams.tup
@@ -31,5 +31,6 @@ TS_TUP_BEGIN( trace_streams_type )
     TS_TUP( MEMORY_PARTITION_UNIT ),
     TS_TUP( MEMORY_SUBPARTITION_UNIT ),
     TS_TUP( INTERCONNECT ),
+    TS_TUP( LIVENESS ),
     TS_TUP( NUM_TRACE_STREAMS )
 TS_TUP_END( trace_streams_type )