Merge branch 'dev' into fix_different_latencies_to_same_ex_unit

133 files changed, 4758 insertions, 2177 deletions

diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index 5ad6f10..fda84e8 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Inderpreet Singh, Timothy Rogers,
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Inderpreet Singh, Timothy Rogers, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -26,6 +28,7 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
+
 #include "abstract_hardware_model.h"
 #include <sys/stat.h>
 #include <algorithm>
@@ -205,8 +208,8 @@ gpgpu_t::gpgpu_t(const gpgpu_functional_sim_config &config, gpgpu_context *ctx)
   gpu_tot_sim_cycle = 0;
 }
 
-address_type line_size_based_tag_func(new_addr_type address,
-                                      new_addr_type line_size) {
+new_addr_type line_size_based_tag_func(new_addr_type address,
+                                       new_addr_type line_size) {
   // gives the tag for an address based on a given line size
   return address & ~(line_size - 1);
 }
@@ -281,14 +284,16 @@ void warp_inst_t::broadcast_barrier_reduction(
 void warp_inst_t::generate_mem_accesses() {
   if (empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created) return;
   if (!((op == LOAD_OP) || (op == TENSOR_CORE_LOAD_OP) || (op == STORE_OP) ||
-        (op == TENSOR_CORE_STORE_OP)))
+        (op == TENSOR_CORE_STORE_OP) ))
     return;
   if (m_warp_active_mask.count() == 0) return;  // predicated off
 
   const size_t starting_queue_size = m_accessq.size();
 
   assert(is_load() || is_store());
-  assert(m_per_scalar_thread_valid);  // need address information per thread
+
+  //if((space.get_type() != tex_space) && (space.get_type() != const_space))
+    assert(m_per_scalar_thread_valid);  // need address information per thread
 
   bool is_write = is_store();
 
@@ -448,7 +453,8 @@ void warp_inst_t::generate_mem_accesses() {
     for (unsigned thread = 0; thread < m_config->warp_size; thread++) {
       if (!active(thread)) continue;
       new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0];
-      unsigned block_address = line_size_based_tag_func(addr, cache_block_size);
+      new_addr_type block_address =
+          line_size_based_tag_func(addr, cache_block_size);
       accesses[block_address].set(thread);
       unsigned idx = addr - block_address;
       for (unsigned i = 0; i < data_size; i++) byte_mask.set(idx + i);
@@ -530,7 +536,8 @@ void warp_inst_t::memory_coalescing_arch(bool is_write,
            (m_per_scalar_thread[thread].memreqaddr[access] != 0);
            access++) {
         new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access];
-        unsigned block_address = line_size_based_tag_func(addr, segment_size);
+        new_addr_type block_address =
+            line_size_based_tag_func(addr, segment_size);
         unsigned chunk =
             (addr & 127) / 32;  // which 32-byte chunk within in a 128-byte
                                 // chunk does this thread access?
@@ -552,7 +559,8 @@ void warp_inst_t::memory_coalescing_arch(bool is_write,
         if (block_address != line_size_based_tag_func(
                                  addr + data_size_coales - 1, segment_size)) {
           addr = addr + data_size_coales - 1;
-          unsigned block_address = line_size_based_tag_func(addr, segment_size);
+          new_addr_type block_address =
+              line_size_based_tag_func(addr, segment_size);
           unsigned chunk = (addr & 127) / 32;
           transaction_info &info = subwarp_transactions[block_address];
           info.chunks.set(chunk);
@@ -625,7 +633,8 @@ void warp_inst_t::memory_coalescing_arch_atomic(bool is_write,
       if (!active(thread)) continue;
 
       new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[0];
-      unsigned block_address = line_size_based_tag_func(addr, segment_size);
+      new_addr_type block_address =
+          line_size_based_tag_func(addr, segment_size);
       unsigned chunk =
           (addr & 127) / 32;  // which 32-byte chunk within in a 128-byte chunk
                               // does this thread access?
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 49f3e9f..6e4a87d 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Inderpreet Singh,
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Inderpreet Singh, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -59,13 +61,37 @@ enum _memory_space_t {
   instruction_space
 };
 
+#ifndef COEFF_STRUCT
+#define COEFF_STRUCT
+
+struct PowerscalingCoefficients{
+    double int_coeff;
+    double int_mul_coeff;
+    double int_mul24_coeff;
+    double int_mul32_coeff;
+    double int_div_coeff;
+    double fp_coeff;
+    double dp_coeff;
+    double fp_mul_coeff;
+    double fp_div_coeff;
+    double dp_mul_coeff;
+    double dp_div_coeff;
+    double sqrt_coeff;
+    double log_coeff;
+    double sin_coeff;
+    double exp_coeff;
+    double tensor_coeff;
+    double tex_coeff;
+};
+#endif
+
 enum FuncCache {
   FuncCachePreferNone = 0,
   FuncCachePreferShared = 1,
   FuncCachePreferL1 = 2
 };
 
-enum AdaptiveCache { FIXED = 0, ADAPTIVE_VOLTA = 1 };
+enum AdaptiveCache { FIXED = 0, ADAPTIVE_CACHE = 1 };
 
 #ifdef __cplusplus
 
@@ -75,8 +101,8 @@ enum AdaptiveCache { FIXED = 0, ADAPTIVE_VOLTA = 1 };
 
 typedef unsigned long long new_addr_type;
 typedef unsigned long long cudaTextureObject_t;
-typedef unsigned address_type;
-typedef unsigned addr_t;
+typedef unsigned long long address_type;
+typedef unsigned long long addr_t;
 
 // the following are operations the timing model can see
 #define SPECIALIZED_UNIT_NUM 8
@@ -134,8 +160,14 @@ enum special_operations_t {
   FP_SQRT_OP,
   FP_LG_OP,
   FP_SIN_OP,
-  FP_EXP_OP
+  FP_EXP_OP,
+  DP_MUL_OP,
+  DP_DIV_OP,
+  DP___OP,
+  TENSOR__OP,
+  TEX__OP
 };
+
 typedef enum special_operations_t
     special_ops;  // Required to identify for the power model
 enum operation_pipeline_t {
@@ -259,6 +291,7 @@ class kernel_info_t {
            m_next_tid.x < m_block_dim.x;
   }
   unsigned get_uid() const { return m_uid; }
+  std::string get_name() const { return name(); }
   std::string name() const;
 
   std::list<class ptx_thread_info *> &active_threads() {
@@ -373,6 +406,8 @@ class core_config {
   }
   unsigned mem_warp_parts;
   mutable unsigned gpgpu_shmem_size;
+  char *gpgpu_shmem_option;
+  std::vector<unsigned> shmem_opt_list;
   unsigned gpgpu_shmem_sizeDefault;
   unsigned gpgpu_shmem_sizePrefL1;
   unsigned gpgpu_shmem_sizePrefShared;
@@ -869,6 +904,13 @@ class mem_fetch_allocator {
   virtual mem_fetch *alloc(const class warp_inst_t &inst,
                            const mem_access_t &access,
                            unsigned long long cycle) const = 0;
+  virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type,
+                           const active_mask_t &active_mask,
+                           const mem_access_byte_mask_t &byte_mask,
+                           const mem_access_sector_mask_t &sector_mask,
+                           unsigned size, bool wr, unsigned long long cycle,
+                           unsigned wid, unsigned sid, unsigned tpc,
+                           mem_fetch *original_mf) const = 0;
 };
 
 // the maximum number of destination, source, or address uarch operands in a
@@ -902,6 +944,7 @@ class inst_t {
     sp_op = OTHER_OP;
     op_pipe = UNKOWN_OP;
     mem_op = NOT_TEX;
+    const_cache_operand = 0;
     num_operands = 0;
     num_regs = 0;
     memset(out, 0, sizeof(unsigned));
@@ -930,6 +973,20 @@ class inst_t {
     return (op == STORE_OP || op == TENSOR_CORE_STORE_OP ||
             memory_op == memory_store);
   }
+
+  bool is_fp() const { return ((sp_op == FP__OP));}    //VIJAY
+  bool is_fpdiv() const { return ((sp_op == FP_DIV_OP));} 
+  bool is_fpmul() const { return ((sp_op == FP_MUL_OP));} 
+  bool is_dp() const { return ((sp_op == DP___OP));}    
+  bool is_dpdiv() const { return ((sp_op == DP_DIV_OP));} 
+  bool is_dpmul() const { return ((sp_op == DP_MUL_OP));}
+  bool is_imul() const { return ((sp_op == INT_MUL_OP));} 
+  bool is_imul24() const { return ((sp_op == INT_MUL24_OP));} 
+  bool is_imul32() const { return ((sp_op == INT_MUL32_OP));} 
+  bool is_idiv() const { return ((sp_op == INT_DIV_OP));}   
+  bool is_sfu() const {return ((sp_op == FP_SQRT_OP) || (sp_op == FP_LG_OP)  || (sp_op == FP_SIN_OP)  || (sp_op == FP_EXP_OP) || (sp_op == TENSOR__OP));}
+  bool is_alu() const {return (sp_op == INT__OP);}
+
   unsigned get_num_operands() const { return num_operands; }
   unsigned get_num_regs() const { return num_regs; }
   void set_num_regs(unsigned num) { num_regs = num; }
@@ -953,6 +1010,7 @@ class inst_t {
   operation_pipeline op_pipe;  // code (uarch visible) identify the pipeline of
                                // the operation (SP, SFU or MEM)
   mem_operation mem_op;        // code (uarch visible) identify memory type
+  bool const_cache_operand;   // has a load from constant memory as an operand
   _memory_op_t memory_op;      // memory_op used by ptxplus
   unsigned num_operands;
   unsigned num_regs;  // count vector operand as one register operand
@@ -1291,6 +1349,7 @@ class register_set {
     }
     m_name = name;
   }
+  const char *get_name() { return m_name; }
   bool has_free() {
     for (unsigned i = 0; i < regs.size(); i++) {
       if (regs[i]->empty()) {
@@ -1315,7 +1374,35 @@ class register_set {
     }
     return false;
   }
+  bool has_ready(bool sub_core_model, unsigned reg_id) {
+    if (!sub_core_model) return has_ready();
+    assert(reg_id < regs.size());
+    return (not regs[reg_id]->empty());
+  }
 
+  unsigned get_ready_reg_id() {
+    // for sub core model we need to figure which reg_id has the ready warp
+    // this function should only be called if has_ready() was true
+    assert(has_ready());
+    warp_inst_t **ready;
+    ready = NULL;
+    unsigned reg_id;
+    for (unsigned i = 0; i < regs.size(); i++) {
+      if (not regs[i]->empty()) {
+        if (ready and (*ready)->get_uid() < regs[i]->get_uid()) {
+          // ready is oldest
+        } else {
+          ready = &regs[i];
+          reg_id = i;
+        }
+      }
+    }
+    return reg_id;
+  }
+  unsigned get_schd_id(unsigned reg_id) {
+    assert(not regs[reg_id]->empty());
+    return regs[reg_id]->get_schd_id();
+  }
   void move_in(warp_inst_t *&src) {
     warp_inst_t **free = get_free();
     move_warp(*free, src);
@@ -1323,10 +1410,29 @@ class register_set {
   // void copy_in( warp_inst_t* src ){
   //   src->copy_contents_to(*get_free());
   //}
+  void move_in(bool sub_core_model, unsigned reg_id, warp_inst_t *&src) {
+    warp_inst_t **free;
+    if (!sub_core_model) {
+      free = get_free();
+    } else {
+      assert(reg_id < regs.size());
+      free = get_free(sub_core_model, reg_id);
+    }
+    move_warp(*free, src);
+  }
+
   void move_out_to(warp_inst_t *&dest) {
     warp_inst_t **ready = get_ready();
     move_warp(dest, *ready);
   }
+  void move_out_to(bool sub_core_model, unsigned reg_id, warp_inst_t *&dest) {
+    if (!sub_core_model) {
+      return move_out_to(dest);
+    }
+    warp_inst_t **ready = get_ready(sub_core_model, reg_id);
+    assert(ready != NULL);
+    move_warp(dest, *ready);
+  }
 
   warp_inst_t **get_ready() {
     warp_inst_t **ready;
@@ -1342,6 +1448,14 @@ class register_set {
     }
     return ready;
   }
+  warp_inst_t **get_ready(bool sub_core_model, unsigned reg_id) {
+    if (!sub_core_model) return get_ready();
+    warp_inst_t **ready;
+    ready = NULL;
+    assert(reg_id < regs.size());
+    if (not regs[reg_id]->empty()) ready = &regs[reg_id];
+    return ready;
+  }
 
   void print(FILE *fp) const {
     fprintf(fp, "%s : @%p\n", m_name, this);
diff --git a/src/gpuwattch/Alpha21364.xml b/src/accelwattch/Alpha21364.xml
index c40c4f5..c40c4f5 100644
--- a/src/gpuwattch/Alpha21364.xml
+++ b/src/accelwattch/Alpha21364.xml
diff --git a/src/gpuwattch/Niagara1.xml b/src/accelwattch/Niagara1.xml
index ae748e2..ae748e2 100644
--- a/src/gpuwattch/Niagara1.xml
+++ b/src/accelwattch/Niagara1.xml
diff --git a/src/gpuwattch/Niagara1_sharing.xml b/src/accelwattch/Niagara1_sharing.xml
index 93531ae..93531ae 100644
--- a/src/gpuwattch/Niagara1_sharing.xml
+++ b/src/accelwattch/Niagara1_sharing.xml
diff --git a/src/gpuwattch/Niagara1_sharing_DC.xml b/src/accelwattch/Niagara1_sharing_DC.xml
index 574ec81..574ec81 100644
--- a/src/gpuwattch/Niagara1_sharing_DC.xml
+++ b/src/accelwattch/Niagara1_sharing_DC.xml
diff --git a/src/gpuwattch/Niagara1_sharing_SBT.xml b/src/accelwattch/Niagara1_sharing_SBT.xml
index 32eeca3..32eeca3 100644
--- a/src/gpuwattch/Niagara1_sharing_SBT.xml
+++ b/src/accelwattch/Niagara1_sharing_SBT.xml
diff --git a/src/gpuwattch/Niagara1_sharing_ST.xml b/src/accelwattch/Niagara1_sharing_ST.xml
index 3f0573f..3f0573f 100644
--- a/src/gpuwattch/Niagara1_sharing_ST.xml
+++ b/src/accelwattch/Niagara1_sharing_ST.xml
diff --git a/src/gpuwattch/Niagara2.xml b/src/accelwattch/Niagara2.xml
index c7e311f..c7e311f 100644
--- a/src/gpuwattch/Niagara2.xml
+++ b/src/accelwattch/Niagara2.xml
diff --git a/src/gpuwattch/Penryn.xml b/src/accelwattch/Penryn.xml
index fe9715b..fe9715b 100644
--- a/src/gpuwattch/Penryn.xml
+++ b/src/accelwattch/Penryn.xml
diff --git a/src/gpuwattch/README b/src/accelwattch/README
index 4887b10..4887b10 100644
--- a/src/gpuwattch/README
+++ b/src/accelwattch/README
diff --git a/src/gpuwattch/XML_Parse.cc b/src/accelwattch/XML_Parse.cc
index 1b9a38a..eaec748 100644
--- a/src/gpuwattch/XML_Parse.cc
+++ b/src/accelwattch/XML_Parse.cc
@@ -30,12 +30,14 @@
  ***************************************************************************/

 /********************************************************************

  *      Modified by:

- ** Jingwen Leng, Univeristy of Texas, Austin                   * Syed Gilani,

- *University of Wisconsin–Madison                * Tayler Hetherington,

- *University of British Columbia         * Ahmed ElTantawy, University of

- *British Columbia             *

+ * Jingwen Leng, University of Texas, Austin                

+ * Syed Gilani, University of Wisconsin–Madison         

+ * Tayler Hetherington, University of British Columbia

+ * Ahmed ElTantawy, University of British Columbia

+ * Vijay Kandiah, Northwestern University

  ********************************************************************/

 

+

 #include "XML_Parse.h"

 #include <stdio.h>

 #include <string>

@@ -43,13 +45,14 @@
 

 using namespace std;

 

-const char* perf_count_label[] = {

-    "TOT_INST,",    "FP_INT,",  "IC_H,",     "IC_M,",        "DC_RH,",

-    "DC_RM,",       "DC_WH,",   "DC_WM,",    "TC_H,",        "TC_M,",

-    "CC_H,",        "CC_M,",    "SHRD_ACC,", "REG_RD,",      "REG_WR,",

-    "NON_REG_OPs,", "SP_ACC,",  "SFU_ACC,",  "FPU_ACC,",     "MEM_RD,",

-    "MEM_WR,",      "MEM_PRE,", "L2_RH,",    "L2_RM,",       "L2_WH,",

-    "L2_WM,",       "NOC_A,",   "PIPE_A,",   "IDLE_CORE_N,", "CONST_DYNAMICN"};

+const char * perf_count_label[] = {

+  "TOT_INST,", "FP_INT,", "IC_H,", "IC_M,", "DC_RH,", "DC_RM,", "DC_WH,", "DC_WM,",

+  "TC_H,", "TC_M,", "CC_H,", "CC_M,", "SHRD_ACC,", "REG_RD,", "REG_WR,", "NON_REG_OPs,",

+  "INT_ACC,", "FPU_ACC,", "DPU_ACC,", "INT_MUL24_ACC,", "INT_MUL32_ACC,", "INT_MUL_ACC,","INT_DIV_ACC,", 

+  "FP_MUL_ACC,", "FP_DIV_ACC,", "FP_SQRT_ACC,", "FP_LG_ACC,", "FP_SIN_ACC,", "FP_EXP_ACC,", "DP_MUL_ACC,", 

+  "DP_DIV_ACC,", "TENSOR_ACC,", "TEX_ACC,", "MEM_RD,","MEM_WR,", "MEM_PRE,", "L2_RH,", "L2_RM,", "L2_WH,",

+  "L2_WM,", "NOC_A,", "PIPE_A,", "IDLE_CORE_N,", "constant_power"};

+

 

 void ParseXML::parse(char* filepath) {

   unsigned int i, j, k, m, n;

@@ -160,6 +163,199 @@ void ParseXML::parse(char* filepath) {
           atoi(xNode2.getChildNode("param", i).getAttribute("value"));

       continue;

     }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "modeled_chip_voltage_ref") == 0) {

+      sys.modeled_chip_voltage_ref =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat1_flane") == 0) {

+      sys.static_cat1_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat2_flane") == 0) {

+      sys.static_cat2_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat3_flane") == 0) {

+      sys.static_cat3_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat4_flane") == 0) {

+      sys.static_cat4_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat5_flane") == 0) {

+      sys.static_cat5_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat6_flane") == 0) {

+      sys.static_cat6_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_shared_flane") == 0) {

+      sys.static_shared_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_l1_flane") == 0) {

+      sys.static_l1_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_l2_flane") == 0) {

+      sys.static_l2_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_light_flane") == 0) {

+      sys.static_light_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_intadd_flane") == 0) {

+      sys.static_intadd_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_intmul_flane") == 0) {

+      sys.static_intmul_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_geomean_flane") == 0) {

+      sys.static_geomean_flane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat1_addlane") == 0) {

+      sys.static_cat1_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat2_addlane") == 0) {

+      sys.static_cat2_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat3_addlane") == 0) {

+      sys.static_cat3_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat4_addlane") == 0) {

+      sys.static_cat4_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat5_addlane") == 0) {

+      sys.static_cat5_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_cat6_addlane") == 0) {

+      sys.static_cat6_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_shared_addlane") == 0) {

+      sys.static_shared_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_l1_addlane") == 0) {

+      sys.static_l1_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_l2_addlane") == 0) {

+      sys.static_l2_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_light_addlane") == 0) {

+      sys.static_light_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_intadd_addlane") == 0) {

+      sys.static_intadd_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_intmul_addlane") == 0) {

+      sys.static_intmul_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

+               "static_geomean_addlane") == 0) {

+      sys.static_geomean_addlane =

+          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+      continue;

+    }

+

+

     if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

                "target_chip_area") == 0) {

       sys.target_chip_area =

@@ -419,22 +615,106 @@ void ParseXML::parse(char* filepath) {
           atof(xNode2.getChildNode("param", i).getAttribute("value"));

       continue;

     }

-    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

-               "SP_ACC") == 0) {

-      sys.scaling_coefficients[SP_ACC] =

-          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "INT_ACC")==0) {

+      sys.scaling_coefficients[INT_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

       continue;

     }

-    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

-               "SFU_ACC") == 0) {

-      sys.scaling_coefficients[SFU_ACC] =

-          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_ACC")==0) {

+      sys.scaling_coefficients[FP_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

       continue;

     }

-    if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

-               "FPU_ACC") == 0) {

-      sys.scaling_coefficients[FPU_ACC] =

-          atof(xNode2.getChildNode("param", i).getAttribute("value"));

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "DP_ACC")==0) {

+      sys.scaling_coefficients[DP_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "INT_MUL24_ACC")==0) {

+      sys.scaling_coefficients[INT_MUL24_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "INT_MUL32_ACC")==0) {

+      sys.scaling_coefficients[INT_MUL32_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "INT_MUL_ACC")==0) {

+      sys.scaling_coefficients[INT_MUL_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "INT_DIV_ACC")==0) {

+      sys.scaling_coefficients[INT_DIV_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_MUL_ACC")==0) {

+      sys.scaling_coefficients[FP_MUL_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_DIV_ACC")==0) {

+      sys.scaling_coefficients[FP_DIV_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_SQRT_ACC")==0) {

+      sys.scaling_coefficients[FP_SQRT_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_LG_ACC")==0) {

+      sys.scaling_coefficients[FP_LG_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_SIN_ACC")==0) {

+      sys.scaling_coefficients[FP_SIN_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "FP_EXP_ACC")==0) {

+      sys.scaling_coefficients[FP_EXP_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "DP_MUL_ACC")==0) {

+      sys.scaling_coefficients[DP_MUL_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "DP_DIV_ACC")==0) {

+      sys.scaling_coefficients[DP_DIV_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "TENSOR_ACC")==0) {

+      sys.scaling_coefficients[TENSOR_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

+      continue;

+    }

+    if (strcmp(xNode2.getChildNode("param",i).getAttribute("name"),

+                "TEX_ACC")==0) {

+      sys.scaling_coefficients[TEX_ACC] =

+          atof(xNode2.getChildNode("param",i).getAttribute("value"));

       continue;

     }

     if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

@@ -498,8 +778,8 @@ void ParseXML::parse(char* filepath) {
       continue;

     }

     if (strcmp(xNode2.getChildNode("param", i).getAttribute("name"),

-               "CONST_DYNAMICN") == 0) {

-      sys.scaling_coefficients[CONST_DYNAMICN] =

+               "constant_power") == 0) {

+      sys.scaling_coefficients[constant_power] =

           atof(xNode2.getChildNode("param", i).getAttribute("value"));

       continue;

     }

@@ -4187,8 +4467,9 @@ void ParseXML::initialize()  // Initialize all
   // strcpy(sys.homogeneous_cores,"default");

   sys.core_tech_node = 1;

   sys.target_core_clockrate = 1;

+  sys.modeled_chip_voltage_ref = 1;

   sys.target_chip_area = 1;

-  sys.temperature = 1;

+  sys.temperature = 340;

   sys.number_cache_levels = 1;

   sys.homogeneous_cores = 1;

   sys.homogeneous_L1Directories = 1;

@@ -4198,6 +4479,34 @@ void ParseXML::initialize()  // Initialize all
   sys.homogeneous_NoCs = 1;

   sys.homogeneous_ccs = 1;

 

+  sys.static_cat1_flane = 0;

+  sys.static_cat2_flane = 0;

+  sys.static_cat3_flane = 0;

+  sys.static_cat4_flane = 0;

+  sys.static_cat5_flane = 0;

+  sys.static_cat6_flane = 0;

+  sys.static_shared_flane = 0;

+  sys.static_l1_flane = 0;

+  sys.static_l2_flane = 0;

+  sys.static_light_flane = 0;

+  sys.static_intadd_flane = 0;

+  sys.static_intmul_flane = 0;

+  sys.static_geomean_flane = 0;

+

+  sys.static_cat1_addlane = 0;

+  sys.static_cat2_addlane = 0;

+  sys.static_cat3_addlane = 0;

+  sys.static_cat4_addlane = 0;

+  sys.static_cat5_addlane = 0;

+  sys.static_cat6_addlane = 0;

+  sys.static_shared_addlane = 0;

+  sys.static_l1_addlane = 0;

+  sys.static_l2_addlane = 0;

+  sys.static_light_addlane = 0;

+  sys.static_intadd_addlane = 0;

+  sys.static_intmul_addlane = 0;

+  sys.static_geomean_addlane = 0;

+

   sys.Max_area_deviation = 1;

   sys.Max_power_deviation = 1;

   sys.device_type = 1;

diff --git a/src/gpuwattch/XML_Parse.h b/src/accelwattch/XML_Parse.h
index 30c4e4b..c82359f 100644
--- a/src/gpuwattch/XML_Parse.h
+++ b/src/accelwattch/XML_Parse.h
@@ -30,10 +30,11 @@
  ***************************************************************************/

 /********************************************************************

  *      Modified by:

- ** Jingwen Leng, Univeristy of Texas, Austin                   * Syed Gilani,

- *University of Wisconsin–Madison                * Tayler Hetherington,

- *University of British Columbia         * Ahmed ElTantawy, University of

- *British Columbia             *

+ * Jingwen Leng, University of Texas, Austin                

+ * Syed Gilani, University of Wisconsin–Madison         

+ * Tayler Hetherington, University of British Columbia

+ * Ahmed ElTantawy, University of British Columbia

+ * Vijay Kandiah, Northwestern University

  ********************************************************************/

 

 #ifndef XML_PARSE_H_

@@ -69,7 +70,7 @@ ToXMLStringTool tx,tx2;
 extern const char* perf_count_label[];

 

 enum perf_count_t {

-  TOT_INST = 0,

+  TOT_INST=0,

   FP_INT,

   IC_H,

   IC_M,

@@ -85,9 +86,23 @@ enum perf_count_t {
   REG_RD,

   REG_WR,

   NON_REG_OPs,

-  SP_ACC,

-  SFU_ACC,

-  FPU_ACC,

+  INT_ACC, //SPU

+  FP_ACC, //FPU

+  DP_ACC, //FPU

+  INT_MUL24_ACC, //SFU

+  INT_MUL32_ACC, //SFU

+  INT_MUL_ACC, //SFU 

+  INT_DIV_ACC, //SFU

+  FP_MUL_ACC, //SFU

+  FP_DIV_ACC, //SFU

+  FP_SQRT_ACC, //SFU

+  FP_LG_ACC, //SFU

+  FP_SIN_ACC, //SFU

+  FP_EXP_ACC, //SFU

+  DP_MUL_ACC, //SFU

+  DP_DIV_ACC, //SFU 

+  TENSOR_ACC, //SFU

+  TEX_ACC, //SFU 

   MEM_RD,

   MEM_WR,

   MEM_PRE,

@@ -98,7 +113,7 @@ enum perf_count_t {
   NOC_A,

   PIPE_A,

   IDLE_CORE_N,

-  CONST_DYNAMICN,

+  constant_power,

   NUM_PERFORMANCE_COUNTERS

 };

 

@@ -635,6 +650,33 @@ typedef struct {
   int homogeneous_L2Directories;

   double core_tech_node;

   int target_core_clockrate;

+  double modeled_chip_voltage_ref;

+  double static_cat1_flane;

+  double static_cat2_flane;

+  double static_cat3_flane;

+  double static_cat4_flane;

+  double static_cat5_flane;

+  double static_cat6_flane;

+  double static_shared_flane;

+  double static_l1_flane;

+  double static_l2_flane;

+  double static_light_flane;

+  double static_intadd_flane;

+  double static_intmul_flane;

+  double static_geomean_flane;

+  double static_cat1_addlane;

+  double static_cat2_addlane;

+  double static_cat3_addlane;

+  double static_cat4_addlane;

+  double static_cat5_addlane;

+  double static_cat6_addlane;

+  double static_shared_addlane;

+  double static_l1_addlane;

+  double static_l2_addlane;

+  double static_light_addlane;

+  double static_intadd_addlane;

+  double static_intmul_addlane;

+  double static_geomean_addlane;

   int target_chip_area;

   int temperature;

   int number_cache_levels;

diff --git a/src/gpuwattch/Xeon.xml b/src/accelwattch/Xeon.xml
index 5342104..5342104 100644
--- a/src/gpuwattch/Xeon.xml
+++ b/src/accelwattch/Xeon.xml
diff --git a/src/gpuwattch/arch_const.h b/src/accelwattch/arch_const.h
index 555cc17..555cc17 100644
--- a/src/gpuwattch/arch_const.h
+++ b/src/accelwattch/arch_const.h
diff --git a/src/gpuwattch/array.cc b/src/accelwattch/array.cc
index 108a16b..108a16b 100644
--- a/src/gpuwattch/array.cc
+++ b/src/accelwattch/array.cc
diff --git a/src/gpuwattch/array.h b/src/accelwattch/array.h
index e2dc3d1..e2dc3d1 100644
--- a/src/gpuwattch/array.h
+++ b/src/accelwattch/array.h
diff --git a/src/gpuwattch/basic_components.cc b/src/accelwattch/basic_components.cc
index d58023a..d58023a 100644
--- a/src/gpuwattch/basic_components.cc
+++ b/src/accelwattch/basic_components.cc
diff --git a/src/gpuwattch/basic_components.h b/src/accelwattch/basic_components.h
index 2e9907a..2e9907a 100644
--- a/src/gpuwattch/basic_components.h
+++ b/src/accelwattch/basic_components.h
diff --git a/src/gpuwattch/cacti/README b/src/accelwattch/cacti/README
index de429d2..de429d2 100644
--- a/src/gpuwattch/cacti/README
+++ b/src/accelwattch/cacti/README
diff --git a/src/gpuwattch/cacti/Ucache.cc b/src/accelwattch/cacti/Ucache.cc
index 8f733f7..e92e67b 100644
--- a/src/gpuwattch/cacti/Ucache.cc
+++ b/src/accelwattch/cacti/Ucache.cc
@@ -223,7 +223,7 @@ void * calc_time_mt_wrapper(void * void_obj)
   delete tag_arr.back();
   data_arr.pop_back();
   tag_arr.pop_back();
-
+  pthread_exit(NULL);
 }
 
 
@@ -246,7 +246,7 @@ bool calculate_time(
 {
   DynamicParameter dyn_p(is_tag, pure_ram, pure_cam, Nspd, Ndwl, Ndbl, Ndcm, Ndsam_lev_1, Ndsam_lev_2, is_main_mem);
 
-  if (dyn_p.is_valid == false)
+  if (dyn_p.is_valid != true)
   {
     return false;
   }
diff --git a/src/gpuwattch/cacti/Ucache.h b/src/accelwattch/cacti/Ucache.h
index cbd578f..cbd578f 100644
--- a/src/gpuwattch/cacti/Ucache.h
+++ b/src/accelwattch/cacti/Ucache.h
diff --git a/src/gpuwattch/cacti/arbiter.cc b/src/accelwattch/cacti/arbiter.cc
index f4f5204..f4f5204 100644
--- a/src/gpuwattch/cacti/arbiter.cc
+++ b/src/accelwattch/cacti/arbiter.cc
diff --git a/src/gpuwattch/cacti/arbiter.h b/src/accelwattch/cacti/arbiter.h
index 0884a04..0884a04 100644
--- a/src/gpuwattch/cacti/arbiter.h
+++ b/src/accelwattch/cacti/arbiter.h
diff --git a/src/gpuwattch/cacti/area.cc b/src/accelwattch/cacti/area.cc
index 0d8d4b7..0d8d4b7 100644
--- a/src/gpuwattch/cacti/area.cc
+++ b/src/accelwattch/cacti/area.cc
diff --git a/src/gpuwattch/cacti/area.h b/src/accelwattch/cacti/area.h
index 92272f0..92272f0 100644
--- a/src/gpuwattch/cacti/area.h
+++ b/src/accelwattch/cacti/area.h
diff --git a/src/gpuwattch/cacti/bank.cc b/src/accelwattch/cacti/bank.cc
index 2e58080..2e58080 100755
--- a/src/gpuwattch/cacti/bank.cc
+++ b/src/accelwattch/cacti/bank.cc
diff --git a/src/gpuwattch/cacti/bank.h b/src/accelwattch/cacti/bank.h
index 47300da..47300da 100755
--- a/src/gpuwattch/cacti/bank.h
+++ b/src/accelwattch/cacti/bank.h
diff --git a/src/gpuwattch/cacti/basic_circuit.cc b/src/accelwattch/cacti/basic_circuit.cc
index a8ea501..a8ea501 100644
--- a/src/gpuwattch/cacti/basic_circuit.cc
+++ b/src/accelwattch/cacti/basic_circuit.cc
diff --git a/src/gpuwattch/cacti/basic_circuit.h b/src/accelwattch/cacti/basic_circuit.h
index bafd3d3..bafd3d3 100644
--- a/src/gpuwattch/cacti/basic_circuit.h
+++ b/src/accelwattch/cacti/basic_circuit.h
diff --git a/src/gpuwattch/cacti/batch_tests b/src/accelwattch/cacti/batch_tests
index 45a0389..45a0389 100755
--- a/src/gpuwattch/cacti/batch_tests
+++ b/src/accelwattch/cacti/batch_tests
diff --git a/src/gpuwattch/cacti/cache.cfg b/src/accelwattch/cacti/cache.cfg
index 03de34a..03de34a 100755
--- a/src/gpuwattch/cacti/cache.cfg
+++ b/src/accelwattch/cacti/cache.cfg
diff --git a/src/gpuwattch/cacti/cacti.i b/src/accelwattch/cacti/cacti.i
index 7964138..7964138 100644
--- a/src/gpuwattch/cacti/cacti.i
+++ b/src/accelwattch/cacti/cacti.i
diff --git a/src/gpuwattch/cacti/cacti.mk b/src/accelwattch/cacti/cacti.mk
index 7f3c573..41f9218 100644
--- a/src/gpuwattch/cacti/cacti.mk
+++ b/src/accelwattch/cacti/cacti.mk
@@ -1,5 +1,5 @@
 
-OUTPUT_DIR=$(SIM_OBJ_FILES_DIR)/gpuwattch/cacti
+OUTPUT_DIR=$(SIM_OBJ_FILES_DIR)/accelwattch/cacti
 TARGET = cacti
 SHELL = /bin/sh
 .PHONY: all depend clean
diff --git a/src/gpuwattch/cacti/cacti_interface.cc b/src/accelwattch/cacti/cacti_interface.cc
index 99d734d..99d734d 100644
--- a/src/gpuwattch/cacti/cacti_interface.cc
+++ b/src/accelwattch/cacti/cacti_interface.cc
diff --git a/src/gpuwattch/cacti/cacti_interface.h b/src/accelwattch/cacti/cacti_interface.h
index ff639da..ff639da 100644
--- a/src/gpuwattch/cacti/cacti_interface.h
+++ b/src/accelwattch/cacti/cacti_interface.h
diff --git a/src/gpuwattch/cacti/component.cc b/src/accelwattch/cacti/component.cc
index abe5cb9..abe5cb9 100644
--- a/src/gpuwattch/cacti/component.cc
+++ b/src/accelwattch/cacti/component.cc
diff --git a/src/gpuwattch/cacti/component.h b/src/accelwattch/cacti/component.h
index 74a089d..74a089d 100644
--- a/src/gpuwattch/cacti/component.h
+++ b/src/accelwattch/cacti/component.h
diff --git a/src/gpuwattch/cacti/const.h b/src/accelwattch/cacti/const.h
index ada91f9..ada91f9 100644
--- a/src/gpuwattch/cacti/const.h
+++ b/src/accelwattch/cacti/const.h
diff --git a/src/gpuwattch/cacti/contention.dat b/src/accelwattch/cacti/contention.dat
index 826553e..826553e 100755
--- a/src/gpuwattch/cacti/contention.dat
+++ b/src/accelwattch/cacti/contention.dat
diff --git a/src/gpuwattch/cacti/crossbar.cc b/src/accelwattch/cacti/crossbar.cc
index d7386a8..d7386a8 100644
--- a/src/gpuwattch/cacti/crossbar.cc
+++ b/src/accelwattch/cacti/crossbar.cc
diff --git a/src/gpuwattch/cacti/crossbar.h b/src/accelwattch/cacti/crossbar.h
index 47339c0..47339c0 100644
--- a/src/gpuwattch/cacti/crossbar.h
+++ b/src/accelwattch/cacti/crossbar.h
diff --git a/src/gpuwattch/cacti/decoder.cc b/src/accelwattch/cacti/decoder.cc
index 32644d9..32644d9 100644
--- a/src/gpuwattch/cacti/decoder.cc
+++ b/src/accelwattch/cacti/decoder.cc
diff --git a/src/gpuwattch/cacti/decoder.h b/src/accelwattch/cacti/decoder.h
index 6366e16..6366e16 100644
--- a/src/gpuwattch/cacti/decoder.h
+++ b/src/accelwattch/cacti/decoder.h
diff --git a/src/gpuwattch/cacti/highradix.cc b/src/accelwattch/cacti/highradix.cc
index e06cad7..e06cad7 100644
--- a/src/gpuwattch/cacti/highradix.cc
+++ b/src/accelwattch/cacti/highradix.cc
diff --git a/src/gpuwattch/cacti/highradix.h b/src/accelwattch/cacti/highradix.h
index 7b41801..7b41801 100644
--- a/src/gpuwattch/cacti/highradix.h
+++ b/src/accelwattch/cacti/highradix.h
diff --git a/src/gpuwattch/cacti/htree2.cc b/src/accelwattch/cacti/htree2.cc
index c40a880..c40a880 100644
--- a/src/gpuwattch/cacti/htree2.cc
+++ b/src/accelwattch/cacti/htree2.cc
diff --git a/src/gpuwattch/cacti/htree2.h b/src/accelwattch/cacti/htree2.h
index 6700e80..6700e80 100644
--- a/src/gpuwattch/cacti/htree2.h
+++ b/src/accelwattch/cacti/htree2.h
diff --git a/src/gpuwattch/cacti/io.cc b/src/accelwattch/cacti/io.cc
index c513dbf..c513dbf 100644
--- a/src/gpuwattch/cacti/io.cc
+++ b/src/accelwattch/cacti/io.cc
diff --git a/src/gpuwattch/cacti/io.h b/src/accelwattch/cacti/io.h
index 1bd06cf..1bd06cf 100644
--- a/src/gpuwattch/cacti/io.h
+++ b/src/accelwattch/cacti/io.h
diff --git a/src/gpuwattch/cacti/main.cc b/src/accelwattch/cacti/main.cc
index 58c2fe1..58c2fe1 100644
--- a/src/gpuwattch/cacti/main.cc
+++ b/src/accelwattch/cacti/main.cc
diff --git a/src/gpuwattch/cacti/makefile b/src/accelwattch/cacti/makefile
index 6cc36db..6cc36db 100644
--- a/src/gpuwattch/cacti/makefile
+++ b/src/accelwattch/cacti/makefile
diff --git a/src/gpuwattch/cacti/mat.cc b/src/accelwattch/cacti/mat.cc
index dc903da..dc903da 100755
--- a/src/gpuwattch/cacti/mat.cc
+++ b/src/accelwattch/cacti/mat.cc
diff --git a/src/gpuwattch/cacti/mat.h b/src/accelwattch/cacti/mat.h
index e408a7d..e408a7d 100755
--- a/src/gpuwattch/cacti/mat.h
+++ b/src/accelwattch/cacti/mat.h
diff --git a/src/gpuwattch/cacti/nuca.cc b/src/accelwattch/cacti/nuca.cc
index f31b0c8..f31b0c8 100644
--- a/src/gpuwattch/cacti/nuca.cc
+++ b/src/accelwattch/cacti/nuca.cc
diff --git a/src/gpuwattch/cacti/nuca.h b/src/accelwattch/cacti/nuca.h
index bbdee7d..bbdee7d 100644
--- a/src/gpuwattch/cacti/nuca.h
+++ b/src/accelwattch/cacti/nuca.h
diff --git a/src/gpuwattch/cacti/out_batch_test_result.csv b/src/accelwattch/cacti/out_batch_test_result.csv
index 74cd5f2..74cd5f2 100644
--- a/src/gpuwattch/cacti/out_batch_test_result.csv
+++ b/src/accelwattch/cacti/out_batch_test_result.csv
diff --git a/src/gpuwattch/cacti/parameter.cc b/src/accelwattch/cacti/parameter.cc
index 3f5a802..3f5a802 100644
--- a/src/gpuwattch/cacti/parameter.cc
+++ b/src/accelwattch/cacti/parameter.cc
diff --git a/src/gpuwattch/cacti/parameter.h b/src/accelwattch/cacti/parameter.h
index 2c977ee..2c977ee 100644
--- a/src/gpuwattch/cacti/parameter.h
+++ b/src/accelwattch/cacti/parameter.h
diff --git a/src/gpuwattch/cacti/router.cc b/src/accelwattch/cacti/router.cc
index 9826c69..9826c69 100644
--- a/src/gpuwattch/cacti/router.cc
+++ b/src/accelwattch/cacti/router.cc
diff --git a/src/gpuwattch/cacti/router.h b/src/accelwattch/cacti/router.h
index 21023c4..21023c4 100644
--- a/src/gpuwattch/cacti/router.h
+++ b/src/accelwattch/cacti/router.h
diff --git a/src/gpuwattch/cacti/subarray.cc b/src/accelwattch/cacti/subarray.cc
index ef5737d..ef5737d 100755
--- a/src/gpuwattch/cacti/subarray.cc
+++ b/src/accelwattch/cacti/subarray.cc
diff --git a/src/gpuwattch/cacti/subarray.h b/src/accelwattch/cacti/subarray.h
index 5fb0624..5fb0624 100755
--- a/src/gpuwattch/cacti/subarray.h
+++ b/src/accelwattch/cacti/subarray.h
diff --git a/src/gpuwattch/cacti/technology.cc b/src/accelwattch/cacti/technology.cc
index 7067470..7067470 100644
--- a/src/gpuwattch/cacti/technology.cc
+++ b/src/accelwattch/cacti/technology.cc
diff --git a/src/gpuwattch/cacti/uca.cc b/src/accelwattch/cacti/uca.cc
index ed9be49..ed9be49 100755
--- a/src/gpuwattch/cacti/uca.cc
+++ b/src/accelwattch/cacti/uca.cc
diff --git a/src/gpuwattch/cacti/uca.h b/src/accelwattch/cacti/uca.h
index 93ae3c5..93ae3c5 100755
--- a/src/gpuwattch/cacti/uca.h
+++ b/src/accelwattch/cacti/uca.h
diff --git a/src/gpuwattch/cacti/wire.cc b/src/accelwattch/cacti/wire.cc
index 3da3e84..3da3e84 100644
--- a/src/gpuwattch/cacti/wire.cc
+++ b/src/accelwattch/cacti/wire.cc
diff --git a/src/gpuwattch/cacti/wire.h b/src/accelwattch/cacti/wire.h
index ace7c9c..ace7c9c 100644
--- a/src/gpuwattch/cacti/wire.h
+++ b/src/accelwattch/cacti/wire.h
diff --git a/src/gpuwattch/core.cc b/src/accelwattch/core.cc
index cbaefc7..cbaefc7 100644
--- a/src/gpuwattch/core.cc
+++ b/src/accelwattch/core.cc
diff --git a/src/gpuwattch/core.h b/src/accelwattch/core.h
index 67ff4b1..67ff4b1 100644
--- a/src/gpuwattch/core.h
+++ b/src/accelwattch/core.h
diff --git a/src/gpuwattch/fermi.xml b/src/accelwattch/fermi.xml
index 5b09d45..5b09d45 100755
--- a/src/gpuwattch/fermi.xml
+++ b/src/accelwattch/fermi.xml
diff --git a/src/gpuwattch/globalvar.h b/src/accelwattch/globalvar.h
index e0b76c2..e0b76c2 100644
--- a/src/gpuwattch/globalvar.h
+++ b/src/accelwattch/globalvar.h
diff --git a/src/gpuwattch/gpgpu.xml b/src/accelwattch/gpgpu.xml
index 313527c..313527c 100644
--- a/src/gpuwattch/gpgpu.xml
+++ b/src/accelwattch/gpgpu.xml
diff --git a/src/gpuwattch/gpgpu_sim.verify b/src/accelwattch/gpgpu_sim.verify
index 3b32656..3b32656 100644
--- a/src/gpuwattch/gpgpu_sim.verify
+++ b/src/accelwattch/gpgpu_sim.verify
diff --git a/src/accelwattch/gpgpu_sim_wrapper.cc b/src/accelwattch/gpgpu_sim_wrapper.cc
new file mode 100644
index 0000000..67d9daa
--- /dev/null
+++ b/src/accelwattch/gpgpu_sim_wrapper.cc
@@ -0,0 +1,1143 @@
+// Copyright (c) 2009-2021, Tor M. Aamodt, Tayler Hetherington, Ahmed ElTantawy, Vijay Kandiah, Nikos Hardavellas
+// The University of British Columbia, Northwestern University
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "gpgpu_sim_wrapper.h"
+#include <sys/stat.h>
+#define SP_BASE_POWER 0
+#define SFU_BASE_POWER 0
+
+static const char* pwr_cmp_label[] = {
+    "IBP,", "ICP,", "DCP,", "TCP,", "CCP,", "SHRDP,", "RFP,", "INTP,", 
+    "FPUP,", "DPUP,", "INT_MUL24P,", "INT_MUL32P,", "INT_MULP,", "INT_DIVP,", 
+    "FP_MULP,", "FP_DIVP,", "FP_SQRTP,", "FP_LGP,", "FP_SINP,", "FP_EXP,", 
+    "DP_MULP,", "DP_DIVP,", "TENSORP,", "TEXP,", "SCHEDP,", "L2CP,", "MCP,", "NOCP,", 
+    "DRAMP,", "PIPEP,", "IDLE_COREP,", "CONSTP", "STATICP"};
+
+enum pwr_cmp_t {
+  IBP=0,
+  ICP,
+  DCP,
+  TCP,
+  CCP,
+  SHRDP,
+  RFP,
+  INTP,
+  FPUP,
+  DPUP,
+  INT_MUL24P,
+  INT_MUL32P,
+  INT_MULP,
+  INT_DIVP,
+  FP_MULP,
+  FP_DIVP,
+  FP_SQRTP,
+  FP_LGP,
+  FP_SINP,
+  FP_EXP,
+  DP_MULP,
+  DP_DIVP,
+  TENSORP,
+  TEXP,
+  SCHEDP,
+  L2CP,
+  MCP,
+  NOCP,
+  DRAMP,
+  PIPEP,
+  IDLE_COREP,
+  CONSTP,
+  STATICP,
+  NUM_COMPONENTS_MODELLED
+};
+
+gpgpu_sim_wrapper::gpgpu_sim_wrapper(bool power_simulation_enabled,
+                                     char* xmlfile, int power_simulation_mode, bool dvfs_enabled) {
+  kernel_sample_count = 0;
+  total_sample_count = 0;
+
+  kernel_tot_power = 0;
+  avg_threads_per_warp_tot = 0;
+  num_pwr_cmps = NUM_COMPONENTS_MODELLED;
+  num_perf_counters = NUM_PERFORMANCE_COUNTERS;
+
+  // Initialize per-component counter/power vectors
+  avg_max_min_counters<double> init;
+  kernel_cmp_pwr.resize(NUM_COMPONENTS_MODELLED, init);
+  kernel_cmp_perf_counters.resize(NUM_PERFORMANCE_COUNTERS, init);
+
+  kernel_power = init;   // Per-kernel powers
+  gpu_tot_power = init;  // Global powers
+
+  sample_cmp_pwr.resize(NUM_COMPONENTS_MODELLED, 0);
+
+  sample_perf_counters.resize(NUM_PERFORMANCE_COUNTERS, 0);
+  initpower_coeff.resize(NUM_PERFORMANCE_COUNTERS, 0);
+  effpower_coeff.resize(NUM_PERFORMANCE_COUNTERS, 0);
+
+  const_dynamic_power = 0;
+  proc_power = 0;
+
+  g_power_filename = NULL;
+  g_power_trace_filename = NULL;
+  g_metric_trace_filename = NULL;
+  g_steady_state_tracking_filename = NULL;
+  xml_filename = xmlfile;
+  g_power_simulation_enabled = power_simulation_enabled;
+  g_power_simulation_mode = power_simulation_mode;
+  g_dvfs_enabled = dvfs_enabled;
+  g_power_trace_enabled = false;
+  g_steady_power_levels_enabled = false;
+  g_power_trace_zlevel = 0;
+  g_power_per_cycle_dump = false;
+  gpu_steady_power_deviation = 0;
+  gpu_steady_min_period = 0;
+
+  gpu_stat_sample_freq = 0;
+  p = new ParseXML();
+  if (g_power_simulation_enabled) {
+    p->parse(xml_filename);
+  }
+  proc = new Processor(p);
+  power_trace_file = NULL;
+  metric_trace_file = NULL;
+  steady_state_tacking_file = NULL;
+  has_written_avg = false;
+  init_inst_val = false;
+}
+
+gpgpu_sim_wrapper::~gpgpu_sim_wrapper() {}
+
+bool gpgpu_sim_wrapper::sanity_check(double a, double b) {
+  if (b == 0)
+    return (abs(a - b) < 0.00001);
+  else
+    return (abs(a - b) / abs(b) < 0.00001);
+
+  return false;
+}
+void gpgpu_sim_wrapper::init_mcpat_hw_mode(unsigned gpu_sim_cycle) {
+   p->sys.total_cycles = gpu_sim_cycle; //total simulated cycles for current kernel
+}
+
+void gpgpu_sim_wrapper::init_mcpat(
+    char* xmlfile, char* powerfilename, char* power_trace_filename,
+    char* metric_trace_filename, char* steady_state_filename,
+    bool power_sim_enabled, bool trace_enabled, bool steady_state_enabled,
+    bool power_per_cycle_dump, double steady_power_deviation,
+    double steady_min_period, int zlevel, double init_val,
+    int stat_sample_freq, int power_sim_mode, bool dvfs_enabled,
+    unsigned clock_freq, unsigned num_shaders) {
+  // Write File Headers for (-metrics trace, -power trace)
+
+  reset_counters();
+  static bool mcpat_init = true;
+
+  // initialize file name if it is not set
+  time_t curr_time;
+  time(&curr_time);
+  char* date = ctime(&curr_time);
+  char* s = date;
+  while (*s) {
+    if (*s == ' ' || *s == '\t' || *s == ':') *s = '-';
+    if (*s == '\n' || *s == '\r') *s = 0;
+    s++;
+  }
+
+  if (mcpat_init) {
+    g_power_filename = powerfilename;
+    g_power_trace_filename = power_trace_filename;
+    g_metric_trace_filename = metric_trace_filename;
+    g_steady_state_tracking_filename = steady_state_filename;
+    xml_filename = xmlfile;
+    g_power_simulation_enabled = power_sim_enabled;
+    g_power_simulation_mode = power_sim_mode;
+    g_dvfs_enabled = dvfs_enabled;
+    g_power_trace_enabled = trace_enabled;
+    g_steady_power_levels_enabled = steady_state_enabled;
+    g_power_trace_zlevel = zlevel;
+    g_power_per_cycle_dump = power_per_cycle_dump;
+    gpu_steady_power_deviation = steady_power_deviation;
+    gpu_steady_min_period = steady_min_period;
+
+    gpu_stat_sample_freq = stat_sample_freq;
+
+    // p->sys.total_cycles=gpu_stat_sample_freq*4;
+    p->sys.total_cycles = gpu_stat_sample_freq;
+    p->sys.target_core_clockrate = clock_freq;
+    p->sys.number_of_cores = num_shaders;
+    p->sys.core[0].clock_rate = clock_freq;
+    power_trace_file = NULL;
+    metric_trace_file = NULL;
+    steady_state_tacking_file = NULL;
+
+    if (g_power_trace_enabled) {
+      power_trace_file = gzopen(g_power_trace_filename, "w");
+      metric_trace_file = gzopen(g_metric_trace_filename, "w");
+      if ((power_trace_file == NULL) || (metric_trace_file == NULL)) {
+        printf("error - could not open trace files \n");
+        exit(1);
+      }
+      gzsetparams(power_trace_file, g_power_trace_zlevel, Z_DEFAULT_STRATEGY);
+
+      gzprintf(power_trace_file, "power,");
+      for (unsigned i = 0; i < num_pwr_cmps; i++) {
+        gzprintf(power_trace_file, pwr_cmp_label[i]);
+      }
+      gzprintf(power_trace_file, "\n");
+
+      gzsetparams(metric_trace_file, g_power_trace_zlevel, Z_DEFAULT_STRATEGY);
+      for (unsigned i = 0; i < num_perf_counters; i++) {
+        gzprintf(metric_trace_file, perf_count_label[i]);
+      }
+      gzprintf(metric_trace_file, "\n");
+
+      gzclose(power_trace_file);
+      gzclose(metric_trace_file);
+    }
+    if (g_steady_power_levels_enabled) {
+      steady_state_tacking_file = gzopen(g_steady_state_tracking_filename, "w");
+      if ((steady_state_tacking_file == NULL)) {
+        printf("error - could not open trace files \n");
+        exit(1);
+      }
+      gzsetparams(steady_state_tacking_file, g_power_trace_zlevel,
+                  Z_DEFAULT_STRATEGY);
+      gzprintf(steady_state_tacking_file, "start,end,power,IPC,");
+      for (unsigned i = 0; i < num_perf_counters; i++) {
+        gzprintf(steady_state_tacking_file, perf_count_label[i]);
+      }
+      gzprintf(steady_state_tacking_file, "\n");
+
+      gzclose(steady_state_tacking_file);
+    }
+
+    mcpat_init = false;
+    has_written_avg = false;
+    powerfile.open(g_power_filename);
+    int flg = chmod(g_power_filename, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    assert(flg == 0);
+  }
+  sample_val = 0;
+  init_inst_val = init_val;  // gpu_tot_sim_insn+gpu_sim_insn;
+}
+
+void gpgpu_sim_wrapper::reset_counters() {
+  avg_max_min_counters<double> init;
+  for (unsigned i = 0; i < num_perf_counters; ++i) {
+    sample_perf_counters[i] = 0;
+    kernel_cmp_perf_counters[i] = init;
+  }
+  for (unsigned i = 0; i < num_pwr_cmps; ++i) {
+    sample_cmp_pwr[i] = 0;
+    kernel_cmp_pwr[i] = init;
+  }
+
+  // Reset per-kernel counters
+  kernel_sample_count = 0;
+  kernel_tot_power = 0;
+  kernel_power = init;
+  avg_threads_per_warp_tot = 0;
+  return;
+}
+
+void gpgpu_sim_wrapper::set_inst_power(bool clk_gated_lanes, double tot_cycles,
+                                       double busy_cycles, double tot_inst,
+                                       double int_inst, double fp_inst,
+                                       double load_inst, double store_inst,
+                                       double committed_inst) {
+  p->sys.core[0].gpgpu_clock_gated_lanes = clk_gated_lanes;
+  p->sys.core[0].total_cycles = tot_cycles;
+  p->sys.core[0].busy_cycles = busy_cycles;
+  p->sys.core[0].total_instructions =
+      tot_inst * p->sys.scaling_coefficients[TOT_INST];
+  p->sys.core[0].int_instructions =
+      int_inst * p->sys.scaling_coefficients[FP_INT];
+  p->sys.core[0].fp_instructions =
+      fp_inst * p->sys.scaling_coefficients[FP_INT];
+  p->sys.core[0].load_instructions = load_inst;
+  p->sys.core[0].store_instructions = store_inst;
+  p->sys.core[0].committed_instructions = committed_inst;
+  sample_perf_counters[FP_INT] = int_inst + fp_inst;
+  sample_perf_counters[TOT_INST] = tot_inst;
+}
+
+void gpgpu_sim_wrapper::set_regfile_power(double reads, double writes,
+                                          double ops) {
+  p->sys.core[0].int_regfile_reads =
+      reads * p->sys.scaling_coefficients[REG_RD];
+  p->sys.core[0].int_regfile_writes =
+      writes * p->sys.scaling_coefficients[REG_WR];
+  p->sys.core[0].non_rf_operands =
+      ops * p->sys.scaling_coefficients[NON_REG_OPs];
+  sample_perf_counters[REG_RD] = reads;
+  sample_perf_counters[REG_WR] = writes;
+  sample_perf_counters[NON_REG_OPs] = ops;
+}
+
+void gpgpu_sim_wrapper::set_icache_power(double hits, double misses) {
+  p->sys.core[0].icache.read_accesses =
+      hits * p->sys.scaling_coefficients[IC_H] +
+      misses * p->sys.scaling_coefficients[IC_M];
+  p->sys.core[0].icache.read_misses =
+      misses * p->sys.scaling_coefficients[IC_M];
+  sample_perf_counters[IC_H] = hits;
+  sample_perf_counters[IC_M] = misses;
+}
+
+void gpgpu_sim_wrapper::set_ccache_power(double hits, double misses) {
+  p->sys.core[0].ccache.read_accesses =
+      hits * p->sys.scaling_coefficients[CC_H] +
+      misses * p->sys.scaling_coefficients[CC_M];
+  p->sys.core[0].ccache.read_misses =
+      misses * p->sys.scaling_coefficients[CC_M];
+  sample_perf_counters[CC_H] = hits;
+  sample_perf_counters[CC_M] = misses;
+  // TODO: coalescing logic is counted as part of the caches power (this is not
+  // valid for no-caches architectures)
+}
+
+void gpgpu_sim_wrapper::set_tcache_power(double hits, double misses) {
+  p->sys.core[0].tcache.read_accesses =
+      hits * p->sys.scaling_coefficients[TC_H] +
+      misses * p->sys.scaling_coefficients[TC_M];
+  p->sys.core[0].tcache.read_misses =
+      misses * p->sys.scaling_coefficients[TC_M];
+  sample_perf_counters[TC_H] = hits;
+  sample_perf_counters[TC_M] = misses;
+  // TODO: coalescing logic is counted as part of the caches power (this is not
+  // valid for no-caches architectures)
+}
+
+void gpgpu_sim_wrapper::set_shrd_mem_power(double accesses) {
+  p->sys.core[0].sharedmemory.read_accesses =
+      accesses * p->sys.scaling_coefficients[SHRD_ACC];
+  sample_perf_counters[SHRD_ACC] = accesses;
+}
+
+void gpgpu_sim_wrapper::set_l1cache_power(double read_hits, double read_misses,
+                                          double write_hits,
+                                          double write_misses) {
+  p->sys.core[0].dcache.read_accesses =
+      read_hits * p->sys.scaling_coefficients[DC_RH] +
+      read_misses * p->sys.scaling_coefficients[DC_RM];
+  p->sys.core[0].dcache.read_misses =
+      read_misses * p->sys.scaling_coefficients[DC_RM];
+  p->sys.core[0].dcache.write_accesses =
+      write_hits * p->sys.scaling_coefficients[DC_WH] +
+      write_misses * p->sys.scaling_coefficients[DC_WM];
+  p->sys.core[0].dcache.write_misses =
+      write_misses * p->sys.scaling_coefficients[DC_WM];
+  sample_perf_counters[DC_RH] = read_hits;
+  sample_perf_counters[DC_RM] = read_misses;
+  sample_perf_counters[DC_WH] = write_hits;
+  sample_perf_counters[DC_WM] = write_misses;
+  // TODO: coalescing logic is counted as part of the caches power (this is not
+  // valid for no-caches architectures)
+}
+
+void gpgpu_sim_wrapper::set_l2cache_power(double read_hits, double read_misses,
+                                          double write_hits,
+                                          double write_misses) {
+  p->sys.l2.total_accesses = read_hits * p->sys.scaling_coefficients[L2_RH] +
+                             read_misses * p->sys.scaling_coefficients[L2_RM] +
+                             write_hits * p->sys.scaling_coefficients[L2_WH] +
+                             write_misses * p->sys.scaling_coefficients[L2_WM];
+  p->sys.l2.read_accesses = read_hits * p->sys.scaling_coefficients[L2_RH] +
+                            read_misses * p->sys.scaling_coefficients[L2_RM];
+  p->sys.l2.write_accesses = write_hits * p->sys.scaling_coefficients[L2_WH] +
+                             write_misses * p->sys.scaling_coefficients[L2_WM];
+  p->sys.l2.read_hits = read_hits * p->sys.scaling_coefficients[L2_RH];
+  p->sys.l2.read_misses = read_misses * p->sys.scaling_coefficients[L2_RM];
+  p->sys.l2.write_hits = write_hits * p->sys.scaling_coefficients[L2_WH];
+  p->sys.l2.write_misses = write_misses * p->sys.scaling_coefficients[L2_WM];
+  sample_perf_counters[L2_RH] = read_hits;
+  sample_perf_counters[L2_RM] = read_misses;
+  sample_perf_counters[L2_WH] = write_hits;
+  sample_perf_counters[L2_WM] = write_misses;
+}
+
+void gpgpu_sim_wrapper::set_num_cores(double num_core) {
+  
+  num_cores = num_core;
+}
+
+void gpgpu_sim_wrapper::set_idle_core_power(double num_idle_core) {
+  p->sys.num_idle_cores = num_idle_core;
+  sample_perf_counters[IDLE_CORE_N] = num_idle_core;
+  num_idle_cores = num_idle_core;
+}
+
+void gpgpu_sim_wrapper::set_duty_cycle_power(double duty_cycle) {
+  p->sys.core[0].pipeline_duty_cycle =
+      duty_cycle * p->sys.scaling_coefficients[PIPE_A];
+  sample_perf_counters[PIPE_A] = duty_cycle;
+}
+
+void gpgpu_sim_wrapper::set_mem_ctrl_power(double reads, double writes,
+                                           double dram_precharge) {
+  p->sys.mc.memory_accesses = reads * p->sys.scaling_coefficients[MEM_RD] +
+                              writes * p->sys.scaling_coefficients[MEM_WR];
+  p->sys.mc.memory_reads = reads * p->sys.scaling_coefficients[MEM_RD];
+  p->sys.mc.memory_writes = writes * p->sys.scaling_coefficients[MEM_WR];
+  p->sys.mc.dram_pre = dram_precharge * p->sys.scaling_coefficients[MEM_PRE];
+  sample_perf_counters[MEM_RD] = reads;
+  sample_perf_counters[MEM_WR] = writes;
+  sample_perf_counters[MEM_PRE] = dram_precharge;
+}
+
+
+void gpgpu_sim_wrapper::set_model_voltage(double model_voltage) {
+	modeled_chip_voltage = model_voltage;
+}
+
+
+void gpgpu_sim_wrapper::set_exec_unit_power(double fpu_accesses,
+                                            double ialu_accesses,
+                                            double sfu_accesses) {
+  p->sys.core[0].fpu_accesses = fpu_accesses;
+  tot_fpu_accesses = fpu_accesses;
+  //Integer ALU (not present in Tesla)
+  p->sys.core[0].ialu_accesses = ialu_accesses;
+
+  //Sfu accesses
+  p->sys.core[0].mul_accesses = sfu_accesses;
+  tot_sfu_accesses = sfu_accesses;
+}
+
+PowerscalingCoefficients * gpgpu_sim_wrapper::get_scaling_coeffs()
+{
+
+  PowerscalingCoefficients * scalingCoeffs = new PowerscalingCoefficients();
+
+  scalingCoeffs->int_coeff = p->sys.scaling_coefficients[INT_ACC];
+  scalingCoeffs->int_mul_coeff = p->sys.scaling_coefficients[INT_MUL_ACC];
+  scalingCoeffs->int_mul24_coeff = p->sys.scaling_coefficients[INT_MUL24_ACC];
+  scalingCoeffs->int_mul32_coeff = p->sys.scaling_coefficients[INT_MUL32_ACC];
+  scalingCoeffs->int_div_coeff = p->sys.scaling_coefficients[INT_DIV_ACC];
+  scalingCoeffs->fp_coeff = p->sys.scaling_coefficients[FP_ACC];
+  scalingCoeffs->dp_coeff = p->sys.scaling_coefficients[DP_ACC];
+  scalingCoeffs->fp_mul_coeff = p->sys.scaling_coefficients[FP_MUL_ACC];
+  scalingCoeffs->fp_div_coeff = p->sys.scaling_coefficients[FP_DIV_ACC];
+  scalingCoeffs->dp_mul_coeff = p->sys.scaling_coefficients[DP_MUL_ACC];
+  scalingCoeffs->dp_div_coeff = p->sys.scaling_coefficients[DP_DIV_ACC];
+  scalingCoeffs->sqrt_coeff = p->sys.scaling_coefficients[FP_SQRT_ACC];
+  scalingCoeffs->log_coeff = p->sys.scaling_coefficients[FP_LG_ACC];
+  scalingCoeffs->sin_coeff = p->sys.scaling_coefficients[FP_SIN_ACC];
+  scalingCoeffs->exp_coeff = p->sys.scaling_coefficients[FP_EXP_ACC];
+  scalingCoeffs->tensor_coeff = p->sys.scaling_coefficients[TENSOR_ACC];
+  scalingCoeffs->tex_coeff = p->sys.scaling_coefficients[TEX_ACC];
+  return scalingCoeffs;
+
+}
+
+void gpgpu_sim_wrapper::set_int_accesses(double ialu_accesses, 
+                                        double imul24_accesses, 
+                                        double imul32_accesses, 
+                                        double imul_accesses, 
+                                        double idiv_accesses)
+{
+
+  sample_perf_counters[INT_ACC]=ialu_accesses;
+  sample_perf_counters[INT_MUL24_ACC]=imul24_accesses;
+  sample_perf_counters[INT_MUL32_ACC]=imul32_accesses;
+  sample_perf_counters[INT_MUL_ACC]=imul_accesses;
+  sample_perf_counters[INT_DIV_ACC]=idiv_accesses;
+}
+
+void gpgpu_sim_wrapper::set_dp_accesses(double dpu_accesses, 
+                                        double dpmul_accesses, 
+                                        double dpdiv_accesses)
+{
+  sample_perf_counters[DP_ACC]=dpu_accesses;
+  sample_perf_counters[DP_MUL_ACC]=dpmul_accesses;
+  sample_perf_counters[DP_DIV_ACC]=dpdiv_accesses;
+}
+
+void gpgpu_sim_wrapper::set_fp_accesses(double fpu_accesses, 
+                                        double fpmul_accesses, 
+                                        double fpdiv_accesses)
+{
+  sample_perf_counters[FP_ACC]=fpu_accesses;
+  sample_perf_counters[FP_MUL_ACC]=fpmul_accesses;
+  sample_perf_counters[FP_DIV_ACC]=fpdiv_accesses;
+}
+
+void gpgpu_sim_wrapper::set_trans_accesses(double sqrt_accesses, 
+                                           double log_accesses, 
+                                           double sin_accesses, 
+                                           double exp_accesses)
+{
+
+  sample_perf_counters[FP_SQRT_ACC]=sqrt_accesses;
+  sample_perf_counters[FP_LG_ACC]=log_accesses;
+  sample_perf_counters[FP_SIN_ACC]=sin_accesses;
+  sample_perf_counters[FP_EXP_ACC]=exp_accesses;
+
+}
+
+void gpgpu_sim_wrapper::set_tensor_accesses(double tensor_accesses)
+{
+  sample_perf_counters[TENSOR_ACC]=tensor_accesses;
+
+}
+
+void gpgpu_sim_wrapper::set_tex_accesses(double tex_accesses)
+{
+  sample_perf_counters[TEX_ACC]=tex_accesses;
+
+}
+
+void gpgpu_sim_wrapper::set_avg_active_threads(float active_threads)
+{
+  avg_threads_per_warp = (unsigned)ceil(active_threads);
+  avg_threads_per_warp_tot += active_threads;
+}
+
+void gpgpu_sim_wrapper::set_active_lanes_power(double sp_avg_active_lane,
+                                               double sfu_avg_active_lane) {
+  p->sys.core[0].sp_average_active_lanes = sp_avg_active_lane;
+  p->sys.core[0].sfu_average_active_lanes = sfu_avg_active_lane;
+}
+
+void gpgpu_sim_wrapper::set_NoC_power(double noc_tot_acc) {
+  p->sys.NoC[0].total_accesses =
+      noc_tot_acc * p->sys.scaling_coefficients[NOC_A];
+  sample_perf_counters[NOC_A] = noc_tot_acc;
+}
+
+void gpgpu_sim_wrapper::power_metrics_calculations() {
+  total_sample_count++;
+  kernel_sample_count++;
+
+  // Current sample power
+  double sample_power = proc->rt_power.readOp.dynamic + sample_cmp_pwr[CONSTP] + sample_cmp_pwr[STATICP];
+  // double sample_power;
+  // for(unsigned i=0; i<num_pwr_cmps; i++){
+  //   sample_power+=sample_cmp_pwr[i]; //fix for dvfs
+  // }
+
+  // Average power
+  // Previous + new + constant dynamic power (e.g., dynamic clocking power)
+  kernel_tot_power += sample_power;
+  kernel_power.avg = kernel_tot_power / kernel_sample_count;
+  for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
+    kernel_cmp_pwr[ind].avg += (double)sample_cmp_pwr[ind];
+  }
+
+  for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
+    kernel_cmp_perf_counters[ind].avg += (double)sample_perf_counters[ind];
+  }
+
+  // Max Power
+  if (sample_power > kernel_power.max) {
+    kernel_power.max = sample_power;
+    for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
+      kernel_cmp_pwr[ind].max = (double)sample_cmp_pwr[ind];
+    }
+    for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
+      kernel_cmp_perf_counters[ind].max = sample_perf_counters[ind];
+    }
+  }
+
+  // Min Power
+  if (sample_power < kernel_power.min || (kernel_power.min == 0)) {
+    kernel_power.min = sample_power;
+    for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
+      kernel_cmp_pwr[ind].min = (double)sample_cmp_pwr[ind];
+    }
+    for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
+      kernel_cmp_perf_counters[ind].min = sample_perf_counters[ind];
+    }
+  }
+
+  gpu_tot_power.avg = (gpu_tot_power.avg + sample_power);
+  gpu_tot_power.max =
+      (sample_power > gpu_tot_power.max) ? sample_power : gpu_tot_power.max;
+  gpu_tot_power.min =
+      ((sample_power < gpu_tot_power.min) || (gpu_tot_power.min == 0))
+          ? sample_power
+          : gpu_tot_power.min;
+}
+
+void gpgpu_sim_wrapper::print_trace_files() {
+  open_files();
+
+  for (unsigned i = 0; i < num_perf_counters; ++i) {
+    gzprintf(metric_trace_file, "%f,", sample_perf_counters[i]);
+  }
+  gzprintf(metric_trace_file, "\n");
+
+  gzprintf(power_trace_file, "%f,", proc_power);
+  for (unsigned i = 0; i < num_pwr_cmps; ++i) {
+    gzprintf(power_trace_file, "%f,", sample_cmp_pwr[i]);
+  }
+  gzprintf(power_trace_file, "\n");
+
+  close_files();
+}
+
+void gpgpu_sim_wrapper::update_coefficients()
+{
+
+  initpower_coeff[FP_INT]=proc->cores[0]->get_coefficient_fpint_insts();
+  effpower_coeff[FP_INT]=initpower_coeff[FP_INT] * p->sys.scaling_coefficients[FP_INT];
+
+  initpower_coeff[TOT_INST]=proc->cores[0]->get_coefficient_tot_insts();
+  effpower_coeff[TOT_INST]=initpower_coeff[TOT_INST] * p->sys.scaling_coefficients[TOT_INST];
+
+  initpower_coeff[REG_RD]=proc->cores[0]->get_coefficient_regreads_accesses()*(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+  initpower_coeff[REG_WR]=proc->cores[0]->get_coefficient_regwrites_accesses()*(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+  initpower_coeff[NON_REG_OPs]=proc->cores[0]->get_coefficient_noregfileops_accesses()*(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+  effpower_coeff[REG_RD]=initpower_coeff[REG_RD]*p->sys.scaling_coefficients[REG_RD];
+  effpower_coeff[REG_WR]=initpower_coeff[REG_WR]*p->sys.scaling_coefficients[REG_WR];
+  effpower_coeff[NON_REG_OPs]=initpower_coeff[NON_REG_OPs]*p->sys.scaling_coefficients[NON_REG_OPs];
+
+  initpower_coeff[IC_H]=proc->cores[0]->get_coefficient_icache_hits();
+  initpower_coeff[IC_M]=proc->cores[0]->get_coefficient_icache_misses();
+  effpower_coeff[IC_H]=initpower_coeff[IC_H]*p->sys.scaling_coefficients[IC_H];
+  effpower_coeff[IC_M]=initpower_coeff[IC_M]*p->sys.scaling_coefficients[IC_M];
+
+  initpower_coeff[CC_H]=(proc->cores[0]->get_coefficient_ccache_readhits()+proc->get_coefficient_readcoalescing());
+  initpower_coeff[CC_M]=(proc->cores[0]->get_coefficient_ccache_readmisses()+proc->get_coefficient_readcoalescing());
+  effpower_coeff[CC_H]=initpower_coeff[CC_H]*p->sys.scaling_coefficients[CC_H];
+  effpower_coeff[CC_M]=initpower_coeff[CC_M]*p->sys.scaling_coefficients[CC_M];
+
+  initpower_coeff[TC_H]=(proc->cores[0]->get_coefficient_tcache_readhits()+proc->get_coefficient_readcoalescing());
+  initpower_coeff[TC_M]=(proc->cores[0]->get_coefficient_tcache_readmisses()+proc->get_coefficient_readcoalescing());
+  effpower_coeff[TC_H]=initpower_coeff[TC_H]*p->sys.scaling_coefficients[TC_H];
+  effpower_coeff[TC_M]=initpower_coeff[TC_M]*p->sys.scaling_coefficients[TC_M];
+
+  initpower_coeff[SHRD_ACC]=proc->cores[0]->get_coefficient_sharedmemory_readhits();
+  effpower_coeff[SHRD_ACC]=initpower_coeff[SHRD_ACC]*p->sys.scaling_coefficients[SHRD_ACC];
+
+  initpower_coeff[DC_RH]=(proc->cores[0]->get_coefficient_dcache_readhits() + proc->get_coefficient_readcoalescing());
+  initpower_coeff[DC_RM]=(proc->cores[0]->get_coefficient_dcache_readmisses() + proc->get_coefficient_readcoalescing());
+  initpower_coeff[DC_WH]=(proc->cores[0]->get_coefficient_dcache_writehits() + proc->get_coefficient_writecoalescing());
+  initpower_coeff[DC_WM]=(proc->cores[0]->get_coefficient_dcache_writemisses() + proc->get_coefficient_writecoalescing());
+  effpower_coeff[DC_RH]=initpower_coeff[DC_RH]*p->sys.scaling_coefficients[DC_RH];
+  effpower_coeff[DC_RM]=initpower_coeff[DC_RM]*p->sys.scaling_coefficients[DC_RM];
+  effpower_coeff[DC_WH]=initpower_coeff[DC_WH]*p->sys.scaling_coefficients[DC_WH];
+  effpower_coeff[DC_WM]=initpower_coeff[DC_WM]*p->sys.scaling_coefficients[DC_WM];
+
+  initpower_coeff[L2_RH]=proc->get_coefficient_l2_read_hits();
+  initpower_coeff[L2_RM]=proc->get_coefficient_l2_read_misses();
+  initpower_coeff[L2_WH]=proc->get_coefficient_l2_write_hits();
+  initpower_coeff[L2_WM]=proc->get_coefficient_l2_write_misses();
+  effpower_coeff[L2_RH]=initpower_coeff[L2_RH]*p->sys.scaling_coefficients[L2_RH];
+  effpower_coeff[L2_RM]=initpower_coeff[L2_RM]*p->sys.scaling_coefficients[L2_RM];
+  effpower_coeff[L2_WH]=initpower_coeff[L2_WH]*p->sys.scaling_coefficients[L2_WH];
+  effpower_coeff[L2_WM]=initpower_coeff[L2_WM]*p->sys.scaling_coefficients[L2_WM];
+
+  initpower_coeff[IDLE_CORE_N]=p->sys.idle_core_power * proc->cores[0]->executionTime;
+  effpower_coeff[IDLE_CORE_N]=initpower_coeff[IDLE_CORE_N]*p->sys.scaling_coefficients[IDLE_CORE_N];
+
+  initpower_coeff[PIPE_A]=proc->cores[0]->get_coefficient_duty_cycle();
+  effpower_coeff[PIPE_A]=initpower_coeff[PIPE_A]*p->sys.scaling_coefficients[PIPE_A];
+
+  initpower_coeff[MEM_RD]=proc->get_coefficient_mem_reads();
+  initpower_coeff[MEM_WR]=proc->get_coefficient_mem_writes();
+  initpower_coeff[MEM_PRE]=proc->get_coefficient_mem_pre();
+  effpower_coeff[MEM_RD]=initpower_coeff[MEM_RD]*p->sys.scaling_coefficients[MEM_RD];
+  effpower_coeff[MEM_WR]=initpower_coeff[MEM_WR]*p->sys.scaling_coefficients[MEM_WR];
+  effpower_coeff[MEM_PRE]=initpower_coeff[MEM_PRE]*p->sys.scaling_coefficients[MEM_PRE];
+  
+  double fp_coeff = proc->cores[0]->get_coefficient_fpu_accesses();
+  double sfu_coeff = proc->cores[0]->get_coefficient_sfu_accesses();
+
+  initpower_coeff[INT_ACC]= proc->cores[0]->get_coefficient_ialu_accesses()*(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+
+  if(tot_fpu_accesses != 0){
+    initpower_coeff[FP_ACC]= fp_coeff * sample_perf_counters[FP_ACC]/tot_fpu_accesses;
+    initpower_coeff[DP_ACC]= fp_coeff * sample_perf_counters[DP_ACC]/tot_fpu_accesses;
+  }
+  else{
+    initpower_coeff[FP_ACC]= 0;
+    initpower_coeff[DP_ACC]= 0;
+  }
+
+  if(tot_sfu_accesses != 0){
+    initpower_coeff[INT_MUL24_ACC]= sfu_coeff * sample_perf_counters[INT_MUL24_ACC]/tot_sfu_accesses;
+    initpower_coeff[INT_MUL32_ACC]= sfu_coeff * sample_perf_counters[INT_MUL32_ACC]/tot_sfu_accesses;
+    initpower_coeff[INT_MUL_ACC]= sfu_coeff * sample_perf_counters[INT_MUL_ACC]/tot_sfu_accesses;
+    initpower_coeff[INT_DIV_ACC]= sfu_coeff * sample_perf_counters[INT_DIV_ACC]/tot_sfu_accesses;
+    initpower_coeff[DP_MUL_ACC]= sfu_coeff * sample_perf_counters[DP_MUL_ACC]/tot_sfu_accesses;
+    initpower_coeff[DP_DIV_ACC]= sfu_coeff * sample_perf_counters[DP_DIV_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_MUL_ACC]= sfu_coeff * sample_perf_counters[FP_MUL_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_DIV_ACC]= sfu_coeff * sample_perf_counters[FP_DIV_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_SQRT_ACC]= sfu_coeff * sample_perf_counters[FP_SQRT_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_LG_ACC]= sfu_coeff * sample_perf_counters[FP_LG_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_SIN_ACC]= sfu_coeff * sample_perf_counters[FP_SIN_ACC]/tot_sfu_accesses;
+    initpower_coeff[FP_EXP_ACC]= sfu_coeff * sample_perf_counters[FP_EXP_ACC]/tot_sfu_accesses;
+    initpower_coeff[TENSOR_ACC]= sfu_coeff * sample_perf_counters[TENSOR_ACC]/tot_sfu_accesses;
+    initpower_coeff[TEX_ACC]= sfu_coeff * sample_perf_counters[TEX_ACC]/tot_sfu_accesses;
+  }
+  else{
+    initpower_coeff[INT_MUL24_ACC]= 0;
+    initpower_coeff[INT_MUL32_ACC]= 0;
+    initpower_coeff[INT_MUL_ACC]= 0;
+    initpower_coeff[INT_DIV_ACC]= 0;
+    initpower_coeff[DP_MUL_ACC]= 0;
+    initpower_coeff[DP_DIV_ACC]= 0;
+    initpower_coeff[FP_MUL_ACC]= 0;
+    initpower_coeff[FP_DIV_ACC]= 0;
+    initpower_coeff[FP_SQRT_ACC]= 0;
+    initpower_coeff[FP_LG_ACC]= 0;
+    initpower_coeff[FP_SIN_ACC]= 0;
+    initpower_coeff[FP_EXP_ACC]= 0;
+    initpower_coeff[TENSOR_ACC]= 0;
+    initpower_coeff[TEX_ACC]= 0;
+  }
+
+  effpower_coeff[INT_ACC]= initpower_coeff[INT_ACC];
+  effpower_coeff[FP_ACC]= initpower_coeff[FP_ACC];
+  effpower_coeff[DP_ACC]= initpower_coeff[DP_ACC];
+  effpower_coeff[INT_MUL24_ACC]= initpower_coeff[INT_MUL24_ACC];
+  effpower_coeff[INT_MUL32_ACC]= initpower_coeff[INT_MUL32_ACC];
+  effpower_coeff[INT_MUL_ACC]= initpower_coeff[INT_MUL_ACC];
+  effpower_coeff[INT_DIV_ACC]= initpower_coeff[INT_DIV_ACC];
+  effpower_coeff[DP_MUL_ACC]= initpower_coeff[DP_MUL_ACC];
+  effpower_coeff[DP_DIV_ACC]= initpower_coeff[DP_DIV_ACC];
+  effpower_coeff[FP_MUL_ACC]= initpower_coeff[FP_MUL_ACC];
+  effpower_coeff[FP_DIV_ACC]= initpower_coeff[FP_DIV_ACC];
+  effpower_coeff[FP_SQRT_ACC]= initpower_coeff[FP_SQRT_ACC];
+  effpower_coeff[FP_LG_ACC]= initpower_coeff[FP_LG_ACC];
+  effpower_coeff[FP_SIN_ACC]= initpower_coeff[FP_SIN_ACC];
+  effpower_coeff[FP_EXP_ACC]= initpower_coeff[FP_EXP_ACC];
+  effpower_coeff[TENSOR_ACC]= initpower_coeff[TENSOR_ACC];
+  effpower_coeff[TEX_ACC]= initpower_coeff[TEX_ACC];
+
+  initpower_coeff[NOC_A]=proc->get_coefficient_noc_accesses();
+  effpower_coeff[NOC_A]=initpower_coeff[NOC_A]*p->sys.scaling_coefficients[NOC_A];
+
+  //const_dynamic_power=proc->get_const_dynamic_power()/(proc->cores[0]->executionTime);
+
+  for(unsigned i=0; i<num_perf_counters; i++){
+    initpower_coeff[i]/=(proc->cores[0]->executionTime);
+    effpower_coeff[i]/=(proc->cores[0]->executionTime);
+  }
+}
+
+double gpgpu_sim_wrapper::calculate_static_power(){ 
+	double int_accesses = initpower_coeff[INT_ACC] + initpower_coeff[INT_MUL24_ACC] + initpower_coeff[INT_MUL32_ACC] + initpower_coeff[INT_MUL_ACC] + initpower_coeff[INT_DIV_ACC];
+	double int_add_accesses = initpower_coeff[INT_ACC];
+	double int_mul_accesses = initpower_coeff[INT_MUL24_ACC] + initpower_coeff[INT_MUL32_ACC] + initpower_coeff[INT_MUL_ACC] + initpower_coeff[INT_DIV_ACC];
+	double fp_accesses = initpower_coeff[FP_ACC] + initpower_coeff[FP_MUL_ACC] + initpower_coeff[FP_DIV_ACC];
+	double dp_accesses = initpower_coeff[DP_ACC] + initpower_coeff[DP_MUL_ACC] + initpower_coeff[DP_DIV_ACC];
+	double sfu_accesses = initpower_coeff[FP_SQRT_ACC] + initpower_coeff[FP_LG_ACC] + initpower_coeff[FP_SIN_ACC] + initpower_coeff[FP_EXP_ACC];
+	double tensor_accesses = initpower_coeff[TENSOR_ACC];
+	double tex_accesses = initpower_coeff[TEX_ACC];
+	double total_static_power = 0.0;
+	double base_static_power = 0.0; 
+	double lane_static_power = 0.0;
+	double per_active_core = (num_cores - num_idle_cores)/num_cores;
+
+
+	double l1_accesses = initpower_coeff[DC_RH] + initpower_coeff[DC_RM] + initpower_coeff[DC_WH] + initpower_coeff[DC_WM];
+	double l2_accesses = initpower_coeff[L2_RH] + initpower_coeff[L2_RM] + initpower_coeff[L2_WH] + initpower_coeff[L2_WM];
+	double shared_accesses = initpower_coeff[SHRD_ACC];
+
+
+	if(avg_threads_per_warp == 0){ //no functional unit threads, check for memory or a 'LIGHT_SM'
+		if(l1_accesses != 0.0)
+			return (p->sys.static_l1_flane*per_active_core);
+		else if(shared_accesses != 0.0)
+			return (p->sys.static_shared_flane*per_active_core);
+		else if(l2_accesses != 0.0)
+			return (p->sys.static_l2_flane*per_active_core);
+		else //LIGHT_SM
+			return (p->sys.static_light_flane*per_active_core); //return LIGHT_SM base static power
+	}
+
+	/* using a linear model for thread divergence */
+	if((int_accesses != 0.0) && (fp_accesses != 0.0) && (dp_accesses != 0.0) && (sfu_accesses == 0.0) && (tensor_accesses == 0.0) && (tex_accesses == 0.0)){
+		/* INT_FP_DP */
+		base_static_power = p->sys.static_cat3_flane;
+		lane_static_power = p->sys.static_cat3_addlane;
+	}
+
+	else if((int_accesses != 0.0) && (fp_accesses != 0.0) && (dp_accesses == 0.0) && (sfu_accesses == 0.0) && (tensor_accesses != 0.0) && (tex_accesses == 0.0)){
+		/* INT_FP_TENSOR */
+		base_static_power = p->sys.static_cat6_flane;
+		lane_static_power = p->sys.static_cat6_addlane;
+	}
+
+	else if((int_accesses != 0.0) && (fp_accesses != 0.0) && (dp_accesses == 0.0) && (sfu_accesses != 0.0) && (tensor_accesses == 0.0) && (tex_accesses == 0.0)){
+		/* INT_FP_SFU */
+		base_static_power = p->sys.static_cat4_flane;
+		lane_static_power = p->sys.static_cat4_addlane;
+	}
+
+	else if((int_accesses != 0.0) && (fp_accesses != 0.0) && (dp_accesses == 0.0) && (sfu_accesses == 0.0) && (tensor_accesses == 0.0) && (tex_accesses != 0.0)){
+		/* INT_FP_TEX */
+		base_static_power = p->sys.static_cat5_flane;
+		lane_static_power = p->sys.static_cat5_addlane;
+	}
+
+	else if((int_accesses != 0.0) && (fp_accesses != 0.0) && (dp_accesses == 0.0) && (sfu_accesses == 0.0) && (tensor_accesses == 0.0) && (tex_accesses == 0.0)){
+		/* INT_FP */
+		base_static_power = p->sys.static_cat2_flane;
+		lane_static_power = p->sys.static_cat2_addlane;
+	}
+
+	else if((int_accesses != 0.0) && (fp_accesses == 0.0) && (dp_accesses == 0.0) && (sfu_accesses == 0.0) && (tensor_accesses == 0.0) && (tex_accesses == 0.0)){
+		/* INT */
+		/* Seperating INT_ADD only and INT_MUL only from mix of INT instructions */
+		if((int_add_accesses != 0.0) && (int_mul_accesses == 0.0)){ //INT_ADD
+			base_static_power = p->sys.static_intadd_flane;
+			lane_static_power = p->sys.static_intadd_addlane;
+		}
+		else if((int_add_accesses == 0.0) && (int_mul_accesses != 0.0)){ //INT_MUL
+			base_static_power = p->sys.static_intmul_flane;
+			lane_static_power = p->sys.static_intmul_addlane;
+		}
+		else{ //INT_ADD+MUL
+			base_static_power = p->sys.static_cat1_flane;
+			lane_static_power = p->sys.static_cat1_addlane;
+		}
+	}
+
+	else if((int_accesses == 0.0) && (fp_accesses == 0.0) && (dp_accesses == 0.0) && (sfu_accesses == 0.0) && (tensor_accesses == 0.0) && (tex_accesses == 0.0)){
+		/* LIGHT_SM or memory only sample */
+		lane_static_power = 0.0; //addlane static power is 0 for l1/l2/shared memory only accesses
+		if(l1_accesses != 0.0)
+			base_static_power = p->sys.static_l1_flane;
+		else if(shared_accesses != 0.0)
+			base_static_power = p->sys.static_shared_flane;
+		else if(l2_accesses != 0.0)
+			base_static_power = p->sys.static_l2_flane;
+		else{
+			base_static_power = p->sys.static_light_flane;
+			lane_static_power = p->sys.static_light_addlane;
+		}
+	}
+	else{
+		base_static_power = p->sys.static_geomean_flane; //GEOMEAN except LIGHT_SM if we don't fall into any of the categories above
+		lane_static_power = p->sys.static_geomean_addlane;
+	}
+
+	total_static_power = base_static_power + (((double)avg_threads_per_warp-1.0)*lane_static_power); //Linear Model
+	return (total_static_power*per_active_core);
+}
+
+void gpgpu_sim_wrapper::update_components_power()
+{
+
+  update_coefficients();
+
+  proc_power=proc->rt_power.readOp.dynamic;
+  sample_cmp_pwr[IBP]=(proc->cores[0]->ifu->IB->rt_power.readOp.dynamic
+          +proc->cores[0]->ifu->IB->rt_power.writeOp.dynamic
+          +proc->cores[0]->ifu->ID_misc->rt_power.readOp.dynamic
+          +proc->cores[0]->ifu->ID_operand->rt_power.readOp.dynamic
+          +proc->cores[0]->ifu->ID_inst->rt_power.readOp.dynamic)/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[ICP]=proc->cores[0]->ifu->icache.rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[DCP]=proc->cores[0]->lsu->dcache.rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[TCP]=proc->cores[0]->lsu->tcache.rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[CCP]=proc->cores[0]->lsu->ccache.rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[SHRDP]=proc->cores[0]->lsu->sharedmemory.rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[RFP]=(proc->cores[0]->exu->rfu->rt_power.readOp.dynamic/(proc->cores[0]->executionTime))
+         *(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+
+  double sample_fp_pwr = (proc->cores[0]->exu->fp_u->rt_power.readOp.dynamic/(proc->cores[0]->executionTime));
+
+  double sample_sfu_pwr = (proc->cores[0]->exu->mul->rt_power.readOp.dynamic/(proc->cores[0]->executionTime));
+
+  sample_cmp_pwr[INTP]=(proc->cores[0]->exu->exeu->rt_power.readOp.dynamic/(proc->cores[0]->executionTime))
+         *(proc->cores[0]->exu->rf_fu_clockRate/proc->cores[0]->exu->clockRate);
+
+  
+  if(tot_fpu_accesses != 0){
+    sample_cmp_pwr[FPUP]= sample_fp_pwr * sample_perf_counters[FP_ACC]/tot_fpu_accesses;
+    sample_cmp_pwr[DPUP]= sample_fp_pwr * sample_perf_counters[DP_ACC]/tot_fpu_accesses;
+  }
+  else{
+    sample_cmp_pwr[FPUP]= 0;
+    sample_cmp_pwr[DPUP]= 0;
+  }
+  if(tot_sfu_accesses != 0){
+    sample_cmp_pwr[INT_MUL24P]= sample_sfu_pwr * sample_perf_counters[INT_MUL24_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[INT_MUL32P]= sample_sfu_pwr * sample_perf_counters[INT_MUL32_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[INT_MULP]= sample_sfu_pwr * sample_perf_counters[INT_MUL_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[INT_DIVP]= sample_sfu_pwr * sample_perf_counters[INT_DIV_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_MULP]= sample_sfu_pwr * sample_perf_counters[FP_MUL_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_DIVP]= sample_sfu_pwr * sample_perf_counters[FP_DIV_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_SQRTP]= sample_sfu_pwr * sample_perf_counters[FP_SQRT_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_LGP]= sample_sfu_pwr * sample_perf_counters[FP_LG_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_SINP]= sample_sfu_pwr * sample_perf_counters[FP_SIN_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[FP_EXP]= sample_sfu_pwr * sample_perf_counters[FP_EXP_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[DP_MULP]= sample_sfu_pwr * sample_perf_counters[DP_MUL_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[DP_DIVP]= sample_sfu_pwr * sample_perf_counters[DP_DIV_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[TENSORP]= sample_sfu_pwr * sample_perf_counters[TENSOR_ACC]/tot_sfu_accesses;
+    sample_cmp_pwr[TEXP]= sample_sfu_pwr * sample_perf_counters[TEX_ACC]/tot_sfu_accesses;
+  }
+  else{
+    sample_cmp_pwr[INT_MUL24P]= 0;
+    sample_cmp_pwr[INT_MUL32P]= 0;
+    sample_cmp_pwr[INT_MULP]= 0;
+    sample_cmp_pwr[INT_DIVP]= 0;
+    sample_cmp_pwr[FP_MULP]= 0;
+    sample_cmp_pwr[FP_DIVP]= 0;
+    sample_cmp_pwr[FP_SQRTP]= 0;
+    sample_cmp_pwr[FP_LGP]= 0;
+    sample_cmp_pwr[FP_SINP]= 0;
+    sample_cmp_pwr[FP_EXP]= 0;
+    sample_cmp_pwr[DP_MULP]= 0;
+    sample_cmp_pwr[DP_DIVP]= 0;
+    sample_cmp_pwr[TENSORP]= 0;
+    sample_cmp_pwr[TEXP]= 0;
+  }
+
+  sample_cmp_pwr[SCHEDP]=proc->cores[0]->exu->scheu->rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[L2CP]=(proc->XML->sys.number_of_L2s>0)? proc->l2array[0]->rt_power.readOp.dynamic/(proc->cores[0]->executionTime):0;
+
+  sample_cmp_pwr[MCP]=(proc->mc->rt_power.readOp.dynamic-proc->mc->dram->rt_power.readOp.dynamic)/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[NOCP]=proc->nocs[0]->rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[DRAMP]=proc->mc->dram->rt_power.readOp.dynamic/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[PIPEP]=proc->cores[0]->Pipeline_energy/(proc->cores[0]->executionTime);
+
+  sample_cmp_pwr[IDLE_COREP]=proc->cores[0]->IdleCoreEnergy/(proc->cores[0]->executionTime);
+
+  // This constant dynamic power (e.g., clock power) part is estimated via regression model.
+  sample_cmp_pwr[CONSTP]=0;
+  sample_cmp_pwr[STATICP]=0;
+  // double cnst_dyn = proc->get_const_dynamic_power()/(proc->cores[0]->executionTime);
+  // // If the regression scaling term is greater than the recorded constant dynamic power
+  // // then use the difference (other portion already added to dynamic power). Else,
+  // // all the constant dynamic power is accounted for, add nothing.
+  // if(p->sys.scaling_coefficients[constant_power] > cnst_dyn)
+  //   sample_cmp_pwr[CONSTP] = (p->sys.scaling_coefficients[constant_power]-cnst_dyn);
+  sample_cmp_pwr[CONSTP] = p->sys.scaling_coefficients[constant_power];
+  sample_cmp_pwr[STATICP] = calculate_static_power();
+
+  if(g_dvfs_enabled){
+  	double voltage_ratio = modeled_chip_voltage/p->sys.modeled_chip_voltage_ref; 
+  	sample_cmp_pwr[IDLE_COREP] *= voltage_ratio; // static power scaled by voltage_ratio
+  	sample_cmp_pwr[STATICP] *= voltage_ratio;  // static power scaled by voltage_ratio
+  	for(unsigned i=0; i<num_pwr_cmps; i++){
+    	if((i != IDLE_COREP) && (i != STATICP)){ 
+    		sample_cmp_pwr[i] *= voltage_ratio*voltage_ratio; // dynamic power scaled by square of voltage_ratio
+    	}
+  	}
+  }
+  
+  proc_power+=sample_cmp_pwr[CONSTP]+sample_cmp_pwr[STATICP];
+  if(!g_dvfs_enabled){ // sanity check will fail when voltage scaling is applied, fix later
+	  double sum_pwr_cmp=0;
+	  for(unsigned i=0; i<num_pwr_cmps; i++){
+	    sum_pwr_cmp+=sample_cmp_pwr[i];
+	  }
+	  bool check=false;
+	  check=sanity_check(sum_pwr_cmp,proc_power);
+	  if(!check)
+	    printf("sum_pwr_cmp %f : proc_power %f \n",sum_pwr_cmp,proc_power);
+	  assert("Total Power does not equal the sum of the components\n" && (check));
+  }
+}
+
+void gpgpu_sim_wrapper::compute() { proc->compute(); }
+void gpgpu_sim_wrapper::print_power_kernel_stats(
+    double gpu_sim_cycle, double gpu_tot_sim_cycle, double init_value,
+    const std::string& kernel_info_string, bool print_trace) {
+  detect_print_steady_state(1, init_value);
+  if (g_power_simulation_enabled) {
+    powerfile << kernel_info_string << std::endl;
+
+    sanity_check((kernel_power.avg * kernel_sample_count), kernel_tot_power);
+    powerfile << "Kernel Average Power Data:" << std::endl;
+    powerfile << "kernel_avg_power = " << kernel_power.avg << std::endl;
+
+    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
+      powerfile << "gpu_avg_" << pwr_cmp_label[i] << " = "
+                << kernel_cmp_pwr[i].avg / kernel_sample_count << std::endl;
+    }
+    for (unsigned i = 0; i < num_perf_counters; ++i) {
+      powerfile << "gpu_avg_" << perf_count_label[i] << " = "
+                << kernel_cmp_perf_counters[i].avg / kernel_sample_count
+                << std::endl;
+    }
+
+    powerfile << "gpu_avg_threads_per_warp = "
+                << avg_threads_per_warp_tot / (double)kernel_sample_count
+                << std::endl;
+
+    for (unsigned i = 0; i < num_perf_counters; ++i) {
+      powerfile << "gpu_tot_" << perf_count_label[i] << " = "
+                << kernel_cmp_perf_counters[i].avg
+                << std::endl;
+    }
+
+    powerfile << std::endl << "Kernel Maximum Power Data:" << std::endl;
+    powerfile << "kernel_max_power = " << kernel_power.max << std::endl;
+    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
+      powerfile << "gpu_max_" << pwr_cmp_label[i] << " = "
+                << kernel_cmp_pwr[i].max << std::endl;
+    }
+    for (unsigned i = 0; i < num_perf_counters; ++i) {
+      powerfile << "gpu_max_" << perf_count_label[i] << " = "
+                << kernel_cmp_perf_counters[i].max << std::endl;
+    }
+
+    powerfile << std::endl << "Kernel Minimum Power Data:" << std::endl;
+    powerfile << "kernel_min_power = " << kernel_power.min << std::endl;
+    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
+      powerfile << "gpu_min_" << pwr_cmp_label[i] << " = "
+                << kernel_cmp_pwr[i].min << std::endl;
+    }
+    for (unsigned i = 0; i < num_perf_counters; ++i) {
+      powerfile << "gpu_min_" << perf_count_label[i] << " = "
+                << kernel_cmp_perf_counters[i].min << std::endl;
+    }
+
+    powerfile << std::endl
+              << "Accumulative Power Statistics Over Previous Kernels:"
+              << std::endl;
+    powerfile << "gpu_tot_avg_power = "
+              << gpu_tot_power.avg / total_sample_count << std::endl;
+    powerfile << "gpu_tot_max_power = " << gpu_tot_power.max << std::endl;
+    powerfile << "gpu_tot_min_power = " << gpu_tot_power.min << std::endl;
+    powerfile << std::endl << std::endl;
+    powerfile.flush();
+
+    if (print_trace) {
+      print_trace_files();
+    }
+  }
+}
+void gpgpu_sim_wrapper::dump() {
+  if (g_power_per_cycle_dump) proc->displayEnergy(2, 5);
+}
+
+void gpgpu_sim_wrapper::print_steady_state(int position, double init_val) {
+  double temp_avg = sample_val / (double)samples.size();
+  double temp_ipc = (init_val - init_inst_val) /
+                    (double)(samples.size() * gpu_stat_sample_freq);
+
+  if ((samples.size() >
+       gpu_steady_min_period)) {  // If steady state occurred for some time,
+                                  // print to file
+    has_written_avg = true;
+    gzprintf(steady_state_tacking_file, "%u,%d,%f,%f,", sample_start,
+             total_sample_count, temp_avg, temp_ipc);
+    for (unsigned i = 0; i < num_perf_counters; ++i) {
+      gzprintf(steady_state_tacking_file, "%f,",
+               samples_counter.at(i) / ((double)samples.size()));
+    }
+    gzprintf(steady_state_tacking_file, "\n");
+  } else {
+    if (!has_written_avg && position)
+      gzprintf(steady_state_tacking_file,
+               "ERROR! Not enough steady state points to generate average\n");
+  }
+
+  sample_start = 0;
+  sample_val = 0;
+  init_inst_val = init_val;
+  samples.clear();
+  samples_counter.clear();
+  pwr_counter.clear();
+  assert(samples.size() == 0);
+}
+
+void gpgpu_sim_wrapper::detect_print_steady_state(int position,
+                                                  double init_val) {
+  // Calculating Average
+  if (g_power_simulation_enabled && g_steady_power_levels_enabled) {
+    steady_state_tacking_file = gzopen(g_steady_state_tracking_filename, "a");
+    if (position == 0) {
+      if (samples.size() == 0) {
+        // First sample
+        sample_start = total_sample_count;
+        sample_val = proc->rt_power.readOp.dynamic;
+        init_inst_val = init_val;
+        samples.push_back(proc->rt_power.readOp.dynamic);
+        assert(samples_counter.size() == 0);
+        assert(pwr_counter.size() == 0);
+
+        for (unsigned i = 0; i < (num_perf_counters); ++i) {
+          samples_counter.push_back(sample_perf_counters[i]);
+        }
+
+        for (unsigned i = 0; i < (num_pwr_cmps); ++i) {
+          pwr_counter.push_back(sample_cmp_pwr[i]);
+        }
+        assert(pwr_counter.size() == (double)num_pwr_cmps);
+        assert(samples_counter.size() == (double)num_perf_counters);
+      } else {
+        // Get current average
+        double temp_avg = sample_val / (double)samples.size();
+
+        if (abs(proc->rt_power.readOp.dynamic - temp_avg) <
+            gpu_steady_power_deviation) {  // Value is within threshold
+          sample_val += proc->rt_power.readOp.dynamic;
+          samples.push_back(proc->rt_power.readOp.dynamic);
+          for (unsigned i = 0; i < (num_perf_counters); ++i) {
+            samples_counter.at(i) += sample_perf_counters[i];
+          }
+
+          for (unsigned i = 0; i < (num_pwr_cmps); ++i) {
+            pwr_counter.at(i) += sample_cmp_pwr[i];
+          }
+
+        } else {  // Value exceeds threshold, not considered steady state
+          print_steady_state(position, init_val);
+        }
+      }
+    } else {
+      print_steady_state(position, init_val);
+    }
+    gzclose(steady_state_tacking_file);
+  }
+}
+
+void gpgpu_sim_wrapper::open_files() {
+  if (g_power_simulation_enabled) {
+    if (g_power_trace_enabled) {
+      power_trace_file = gzopen(g_power_trace_filename, "a");
+      metric_trace_file = gzopen(g_metric_trace_filename, "a");
+    }
+  }
+}
+void gpgpu_sim_wrapper::close_files() {
+  if (g_power_simulation_enabled) {
+    if (g_power_trace_enabled) {
+      gzclose(power_trace_file);
+      gzclose(metric_trace_file);
+    }
+  }
+}
diff --git a/src/gpuwattch/gpgpu_sim_wrapper.h b/src/accelwattch/gpgpu_sim_wrapper.h
index 00e4f07..33c4b72 100644
--- a/src/gpuwattch/gpgpu_sim_wrapper.h
+++ b/src/accelwattch/gpgpu_sim_wrapper.h
@@ -1,18 +1,19 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Tayler Hetherington, Ahmed ElTantawy,
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Tayler Hetherington, Ahmed ElTantawy, Vijay Kandiah, Nikos Hardavellas
+// The University of British Columbia, Northwestern University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -54,9 +55,34 @@ struct avg_max_min_counters {
   }
 };
 
+#ifndef COEFF_STRUCT
+#define COEFF_STRUCT
+
+struct PowerscalingCoefficients{
+    double int_coeff;
+    double int_mul_coeff;
+    double int_mul24_coeff;
+    double int_mul32_coeff;
+    double int_div_coeff;
+    double fp_coeff;
+    double dp_coeff;
+    double fp_mul_coeff;
+    double fp_div_coeff;
+    double dp_mul_coeff;
+    double dp_div_coeff;
+    double sqrt_coeff;
+    double log_coeff;
+    double sin_coeff;
+    double exp_coeff;
+    double tensor_coeff;
+    double tex_coeff;
+};
+
+#endif
+
 class gpgpu_sim_wrapper {
  public:
-  gpgpu_sim_wrapper(bool power_simulation_enabled, char* xmlfile);
+  gpgpu_sim_wrapper(bool power_simulation_enabled, char* xmlfile, int power_simulation_mode, bool dvfs_enabled);
   ~gpgpu_sim_wrapper();
 
   void init_mcpat(char* xmlfile, char* powerfile, char* power_trace_file,
@@ -64,7 +90,9 @@ class gpgpu_sim_wrapper {
                   bool power_sim_enabled, bool trace_enabled,
                   bool steady_state_enabled, bool power_per_cycle_dump,
                   double steady_power_deviation, double steady_min_period,
-                  int zlevel, double init_val, int stat_sample_freq);
+                  int zlevel, double init_val, int stat_sample_freq, int power_sim_mode, 
+                  bool dvfs_enabled, unsigned clock_freq, unsigned num_shaders);
+  void init_mcpat_hw_mode(unsigned gpu_sim_cycle);
   void detect_print_steady_state(int position, double init_val);
   void close_files();
   void open_files();
@@ -72,6 +100,7 @@ class gpgpu_sim_wrapper {
   void dump();
   void print_trace_files();
   void update_components_power();
+  double calculate_static_power();
   void update_coefficients();
   void reset_counters();
   void print_power_kernel_stats(double gpu_sim_cycle, double gpu_tot_sim_cycle,
@@ -79,6 +108,7 @@ class gpgpu_sim_wrapper {
                                 const std::string& kernel_info_string,
                                 bool print_trace);
   void power_metrics_calculations();
+  void set_model_voltage(double model_voltage);
   void set_inst_power(bool clk_gated_lanes, double tot_cycles,
                       double busy_cycles, double tot_inst, double int_inst,
                       double fp_inst, double load_inst, double store_inst,
@@ -92,16 +122,31 @@ class gpgpu_sim_wrapper {
                          double write_accesses, double write_misses);
   void set_l2cache_power(double read_accesses, double read_misses,
                          double write_accesses, double write_misses);
+  void set_num_cores(double num_core);
   void set_idle_core_power(double num_idle_core);
   void set_duty_cycle_power(double duty_cycle);
   void set_mem_ctrl_power(double reads, double writes, double dram_precharge);
   void set_exec_unit_power(double fpu_accesses, double ialu_accesses,
                            double sfu_accesses);
+  void set_int_accesses(double ialu_accesses, double imul24_accesses, 
+                        double imul32_accesses, double imul_accesses, 
+                        double idiv_accesses);
+  void set_dp_accesses(double dpu_accesses, double dpmul_accesses, 
+                       double dpdiv_accesses);
+  void set_fp_accesses(double fpu_accesses, double fpmul_accesses, 
+                       double fpdiv_accesses);
+  void set_trans_accesses(double sqrt_accesses, double log_accesses, 
+                       double sin_accesses, double exp_accesses);
+  void set_tensor_accesses(double tensor_accesses);
+  void set_tex_accesses(double tex_accesses);
+  void set_avg_active_threads(float active_threads);
   void set_active_lanes_power(double sp_avg_active_lane,
                               double sfu_avg_active_lane);
-  void set_NoC_power(double noc_tot_reads, double noc_tot_write);
+  void set_NoC_power(double noc_tot_acc);
   bool sanity_check(double a, double b);
 
+  PowerscalingCoefficients * get_scaling_coeffs();
+
  private:
   void print_steady_state(int position, double init_val);
 
@@ -109,8 +154,10 @@ class gpgpu_sim_wrapper {
   ParseXML* p;
   // power parameters
   double const_dynamic_power;
+  double avg_threads_per_warp_tot;
   double proc_power;
-
+  double num_cores;
+  double num_idle_cores;
   unsigned num_perf_counters;  // # of performance counters
   unsigned num_pwr_cmps;       // # of components modelled
   int kernel_sample_count;     // # of samples per kernel
@@ -140,6 +187,10 @@ class gpgpu_sim_wrapper {
   unsigned sample_start;
   double sample_val;
   double init_inst_val;
+  double tot_sfu_accesses;
+  double tot_fpu_accesses;
+  double modeled_chip_voltage;
+  unsigned avg_threads_per_warp;
   std::vector<double> samples;
   std::vector<double> samples_counter;
   std::vector<double> pwr_counter;
@@ -150,6 +201,8 @@ class gpgpu_sim_wrapper {
   char* g_metric_trace_filename;
   char* g_steady_state_tracking_filename;
   bool g_power_simulation_enabled;
+  int g_power_simulation_mode;
+  bool g_dvfs_enabled;
   bool g_steady_power_levels_enabled;
   bool g_power_trace_enabled;
   bool g_power_per_cycle_dump;
diff --git a/src/gpuwattch/gpgpu_static.xml b/src/accelwattch/gpgpu_static.xml
index befa95e..befa95e 100644
--- a/src/gpuwattch/gpgpu_static.xml
+++ b/src/accelwattch/gpgpu_static.xml
diff --git a/src/gpuwattch/interconnect.cc b/src/accelwattch/interconnect.cc
index caf66b3..caf66b3 100644
--- a/src/gpuwattch/interconnect.cc
+++ b/src/accelwattch/interconnect.cc
diff --git a/src/gpuwattch/interconnect.h b/src/accelwattch/interconnect.h
index b725c1d..b725c1d 100644
--- a/src/gpuwattch/interconnect.h
+++ b/src/accelwattch/interconnect.h
diff --git a/src/gpuwattch/iocontrollers.cc b/src/accelwattch/iocontrollers.cc
index f5e2502..f5e2502 100644
--- a/src/gpuwattch/iocontrollers.cc
+++ b/src/accelwattch/iocontrollers.cc
diff --git a/src/gpuwattch/iocontrollers.h b/src/accelwattch/iocontrollers.h
index 9d6c48a..9d6c48a 100644
--- a/src/gpuwattch/iocontrollers.h
+++ b/src/accelwattch/iocontrollers.h
diff --git a/src/gpuwattch/logic.cc b/src/accelwattch/logic.cc
index 7f40189..7f40189 100644
--- a/src/gpuwattch/logic.cc
+++ b/src/accelwattch/logic.cc
diff --git a/src/gpuwattch/logic.h b/src/accelwattch/logic.h
index 1e7a23d..1e7a23d 100644
--- a/src/gpuwattch/logic.h
+++ b/src/accelwattch/logic.h
diff --git a/src/gpuwattch/main.cc b/src/accelwattch/main.cc
index 03d3e7c..03d3e7c 100644
--- a/src/gpuwattch/main.cc
+++ b/src/accelwattch/main.cc
diff --git a/src/gpuwattch/makefile b/src/accelwattch/makefile
index 354c9ec..354c9ec 100644
--- a/src/gpuwattch/makefile
+++ b/src/accelwattch/makefile
diff --git a/src/gpuwattch/mcpat.mk b/src/accelwattch/mcpat.mk
index a09c23b..ad2d6c2 100644
--- a/src/gpuwattch/mcpat.mk
+++ b/src/accelwattch/mcpat.mk
@@ -1,5 +1,5 @@
 
-OUTPUT_DIR=$(SIM_OBJ_FILES_DIR)/gpuwattch
+OUTPUT_DIR=$(SIM_OBJ_FILES_DIR)/accelwattch
 TARGET = mcpat
 SHELL = /bin/sh
 .PHONY: all depend clean
diff --git a/src/gpuwattch/mcpatXeonCore.mk b/src/accelwattch/mcpatXeonCore.mk
index 20cf0dd..20cf0dd 100644
--- a/src/gpuwattch/mcpatXeonCore.mk
+++ b/src/accelwattch/mcpatXeonCore.mk
diff --git a/src/gpuwattch/memoryctrl.cc b/src/accelwattch/memoryctrl.cc
index 2abb1d9..2abb1d9 100644
--- a/src/gpuwattch/memoryctrl.cc
+++ b/src/accelwattch/memoryctrl.cc
diff --git a/src/gpuwattch/memoryctrl.h b/src/accelwattch/memoryctrl.h
index 4ac55fc..4ac55fc 100644
--- a/src/gpuwattch/memoryctrl.h
+++ b/src/accelwattch/memoryctrl.h
diff --git a/src/gpuwattch/noc.cc b/src/accelwattch/noc.cc
index b7ccdb3..b7ccdb3 100644
--- a/src/gpuwattch/noc.cc
+++ b/src/accelwattch/noc.cc
diff --git a/src/gpuwattch/noc.h b/src/accelwattch/noc.h
index 3dc3ac2..3dc3ac2 100644
--- a/src/gpuwattch/noc.h
+++ b/src/accelwattch/noc.h
diff --git a/src/gpuwattch/processor.cc b/src/accelwattch/processor.cc
index fc6db46..9e7f5b2 100644
--- a/src/gpuwattch/processor.cc
+++ b/src/accelwattch/processor.cc
@@ -30,11 +30,13 @@
  ***************************************************************************/
 /********************************************************************
  *      Modified by:
- ** Jingwen Leng, Univeristy of Texas, Austin                   * Syed Gilani,
- *University of Wisconsin–Madison                * Tayler Hetherington,
- *University of British Columbia         * Ahmed ElTantawy, University of
- *British Columbia             *
+ * Jingwen Leng, University of Texas, Austin                
+ * Syed Gilani, University of Wisconsin–Madison         
+ * Tayler Hetherington, University of British Columbia
+ * Ahmed ElTantawy, University of British Columbia
+ * Vijay Kandiah, Northwestern University
  ********************************************************************/
+
 #include "processor.h"
 #include <assert.h>
 #include <stdio.h>
@@ -118,7 +120,7 @@ Processor::Processor(ParseXML *XML_interface)
       set_pppm(pppm_t, cores[i]->clockRate * procdynp.numCore, procdynp.numCore,
                procdynp.numCore, procdynp.numCore);
       // set the exClockRate
-      exClockRate = cores[0]->clockRate * 2;  // TODO; get from XML file
+      exClockRate = cores[0]->clockRate;  // TODO; get from XML file
       // cout<<"****EX clock rate:"<<exClockRate<<endl;
       core.power = core.power + cores[i]->power * pppm_t;
       set_pppm(pppm_t, 1 / cores[i]->executionTime, procdynp.numCore,
diff --git a/src/gpuwattch/processor.h b/src/accelwattch/processor.h
index 888ba31..888ba31 100644
--- a/src/gpuwattch/processor.h
+++ b/src/accelwattch/processor.h
diff --git a/src/gpuwattch/quadro.xml b/src/accelwattch/quadro.xml
index 934e027..934e027 100644
--- a/src/gpuwattch/quadro.xml
+++ b/src/accelwattch/quadro.xml
diff --git a/src/gpuwattch/results/Alpha21364 b/src/accelwattch/results/Alpha21364
index 1b3d9e4..1b3d9e4 100644
--- a/src/gpuwattch/results/Alpha21364
+++ b/src/accelwattch/results/Alpha21364
diff --git a/src/gpuwattch/results/Alpha21364_90nm b/src/accelwattch/results/Alpha21364_90nm
index 2a97d77..2a97d77 100644
--- a/src/gpuwattch/results/Alpha21364_90nm
+++ b/src/accelwattch/results/Alpha21364_90nm
diff --git a/src/gpuwattch/results/Penryn b/src/accelwattch/results/Penryn
index af39390..af39390 100644
--- a/src/gpuwattch/results/Penryn
+++ b/src/accelwattch/results/Penryn
diff --git a/src/gpuwattch/results/T1 b/src/accelwattch/results/T1
index f63e51c..f63e51c 100644
--- a/src/gpuwattch/results/T1
+++ b/src/accelwattch/results/T1
diff --git a/src/gpuwattch/results/T1_DC_64 b/src/accelwattch/results/T1_DC_64
index cdb0a1b..cdb0a1b 100644
--- a/src/gpuwattch/results/T1_DC_64
+++ b/src/accelwattch/results/T1_DC_64
diff --git a/src/gpuwattch/results/T1_SBT_64 b/src/accelwattch/results/T1_SBT_64
index ec8968a..ec8968a 100644
--- a/src/gpuwattch/results/T1_SBT_64
+++ b/src/accelwattch/results/T1_SBT_64
diff --git a/src/gpuwattch/results/T1_ST_64 b/src/accelwattch/results/T1_ST_64
index f3d95b5..f3d95b5 100644
--- a/src/gpuwattch/results/T1_ST_64
+++ b/src/accelwattch/results/T1_ST_64
diff --git a/src/gpuwattch/results/T2 b/src/accelwattch/results/T2
index e24701a..e24701a 100644
--- a/src/gpuwattch/results/T2
+++ b/src/accelwattch/results/T2
diff --git a/src/gpuwattch/results/Xeon_core b/src/accelwattch/results/Xeon_core
index 0cc9ae6..0cc9ae6 100644
--- a/src/gpuwattch/results/Xeon_core
+++ b/src/accelwattch/results/Xeon_core
diff --git a/src/gpuwattch/results/Xeon_uncore b/src/accelwattch/results/Xeon_uncore
index 558331c..558331c 100644
--- a/src/gpuwattch/results/Xeon_uncore
+++ b/src/accelwattch/results/Xeon_uncore
diff --git a/src/gpuwattch/sharedcache.cc b/src/accelwattch/sharedcache.cc
index e28202d..e28202d 100644
--- a/src/gpuwattch/sharedcache.cc
+++ b/src/accelwattch/sharedcache.cc
diff --git a/src/gpuwattch/sharedcache.h b/src/accelwattch/sharedcache.h
index c5aba19..c5aba19 100644
--- a/src/gpuwattch/sharedcache.h
+++ b/src/accelwattch/sharedcache.h
diff --git a/src/gpuwattch/technology_xeon_core.cc b/src/accelwattch/technology_xeon_core.cc
index 6f95cc2..6f95cc2 100644
--- a/src/gpuwattch/technology_xeon_core.cc
+++ b/src/accelwattch/technology_xeon_core.cc
diff --git a/src/gpuwattch/version.h b/src/accelwattch/version.h
index 41bd5b9..41bd5b9 100644
--- a/src/gpuwattch/version.h
+++ b/src/accelwattch/version.h
diff --git a/src/gpuwattch/xmlParser.cc b/src/accelwattch/xmlParser.cc
index 8f49b39..8f49b39 100644
--- a/src/gpuwattch/xmlParser.cc
+++ b/src/accelwattch/xmlParser.cc
diff --git a/src/gpuwattch/xmlParser.h b/src/accelwattch/xmlParser.h
index 71a1f57..71a1f57 100644
--- a/src/gpuwattch/xmlParser.h
+++ b/src/accelwattch/xmlParser.h
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 71f0703..680ce79 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung,
-// George L. Yuan, Jimmy Kwa
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung,
+// George L. Yuan, Jimmy Kwa, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -588,65 +590,119 @@ void ptx_instruction::set_fp_or_int_archop() {
       oprnd_type = INT_OP;
   }
 }
-void ptx_instruction::set_mul_div_or_other_archop() {
-  sp_op = OTHER_OP;
-  if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) &&
-      (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) &&
-      (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) &&
-      (m_opcode != CALL_OP)) {
-    if (get_type() == F32_TYPE || get_type() == F64_TYPE ||
-        get_type() == FF64_TYPE) {
-      switch (get_opcode()) {
-        case MUL_OP:
-        case MAD_OP:
-          sp_op = FP_MUL_OP;
-          break;
-        case DIV_OP:
-          sp_op = FP_DIV_OP;
-          break;
-        case LG2_OP:
-          sp_op = FP_LG_OP;
-          break;
-        case RSQRT_OP:
-        case SQRT_OP:
-          sp_op = FP_SQRT_OP;
-          break;
-        case RCP_OP:
-          sp_op = FP_DIV_OP;
-          break;
-        case SIN_OP:
-        case COS_OP:
-          sp_op = FP_SIN_OP;
-          break;
-        case EX2_OP:
-          sp_op = FP_EXP_OP;
-          break;
-        default:
-          if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP;
-          break;
+
+void ptx_instruction::set_mul_div_or_other_archop(){
+  sp_op=OTHER_OP;
+  if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){
+    if(get_type() == F64_TYPE || get_type() == FF64_TYPE){
+         switch(get_opcode()){
+            case MUL_OP:
+            case MAD_OP:
+            case FMA_OP:
+                sp_op=DP_MUL_OP;
+               break;
+            case DIV_OP:
+            case REM_OP:
+                sp_op=DP_DIV_OP;
+               break;
+            case RCP_OP:
+                sp_op=DP_DIV_OP;
+               break;
+            case LG2_OP:
+                sp_op=FP_LG_OP;
+               break;
+            case RSQRT_OP:
+            case SQRT_OP:
+                sp_op=FP_SQRT_OP;
+               break;            
+            case SIN_OP:
+            case COS_OP:
+                sp_op=FP_SIN_OP;
+               break;
+            case EX2_OP:
+                sp_op=FP_EXP_OP;
+               break;
+            case MMA_OP:
+                sp_op=TENSOR__OP;
+            break;
+            case TEX_OP:
+                sp_op=TEX__OP;
+            break;
+            default:
+               if((op==DP_OP) || (op==ALU_OP))
+                  sp_op=DP___OP;
+               break;
+         }
       }
-    } else {
-      switch (get_opcode()) {
-        case MUL24_OP:
-        case MAD24_OP:
-          sp_op = INT_MUL24_OP;
-          break;
-        case MUL_OP:
-        case MAD_OP:
-          if (get_type() == U32_TYPE || get_type() == S32_TYPE ||
-              get_type() == B32_TYPE)
-            sp_op = INT_MUL32_OP;
-          else
-            sp_op = INT_MUL_OP;
-          break;
-        case DIV_OP:
-          sp_op = INT_DIV_OP;
-          break;
-        default:
-          if ((op == ALU_OP)) sp_op = INT__OP;
-          break;
+      else if(get_type()==F16_TYPE || get_type()==F32_TYPE){
+         switch(get_opcode()){
+            case MUL_OP:
+            case MAD_OP:
+            case FMA_OP:
+                sp_op=FP_MUL_OP;
+               break;
+            case DIV_OP:
+            case REM_OP:
+                sp_op=FP_DIV_OP;
+               break;
+            case RCP_OP:
+                sp_op=FP_DIV_OP;
+               break;
+            case LG2_OP:
+                sp_op=FP_LG_OP;
+               break;
+            case RSQRT_OP:
+            case SQRT_OP:
+                sp_op=FP_SQRT_OP;
+               break;            
+            case SIN_OP:
+            case COS_OP:
+                sp_op=FP_SIN_OP;
+               break;
+            case EX2_OP:
+                sp_op=FP_EXP_OP;
+               break;
+            case MMA_OP:
+                sp_op=TENSOR__OP;
+            break;
+            case TEX_OP:
+                sp_op=TEX__OP;
+            break;
+            default:
+               if((op==SP_OP) || (op==ALU_OP))
+                  sp_op=FP__OP;
+               break;
+         }
+      }else {
+         switch(get_opcode()){
+            case MUL24_OP:
+            case MAD24_OP:
+                sp_op=INT_MUL24_OP;
+            break;
+            case MUL_OP:
+            case MAD_OP:
+            case FMA_OP:
+               if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE)
+                   sp_op=INT_MUL32_OP;
+               else
+                   sp_op=INT_MUL_OP;
+            break;
+            case DIV_OP:
+            case REM_OP:
+                sp_op=INT_DIV_OP;
+            break;
+            case MMA_OP:
+                sp_op=TENSOR__OP;
+            break;
+            case TEX_OP:
+                sp_op=TEX__OP;
+            break;
+            default:
+               if((op==INTP_OP) || (op==ALU_OP))
+                   sp_op=INT__OP;
+               break;
+         }
       }
-    }
   }
 }
 
@@ -880,6 +936,7 @@ void ptx_instruction::set_opcode_and_latency() {
     case MAD_OP:
     case MADC_OP:
     case MADP_OP:
+    case FMA_OP:
       // MAD latency
       switch (get_type()) {
         case F32_TYPE:
@@ -903,7 +960,18 @@ void ptx_instruction::set_opcode_and_latency() {
           break;
       }
       break;
+    case MUL24_OP: //MUL24 is performed on mul32 units (with additional instructions for bitmasking) on devices with compute capability >1.x
+      latency = int_latency[2]+1;
+      initiation_interval = int_init[2]+1;
+      op = INTP_OP;
+      break;
+    case MAD24_OP:
+      latency = int_latency[3]+1;
+      initiation_interval = int_init[3]+1;
+      op = INTP_OP;
+      break;
     case DIV_OP:
+    case REM_OP:
       // Floating point only
       op = SFU_OP;
       switch (get_type()) {
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 8936fa8..e22d88a 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
-// Jimmy Kwa, George L. Yuan
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
+// Jimmy Kwa, George L. Yuan, Vijay Kandiah, Nikos Hardavellas,
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -26,6 +28,7 @@
 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
+
 #include "instructions.h"
 #include "half.h"
 #include "half.hpp"
@@ -166,8 +169,9 @@ void inst_not_implemented(const ptx_instruction *pI);
 ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo,
                               operand_info dstInfo, unsigned type,
                               ptx_thread_info *thread);
-                              
-void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, int op_code);
+
+void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread,
+                           int op_code);
 
 void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst);
 
@@ -1711,40 +1715,50 @@ void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
   }
   thread->set_operand_value(dst, data, i_type, thread, pI);
 }
-void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread)
-{
-  const operand_info &dst  = pI->dst();
+void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
+  const operand_info &dst = pI->dst();
   const operand_info &src1 = pI->src1();
   const unsigned i_type = pI->get_type();
 
-  const ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1);
-  const int msb = ( i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63;
+  const ptx_reg_t src1_data =
+      thread->get_operand_value(src1, dst, i_type, thread, 1);
+  const int msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63;
 
   unsigned long a = 0;
-  switch (i_type)
-  {
-    case S32_TYPE: a = src1_data.s32; break;
-    case U32_TYPE: a = src1_data.u32; break;
-    case S64_TYPE: a = src1_data.s64; break;
-    case U64_TYPE: a = src1_data.u64; break;
-    default: assert(false); abort();
+  switch (i_type) {
+    case S32_TYPE:
+      a = src1_data.s32;
+      break;
+    case U32_TYPE:
+      a = src1_data.u32;
+      break;
+    case S64_TYPE:
+      a = src1_data.s64;
+      break;
+    case U64_TYPE:
+      a = src1_data.u64;
+      break;
+    default:
+      assert(false);
+      abort();
   }
 
   // negate negative signed inputs
-  if ( ( i_type == S32_TYPE || i_type == S64_TYPE ) && ( a & ( 1 << msb ) ) ) {
-      a = ~a;
+  if ((i_type == S32_TYPE || i_type == S64_TYPE) && (a & (1 << msb))) {
+    a = ~a;
   }
   uint32_t d_data = 0xffffffff;
   for (uint32_t i = msb; i >= 0; i--) {
-      if (a & (1<<i))  { d_data = i; break; }
+    if (a & (1 << i)) {
+      d_data = i;
+      break;
+    }
   }
 
   // if (.shiftamt && d != 0xffffffff)  { d = msb - d; }
 
   // store d
   thread->set_operand_value(dst, d_data, U32_TYPE, thread, pI);
-
-
 }
 
 void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
@@ -3966,7 +3980,7 @@ void mad_def(const ptx_instruction *pI, ptx_thread_info *thread,
           fesetround(FE_TOWARDZERO);
           break;
         default:
-          assert(0);
+          //assert(0);
           break;
       }
       d.f32 = a.f32 * b.f32 + c.f32;
@@ -4312,11 +4326,8 @@ void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
     case S64_TYPE:
       t.s64 = a.s64 * b.s64;
       assert(!pI->is_wide());
-      assert(!pI->is_hi());
-      if (pI->is_lo())
-        d.s64 = t.s64;
-      else
-        assert(0);
+      //assert(!pI->is_hi());
+      d.s64 = t.s64;
       break;
     case U16_TYPE:
       t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16);
@@ -6339,12 +6350,10 @@ void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
 #define VMAX 0
 #define VMIN 1
 
-void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread)
-{
-   video_mem_instruction(pI, thread, VMAX);
+void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
+  video_mem_instruction(pI, thread, VMAX);
 }
-void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread)
-{
+void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
   video_mem_instruction(pI, thread, VMIN);
 }
 void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
@@ -6440,12 +6449,12 @@ void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
   }
 }
 
-void activemask_impl( const ptx_instruction *pI, ptx_thread_info *thread )
-{
+void activemask_impl(const ptx_instruction *pI, ptx_thread_info *thread) {
   active_mask_t l_activemask_bitset = pI->get_warp_active_mask();
-  uint32_t l_activemask_uint = static_cast<uint32_t>(l_activemask_bitset.to_ulong());
+  uint32_t l_activemask_uint =
+      static_cast<uint32_t>(l_activemask_bitset.to_ulong());
 
-  const operand_info &dst  = pI->dst();
+  const operand_info &dst = pI->dst();
   thread->set_operand_value(dst, l_activemask_uint, U32_TYPE, thread, pI);
 }
 
@@ -6527,12 +6536,12 @@ ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo,
   return result;
 }
 
-void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, int op_code)
-{
-  const operand_info &dst  = pI->dst(); // d
-  const operand_info &src1 = pI->src1(); // a
-  const operand_info &src2 = pI->src2(); // b
-  const operand_info &src3 = pI->src3(); // c
+void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread,
+                           int op_code) {
+  const operand_info &dst = pI->dst();    // d
+  const operand_info &src1 = pI->src1();  // a
+  const operand_info &src2 = pI->src2();  // b
+  const operand_info &src3 = pI->src3();  // c
 
   const unsigned i_type = pI->get_type();
 
@@ -6557,19 +6566,18 @@ void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, i
   auto option = options.begin();
   assert(*option == ATOMIC_MAX || *option == ATOMIC_MIN);
 
-  switch ( i_type ) {
+  switch (i_type) {
     case S32_TYPE: {
       // assert all operands are S32_TYPE:
       scalar_type = pI->get_scalar_type();
-      for (std::list<int>::iterator scalar = scalar_type.begin(); scalar != scalar_type.end(); scalar++)
-      {
+      for (std::list<int>::iterator scalar = scalar_type.begin();
+           scalar != scalar_type.end(); scalar++) {
         assert(*scalar == S32_TYPE);
       }
       assert(scalar_type.size() == 3);
       scalar_type.clear();
 
-      switch (op_code)
-      {
+      switch (op_code) {
         case VMAX:
           data.s32 = MY_MAX_I(ta.s32, tb.s32);
           break;
@@ -6580,26 +6588,23 @@ void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, i
           assert(0);
       }
 
-      switch (*option)
-      {
+      switch (*option) {
         case ATOMIC_MAX:
           data.s32 = MY_MAX_I(data.s32, c.s32);
-        break;
+          break;
         case ATOMIC_MIN:
           data.s32 = MY_MIN_I(data.s32, c.s32);
-        break;
+          break;
         default:
-          assert(0); // not yet implemented
+          assert(0);  // not yet implemented
       }
       break;
-
     }
     default:
-      assert(0); // not yet implemented
+      assert(0);  // not yet implemented
   }
 
   thread->set_operand_value(dst, data, i_type, thread, pI);
 
   return;
 }
-
diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l
index 6754045..15b3cf7 100644
--- a/src/cuda-sim/ptx.l
+++ b/src/cuda-sim/ptx.l
@@ -1,32 +1,35 @@
 /*
-Copyright (c) 2009-2011, Tor M. Aamodt
-The University of British Columbia
+Copyright (c) 2009-2021, Tor M. Aamodt, Vijay Kandiah, Nikos Hardavellas, 
+Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+The University of British Columbia, Northwestern University, Purdue University
 All rights reserved.
-
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-Redistributions in binary form must reproduce the above copyright notice, this
-list of conditions and the following disclaimer in the documentation and/or
-other materials provided with the distribution.
-Neither the name of The University of British Columbia nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer;
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution;
+3. Neither the names of The University of British Columbia, Northwestern 
+   University nor the names of their contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
 */
 
+
 %option nounput
 %option noyywrap
 %option yylineno
@@ -69,6 +72,7 @@ andn	TC; yylval->int_value = ANDN_OP; return OPCODE;
 atom	TC; yylval->int_value = ATOM_OP; return OPCODE;
 bar.warp 	TC; yylval->int_value = NOP_OP; return OPCODE;
 bar 	TC; yylval->int_value = BAR_OP; return OPCODE;
+barrier	TC; yylval->int_value = BAR_OP; return OPCODE;
 bfe     TC; yylval->int_value = BFE_OP; return OPCODE;
 bfi     TC; yylval->int_value = BFI_OP; return OPCODE;
 bfind   TC; yylval->int_value = BFIND_OP; return OPCODE;
@@ -167,14 +171,22 @@ breakaddr  TC; yylval->int_value = BREAKADDR_OP; return OPCODE;
 "CPTX_END"	printf("ENDING CUSTOM PTX.\n"); BEGIN(IN_COMMENT);
 
 <INITIAL,NOT_OPCODE,IN_INST,IN_FUNC_DECL>{
-\.a\.sync TC; yylval->int_value = LOAD_A; return WMMA_DIRECTIVE;
-\.b\.sync TC; yylval->int_value = LOAD_B; return WMMA_DIRECTIVE;
-\.c\.sync TC; yylval->int_value = LOAD_C; return WMMA_DIRECTIVE;
-\.d\.sync TC; yylval->int_value = STORE_D; return WMMA_DIRECTIVE;
-\.mma\.sync TC;yylval->int_value=MMA; return WMMA_DIRECTIVE;
+\.a\.sync\.aligned TC; yylval->int_value = LOAD_A; return WMMA_DIRECTIVE;
+\.b\.sync\.aligned TC; yylval->int_value = LOAD_B; return WMMA_DIRECTIVE;
+\.c\.sync\.aligned TC; yylval->int_value = LOAD_C; return WMMA_DIRECTIVE;
+\.d\.sync\.aligned TC; yylval->int_value = STORE_D; return WMMA_DIRECTIVE;
+\.mma\.sync\.aligned TC;yylval->int_value=MMA; return WMMA_DIRECTIVE;
 
 \.row TC; yylval->int_value = ROW; return LAYOUT;
 \.col TC; yylval->int_value = COL; return LAYOUT;
+\.m16n16k16\.global TC; yylval->int_value = M16N16K16; return CONFIGURATION;
+\.m32n8k16\.global TC; yylval->int_value = M32N8K16; return CONFIGURATION;
+\.m8n32k16\.global TC;  yylval->int_value = M8N32K16; return CONFIGURATION;
+
+\.m16n16k16\.shared TC; yylval->int_value = M16N16K16; return CONFIGURATION;
+\.m32n8k16\.shared TC; yylval->int_value = M32N8K16; return CONFIGURATION;
+\.m8n32k16\.shared TC;  yylval->int_value = M8N32K16; return CONFIGURATION;
+
 \.m16n16k16 TC; yylval->int_value = M16N16K16; return CONFIGURATION;
 \.m32n8k16 TC; yylval->int_value = M32N8K16; return CONFIGURATION;
 \.m8n32k16 TC;  yylval->int_value = M8N32K16; return CONFIGURATION;
@@ -476,4 +488,4 @@ int ptx_error( yyscan_t yyscanner, ptx_recognizer* recognizer, const char *s )
 	fflush(stdout);
 	//exit(1);
 	return 0;
-}
+}
\ No newline at end of file
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index e5b5fb7..029cf73 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung,
-// George L. Yuan
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung,
+// George L. Yuan, Vijay Kandiah, Nikos Hardavellas,
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -1147,8 +1149,8 @@ static std::list<operand_info> check_operands(
     const std::list<operand_info> &operands, gpgpu_context *ctx) {
   static int g_warn_literal_operands_two_type_inst;
   if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) ||
-      (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP) || 
-      (opcode == VMIN_OP) || (opcode == VMAX_OP) ) {
+      (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP) ||
+      (opcode == VMIN_OP) || (opcode == VMAX_OP)) {
     // just make sure these do not have have const operands...
     if (!g_warn_literal_operands_two_type_inst) {
       std::list<operand_info>::const_iterator o;
@@ -1384,6 +1386,8 @@ ptx_instruction::ptx_instruction(
       case CS_OPTION:
       case LU_OPTION:
       case CV_OPTION:
+      case WB_OPTION: 
+      case WT_OPTION:
         m_cache_option = last_ptx_inst_option;
         break;
       case HALF_OPTION:
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 4243941..8251759 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -966,8 +966,8 @@ class ptx_instruction : public warp_inst_t {
   int get_pred_mod() const { return m_pred_mod; }
   const char *get_source() const { return m_source.c_str(); }
 
-  const std::list<int> get_scalar_type() const {return m_scalar_type;}
-  const std::list<int> get_options() const {return m_options;}
+  const std::list<int> get_scalar_type() const { return m_scalar_type; }
+  const std::list<int> get_options() const { return m_options; }
 
   typedef std::vector<operand_info>::const_iterator const_iterator;
 
diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc
index afdb41b..86a33c2 100644
--- a/src/cuda-sim/ptx_parser.cc
+++ b/src/cuda-sim/ptx_parser.cc
@@ -622,13 +622,13 @@ void ptx_recognizer::add_scalar_type_spec(int type_spec) {
                     g_ptx_token_decode[type_spec].c_str());
   g_scalar_type.push_back(type_spec);
   if (g_scalar_type.size() > 1) {
-    parse_assert((g_opcode == -1) || (g_opcode == CVT_OP) ||
-                     (g_opcode == SET_OP) || (g_opcode == SLCT_OP) ||
-                     (g_opcode == TEX_OP) || (g_opcode == MMA_OP) ||
-                     (g_opcode == DP4A_OP) || (g_opcode == VMIN_OP) || 
-                     (g_opcode == VMAX_OP),
-                 "only cvt, set, slct, tex, vmin, vmax and dp4a can have more than one "
-                 "type specifier.");
+    parse_assert(
+        (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) ||
+            (g_opcode == SLCT_OP) || (g_opcode == TEX_OP) ||
+            (g_opcode == MMA_OP) || (g_opcode == DP4A_OP) ||
+            (g_opcode == VMIN_OP) || (g_opcode == VMAX_OP),
+        "only cvt, set, slct, tex, vmin, vmax and dp4a can have more than one "
+        "type specifier.");
   }
   g_scalar_type_spec = type_spec;
 }
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index ca47c46..662c2ed 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
-// Ivan Sham, George L. Yuan,
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
+// Ivan Sham, George L. Yuan, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -855,7 +857,7 @@ void dram_t::visualizer_print(gzFile visualizer_file) {
 
 void dram_t::set_dram_power_stats(unsigned &cmd, unsigned &activity,
                                   unsigned &nop, unsigned &act, unsigned &pre,
-                                  unsigned &rd, unsigned &wr,
+                                  unsigned &rd, unsigned &wr, unsigned &wr_WB,
                                   unsigned &req) const {
   // Point power performance counters to low-level DRAM counters
   cmd = n_cmd;
@@ -865,6 +867,7 @@ void dram_t::set_dram_power_stats(unsigned &cmd, unsigned &activity,
   pre = n_pre;
   rd = n_rd;
   wr = n_wr;
+  wr_WB = n_wr_WB;
   req = n_req;
 }
 
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
index 6c212e9..90ea3e4 100644
--- a/src/gpgpu-sim/dram.h
+++ b/src/gpgpu-sim/dram.h
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ivan Sham, Ali Bakhoda,
-// George L. Yuan, Wilson W.L. Fung
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Ivan Sham, Ali Bakhoda,
+// George L. Yuan, Wilson W.L. Fung, Vijay Kandiah, Nikos Hardavellas,
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -135,7 +137,7 @@ class dram_t {
   // Power Model
   void set_dram_power_stats(unsigned &cmd, unsigned &activity, unsigned &nop,
                             unsigned &act, unsigned &pre, unsigned &rd,
-                            unsigned &wr, unsigned &req) const;
+                            unsigned &wr, unsigned &wr_WB, unsigned &req) const;
 
   const memory_config *m_config;
 
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index 75c3691..3a5a67d 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -1,18 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Tayler Hetherington, 
+// Vijay Kandiah, Nikos Hardavellas, Mahmoud Khairy, Junrui Pan,
+// Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -37,7 +40,8 @@
 
 const char *cache_request_status_str(enum cache_request_status status) {
   static const char *static_cache_request_status_str[] = {
-      "HIT", "HIT_RESERVED", "MISS", "RESERVATION_FAIL", "SECTOR_MISS"};
+      "HIT",         "HIT_RESERVED", "MISS", "RESERVATION_FAIL",
+      "SECTOR_MISS", "MSHR_HIT"};
 
   assert(sizeof(static_cache_request_status_str) / sizeof(const char *) ==
          NUM_CACHE_REQUEST_STATUS);
@@ -63,9 +67,9 @@ unsigned l1d_cache_config::set_bank(new_addr_type addr) const {
   // For sector cache, we select one sector per bank (sector interleaving)
   // This is what was found in Volta (one sector per bank, sector interleaving)
   // otherwise, line interleaving
-  return cache_config::hash_function(addr, l1_banks, l1_banks_byte_interleaving,
-                                     m_l1_banks_log2,
-                                     l1_banks_hashing_function);
+  return cache_config::hash_function(addr, l1_banks,
+                                     l1_banks_byte_interleaving_log2,
+                                     l1_banks_log2, l1_banks_hashing_function);
 }
 
 unsigned cache_config::set_index(new_addr_type addr) const {
@@ -210,6 +214,7 @@ void tag_array::init(int core_id, int type_id) {
   m_core_id = core_id;
   m_type_id = type_id;
   is_used = false;
+  m_dirty = 0;
 }
 
 void tag_array::add_pending_line(mem_fetch *mf) {
@@ -231,15 +236,15 @@ void tag_array::remove_pending_line(mem_fetch *mf) {
 }
 
 enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
-                                           mem_fetch *mf,
+                                           mem_fetch *mf, bool is_write,
                                            bool probe_mode) const {
   mem_access_sector_mask_t mask = mf->get_access_sector_mask();
-  return probe(addr, idx, mask, probe_mode, mf);
+  return probe(addr, idx, mask, is_write, probe_mode, mf);
 }
 
 enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
                                            mem_access_sector_mask_t mask,
-                                           bool probe_mode,
+                                           bool is_write, bool probe_mode,
                                            mem_fetch *mf) const {
   // assert( m_config.m_write_policy == READ_ONLY );
   unsigned set_index = m_config.set_index(addr);
@@ -250,7 +255,6 @@ enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
   unsigned long long valid_timestamp = (unsigned)-1;
 
   bool all_reserved = true;
-
   // check for hit or pending hit
   for (unsigned way = 0; way < m_config.m_assoc; way++) {
     unsigned index = set_index * m_config.m_assoc + way;
@@ -263,7 +267,7 @@ enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
         idx = index;
         return HIT;
       } else if (line->get_status(mask) == MODIFIED) {
-        if (line->is_readable(mask)) {
+        if ((!is_write && line->is_readable(mask)) || is_write) {
           idx = index;
           return HIT;
         } else {
@@ -279,20 +283,31 @@ enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
       }
     }
     if (!line->is_reserved_line()) {
-      all_reserved = false;
-      if (line->is_invalid_line()) {
-        invalid_line = index;
-      } else {
-        // valid line : keep track of most appropriate replacement candidate
-        if (m_config.m_replacement_policy == LRU) {
-          if (line->get_last_access_time() < valid_timestamp) {
-            valid_timestamp = line->get_last_access_time();
-            valid_line = index;
-          }
-        } else if (m_config.m_replacement_policy == FIFO) {
-          if (line->get_alloc_time() < valid_timestamp) {
-            valid_timestamp = line->get_alloc_time();
-            valid_line = index;
+      // percentage of dirty lines in the cache
+      // number of dirty lines / total lines in the cache
+      float dirty_line_percentage =
+          ((float)m_dirty / (m_config.m_nset * m_config.m_assoc)) * 100;
+      // If the cacheline is from a load op (not modified), 
+      // or the total dirty cacheline is above a specific value,
+      // Then this cacheline is eligible to be considered for replacement candidate
+      // i.e. Only evict clean cachelines until total dirty cachelines reach the limit.
+      if (!line->is_modified_line() ||
+          dirty_line_percentage >= m_config.m_wr_percent) {
+        all_reserved = false;
+        if (line->is_invalid_line()) {
+          invalid_line = index;
+        } else {
+          // valid line : keep track of most appropriate replacement candidate
+          if (m_config.m_replacement_policy == LRU) {
+            if (line->get_last_access_time() < valid_timestamp) {
+              valid_timestamp = line->get_last_access_time();
+              valid_line = index;
+            }
+          } else if (m_config.m_replacement_policy == FIFO) {
+            if (line->get_alloc_time() < valid_timestamp) {
+              valid_timestamp = line->get_alloc_time();
+              valid_line = index;
+            }
           }
         }
       }
@@ -312,15 +327,6 @@ enum cache_request_status tag_array::probe(new_addr_type addr, unsigned &idx,
     abort();  // if an unreserved block exists, it is either invalid or
               // replaceable
 
-  if (probe_mode && m_config.is_streaming()) {
-    line_table::const_iterator i =
-        pending_lines.find(m_config.block_addr(addr));
-    assert(mf);
-    if (!mf->is_write() && i != pending_lines.end()) {
-      if (i->second != mf->get_inst().get_uid()) return SECTOR_MISS;
-    }
-  }
-
   return MISS;
 }
 
@@ -340,7 +346,7 @@ enum cache_request_status tag_array::access(new_addr_type addr, unsigned time,
   m_access++;
   is_used = true;
   shader_cache_access_log(m_core_id, m_type_id, 0);  // log accesses to cache
-  enum cache_request_status status = probe(addr, idx, mf);
+  enum cache_request_status status = probe(addr, idx, mf, mf->is_write());
   switch (status) {
     case HIT_RESERVED:
       m_pending_hit++;
@@ -353,8 +359,12 @@ enum cache_request_status tag_array::access(new_addr_type addr, unsigned time,
       if (m_config.m_alloc_policy == ON_MISS) {
         if (m_lines[idx]->is_modified_line()) {
           wb = true;
+          // m_lines[idx]->set_byte_mask(mf);
           evicted.set_info(m_lines[idx]->m_block_addr,
-                           m_lines[idx]->get_modified_size());
+                           m_lines[idx]->get_modified_size(),
+                           m_lines[idx]->get_dirty_byte_mask(),
+                           m_lines[idx]->get_dirty_sector_mask());
+          m_dirty--;
         }
         m_lines[idx]->allocate(m_config.tag(addr), m_config.block_addr(addr),
                                time, mf->get_access_sector_mask());
@@ -365,8 +375,12 @@ enum cache_request_status tag_array::access(new_addr_type addr, unsigned time,
       m_sector_miss++;
       shader_cache_access_log(m_core_id, m_type_id, 1);  // log cache misses
       if (m_config.m_alloc_policy == ON_MISS) {
+        bool before = m_lines[idx]->is_modified_line();
         ((sector_cache_block *)m_lines[idx])
             ->allocate_sector(time, mf->get_access_sector_mask());
+        if (before && !m_lines[idx]->is_modified_line()) {
+          m_dirty--;
+        }
       }
       break;
     case RESERVATION_FAIL:
@@ -383,31 +397,45 @@ enum cache_request_status tag_array::access(new_addr_type addr, unsigned time,
   return status;
 }
 
-void tag_array::fill(new_addr_type addr, unsigned time, mem_fetch *mf) {
-  fill(addr, time, mf->get_access_sector_mask());
+void tag_array::fill(new_addr_type addr, unsigned time, mem_fetch *mf,
+                     bool is_write) {
+  fill(addr, time, mf->get_access_sector_mask(), mf->get_access_byte_mask(),
+       is_write);
 }
 
 void tag_array::fill(new_addr_type addr, unsigned time,
-                     mem_access_sector_mask_t mask) {
+                     mem_access_sector_mask_t mask,
+                     mem_access_byte_mask_t byte_mask, bool is_write) {
   // assert( m_config.m_alloc_policy == ON_FILL );
   unsigned idx;
-  enum cache_request_status status = probe(addr, idx, mask);
+  enum cache_request_status status = probe(addr, idx, mask, is_write);
+  bool before = m_lines[idx]->is_modified_line();
   // assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented
   // redundant memory request
-  if (status == MISS)
+  if (status == MISS) {
     m_lines[idx]->allocate(m_config.tag(addr), m_config.block_addr(addr), time,
                            mask);
-  else if (status == SECTOR_MISS) {
+  } else if (status == SECTOR_MISS) {
     assert(m_config.m_cache_type == SECTOR);
     ((sector_cache_block *)m_lines[idx])->allocate_sector(time, mask);
   }
-
-  m_lines[idx]->fill(time, mask);
+  if (before && !m_lines[idx]->is_modified_line()) {
+    m_dirty--;
+  }
+  before = m_lines[idx]->is_modified_line();
+  m_lines[idx]->fill(time, mask, byte_mask);
+  if (m_lines[idx]->is_modified_line() && !before) {
+    m_dirty++;
+  }
 }
 
 void tag_array::fill(unsigned index, unsigned time, mem_fetch *mf) {
   assert(m_config.m_alloc_policy == ON_MISS);
-  m_lines[index]->fill(time, mf->get_access_sector_mask());
+  bool before = m_lines[index]->is_modified_line();
+  m_lines[index]->fill(time, mf->get_access_sector_mask(), mf->get_access_byte_mask());
+  if (m_lines[index]->is_modified_line() && !before) {
+    m_dirty++;
+  }
 }
 
 // TODO: we need write back the flushed data to the upper level
@@ -416,10 +444,12 @@ void tag_array::flush() {
 
   for (unsigned i = 0; i < m_config.get_num_lines(); i++)
     if (m_lines[i]->is_modified_line()) {
-      for (unsigned j = 0; j < SECTOR_CHUNCK_SIZE; j++)
+      for (unsigned j = 0; j < SECTOR_CHUNCK_SIZE; j++) {
         m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j));
+      }
     }
 
+  m_dirty = 0;
   is_used = false;
 }
 
@@ -430,6 +460,7 @@ void tag_array::invalidate() {
     for (unsigned j = 0; j < SECTOR_CHUNCK_SIZE; j++)
       m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j));
 
+  m_dirty = 0;
   is_used = false;
 }
 
@@ -485,8 +516,10 @@ bool was_writeback_sent(const std::list<cache_event> &events,
                         cache_event &wb_event) {
   for (std::list<cache_event>::const_iterator e = events.begin();
        e != events.end(); e++) {
-    if ((*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT) wb_event = *e;
-    return true;
+    if ((*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT) {
+      wb_event = *e;
+      return true;
+    }
   }
   return false;
 }
@@ -612,6 +645,7 @@ void cache_stats::clear() {
   ///
   for (unsigned i = 0; i < NUM_MEM_ACCESS_TYPE; ++i) {
     std::fill(m_stats[i].begin(), m_stats[i].end(), 0);
+    std::fill(m_stats_pw[i].begin(), m_stats_pw[i].end(), 0);
     std::fill(m_fail_stats[i].begin(), m_fail_stats[i].end(), 0);
   }
   m_cache_port_available_cycles = 0;
@@ -771,7 +805,9 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const {
               cache_request_status_str((enum cache_request_status)status),
               m_stats[type][status]);
 
-      if (status != RESERVATION_FAIL)
+      if (status != RESERVATION_FAIL && status != MSHR_HIT)
+        // MSHR_HIT is a special type of SECTOR_MISS
+        // so its already included in the SECTOR_MISS
         total_access[type] += m_stats[type][status];
     }
   }
@@ -1057,8 +1093,7 @@ void baseline_cache::fill(mem_fetch *mf, unsigned time) {
   if (m_config.m_alloc_policy == ON_MISS)
     m_tag_array->fill(e->second.m_cache_index, time, mf);
   else if (m_config.m_alloc_policy == ON_FILL) {
-    m_tag_array->fill(e->second.m_block_addr, time, mf);
-    if (m_config.is_streaming()) m_tag_array->remove_pending_line(mf);
+    m_tag_array->fill(e->second.m_block_addr, time, mf, mf->is_write());
   } else
     abort();
   bool has_atomic = false;
@@ -1066,9 +1101,13 @@ void baseline_cache::fill(mem_fetch *mf, unsigned time) {
   if (has_atomic) {
     assert(m_config.m_alloc_policy == ON_MISS);
     cache_block_t *block = m_tag_array->get_block(e->second.m_cache_index);
+    if (!block->is_modified_line()) {
+      m_tag_array->inc_dirty();
+    }
     block->set_status(MODIFIED,
                       mf->get_access_sector_mask());  // mark line as dirty for
                                                       // atomic operation
+    block->set_byte_mask(mf);
   }
   m_extra_mf_fields.erase(mf);
   m_bandwidth_management.use_fill_port(mf);
@@ -1123,6 +1162,7 @@ void baseline_cache::send_read_request(new_addr_type addr,
       m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf);
 
     m_mshrs.add(mshr_addr, mf);
+    m_stats.inc_stats(mf->get_access_type(), MSHR_HIT);
     do_miss = true;
 
   } else if (!mshr_hit && mshr_avail &&
@@ -1133,9 +1173,6 @@ void baseline_cache::send_read_request(new_addr_type addr,
       m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf);
 
     m_mshrs.add(mshr_addr, mf);
-    if (m_config.is_streaming() && m_config.m_cache_type == SECTOR) {
-      m_tag_array->add_pending_line(mf);
-    }
     m_extra_mf_fields[mf] = extra_mf_fields(
         mshr_addr, mf->get_addr(), cache_index, mf->get_data_size(), m_config);
     mf->set_data_size(m_config.get_atom_sz());
@@ -1162,6 +1199,25 @@ void data_cache::send_write_request(mem_fetch *mf, cache_event request,
   mf->set_status(m_miss_queue_status, time);
 }
 
+void data_cache::update_m_readable(mem_fetch *mf, unsigned cache_index) {
+  cache_block_t *block = m_tag_array->get_block(cache_index);
+  for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; i++) {
+    if (mf->get_access_sector_mask().test(i)) {
+      bool all_set = true;
+      for (unsigned k = i * SECTOR_SIZE; k < (i + 1) * SECTOR_SIZE; k++) {
+        // If any bit in the byte mask (within the sector) is not set, 
+        // the sector is unreadble
+        if (!block->get_dirty_byte_mask().test(k)) {
+          all_set = false;
+          break;
+        }
+      }
+      if (all_set)
+        block->set_m_readable(true, mf->get_access_sector_mask());
+    }
+  }
+}
+
 /****** Write-hit functions (Set by config file) ******/
 
 /// Write-back hit: Mark block as modified
@@ -1173,7 +1229,12 @@ cache_request_status data_cache::wr_hit_wb(new_addr_type addr,
   new_addr_type block_addr = m_config.block_addr(addr);
   m_tag_array->access(block_addr, time, cache_index, mf);  // update LRU state
   cache_block_t *block = m_tag_array->get_block(cache_index);
+  if (!block->is_modified_line()) {
+    m_tag_array->inc_dirty();
+  }
   block->set_status(MODIFIED, mf->get_access_sector_mask());
+  block->set_byte_mask(mf);
+  update_m_readable(mf,cache_index);
 
   return HIT;
 }
@@ -1192,7 +1253,12 @@ cache_request_status data_cache::wr_hit_wt(new_addr_type addr,
   new_addr_type block_addr = m_config.block_addr(addr);
   m_tag_array->access(block_addr, time, cache_index, mf);  // update LRU state
   cache_block_t *block = m_tag_array->get_block(cache_index);
+  if (!block->is_modified_line()) {
+    m_tag_array->inc_dirty();
+  }
   block->set_status(MODIFIED, mf->get_access_sector_mask());
+  block->set_byte_mask(mf);
+  update_m_readable(mf,cache_index);
 
   // generate a write-through
   send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
@@ -1302,8 +1368,10 @@ enum cache_request_status data_cache::wr_miss_wa_naive(
       assert(status ==
              MISS);  // SECTOR_MISS and HIT_RESERVED should not send write back
       mem_fetch *wb = m_memfetch_creator->alloc(
-          evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true,
-          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
+          evicted.m_block_addr, m_wrbk_type, mf->get_access_warp_mask(),
+          evicted.m_byte_mask, evicted.m_sector_mask, evicted.m_modified_size,
+          true, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, -1, -1, -1,
+          NULL);
       // the evicted block may have wrong chip id when advanced L2 hashing  is
       // used, so set the right chip address from the original mf
       wb->set_chip(mf->get_tlx_addr().chip);
@@ -1340,7 +1408,11 @@ enum cache_request_status data_cache::wr_miss_wa_fetch_on_write(
         m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf);
     assert(status != HIT);
     cache_block_t *block = m_tag_array->get_block(cache_index);
+    if (!block->is_modified_line()) {
+      m_tag_array->inc_dirty();
+    }
     block->set_status(MODIFIED, mf->get_access_sector_mask());
+    block->set_byte_mask(mf);
     if (status == HIT_RESERVED)
       block->set_ignore_on_fill(true, mf->get_access_sector_mask());
 
@@ -1349,8 +1421,10 @@ enum cache_request_status data_cache::wr_miss_wa_fetch_on_write(
       // (already modified lower level)
       if (wb && (m_config.m_write_policy != WRITE_THROUGH)) {
         mem_fetch *wb = m_memfetch_creator->alloc(
-            evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true,
-            m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
+            evicted.m_block_addr, m_wrbk_type, mf->get_access_warp_mask(),
+            evicted.m_byte_mask, evicted.m_sector_mask, evicted.m_modified_size,
+            true, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, -1, -1, -1,
+            NULL);
         // the evicted block may have wrong chip id when advanced L2 hashing  is
         // used, so set the right chip address from the original mf
         wb->set_chip(mf->get_tlx_addr().chip);
@@ -1411,6 +1485,7 @@ enum cache_request_status data_cache::wr_miss_wa_fetch_on_write(
 
     cache_block_t *block = m_tag_array->get_block(cache_index);
     block->set_modified_on_fill(true, mf->get_access_sector_mask());
+    block->set_byte_mask_on_fill(true);
 
     events.push_back(cache_event(WRITE_ALLOCATE_SENT));
 
@@ -1419,8 +1494,10 @@ enum cache_request_status data_cache::wr_miss_wa_fetch_on_write(
       // (already modified lower level)
       if (wb && (m_config.m_write_policy != WRITE_THROUGH)) {
         mem_fetch *wb = m_memfetch_creator->alloc(
-            evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true,
-            m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
+            evicted.m_block_addr, m_wrbk_type, mf->get_access_warp_mask(),
+            evicted.m_byte_mask, evicted.m_sector_mask, evicted.m_modified_size,
+            true, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, -1, -1, -1,
+            NULL);
         // the evicted block may have wrong chip id when advanced L2 hashing  is
         // used, so set the right chip address from the original mf
         wb->set_chip(mf->get_tlx_addr().chip);
@@ -1448,6 +1525,10 @@ enum cache_request_status data_cache::wr_miss_wa_lazy_fetch_on_read(
     return RESERVATION_FAIL;  // cannot handle request this cycle
   }
 
+  if (m_config.m_write_policy == WRITE_THROUGH) {
+    send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
+  }
+
   bool wb = false;
   evicted_block_info evicted;
 
@@ -1455,25 +1536,35 @@ enum cache_request_status data_cache::wr_miss_wa_lazy_fetch_on_read(
       m_tag_array->access(block_addr, time, cache_index, wb, evicted, mf);
   assert(m_status != HIT);
   cache_block_t *block = m_tag_array->get_block(cache_index);
+  if (!block->is_modified_line()) {
+    m_tag_array->inc_dirty();
+  }
   block->set_status(MODIFIED, mf->get_access_sector_mask());
+  block->set_byte_mask(mf);
   if (m_status == HIT_RESERVED) {
     block->set_ignore_on_fill(true, mf->get_access_sector_mask());
     block->set_modified_on_fill(true, mf->get_access_sector_mask());
+    block->set_byte_mask_on_fill(true);
   }
 
   if (mf->get_access_byte_mask().count() == m_config.get_atom_sz()) {
     block->set_m_readable(true, mf->get_access_sector_mask());
   } else {
     block->set_m_readable(false, mf->get_access_sector_mask());
+    if (m_status == HIT_RESERVED)
+      block->set_readable_on_fill(true, mf->get_access_sector_mask());
   }
+  update_m_readable(mf,cache_index);
 
   if (m_status != RESERVATION_FAIL) {
     // If evicted block is modified and not a write-through
     // (already modified lower level)
     if (wb && (m_config.m_write_policy != WRITE_THROUGH)) {
       mem_fetch *wb = m_memfetch_creator->alloc(
-          evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true,
-          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
+          evicted.m_block_addr, m_wrbk_type, mf->get_access_warp_mask(),
+          evicted.m_byte_mask, evicted.m_sector_mask, evicted.m_modified_size,
+          true, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, -1, -1, -1,
+          NULL);
       // the evicted block may have wrong chip id when advanced L2 hashing  is
       // used, so set the right chip address from the original mf
       wb->set_chip(mf->get_tlx_addr().chip);
@@ -1516,8 +1607,12 @@ enum cache_request_status data_cache::rd_hit_base(
   if (mf->isatomic()) {
     assert(mf->get_access_type() == GLOBAL_ACC_R);
     cache_block_t *block = m_tag_array->get_block(cache_index);
+    if (!block->is_modified_line()) {
+      m_tag_array->inc_dirty();
+    }
     block->set_status(MODIFIED,
-                      mf->get_access_sector_mask());  // mark line as dirty
+                      mf->get_access_sector_mask());  // mark line as
+    block->set_byte_mask(mf);
   }
   return HIT;
 }
@@ -1548,8 +1643,10 @@ enum cache_request_status data_cache::rd_miss_base(
     // (already modified lower level)
     if (wb && (m_config.m_write_policy != WRITE_THROUGH)) {
       mem_fetch *wb = m_memfetch_creator->alloc(
-          evicted.m_block_addr, m_wrbk_type, evicted.m_modified_size, true,
-          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
+          evicted.m_block_addr, m_wrbk_type, mf->get_access_warp_mask(),
+          evicted.m_byte_mask, evicted.m_sector_mask, evicted.m_modified_size,
+          true, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, -1, -1, -1,
+          NULL);
       // the evicted block may have wrong chip id when advanced L2 hashing  is
       // used, so set the right chip address from the original mf
       wb->set_chip(mf->get_tlx_addr().chip);
@@ -1572,7 +1669,7 @@ enum cache_request_status read_only_cache::access(
   new_addr_type block_addr = m_config.block_addr(addr);
   unsigned cache_index = (unsigned)-1;
   enum cache_request_status status =
-      m_tag_array->probe(block_addr, cache_index, mf);
+      m_tag_array->probe(block_addr, cache_index, mf, mf->is_write());
   enum cache_request_status cache_status = RESERVATION_FAIL;
 
   if (status == HIT) {
@@ -1659,7 +1756,7 @@ enum cache_request_status data_cache::access(new_addr_type addr, mem_fetch *mf,
   new_addr_type block_addr = m_config.block_addr(addr);
   unsigned cache_index = (unsigned)-1;
   enum cache_request_status probe_status =
-      m_tag_array->probe(block_addr, cache_index, mf, true);
+      m_tag_array->probe(block_addr, cache_index, mf, mf->is_write(), true);
   enum cache_request_status access_status =
       process_tag_probe(wr, probe_status, addr, cache_index, mf, time, events);
   m_stats.inc_stats(mf->get_access_type(),
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index 5c28b41..4bbf7e2 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Tayler Hetherington, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -49,6 +51,7 @@ enum cache_request_status {
   MISS,
   RESERVATION_FAIL,
   SECTOR_MISS,
+  MSHR_HIT,
   NUM_CACHE_REQUEST_STATUS
 };
 
@@ -71,14 +74,26 @@ enum cache_event_type {
 struct evicted_block_info {
   new_addr_type m_block_addr;
   unsigned m_modified_size;
+  mem_access_byte_mask_t m_byte_mask;
+  mem_access_sector_mask_t m_sector_mask;
   evicted_block_info() {
     m_block_addr = 0;
     m_modified_size = 0;
+    m_byte_mask.reset();
+    m_sector_mask.reset();
   }
   void set_info(new_addr_type block_addr, unsigned modified_size) {
     m_block_addr = block_addr;
     m_modified_size = modified_size;
   }
+  void set_info(new_addr_type block_addr, unsigned modified_size,
+                mem_access_byte_mask_t byte_mask,
+                mem_access_sector_mask_t sector_mask) {
+    m_block_addr = block_addr;
+    m_modified_size = modified_size;
+    m_byte_mask = byte_mask;
+    m_sector_mask = sector_mask;
+  }
 };
 
 struct cache_event {
@@ -108,7 +123,8 @@ struct cache_block_t {
   virtual void allocate(new_addr_type tag, new_addr_type block_addr,
                         unsigned time,
                         mem_access_sector_mask_t sector_mask) = 0;
-  virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask) = 0;
+  virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask,
+                    mem_access_byte_mask_t byte_mask) = 0;
 
   virtual bool is_invalid_line() = 0;
   virtual bool is_valid_line() = 0;
@@ -119,7 +135,10 @@ struct cache_block_t {
       mem_access_sector_mask_t sector_mask) = 0;
   virtual void set_status(enum cache_block_state m_status,
                           mem_access_sector_mask_t sector_mask) = 0;
-
+  virtual void set_byte_mask(mem_fetch *mf) = 0;
+  virtual void set_byte_mask(mem_access_byte_mask_t byte_mask) = 0;
+  virtual mem_access_byte_mask_t get_dirty_byte_mask() = 0;
+  virtual mem_access_sector_mask_t get_dirty_sector_mask() = 0;
   virtual unsigned long long get_last_access_time() = 0;
   virtual void set_last_access_time(unsigned long long time,
                                     mem_access_sector_mask_t sector_mask) = 0;
@@ -128,6 +147,9 @@ struct cache_block_t {
                                   mem_access_sector_mask_t sector_mask) = 0;
   virtual void set_modified_on_fill(bool m_modified,
                                     mem_access_sector_mask_t sector_mask) = 0;
+  virtual void set_readable_on_fill(bool readable,
+                                    mem_access_sector_mask_t sector_mask) = 0;
+  virtual void set_byte_mask_on_fill(bool m_modified) = 0;
   virtual unsigned get_modified_size() = 0;
   virtual void set_m_readable(bool readable,
                               mem_access_sector_mask_t sector_mask) = 0;
@@ -147,6 +169,7 @@ struct line_cache_block : public cache_block_t {
     m_status = INVALID;
     m_ignore_on_fill_status = false;
     m_set_modified_on_fill = false;
+    m_set_readable_on_fill = false;
     m_readable = true;
   }
   void allocate(new_addr_type tag, new_addr_type block_addr, unsigned time,
@@ -159,13 +182,19 @@ struct line_cache_block : public cache_block_t {
     m_status = RESERVED;
     m_ignore_on_fill_status = false;
     m_set_modified_on_fill = false;
+    m_set_readable_on_fill = false;
+    m_set_byte_mask_on_fill = false;
   }
-  void fill(unsigned time, mem_access_sector_mask_t sector_mask) {
+  virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask,
+                    mem_access_byte_mask_t byte_mask) {
     // if(!m_ignore_on_fill_status)
     //	assert( m_status == RESERVED );
 
     m_status = m_set_modified_on_fill ? MODIFIED : VALID;
 
+    if (m_set_readable_on_fill) m_readable = true;
+    if (m_set_byte_mask_on_fill) set_byte_mask(byte_mask);
+
     m_fill_time = time;
   }
   virtual bool is_invalid_line() { return m_status == INVALID; }
@@ -181,6 +210,20 @@ struct line_cache_block : public cache_block_t {
                           mem_access_sector_mask_t sector_mask) {
     m_status = status;
   }
+  virtual void set_byte_mask(mem_fetch *mf) {
+    m_dirty_byte_mask = m_dirty_byte_mask | mf->get_access_byte_mask();
+  }
+  virtual void set_byte_mask(mem_access_byte_mask_t byte_mask) {
+    m_dirty_byte_mask = m_dirty_byte_mask | byte_mask;
+  }
+  virtual mem_access_byte_mask_t get_dirty_byte_mask() {
+    return m_dirty_byte_mask;
+  }
+  virtual mem_access_sector_mask_t get_dirty_sector_mask() {
+    mem_access_sector_mask_t sector_mask;
+    if (m_status == MODIFIED) sector_mask.set();
+    return sector_mask;
+  }
   virtual unsigned long long get_last_access_time() {
     return m_last_access_time;
   }
@@ -197,6 +240,13 @@ struct line_cache_block : public cache_block_t {
                                     mem_access_sector_mask_t sector_mask) {
     m_set_modified_on_fill = m_modified;
   }
+  virtual void set_readable_on_fill(bool readable,
+                                    mem_access_sector_mask_t sector_mask) {
+    m_set_readable_on_fill = readable;
+  }
+  virtual void set_byte_mask_on_fill(bool m_modified) {
+    m_set_byte_mask_on_fill = m_modified;
+  }
   virtual unsigned get_modified_size() {
     return SECTOR_CHUNCK_SIZE * SECTOR_SIZE;  // i.e. cache line size
   }
@@ -218,7 +268,10 @@ struct line_cache_block : public cache_block_t {
   cache_block_state m_status;
   bool m_ignore_on_fill_status;
   bool m_set_modified_on_fill;
+  bool m_set_readable_on_fill;
+  bool m_set_byte_mask_on_fill;
   bool m_readable;
+  mem_access_byte_mask_t m_dirty_byte_mask;
 };
 
 struct sector_cache_block : public cache_block_t {
@@ -232,11 +285,13 @@ struct sector_cache_block : public cache_block_t {
       m_status[i] = INVALID;
       m_ignore_on_fill_status[i] = false;
       m_set_modified_on_fill[i] = false;
+      m_set_readable_on_fill[i] = false;
       m_readable[i] = true;
     }
     m_line_alloc_time = 0;
     m_line_last_access_time = 0;
     m_line_fill_time = 0;
+    m_dirty_byte_mask.reset();
   }
 
   virtual void allocate(new_addr_type tag, new_addr_type block_addr,
@@ -261,6 +316,8 @@ struct sector_cache_block : public cache_block_t {
     m_status[sidx] = RESERVED;
     m_ignore_on_fill_status[sidx] = false;
     m_set_modified_on_fill[sidx] = false;
+    m_set_readable_on_fill[sidx] = false;
+    m_set_byte_mask_on_fill = false;
 
     // set line stats
     m_line_alloc_time = time;  // only set this for the first allocated sector
@@ -283,6 +340,8 @@ struct sector_cache_block : public cache_block_t {
     else
       m_set_modified_on_fill[sidx] = false;
 
+    m_set_readable_on_fill[sidx] = false;
+
     m_status[sidx] = RESERVED;
     m_ignore_on_fill_status[sidx] = false;
     // m_set_modified_on_fill[sidx] = false;
@@ -293,14 +352,20 @@ struct sector_cache_block : public cache_block_t {
     m_line_fill_time = 0;
   }
 
-  virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask) {
+  virtual void fill(unsigned time, mem_access_sector_mask_t sector_mask,
+                    mem_access_byte_mask_t byte_mask) {
     unsigned sidx = get_sector_index(sector_mask);
 
     //	if(!m_ignore_on_fill_status[sidx])
     //	         assert( m_status[sidx] == RESERVED );
-
     m_status[sidx] = m_set_modified_on_fill[sidx] ? MODIFIED : VALID;
 
+    if (m_set_readable_on_fill[sidx]) {
+      m_readable[sidx] = true;
+      m_set_readable_on_fill[sidx] = false;
+    }
+    if (m_set_byte_mask_on_fill) set_byte_mask(byte_mask);
+
     m_sector_fill_time[sidx] = time;
     m_line_fill_time = time;
   }
@@ -340,6 +405,22 @@ struct sector_cache_block : public cache_block_t {
     m_status[sidx] = status;
   }
 
+  virtual void set_byte_mask(mem_fetch *mf) {
+    m_dirty_byte_mask = m_dirty_byte_mask | mf->get_access_byte_mask();
+  }
+  virtual void set_byte_mask(mem_access_byte_mask_t byte_mask) {
+    m_dirty_byte_mask = m_dirty_byte_mask | byte_mask;
+  }
+  virtual mem_access_byte_mask_t get_dirty_byte_mask() {
+    return m_dirty_byte_mask;
+  }
+  virtual mem_access_sector_mask_t get_dirty_sector_mask() {
+    mem_access_sector_mask_t sector_mask;
+    for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; i++) {
+      if (m_status[i] == MODIFIED) sector_mask.set(i);
+    }
+    return sector_mask;
+  }
   virtual unsigned long long get_last_access_time() {
     return m_line_last_access_time;
   }
@@ -365,7 +446,15 @@ struct sector_cache_block : public cache_block_t {
     unsigned sidx = get_sector_index(sector_mask);
     m_set_modified_on_fill[sidx] = m_modified;
   }
+  virtual void set_byte_mask_on_fill(bool m_modified) {
+    m_set_byte_mask_on_fill = m_modified;
+  }
 
+  virtual void set_readable_on_fill(bool readable,
+                                    mem_access_sector_mask_t sector_mask) {
+    unsigned sidx = get_sector_index(sector_mask);
+    m_set_readable_on_fill[sidx] = readable;
+  }
   virtual void set_m_readable(bool readable,
                               mem_access_sector_mask_t sector_mask) {
     unsigned sidx = get_sector_index(sector_mask);
@@ -400,7 +489,10 @@ struct sector_cache_block : public cache_block_t {
   cache_block_state m_status[SECTOR_CHUNCK_SIZE];
   bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE];
   bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE];
+  bool m_set_readable_on_fill[SECTOR_CHUNCK_SIZE];
+  bool m_set_byte_mask_on_fill;
   bool m_readable[SECTOR_CHUNCK_SIZE];
+  mem_access_byte_mask_t m_dirty_byte_mask;
 
   unsigned get_sector_index(mem_access_sector_mask_t sector_mask) {
     assert(sector_mask.count() == 1);
@@ -463,6 +555,7 @@ class cache_config {
     m_data_port_width = 0;
     m_set_index_function = LINEAR_SET_FUNCTION;
     m_is_streaming = false;
+    m_wr_percent = 0;
   }
   void init(char *config, FuncCache status) {
     cache_status = status;
@@ -503,16 +596,6 @@ class cache_config {
       default:
         exit_parse_error();
     }
-    switch (rp) {
-      case 'L':
-        m_replacement_policy = LRU;
-        break;
-      case 'F':
-        m_replacement_policy = FIFO;
-        break;
-      default:
-        exit_parse_error();
-    }
     switch (wp) {
       case 'R':
         m_write_policy = READ_ONLY;
@@ -546,22 +629,27 @@ class cache_config {
         exit_parse_error();
     }
     if (m_alloc_policy == STREAMING) {
-      // For streaming cache, we set the alloc policy to be on-fill to remove
-      // all line_alloc_fail stalls we set the MSHRs to be equal to max
-      // allocated cache lines. This is possible by moving TAG to be shared
-      // between cache line and MSHR enrty (i.e. for each cache line, there is
-      // an MSHR rntey associated with it) This is the easiest think we can
-      // think about to model (mimic) L1 streaming cache in Pascal and Volta
-      // Based on our microbenchmakrs, MSHRs entries have been increasing
-      // substantially in Pascal and Volta For more information about streaming
-      // cache, see:
-      // http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf
-      // https://ieeexplore.ieee.org/document/8344474/
+      /*
+      For streaming cache:
+      (1) we set the alloc policy to be on-fill to remove all line_alloc_fail
+      stalls. if the whole memory is allocated to the L1 cache, then make the
+      allocation to be on_MISS otherwise, make it ON_FILL to eliminate line
+      allocation fails. i.e. MSHR throughput is the same, independent on the L1
+      cache size/associativity So, we set the allocation policy per kernel
+      basis, see shader.cc, max_cta() function
+
+      (2) We also set the MSHRs to be equal to max
+      allocated cache lines. This is possible by moving TAG to be shared
+      between cache line and MSHR enrty (i.e. for each cache line, there is
+      an MSHR rntey associated with it). This is the easiest think we can
+      think of to model (mimic) L1 streaming cache in Pascal and Volta
+
+      For more information about streaming cache, see:
+      http://on-demand.gputechconf.com/gtc/2017/presentation/s7798-luke-durant-inside-volta.pdf
+      https://ieeexplore.ieee.org/document/8344474/
+      */
       m_is_streaming = true;
       m_alloc_policy = ON_FILL;
-      m_mshr_entries = m_nset * m_assoc * MAX_DEFAULT_CACHE_SIZE_MULTIBLIER;
-      if (m_cache_type == SECTOR) m_mshr_entries *= SECTOR_CHUNCK_SIZE;
-      m_mshr_max_merge = MAX_WARP_PER_SM;
     }
     switch (mshr_type) {
       case 'F':
@@ -610,7 +698,8 @@ class cache_config {
     }
 
     // detect invalid configuration
-    if (m_alloc_policy == ON_FILL and m_write_policy == WRITE_BACK) {
+    if ((m_alloc_policy == ON_FILL || m_alloc_policy == STREAMING) and
+        m_write_policy == WRITE_BACK) {
       // A writeback cache with allocate-on-fill policy will inevitably lead to
       // deadlock: The deadlock happens when an incoming cache-fill evicts a
       // dirty line, generating a writeback request.  If the memory subsystem is
@@ -656,6 +745,9 @@ class cache_config {
       case 'L':
         m_set_index_function = LINEAR_SET_FUNCTION;
         break;
+      case 'X':
+        m_set_index_function = BITWISE_XORING_FUNCTION;
+        break;
       default:
         exit_parse_error();
     }
@@ -675,11 +767,11 @@ class cache_config {
   }
   unsigned get_max_num_lines() const {
     assert(m_valid);
-    return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER * m_nset * original_m_assoc;
+    return get_max_cache_multiplier() * m_nset * original_m_assoc;
   }
   unsigned get_max_assoc() const {
     assert(m_valid);
-    return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER * original_m_assoc;
+    return get_max_cache_multiplier() * original_m_assoc;
   }
   void print(FILE *fp) const {
     fprintf(fp, "Size = %d B (%d Set x %d-way x %d byte line)\n",
@@ -688,6 +780,10 @@ class cache_config {
 
   virtual unsigned set_index(new_addr_type addr) const;
 
+  virtual unsigned get_max_cache_multiplier() const {
+    return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER;
+  }
+
   unsigned hash_function(new_addr_type addr, unsigned m_nset,
                          unsigned m_line_sz_log2, unsigned m_nset_log2,
                          unsigned m_index_function) const;
@@ -722,10 +818,18 @@ class cache_config {
   }
   bool is_streaming() { return m_is_streaming; }
   FuncCache get_cache_status() { return cache_status; }
+  void set_allocation_policy(enum allocation_policy_t alloc) {
+    m_alloc_policy = alloc;
+  }
   char *m_config_string;
   char *m_config_stringPrefL1;
   char *m_config_stringPrefShared;
   FuncCache cache_status;
+  unsigned m_wr_percent;
+  write_allocate_policy_t get_write_allocate_policy() {
+    return m_write_alloc_policy;
+  }
+  write_policy_t get_write_policy() { return m_write_policy; }
 
  protected:
   void exit_parse_error() {
@@ -789,16 +893,28 @@ class l1d_cache_config : public cache_config {
   l1d_cache_config() : cache_config() {}
   unsigned set_bank(new_addr_type addr) const;
   void init(char *config, FuncCache status) {
-    m_banks_byte_interleaving_log2 = LOGB2(l1_banks_byte_interleaving);
-    m_l1_banks_log2 = LOGB2(l1_banks);
+    l1_banks_byte_interleaving_log2 = LOGB2(l1_banks_byte_interleaving);
+    l1_banks_log2 = LOGB2(l1_banks);
     cache_config::init(config, status);
   }
   unsigned l1_latency;
   unsigned l1_banks;
-  unsigned m_l1_banks_log2;
+  unsigned l1_banks_log2;
   unsigned l1_banks_byte_interleaving;
-  unsigned m_banks_byte_interleaving_log2;
+  unsigned l1_banks_byte_interleaving_log2;
   unsigned l1_banks_hashing_function;
+  unsigned m_unified_cache_size;
+  virtual unsigned get_max_cache_multiplier() const {
+    // set * assoc * cacheline size. Then convert Byte to KB
+    // gpgpu_unified_cache_size is in KB while original_sz is in B
+    if (m_unified_cache_size > 0) {
+      unsigned original_size = m_nset * original_m_assoc * m_line_sz / 1024;
+      assert(m_unified_cache_size % original_size == 0);
+      return m_unified_cache_size / original_size;
+    } else {
+      return MAX_DEFAULT_CACHE_SIZE_MULTIBLIER;
+    }
+  }
 };
 
 class l2_cache_config : public cache_config {
@@ -818,9 +934,10 @@ class tag_array {
   ~tag_array();
 
   enum cache_request_status probe(new_addr_type addr, unsigned &idx,
-                                  mem_fetch *mf, bool probe_mode = false) const;
+                                  mem_fetch *mf, bool is_write,
+                                  bool probe_mode = false) const;
   enum cache_request_status probe(new_addr_type addr, unsigned &idx,
-                                  mem_access_sector_mask_t mask,
+                                  mem_access_sector_mask_t mask, bool is_write,
                                   bool probe_mode = false,
                                   mem_fetch *mf = NULL) const;
   enum cache_request_status access(new_addr_type addr, unsigned time,
@@ -829,9 +946,10 @@ class tag_array {
                                    unsigned &idx, bool &wb,
                                    evicted_block_info &evicted, mem_fetch *mf);
 
-  void fill(new_addr_type addr, unsigned time, mem_fetch *mf);
+  void fill(new_addr_type addr, unsigned time, mem_fetch *mf, bool is_write);
   void fill(unsigned idx, unsigned time, mem_fetch *mf);
-  void fill(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask);
+  void fill(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask,
+            mem_access_byte_mask_t byte_mask, bool is_write);
 
   unsigned size() const { return m_config.get_num_lines(); }
   cache_block_t *get_block(unsigned idx) { return m_lines[idx]; }
@@ -849,6 +967,7 @@ class tag_array {
   void update_cache_parameters(cache_config &config);
   void add_pending_line(mem_fetch *mf);
   void remove_pending_line(mem_fetch *mf);
+  void inc_dirty() { m_dirty++; }
 
  protected:
   // This constructor is intended for use only from derived classes that wish to
@@ -869,6 +988,7 @@ class tag_array {
                            // allocated but not filled
   unsigned m_res_fail;
   unsigned m_sector_miss;
+  unsigned m_dirty;
 
   // performance counters for calculating the amount of misses within a time
   // window
@@ -1214,7 +1334,8 @@ class baseline_cache : public cache_t {
   // something is read or written without doing anything else.
   void force_tag_access(new_addr_type addr, unsigned time,
                         mem_access_sector_mask_t mask) {
-    m_tag_array->fill(addr, time, mask);
+    mem_access_byte_mask_t byte_mask;
+    m_tag_array->fill(addr, time, mask, byte_mask, true);
   }
 
  protected:
@@ -1451,7 +1572,7 @@ class data_cache : public baseline_cache {
   /// Sends write request to lower level memory (write or writeback)
   void send_write_request(mem_fetch *mf, cache_event request, unsigned time,
                           std::list<cache_event> &events);
-
+  void update_m_readable(mem_fetch *mf, unsigned cache_index);
   // Member Function pointers - Set by configuration options
   // to the functions below each grouping
   /******* Write-hit configs *******/
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 1650688..5af244b 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, George L. Yuan,
-// Ali Bakhoda, Andrew Turner, Ivan Sham
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, George L. Yuan,
+// Ali Bakhoda, Andrew Turner, Ivan Sham, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -95,10 +97,11 @@ tr1_hash_map<new_addr_type, unsigned> address_random_interleaving;
 
 #include "mem_latency_stat.h"
 
+
 void power_config::reg_options(class OptionParser *opp) {
-  option_parser_register(opp, "-gpuwattch_xml_file", OPT_CSTR,
-                         &g_power_config_name, "GPUWattch XML file",
-                         "gpuwattch.xml");
+  option_parser_register(opp, "-accelwattch_xml_file", OPT_CSTR,
+                         &g_power_config_name, "AccelWattch XML file",
+                         "accelwattch_sass_sim.xml");
 
   option_parser_register(opp, "-power_simulation_enabled", OPT_BOOL,
                          &g_power_simulation_enabled,
@@ -108,6 +111,92 @@ void power_config::reg_options(class OptionParser *opp) {
                          &g_power_per_cycle_dump,
                          "Dump detailed power output each cycle", "0");
 
+
+
+
+  option_parser_register(opp, "-hw_perf_file_name", OPT_CSTR,
+                         &g_hw_perf_file_name, "Hardware Performance Statistics file",
+                         "hw_perf.csv");
+
+  option_parser_register(opp, "-hw_perf_bench_name", OPT_CSTR,
+                         &g_hw_perf_bench_name, "Kernel Name in Hardware Performance Statistics file",
+                         "");
+
+  option_parser_register(opp, "-power_simulation_mode", OPT_INT32,
+                         &g_power_simulation_mode,
+                         "Switch performance counter input for power simulation (0=Sim, 1=HW, 2=HW-Sim Hybrid)", "0");
+
+  option_parser_register(opp, "-dvfs_enabled", OPT_BOOL,
+                         &g_dvfs_enabled,
+                         "Turn on DVFS for power model", "0");
+  option_parser_register(opp, "-aggregate_power_stats", OPT_BOOL,
+                         &g_aggregate_power_stats,
+                         "Accumulate power across all kernels", "0");
+
+  //Accelwattch Hyrbid Configuration
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L1_RH", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L1_RH],
+                         "Get L1 Read Hits for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L1_RM", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L1_RM],
+                         "Get L1 Read Misses for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L1_WH", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L1_WH],
+                         "Get L1 Write Hits for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L1_WM", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L1_WM],
+                         "Get L1 Write Misses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L2_RH", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L2_RH],
+                         "Get L2 Read Hits for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L2_RM", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L2_RM],
+                         "Get L2 Read Misses for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L2_WH", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L2_WH],
+                         "Get L2 Write Hits for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_L2_WM", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_L2_WM],
+                         "Get L2 Write Misses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_CC_ACC", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_CC_ACC],
+                         "Get Constant Cache Acesses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_SHARED_ACC", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_SHRD_ACC],
+                         "Get Shared Memory Acesses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_DRAM_RD", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_DRAM_RD],
+                         "Get DRAM Reads for Accelwattch-Hybrid from Accel-Sim", "0");
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_DRAM_WR", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_DRAM_WR],
+                         "Get DRAM Writes for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_NOC", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_NOC],
+                         "Get Interconnect Acesses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_PIPE_DUTY", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_PIPE_DUTY],
+                         "Get Pipeline Duty Cycle Acesses for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_NUM_SM_IDLE", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_NUM_SM_IDLE],
+                         "Get Number of Idle SMs for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_CYCLES", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_CYCLES],
+                         "Get Executed Cycles for Accelwattch-Hybrid from Accel-Sim", "0");
+
+  option_parser_register(opp, "-accelwattch_hybrid_perfsim_VOLTAGE", OPT_BOOL,
+                         &accelwattch_hybrid_configuration[HW_VOLTAGE],
+                         "Get Chip Voltage for Accelwattch-Hybrid from Accel-Sim", "0");
+
+
   // Output Data Formats
   option_parser_register(
       opp, "-power_trace_enabled", OPT_BOOL, &g_power_trace_enabled,
@@ -249,6 +338,8 @@ void shader_core_config::reg_options(class OptionParser *opp) {
                          " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_"
                          "alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                          "none");
+  option_parser_register(opp, "-gpgpu_l1_cache_write_ratio", OPT_UINT32,
+                         &m_L1D_config.m_wr_percent, "L1D write ratio", "0");
   option_parser_register(opp, "-gpgpu_l1_banks", OPT_UINT32,
                          &m_L1D_config.l1_banks, "The number of L1 cache banks",
                          "1");
@@ -304,7 +395,7 @@ void shader_core_config::reg_options(class OptionParser *opp) {
                          "gpgpu_ignore_resources_limitation (default 0)", "0");
   option_parser_register(
       opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core,
-      "Maximum number of concurrent CTAs in shader (default 8)", "8");
+      "Maximum number of concurrent CTAs in shader (default 32)", "32");
   option_parser_register(
       opp, "-gpgpu_num_cta_barriers", OPT_UINT32, &max_barriers_per_cta,
       "Maximum number of named barriers per CTA (default 16)", "16");
@@ -326,7 +417,14 @@ void shader_core_config::reg_options(class OptionParser *opp) {
   option_parser_register(
       opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size,
       "Size of shared memory per shader core (default 16kB)", "16384");
-  option_parser_register(opp, "-gpgpu_adaptive_cache_config", OPT_UINT32,
+  option_parser_register(opp, "-gpgpu_shmem_option", OPT_CSTR,
+                         &gpgpu_shmem_option,
+                         "Option list of shared memory sizes", "0");
+  option_parser_register(
+      opp, "-gpgpu_unified_l1d_size", OPT_UINT32,
+      &m_L1D_config.m_unified_cache_size,
+      "Size of unified data cache(L1D + shared memory) in KB", "0");
+  option_parser_register(opp, "-gpgpu_adaptive_cache_config", OPT_BOOL,
                          &adaptive_cache_config, "adaptive_cache_config", "0");
   option_parser_register(
       opp, "-gpgpu_shmem_sizeDefault", OPT_UINT32, &gpgpu_shmem_sizeDefault,
@@ -603,7 +701,8 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) {
                          "500.0:2000.0:2000.0:2000.0");
   option_parser_register(
       opp, "-gpgpu_max_concurrent_kernel", OPT_INT32, &max_concurrent_kernel,
-      "maximum kernels that can run concurrently on GPU", "8");
+      "maximum kernels that can run concurrently on GPU, set this value "
+      "according to max resident grids for your compute capability", "32");
   option_parser_register(
       opp, "-gpgpu_cflog_interval", OPT_INT32, &gpgpu_cflog_interval,
       "Interval between each snapshot in control flow logger", "0");
@@ -826,7 +925,7 @@ gpgpu_sim::gpgpu_sim(const gpgpu_sim_config &config, gpgpu_context *ctx)
 
 #ifdef GPGPUSIM_POWER_MODEL
   m_gpgpusim_wrapper = new gpgpu_sim_wrapper(config.g_power_simulation_enabled,
-                                             config.g_power_config_name);
+                                             config.g_power_config_name, config.g_power_simulation_mode, config.g_dvfs_enabled);
 #endif
 
   m_shader_stats = new shader_core_stats(m_shader_config);
@@ -1001,6 +1100,14 @@ void gpgpu_sim::init() {
   partiton_reqs_in_parallel_util = 0;
   gpu_sim_cycle_parition_util = 0;
 
+// McPAT initialization function. Called on first launch of GPU
+#ifdef GPGPUSIM_POWER_MODEL
+  if (m_config.g_power_simulation_enabled) {
+    init_mcpat(m_config, m_gpgpusim_wrapper, m_config.gpu_stat_sample_freq,
+               gpu_tot_sim_insn, gpu_sim_insn);
+  }
+#endif
+
   reinit_clock_domains();
   gpgpu_ctx->func_sim->set_param_gpgpu_num_shaders(m_config.num_shader());
   for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++)
@@ -1026,14 +1133,6 @@ void gpgpu_sim::init() {
   }
 
   if (g_network_mode) icnt_init();
-
-    // McPAT initialization function. Called on first launch of GPU
-#ifdef GPGPUSIM_POWER_MODEL
-  if (m_config.g_power_simulation_enabled) {
-    init_mcpat(m_config, m_gpgpusim_wrapper, m_config.gpu_stat_sample_freq,
-               gpu_tot_sim_insn, gpu_sim_insn);
-  }
-#endif
 }
 
 void gpgpu_sim::update_stats() {
@@ -1058,6 +1157,11 @@ void gpgpu_sim::update_stats() {
   gpu_occupancy = occupancy_stats();
 }
 
+PowerscalingCoefficients *gpgpu_sim::get_scaling_coeffs()
+{
+  return m_gpgpusim_wrapper->get_scaling_coeffs();
+}
+
 void gpgpu_sim::print_stats() {
   gpgpu_ctx->stats->ptx_file_line_stats_write_file();
   gpu_print_stat();
@@ -1137,6 +1241,18 @@ std::string gpgpu_sim::executed_kernel_info_string() {
 
   return statout.str();
 }
+
+std::string gpgpu_sim::executed_kernel_name() {
+  std::stringstream statout;  
+  if( m_executed_kernel_names.size() == 1)
+     statout << m_executed_kernel_names[0];
+  else{
+    for (unsigned int k = 0; k < m_executed_kernel_names.size(); k++) {
+      statout << m_executed_kernel_names[k] << " ";
+    }
+  }
+  return statout.str();
+}
 void gpgpu_sim::set_cache_config(std::string kernel_name,
                                  FuncCache cacheConfig) {
   m_special_cache_config[kernel_name] = cacheConfig;
@@ -1317,10 +1433,20 @@ void gpgpu_sim::gpu_print_stat() {
   m_shader_stats->print(stdout);
 #ifdef GPGPUSIM_POWER_MODEL
   if (m_config.g_power_simulation_enabled) {
+    if(m_config.g_power_simulation_mode > 0){
+        //if(!m_config.g_aggregate_power_stats)
+          mcpat_reset_perf_count(m_gpgpusim_wrapper);
+        calculate_hw_mcpat(m_config, getShaderCoreConfig(), m_gpgpusim_wrapper,
+                  m_power_stats, m_config.gpu_stat_sample_freq,
+                  gpu_tot_sim_cycle, gpu_sim_cycle, gpu_tot_sim_insn,
+                  gpu_sim_insn, m_config.g_power_simulation_mode, m_config.g_dvfs_enabled, 
+                  m_config.g_hw_perf_file_name, m_config.g_hw_perf_bench_name, executed_kernel_name(), m_config.accelwattch_hybrid_configuration, m_config.g_aggregate_power_stats);
+    }
     m_gpgpusim_wrapper->print_power_kernel_stats(
         gpu_sim_cycle, gpu_tot_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn,
         kernel_info_str, true);
-    mcpat_reset_perf_count(m_gpgpusim_wrapper);
+    //if(!m_config.g_aggregate_power_stats)
+      mcpat_reset_perf_count(m_gpgpusim_wrapper);
   }
 #endif
 
@@ -1514,9 +1640,9 @@ bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t &k,
 
     SHADER_DPRINTF(LIVENESS,
                    "GPGPU-Sim uArch: Occupied %u threads, %u shared mem, %u "
-                   "registers, %u ctas\n",
+                   "registers, %u ctas, on shader %d\n",
                    m_occupied_n_threads, m_occupied_shmem, m_occupied_regs,
-                   m_occupied_ctas);
+                   m_occupied_ctas, m_sid);
   }
 
   return true;
@@ -1682,9 +1808,9 @@ void shader_core_ctx::issue_block2core(kernel_info_t &kernel) {
   shader_CTA_count_log(m_sid, 1);
   SHADER_DPRINTF(LIVENESS,
                  "GPGPU-Sim uArch: cta:%2u, start_tid:%4u, end_tid:%4u, "
-                 "initialized @(%lld,%lld)\n",
+                 "initialized @(%lld,%lld), kernel_uid:%u, kernel_name:%s\n",
                  free_cta_hw_id, start_thread, end_thread, m_gpu->gpu_sim_cycle,
-                 m_gpu->gpu_tot_sim_cycle);
+                 m_gpu->gpu_tot_sim_cycle, kernel.get_uid(), kernel.get_name().c_str());
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////
@@ -1787,6 +1913,7 @@ void gpgpu_sim::cycle() {
           m_power_stats->pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i],
           m_power_stats->pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i],
           m_power_stats->pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i],
+          m_power_stats->pwr_mem_stat->n_wr_WB[CURRENT_STAT_IDX][i],
           m_power_stats->pwr_mem_stat->n_req[CURRENT_STAT_IDX][i]);
     }
   }
@@ -1830,7 +1957,7 @@ void gpgpu_sim::cycle() {
         m_cluster[i]->core_cycle();
         *active_sms += m_cluster[i]->get_n_active_sms();
       }
-      // Update core icnt/cache stats for GPUWattch
+      // Update core icnt/cache stats for AccelWattch
       m_cluster[i]->get_icnt_stats(
           m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i],
           m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]);
@@ -1860,10 +1987,12 @@ void gpgpu_sim::cycle() {
       // McPAT main cycle (interface with McPAT)
 #ifdef GPGPUSIM_POWER_MODEL
     if (m_config.g_power_simulation_enabled) {
+      if(m_config.g_power_simulation_mode == 0){
       mcpat_cycle(m_config, getShaderCoreConfig(), m_gpgpusim_wrapper,
                   m_power_stats, m_config.gpu_stat_sample_freq,
                   gpu_tot_sim_cycle, gpu_sim_cycle, gpu_tot_sim_insn,
-                  gpu_sim_insn);
+                  gpu_sim_insn, m_config.g_dvfs_enabled);
+      }
     }
 #endif
 
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 2e6820d..de69ef8 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Vijay Kandiah, Nikos Hardavellas
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -26,6 +28,7 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
+
 #ifndef GPU_SIM_H
 #define GPU_SIM_H
 
@@ -68,6 +71,29 @@ extern tr1_hash_map<new_addr_type, unsigned> address_random_interleaving;
 
 enum dram_ctrl_t { DRAM_FIFO = 0, DRAM_FRFCFS = 1 };
 
+enum hw_perf_t {
+  HW_BENCH_NAME=0,
+  HW_KERNEL_NAME,
+  HW_L1_RH,
+  HW_L1_RM,
+  HW_L1_WH,
+  HW_L1_WM,
+  HW_CC_ACC,
+  HW_SHRD_ACC,
+  HW_DRAM_RD,
+  HW_DRAM_WR,
+  HW_L2_RH,
+  HW_L2_RM,
+  HW_L2_WH,
+  HW_L2_WM,
+  HW_NOC,
+  HW_PIPE_DUTY,
+  HW_NUM_SM_IDLE,
+  HW_CYCLES,
+  HW_VOLTAGE,
+  HW_TOTAL_STATS
+};
+
 struct power_config {
   power_config() { m_valid = true; }
   void init() {
@@ -82,7 +108,8 @@ struct power_config {
       s++;
     }
     char buf1[1024];
-    snprintf(buf1, 1024, "gpgpusim_power_report__%s.log", date);
+    //snprintf(buf1, 1024, "accelwattch_power_report__%s.log", date);
+    snprintf(buf1, 1024, "accelwattch_power_report.log");
     g_power_filename = strdup(buf1);
     char buf2[1024];
     snprintf(buf2, 1024, "gpgpusim_power_trace_report__%s.log.gz", date);
@@ -94,6 +121,9 @@ struct power_config {
     snprintf(buf4, 1024, "gpgpusim_steady_state_tracking_report__%s.log.gz",
              date);
     g_steady_state_tracking_filename = strdup(buf4);
+    // for(int i =0; i< hw_perf_t::HW_TOTAL_STATS; i++){
+    //   accelwattch_hybrid_configuration[i] = 0;
+    // }
 
     if (g_steady_power_levels_enabled) {
       sscanf(gpu_steady_state_definition, "%lf:%lf",
@@ -125,6 +155,14 @@ struct power_config {
   double gpu_steady_power_deviation;
   double gpu_steady_min_period;
 
+
+  char *g_hw_perf_file_name;
+  char *g_hw_perf_bench_name;
+  int g_power_simulation_mode;
+  bool g_dvfs_enabled;
+  bool g_aggregate_power_stats;
+  bool accelwattch_hybrid_configuration[hw_perf_t::HW_TOTAL_STATS];
+
   // Nonlinear power model
   bool g_use_nonlinear_model;
   char *gpu_nonlinear_model_config;
@@ -357,7 +395,7 @@ class gpgpu_sim_config : public power_config,
 
     m_valid = true;
   }
-
+  unsigned get_core_freq() const { return core_freq; }
   unsigned num_shader() const { return m_shader_config.num_shader(); }
   unsigned num_cluster() const { return m_shader_config.n_simt_clusters; }
   unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; }
@@ -527,6 +565,7 @@ class gpgpu_sim : public gpgpu_t {
   bool kernel_more_cta_left(kernel_info_t *kernel) const;
   bool hit_max_cta_count() const;
   kernel_info_t *select_kernel();
+  PowerscalingCoefficients *get_scaling_coeffs();
   void decrement_kernel_latency();
 
   const gpgpu_sim_config &get_config() const { return m_config; }
@@ -634,6 +673,7 @@ class gpgpu_sim : public gpgpu_t {
 
   std::string executed_kernel_info_string();  //< format the kernel information
                                               // into a string for stat printout
+  std::string executed_kernel_name();
   void clear_executed_kernel_info();  //< clear the kernel information after
                                       // stat printout
   virtual void createSIMTCluster() = 0;
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index ab6e5c2..44d793c 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -57,6 +59,19 @@ mem_fetch *partition_mf_allocator::alloc(new_addr_type addr,
   return mf;
 }
 
+mem_fetch *partition_mf_allocator::alloc(
+    new_addr_type addr, mem_access_type type, const active_mask_t &active_mask,
+    const mem_access_byte_mask_t &byte_mask,
+    const mem_access_sector_mask_t &sector_mask, unsigned size, bool wr,
+    unsigned long long cycle, unsigned wid, unsigned sid, unsigned tpc,
+    mem_fetch *original_mf) const {
+  mem_access_t access(type, addr, size, wr, active_mask, byte_mask, sector_mask,
+                      m_memory_config->gpgpu_ctx);
+  mem_fetch *mf =
+      new mem_fetch(access, NULL, wr ? WRITE_PACKET_SIZE : READ_PACKET_SIZE,
+                    wid, sid, tpc, m_memory_config, cycle, original_mf);
+  return mf;
+}
 memory_partition_unit::memory_partition_unit(unsigned partition_id,
                                              const memory_config *config,
                                              class memory_stats_t *stats,
@@ -375,9 +390,9 @@ void memory_partition_unit::set_done(mem_fetch *mf) {
 
 void memory_partition_unit::set_dram_power_stats(
     unsigned &n_cmd, unsigned &n_activity, unsigned &n_nop, unsigned &n_act,
-    unsigned &n_pre, unsigned &n_rd, unsigned &n_wr, unsigned &n_req) const {
+    unsigned &n_pre, unsigned &n_rd, unsigned &n_wr, unsigned &n_wr_WB, unsigned &n_req) const {
   m_dram->set_dram_power_stats(n_cmd, n_activity, n_nop, n_act, n_pre, n_rd,
-                               n_wr, n_req);
+                               n_wr, n_wr_WB, n_req);
 }
 
 void memory_partition_unit::print(FILE *fp) const {
@@ -541,10 +556,15 @@ void memory_sub_partition::cache_cycle(unsigned cycle) {
                m_config->m_L2_config.m_write_alloc_policy ==
                    LAZY_FETCH_ON_READ) &&
               !was_writeallocate_sent(events)) {
-            mf->set_reply();
-            mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,
-                           m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
-            m_L2_icnt_queue->push(mf);
+            if (mf->get_access_type() == L1_WRBK_ACC) {
+              m_request_tracker.erase(mf);
+              delete mf;
+            } else {
+              mf->set_reply();
+              mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,
+                             m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
+              m_L2_icnt_queue->push(mf);
+            }
           }
           // L2 cache accepted request
           m_icnt_L2_queue->pop();
@@ -646,6 +666,7 @@ void gpgpu_sim::print_dram_stats(FILE *fout) const {
   unsigned pre = 0;
   unsigned rd = 0;
   unsigned wr = 0;
+  unsigned wr_WB = 0;
   unsigned req = 0;
   unsigned tot_cmd = 0;
   unsigned tot_nop = 0;
@@ -657,13 +678,13 @@ void gpgpu_sim::print_dram_stats(FILE *fout) const {
 
   for (unsigned i = 0; i < m_memory_config->m_n_mem; i++) {
     m_memory_partition_unit[i]->set_dram_power_stats(cmd, activity, nop, act,
-                                                     pre, rd, wr, req);
+                                                     pre, rd, wr, wr_WB, req);
     tot_cmd += cmd;
     tot_nop += nop;
     tot_act += act;
     tot_pre += pre;
     tot_rd += rd;
-    tot_wr += wr;
+    tot_wr += wr + wr_WB;
     tot_req += req;
   }
   fprintf(fout, "gpgpu_n_dram_reads = %d\n", tot_rd);
@@ -694,71 +715,68 @@ bool memory_sub_partition::busy() const { return !m_request_tracker.empty(); }
 std::vector<mem_fetch *>
 memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch *mf) {
   std::vector<mem_fetch *> result;
-
+  mem_access_sector_mask_t sector_mask = mf->get_access_sector_mask();
   if (mf->get_data_size() == SECTOR_SIZE &&
       mf->get_access_sector_mask().count() == 1) {
     result.push_back(mf);
-  } else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) {
-    // We only accept 32, 64 and 128 bytes reqs
-    unsigned start = 0, end = 0;
-    if (mf->get_data_size() == 128) {
+  } else if (mf->get_data_size() == MAX_MEMORY_ACCESS_SIZE) {
+    // break down every sector
+    mem_access_byte_mask_t mask;
+    for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; i++) {
+      for (unsigned k = i * SECTOR_SIZE; k < (i + 1) * SECTOR_SIZE; k++) {
+        mask.set(k);
+      }
+      mem_fetch *n_mf = m_mf_allocator->alloc(
+          mf->get_addr() + SECTOR_SIZE * i, mf->get_access_type(),
+          mf->get_access_warp_mask(), mf->get_access_byte_mask() & mask,
+          std::bitset<SECTOR_CHUNCK_SIZE>().set(i), SECTOR_SIZE, mf->is_write(),
+          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, mf->get_wid(),
+          mf->get_sid(), mf->get_tpc(), mf);
+
+      result.push_back(n_mf);
+    }
+    // This is for constant cache
+  } else if (mf->get_data_size() == 64 &&
+             (mf->get_access_sector_mask().all() ||
+              mf->get_access_sector_mask().none())) {
+    unsigned start;
+    if (mf->get_addr() % MAX_MEMORY_ACCESS_SIZE == 0)
       start = 0;
-      end = 3;
-    } else if (mf->get_data_size() == 64 &&
-               mf->get_access_sector_mask().to_string() == "1100") {
+    else
       start = 2;
-      end = 3;
-    } else if (mf->get_data_size() == 64 &&
-               mf->get_access_sector_mask().to_string() == "0011") {
-      start = 0;
-      end = 1;
-    } else if (mf->get_data_size() == 64 &&
-               (mf->get_access_sector_mask().to_string() == "1111" ||
-                mf->get_access_sector_mask().to_string() == "0000")) {
-      if (mf->get_addr() % 128 == 0) {
-        start = 0;
-        end = 1;
-      } else {
-        start = 2;
-        end = 3;
+    mem_access_byte_mask_t mask;
+    for (unsigned i = start; i < start + 2; i++) {
+      for (unsigned k = i * SECTOR_SIZE; k < (i + 1) * SECTOR_SIZE; k++) {
+        mask.set(k);
       }
-    } else {
-      printf(
-          "Invalid sector received, address = 0x%06llx, sector mask = %s, data "
-          "size = %d",
-          mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size());
-      assert(0 && "Undefined sector mask is received");
-    }
-
-    std::bitset<SECTOR_SIZE * SECTOR_CHUNCK_SIZE> byte_sector_mask;
-    byte_sector_mask.reset();
-    for (unsigned k = start * SECTOR_SIZE; k < SECTOR_SIZE; ++k)
-      byte_sector_mask.set(k);
-
-    for (unsigned j = start, i = 0; j <= end; ++j, ++i) {
-      const mem_access_t *ma = new mem_access_t(
-          mf->get_access_type(), mf->get_addr() + SECTOR_SIZE * i, SECTOR_SIZE,
-          mf->is_write(), mf->get_access_warp_mask(),
-          mf->get_access_byte_mask() & byte_sector_mask,
-          std::bitset<SECTOR_CHUNCK_SIZE>().set(j), m_gpu->gpgpu_ctx);
-
-      mem_fetch *n_mf =
-          new mem_fetch(*ma, NULL, mf->get_ctrl_size(), mf->get_wid(),
-                        mf->get_sid(), mf->get_tpc(), mf->get_mem_config(),
-                        m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, mf);
+      mem_fetch *n_mf = m_mf_allocator->alloc(
+          mf->get_addr(), mf->get_access_type(), mf->get_access_warp_mask(),
+          mf->get_access_byte_mask() & mask,
+          std::bitset<SECTOR_CHUNCK_SIZE>().set(i), SECTOR_SIZE, mf->is_write(),
+          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle, mf->get_wid(),
+          mf->get_sid(), mf->get_tpc(), mf);
 
       result.push_back(n_mf);
-      byte_sector_mask <<= SECTOR_SIZE;
     }
   } else {
-    printf(
-        "Invalid sector received, address = 0x%06llx, sector mask = %d, byte "
-        "mask = , data size = %u",
-        mf->get_addr(), mf->get_access_sector_mask().count(),
-        mf->get_data_size());
-    assert(0 && "Undefined data size is received");
-  }
+    for (unsigned i = 0; i < SECTOR_CHUNCK_SIZE; i++) {
+      if (sector_mask.test(i)) {
+        mem_access_byte_mask_t mask;
+        for (unsigned k = i * SECTOR_SIZE; k < (i + 1) * SECTOR_SIZE; k++) {
+          mask.set(k);
+        }
+        mem_fetch *n_mf = m_mf_allocator->alloc(
+            mf->get_addr() + SECTOR_SIZE * i, mf->get_access_type(),
+            mf->get_access_warp_mask(), mf->get_access_byte_mask() & mask,
+            std::bitset<SECTOR_CHUNCK_SIZE>().set(i), SECTOR_SIZE,
+            mf->is_write(), m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle,
+            mf->get_wid(), mf->get_sid(), mf->get_tpc(), mf);
 
+        result.push_back(n_mf);
+      }
+    }
+  }
+  if (result.size() == 0) assert(0 && "no mf sent");
   return result;
 }
 
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
index 3152db3..7fa1f29 100644
--- a/src/gpgpu-sim/l2cache.h
+++ b/src/gpgpu-sim/l2cache.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Vijay Kandiah, Nikos Hardavellas,
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -51,6 +53,13 @@ class partition_mf_allocator : public mem_fetch_allocator {
   virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type,
                            unsigned size, bool wr,
                            unsigned long long cycle) const;
+  virtual mem_fetch *alloc(new_addr_type addr, mem_access_type type,
+                           const active_mask_t &active_mask,
+                           const mem_access_byte_mask_t &byte_mask,
+                           const mem_access_sector_mask_t &sector_mask,
+                           unsigned size, bool wr, unsigned long long cycle,
+                           unsigned wid, unsigned sid, unsigned tpc,
+                           mem_fetch *original_mf) const;
 
  private:
   const memory_config *m_memory_config;
@@ -88,7 +97,7 @@ class memory_partition_unit {
   // Power model
   void set_dram_power_stats(unsigned &n_cmd, unsigned &n_activity,
                             unsigned &n_nop, unsigned &n_act, unsigned &n_pre,
-                            unsigned &n_rd, unsigned &n_wr,
+                            unsigned &n_rd, unsigned &n_wr, unsigned &n_wr_WB,
                             unsigned &n_req) const;
 
   int global_sub_partition_id_to_local_id(int global_sub_partition_id) const;
diff --git a/src/gpgpu-sim/power_interface.cc b/src/gpgpu-sim/power_interface.cc
index c637d84..470f2f9 100644
--- a/src/gpgpu-sim/power_interface.cc
+++ b/src/gpgpu-sim/power_interface.cc
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -26,8 +28,10 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
+
 #include "power_interface.h"
 
+
 void init_mcpat(const gpgpu_sim_config &config,
                 class gpgpu_sim_wrapper *wrapper, unsigned stat_sample_freq,
                 unsigned tot_inst, unsigned inst) {
@@ -38,7 +42,11 @@ void init_mcpat(const gpgpu_sim_config &config,
       config.g_power_simulation_enabled, config.g_power_trace_enabled,
       config.g_steady_power_levels_enabled, config.g_power_per_cycle_dump,
       config.gpu_steady_power_deviation, config.gpu_steady_min_period,
-      config.g_power_trace_zlevel, tot_inst + inst, stat_sample_freq);
+      config.g_power_trace_zlevel, tot_inst + inst, stat_sample_freq,  
+      config.g_power_simulation_mode, 
+      config.g_dvfs_enabled,
+      config.get_core_freq()/1000000,
+      config.num_shader());
 }
 
 void mcpat_cycle(const gpgpu_sim_config &config,
@@ -46,7 +54,7 @@ void mcpat_cycle(const gpgpu_sim_config &config,
                  class gpgpu_sim_wrapper *wrapper,
                  class power_stat_t *power_stats, unsigned stat_sample_freq,
                  unsigned tot_cycle, unsigned cycle, unsigned tot_inst,
-                 unsigned inst) {
+                 unsigned inst, bool dvfs_enabled) {
   static bool mcpat_init = true;
 
   if (mcpat_init) {  // If first cycle, don't have any power numbers yet
@@ -55,41 +63,45 @@ void mcpat_cycle(const gpgpu_sim_config &config,
   }
 
   if ((tot_cycle + cycle) % stat_sample_freq == 0) {
+    if(dvfs_enabled){
+      wrapper->set_model_voltage(1); //performance model needs to support this.
+    }
+
     wrapper->set_inst_power(
         shdr_config->gpgpu_clock_gated_lanes, stat_sample_freq,
-        stat_sample_freq, power_stats->get_total_inst(),
-        power_stats->get_total_int_inst(), power_stats->get_total_fp_inst(),
-        power_stats->get_l1d_read_accesses(),
-        power_stats->get_l1d_write_accesses(),
-        power_stats->get_committed_inst());
+        stat_sample_freq, power_stats->get_total_inst(0),
+        power_stats->get_total_int_inst(0), power_stats->get_total_fp_inst(0),
+        power_stats->get_l1d_read_accesses(0),
+        power_stats->get_l1d_write_accesses(0),
+        power_stats->get_committed_inst(0));
 
     // Single RF for both int and fp ops
-    wrapper->set_regfile_power(power_stats->get_regfile_reads(),
-                               power_stats->get_regfile_writes(),
-                               power_stats->get_non_regfile_operands());
+    wrapper->set_regfile_power(power_stats->get_regfile_reads(0),
+                               power_stats->get_regfile_writes(0),
+                               power_stats->get_non_regfile_operands(0));
 
     // Instruction cache stats
-    wrapper->set_icache_power(power_stats->get_inst_c_hits(),
-                              power_stats->get_inst_c_misses());
+    wrapper->set_icache_power(power_stats->get_inst_c_hits(0),
+                              power_stats->get_inst_c_misses(0));
 
     // Constant Cache, shared memory, texture cache
-    wrapper->set_ccache_power(power_stats->get_constant_c_hits(),
-                              power_stats->get_constant_c_misses());
+    wrapper->set_ccache_power(power_stats->get_const_accessess(0), 0); //assuming all HITS in constant cache for now
     wrapper->set_tcache_power(power_stats->get_texture_c_hits(),
                               power_stats->get_texture_c_misses());
-    wrapper->set_shrd_mem_power(power_stats->get_shmem_read_access());
+    wrapper->set_shrd_mem_power(power_stats->get_shmem_access(0));
 
     wrapper->set_l1cache_power(
-        power_stats->get_l1d_read_hits(), power_stats->get_l1d_read_misses(),
-        power_stats->get_l1d_write_hits(), power_stats->get_l1d_write_misses());
+        power_stats->get_l1d_read_hits(0), power_stats->get_l1d_read_misses(0),
+        power_stats->get_l1d_write_hits(0), power_stats->get_l1d_write_misses(0));
 
     wrapper->set_l2cache_power(
-        power_stats->get_l2_read_hits(), power_stats->get_l2_read_misses(),
-        power_stats->get_l2_write_hits(), power_stats->get_l2_write_misses());
+        power_stats->get_l2_read_hits(0), power_stats->get_l2_read_misses(0),
+        power_stats->get_l2_write_hits(0), power_stats->get_l2_write_misses(0));
 
     float active_sms = (*power_stats->m_active_sms) / stat_sample_freq;
     float num_cores = shdr_config->num_shader();
     float num_idle_core = num_cores - active_sms;
+    wrapper->set_num_cores(num_cores);
     wrapper->set_idle_core_power(num_idle_core);
 
     // pipeline power - pipeline_duty_cycle *= percent_active_sms;
@@ -101,38 +113,64 @@ void mcpat_cycle(const gpgpu_sim_config &config,
     wrapper->set_duty_cycle_power(pipeline_duty_cycle);
 
     // Memory Controller
-    wrapper->set_mem_ctrl_power(power_stats->get_dram_rd(),
-                                power_stats->get_dram_wr(),
-                                power_stats->get_dram_pre());
+    wrapper->set_mem_ctrl_power(power_stats->get_dram_rd(0),
+                                power_stats->get_dram_wr(0),
+                                power_stats->get_dram_pre(0));
 
     // Execution pipeline accesses
     // FPU (SP) accesses, Integer ALU (not present in Tesla), Sfu accesses
-    wrapper->set_exec_unit_power(power_stats->get_tot_fpu_accessess(),
-                                 power_stats->get_ialu_accessess(),
-                                 power_stats->get_tot_sfu_accessess());
+
+    wrapper->set_int_accesses(power_stats->get_ialu_accessess(0), 
+                              power_stats->get_intmul24_accessess(0), 
+                              power_stats->get_intmul32_accessess(0), 
+                              power_stats->get_intmul_accessess(0), 
+                              power_stats->get_intdiv_accessess(0));
+
+    wrapper->set_dp_accesses(power_stats->get_dp_accessess(0), 
+                              power_stats->get_dpmul_accessess(0), 
+                              power_stats->get_dpdiv_accessess(0));
+
+    wrapper->set_fp_accesses(power_stats->get_fp_accessess(0), 
+                            power_stats->get_fpmul_accessess(0), 
+                            power_stats->get_fpdiv_accessess(0));
+
+    wrapper->set_trans_accesses(power_stats->get_sqrt_accessess(0), 
+                                power_stats->get_log_accessess(0), 
+                                power_stats->get_sin_accessess(0), 
+                                power_stats->get_exp_accessess(0));
+
+    wrapper->set_tensor_accesses(power_stats->get_tensor_accessess(0));
+
+    wrapper->set_tex_accesses(power_stats->get_tex_accessess(0));
+
+    wrapper->set_exec_unit_power(power_stats->get_tot_fpu_accessess(0),
+                                 power_stats->get_ialu_accessess(0),
+                                 power_stats->get_tot_sfu_accessess(0));
+
+    wrapper->set_avg_active_threads(power_stats->get_active_threads(0));
 
     // Average active lanes for sp and sfu pipelines
     float avg_sp_active_lanes =
         (power_stats->get_sp_active_lanes()) / stat_sample_freq;
     float avg_sfu_active_lanes =
         (power_stats->get_sfu_active_lanes()) / stat_sample_freq;
+    if(avg_sp_active_lanes >32.0 )
+      avg_sp_active_lanes = 32.0;
+    if(avg_sfu_active_lanes >32.0 )
+      avg_sfu_active_lanes = 32.0;
     assert(avg_sp_active_lanes <= 32);
     assert(avg_sfu_active_lanes <= 32);
-    wrapper->set_active_lanes_power(
-        (power_stats->get_sp_active_lanes()) / stat_sample_freq,
-        (power_stats->get_sfu_active_lanes()) / stat_sample_freq);
+    wrapper->set_active_lanes_power(avg_sp_active_lanes, avg_sfu_active_lanes);
 
     double n_icnt_simt_to_mem =
         (double)
-            power_stats->get_icnt_simt_to_mem();  // # flits from SIMT clusters
+            power_stats->get_icnt_simt_to_mem(0);  // # flits from SIMT clusters
                                                   // to memory partitions
     double n_icnt_mem_to_simt =
         (double)
-            power_stats->get_icnt_mem_to_simt();  // # flits from memory
+            power_stats->get_icnt_mem_to_simt(0);  // # flits from memory
                                                   // partitions to SIMT clusters
-    wrapper->set_NoC_power(
-        n_icnt_mem_to_simt,
-        n_icnt_simt_to_mem);  // Number of flits traversing the interconnect
+    wrapper->set_NoC_power(n_icnt_mem_to_simt + n_icnt_simt_to_mem);  // Number of flits traversing the interconnect
 
     wrapper->compute();
 
@@ -152,3 +190,336 @@ void mcpat_cycle(const gpgpu_sim_config &config,
 void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper) {
   wrapper->reset_counters();
 }
+
+bool parse_hw_file(char* hwpowerfile, bool find_target_kernel, vector<string> &hw_data, char* benchname, std::string executed_kernelname){
+  fstream hw_file;
+  hw_file.open(hwpowerfile, ios::in);
+  string line, word, temp;
+  while(!hw_file.eof()){
+    hw_data.clear();
+    getline(hw_file, line);
+    stringstream s(line);
+    while (getline(s,word,',')){
+      hw_data.push_back(word);
+    }
+    if(hw_data[HW_BENCH_NAME] == std::string(benchname)){
+      if(find_target_kernel){
+        if(hw_data[HW_KERNEL_NAME] == ""){
+          hw_file.close();
+          return true;
+        }
+        else{
+          if(hw_data[HW_KERNEL_NAME] == executed_kernelname){
+            hw_file.close();
+            return true;
+          }
+        }
+      }
+      else{
+        hw_file.close();
+        return true;
+      }
+    } 
+  }
+  hw_file.close();
+  return false;
+}
+
+
+void calculate_hw_mcpat(const gpgpu_sim_config &config,
+                 const shader_core_config *shdr_config,
+                 class gpgpu_sim_wrapper *wrapper,
+                 class power_stat_t *power_stats, unsigned stat_sample_freq,
+                 unsigned tot_cycle, unsigned cycle, unsigned tot_inst,
+                 unsigned inst, int power_simulation_mode, bool dvfs_enabled, char* hwpowerfile, 
+                 char* benchname, std::string executed_kernelname, 
+                 const bool *accelwattch_hybrid_configuration, bool aggregate_power_stats){
+
+  /* Reading HW data from CSV file */
+
+  vector<string> hw_data;
+  bool kernel_found = false;
+  kernel_found = parse_hw_file(hwpowerfile, true, hw_data, benchname, executed_kernelname); //Searching for matching executed_kernelname.
+  if(!kernel_found)
+    kernel_found = parse_hw_file(hwpowerfile, false, hw_data, benchname, executed_kernelname); //Searching for any kernel with same benchname. 
+  assert("Could not find perf stats for the target benchmark in hwpowerfile.\n" && (kernel_found));
+  unsigned perf_cycles = static_cast<unsigned int>(std::stod(hw_data[HW_CYCLES]) + 0.5);
+  if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_CYCLES]))
+    perf_cycles = cycle;
+  wrapper->init_mcpat_hw_mode(perf_cycles); //total PERF MODEL cycles for current kernel
+
+  if(dvfs_enabled){
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_VOLTAGE])) 
+      wrapper->set_model_voltage(1); //performance model needs to support this
+    else  
+      wrapper->set_model_voltage(std::stod(hw_data[HW_VOLTAGE])); //performance model needs to support this
+  }
+
+  double l1_read_hits = std::stod(hw_data[HW_L1_RH]);
+  double l1_read_misses = std::stod(hw_data[HW_L1_RM]);
+  double l1_write_hits = std::stod(hw_data[HW_L1_WH]);
+  double l1_write_misses = std::stod(hw_data[HW_L1_WM]);
+
+  if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L1_RH]))
+    l1_read_hits = power_stats->get_l1d_read_hits(1) - power_stats->l1r_hits_kernel;
+  if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L1_RM]))
+    l1_read_misses = power_stats->get_l1d_read_misses(1) - power_stats->l1r_misses_kernel;
+  if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L1_WH]))
+    l1_write_hits = power_stats->get_l1d_write_hits(1) - power_stats->l1w_hits_kernel;
+  if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L1_WM]))
+    l1_write_misses = power_stats->get_l1d_write_misses(1) - power_stats->l1w_misses_kernel;
+
+    if(aggregate_power_stats){
+      power_stats->tot_inst_execution += power_stats->get_total_inst(1);
+      power_stats->tot_int_inst_execution +=  power_stats->get_total_int_inst(1);
+      power_stats->tot_fp_inst_execution +=  power_stats->get_total_fp_inst(1);
+      power_stats->commited_inst_execution += power_stats->get_committed_inst(1);
+      wrapper->set_inst_power(
+        shdr_config->gpgpu_clock_gated_lanes, cycle, //TODO: core.[0] cycles counts don't matter, remove this
+        cycle, power_stats->tot_inst_execution,
+        power_stats->tot_int_inst_execution, power_stats->tot_fp_inst_execution,
+        l1_read_hits + l1_read_misses,
+        l1_write_hits + l1_write_misses,
+        power_stats->commited_inst_execution);
+    }
+    else{
+    wrapper->set_inst_power(
+        shdr_config->gpgpu_clock_gated_lanes, cycle, //TODO: core.[0] cycles counts don't matter, remove this
+        cycle, power_stats->get_total_inst(1),
+        power_stats->get_total_int_inst(1), power_stats->get_total_fp_inst(1),
+        l1_read_hits + l1_read_misses,
+        l1_write_hits + l1_write_misses,
+        power_stats->get_committed_inst(1));
+    }
+
+    // Single RF for both int and fp ops -- activity factor set to 0 for Accelwattch HW and Accelwattch Hybrid because no HW Perf Stats for register files
+    wrapper->set_regfile_power(power_stats->get_regfile_reads(1),
+                               power_stats->get_regfile_writes(1),
+                               power_stats->get_non_regfile_operands(1));
+
+    // Instruction cache stats -- activity factor set to 0 for Accelwattch HW and Accelwattch Hybrid because no HW Perf Stats for instruction cache
+    wrapper->set_icache_power(power_stats->get_inst_c_hits(1) - power_stats->l1i_hits_kernel,
+                              power_stats->get_inst_c_misses(1) - power_stats->l1i_misses_kernel);
+
+    // Constant Cache, shared memory, texture cache
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_CC_ACC]))
+      wrapper->set_ccache_power(power_stats->get_const_accessess(1) - power_stats->cc_accesses_kernel, 0); //assuming all HITS in constant cache for now
+    else  
+      wrapper->set_ccache_power(std::stod(hw_data[HW_CC_ACC]), 0); //assuming all HITS in constant cache for now
+
+    
+    // wrapper->set_tcache_power(power_stats->get_texture_c_hits(),
+    //                           power_stats->get_texture_c_misses());
+
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_SHRD_ACC]))
+      wrapper->set_shrd_mem_power(power_stats->get_shmem_access(1) - power_stats->shared_accesses_kernel);
+    else  
+      wrapper->set_shrd_mem_power(std::stod(hw_data[HW_SHRD_ACC]));
+
+    wrapper->set_l1cache_power( l1_read_hits,  l1_read_misses, l1_write_hits,  l1_write_misses);
+
+    double l2_read_hits = std::stod(hw_data[HW_L2_RH]);
+    double l2_read_misses = std::stod(hw_data[HW_L2_RM]);
+    double l2_write_hits = std::stod(hw_data[HW_L2_WH]);
+    double l2_write_misses = std::stod(hw_data[HW_L2_WM]);
+
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L2_RH]))
+      l2_read_hits = power_stats->get_l2_read_hits(1) - power_stats->l2r_hits_kernel;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L2_RM]))
+      l2_read_misses = power_stats->get_l2_read_misses(1)  - power_stats->l2r_misses_kernel;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L2_WH]))
+      l2_write_hits = power_stats->get_l2_write_hits(1) - power_stats->l2w_hits_kernel;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_L2_WM]))
+      l2_write_misses = power_stats->get_l2_write_misses(1) - power_stats->l2w_misses_kernel;
+
+    wrapper->set_l2cache_power(l2_read_hits, l2_read_misses, l2_write_hits, l2_write_misses);
+    
+    float active_sms = (*power_stats->m_active_sms) / stat_sample_freq;
+    float num_cores = shdr_config->num_shader();
+    float num_idle_core = num_cores - active_sms;
+    wrapper->set_num_cores(num_cores);
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_NUM_SM_IDLE]))
+      wrapper->set_idle_core_power(num_idle_core);
+    else 
+      wrapper->set_idle_core_power(std::stod(hw_data[HW_NUM_SM_IDLE])); 
+
+    float pipeline_duty_cycle =
+        ((*power_stats->m_average_pipeline_duty_cycle / (stat_sample_freq)) <
+         0.8)
+            ? ((*power_stats->m_average_pipeline_duty_cycle) / stat_sample_freq)
+            : 0.8;
+    
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_PIPE_DUTY]))
+      wrapper->set_duty_cycle_power(pipeline_duty_cycle);
+    else
+      wrapper->set_duty_cycle_power(std::stod(hw_data[HW_PIPE_DUTY]));
+
+    // Memory Controller
+  
+    double dram_reads = std::stod(hw_data[HW_DRAM_RD]);
+    double dram_writes = std::stod(hw_data[HW_DRAM_WR]);
+    double dram_pre = 0;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_DRAM_RD]))
+      dram_reads = power_stats->get_dram_rd(1) - power_stats->dram_rd_kernel;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_DRAM_WR]))
+      dram_writes = power_stats->get_dram_wr(1) - power_stats->dram_wr_kernel;
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_DRAM_RD]))
+      dram_pre = power_stats->get_dram_pre(1) - power_stats->dram_pre_kernel;
+
+
+    wrapper->set_mem_ctrl_power(dram_reads, dram_writes, dram_pre);
+
+    if(aggregate_power_stats){
+      power_stats->ialu_acc_execution += power_stats->get_ialu_accessess(1);
+      power_stats->imul24_acc_execution += power_stats->get_intmul24_accessess(1);
+      power_stats->imul32_acc_execution += power_stats->get_intmul32_accessess(1);
+      power_stats->imul_acc_execution += power_stats->get_intmul_accessess(1);
+      power_stats->idiv_acc_execution += power_stats->get_intdiv_accessess(1);
+      power_stats->dp_acc_execution += power_stats->get_dp_accessess(1);
+      power_stats->dpmul_acc_execution += power_stats->get_dpmul_accessess(1);
+      power_stats->dpdiv_acc_execution += power_stats->get_dpdiv_accessess(1);
+      power_stats->fp_acc_execution += power_stats->get_fp_accessess(1);
+      power_stats->fpmul_acc_execution += power_stats->get_fpmul_accessess(1);
+      power_stats->fpdiv_acc_execution += power_stats->get_fpdiv_accessess(1);
+      power_stats->sqrt_acc_execution += power_stats->get_sqrt_accessess(1);
+      power_stats->log_acc_execution += power_stats->get_log_accessess(1);
+      power_stats->sin_acc_execution += power_stats->get_sin_accessess(1);
+      power_stats->exp_acc_execution += power_stats->get_exp_accessess(1);
+      power_stats->tensor_acc_execution += power_stats->get_tensor_accessess(1);
+      power_stats->tex_acc_execution += power_stats->get_tex_accessess(1);
+      power_stats->tot_fpu_acc_execution += power_stats->get_tot_fpu_accessess(1);
+      power_stats->tot_sfu_acc_execution += power_stats->get_tot_sfu_accessess(1);
+      power_stats->tot_threads_acc_execution += power_stats->get_tot_threads_kernel(1);
+      power_stats->tot_warps_acc_execution += power_stats->get_tot_warps_kernel(1);
+      
+      power_stats->sp_active_lanes_execution += (power_stats->get_sp_active_lanes() * shdr_config->num_shader() * shdr_config->gpgpu_num_sp_units);
+      power_stats->sfu_active_lanes_execution += (power_stats->get_sfu_active_lanes() * shdr_config->num_shader() * shdr_config->gpgpu_num_sp_units);
+
+      wrapper->set_int_accesses(power_stats->ialu_acc_execution, 
+                                power_stats->imul24_acc_execution, 
+                                power_stats->imul32_acc_execution, 
+                                power_stats->imul_acc_execution, 
+                                power_stats->idiv_acc_execution);
+
+      wrapper->set_dp_accesses(power_stats->dp_acc_execution, 
+                                power_stats->dpmul_acc_execution, 
+                                power_stats->dpdiv_acc_execution);
+
+      wrapper->set_fp_accesses(power_stats->fp_acc_execution, 
+                              power_stats->fpmul_acc_execution, 
+                              power_stats->fpdiv_acc_execution);
+
+      wrapper->set_trans_accesses(power_stats->sqrt_acc_execution, 
+                                  power_stats->log_acc_execution, 
+                                  power_stats->sin_acc_execution, 
+                                  power_stats->exp_acc_execution);
+
+      wrapper->set_tensor_accesses(power_stats->tensor_acc_execution);
+
+      wrapper->set_tex_accesses(power_stats->tex_acc_execution);
+
+      wrapper->set_exec_unit_power(power_stats->ialu_acc_execution,
+                                   power_stats->tot_fpu_acc_execution,
+                                   power_stats->tot_sfu_acc_execution);
+
+      wrapper->set_avg_active_threads((double)((double)power_stats->tot_threads_acc_execution / (double)power_stats->tot_warps_acc_execution));
+
+      // Average active lanes for sp and sfu pipelines
+      float avg_sp_active_lanes =
+          (power_stats->sp_active_lanes_execution) / shdr_config->num_shader() / shdr_config->gpgpu_num_sp_units / stat_sample_freq;
+      float avg_sfu_active_lanes =
+          (power_stats->sfu_active_lanes_execution) / shdr_config->num_shader() / shdr_config->gpgpu_num_sp_units / stat_sample_freq;
+      if(avg_sp_active_lanes >32.0 )
+        avg_sp_active_lanes = 32.0;
+      if(avg_sfu_active_lanes >32.0 )
+        avg_sfu_active_lanes = 32.0;
+      assert(avg_sp_active_lanes <= 32);
+      assert(avg_sfu_active_lanes <= 32);
+      wrapper->set_active_lanes_power(avg_sp_active_lanes, avg_sfu_active_lanes);
+    }
+    else{
+      wrapper->set_int_accesses(power_stats->get_ialu_accessess(1), 
+                                power_stats->get_intmul24_accessess(1), 
+                                power_stats->get_intmul32_accessess(1), 
+                                power_stats->get_intmul_accessess(1), 
+                                power_stats->get_intdiv_accessess(1));
+
+      wrapper->set_dp_accesses(power_stats->get_dp_accessess(1), 
+                                power_stats->get_dpmul_accessess(1), 
+                                power_stats->get_dpdiv_accessess(1));
+
+      wrapper->set_fp_accesses(power_stats->get_fp_accessess(1), 
+                              power_stats->get_fpmul_accessess(1), 
+                              power_stats->get_fpdiv_accessess(1));
+
+      wrapper->set_trans_accesses(power_stats->get_sqrt_accessess(1), 
+                                  power_stats->get_log_accessess(1), 
+                                  power_stats->get_sin_accessess(1), 
+                                  power_stats->get_exp_accessess(1));
+
+      wrapper->set_tensor_accesses(power_stats->get_tensor_accessess(1));
+
+      wrapper->set_tex_accesses(power_stats->get_tex_accessess(1));
+
+      wrapper->set_exec_unit_power(power_stats->get_tot_fpu_accessess(1),
+                                   power_stats->get_ialu_accessess(1),
+                                   power_stats->get_tot_sfu_accessess(1));
+
+      wrapper->set_avg_active_threads(power_stats->get_active_threads(1));
+
+      // Average active lanes for sp and sfu pipelines
+      float avg_sp_active_lanes =
+          (power_stats->get_sp_active_lanes()) / stat_sample_freq;
+      float avg_sfu_active_lanes =
+          (power_stats->get_sfu_active_lanes()) / stat_sample_freq;
+      if(avg_sp_active_lanes >32.0 )
+        avg_sp_active_lanes = 32.0;
+      if(avg_sfu_active_lanes >32.0 )
+        avg_sfu_active_lanes = 32.0;
+      assert(avg_sp_active_lanes <= 32);
+      assert(avg_sfu_active_lanes <= 32);
+      wrapper->set_active_lanes_power(avg_sp_active_lanes, avg_sfu_active_lanes);
+    }
+
+  
+    double n_icnt_simt_to_mem =
+      (double)
+          (power_stats->get_icnt_simt_to_mem(1) - power_stats->noc_tr_kernel);  // # flits from SIMT clusters
+                                                // to memory partitions
+    double n_icnt_mem_to_simt =
+      (double)
+          (power_stats->get_icnt_mem_to_simt(1)- power_stats->noc_rc_kernel);  // # flits from memory
+                                                // partitions to SIMT clusters
+    if((power_simulation_mode == 2) && (accelwattch_hybrid_configuration[HW_NOC]))   
+      wrapper->set_NoC_power(n_icnt_mem_to_simt + n_icnt_simt_to_mem);  // Number of flits traversing the interconnect from Accel-Sim
+    else
+      wrapper->set_NoC_power(std::stod(hw_data[HW_NOC]));  // Number of flits traversing the interconnect from HW
+   
+    wrapper->compute();
+
+    wrapper->update_components_power();
+
+    wrapper->power_metrics_calculations();
+
+    wrapper->dump();
+    power_stats->l1r_hits_kernel = power_stats->get_l1d_read_hits(1);
+    power_stats->l1r_misses_kernel = power_stats->get_l1d_read_misses(1);
+    power_stats->l1w_hits_kernel = power_stats->get_l1d_write_hits(1);
+    power_stats->l1w_misses_kernel = power_stats->get_l1d_write_misses(1);
+    power_stats->shared_accesses_kernel = power_stats->get_const_accessess(1);
+    power_stats->cc_accesses_kernel = power_stats->get_shmem_access(1);
+    power_stats->dram_rd_kernel = power_stats->get_dram_rd(1);
+    power_stats->dram_wr_kernel = power_stats->get_dram_wr(1);
+    power_stats->dram_pre_kernel = power_stats->get_dram_pre(1);
+    power_stats->l1i_hits_kernel = power_stats->get_inst_c_hits(1);
+    power_stats->l1i_misses_kernel = power_stats->get_inst_c_misses(1);
+    power_stats->l2r_hits_kernel = power_stats->get_l2_read_hits(1);
+    power_stats->l2r_misses_kernel = power_stats->get_l2_read_misses(1);
+    power_stats->l2w_hits_kernel =  power_stats->get_l2_write_hits(1); 
+    power_stats->l2w_misses_kernel = power_stats->get_l2_write_misses(1);
+    power_stats->noc_tr_kernel = power_stats->get_icnt_simt_to_mem(1);
+    power_stats->noc_rc_kernel =  power_stats->get_icnt_mem_to_simt(1);
+
+
+    power_stats->clear();
+}
\ No newline at end of file
diff --git a/src/gpgpu-sim/power_interface.h b/src/gpgpu-sim/power_interface.h
index 2bfd4d5..1c6c510 100644
--- a/src/gpgpu-sim/power_interface.h
+++ b/src/gpgpu-sim/power_interface.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -43,7 +45,19 @@ void mcpat_cycle(const gpgpu_sim_config &config,
                  class gpgpu_sim_wrapper *wrapper,
                  class power_stat_t *power_stats, unsigned stat_sample_freq,
                  unsigned tot_cycle, unsigned cycle, unsigned tot_inst,
-                 unsigned inst);
+                 unsigned inst, bool dvfs_enabled);
+
+void calculate_hw_mcpat(const gpgpu_sim_config &config,
+                 const shader_core_config *shdr_config,
+                 class gpgpu_sim_wrapper *wrapper,
+                 class power_stat_t *power_stats, unsigned stat_sample_freq,
+                 unsigned tot_cycle, unsigned cycle, unsigned tot_inst,
+                 unsigned inst, int power_simulation_mode, bool dvfs_enabled, 
+                 char* hwpowerfile, char* benchname, std::string executed_kernelname, 
+                 const bool *accelwattch_hybrid_configuration, bool aggregate_power_stats);
+
+bool parse_hw_file(char* hwpowerfile, bool find_target_kernel, vector<string> &hw_data, char* benchname, std::string executed_kernelname);
+
 void mcpat_reset_perf_count(class gpgpu_sim_wrapper *wrapper);
 
 #endif /* POWER_INTERFACE_H_ */
diff --git a/src/gpgpu-sim/power_stat.cc b/src/gpgpu-sim/power_stat.cc
index 7b60ddf..d0e673c 100644
--- a/src/gpgpu-sim/power_stat.cc
+++ b/src/gpgpu-sim/power_stat.cc
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021,  Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -54,10 +56,64 @@ power_mem_stat_t::power_mem_stat_t(const memory_config *mem_config,
   init();
 }
 
+void power_stat_t::clear(){
+  for(unsigned i=0; i< NUM_STAT_IDX; ++i){
+    pwr_mem_stat->core_cache_stats[i].clear();
+    pwr_mem_stat->l2_cache_stats[i].clear();
+    for(unsigned j=0; j<m_config->num_shader(); ++j){
+      pwr_core_stat->m_pipeline_duty_cycle[i][j]=0;                
+      pwr_core_stat->m_num_decoded_insn[i][j]=0;
+      pwr_core_stat->m_num_FPdecoded_insn[i][j]=0;
+      pwr_core_stat->m_num_INTdecoded_insn[i][j]=0;
+      pwr_core_stat->m_num_storequeued_insn[i][j]=0;
+      pwr_core_stat->m_num_loadqueued_insn[i][j]=0;
+      pwr_core_stat->m_num_tex_inst[i][j]=0;
+      pwr_core_stat->m_num_ialu_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_fp_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_imul_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_imul24_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_imul32_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_fpmul_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_idiv_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_fpdiv_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_dp_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_dpmul_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_dpdiv_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_tensor_core_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_const_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_tex_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_sp_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_sfu_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_sqrt_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_log_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_sin_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_exp_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_mem_acesses[i][j]=0;                   
+      pwr_core_stat->m_num_sp_committed[i][j]=0;
+      pwr_core_stat->m_num_sfu_committed[i][j]=0;
+      pwr_core_stat->m_num_mem_committed[i][j]=0;
+      pwr_core_stat->m_read_regfile_acesses[i][j]=0;
+      pwr_core_stat->m_write_regfile_acesses[i][j]=0;
+      pwr_core_stat->m_non_rf_operands[i][j]=0;
+      pwr_core_stat->m_active_sp_lanes[i][j]=0;
+      pwr_core_stat->m_active_sfu_lanes[i][j]=0;
+      pwr_core_stat->m_active_exu_threads[i][j]=0;                   
+      pwr_core_stat->m_active_exu_warps[i][j]=0;
+    }
+    for (unsigned j = 0; j < m_mem_config->m_n_mem; ++j) {
+      pwr_mem_stat->n_rd[i][j]=0;
+      pwr_mem_stat->n_wr[i][j]=0;
+      pwr_mem_stat->n_pre[i][j]=0;
+    }
+  }
+}
+
+
+
 void power_mem_stat_t::init() {
-  shmem_read_access[CURRENT_STAT_IDX] =
+  shmem_access[CURRENT_STAT_IDX] =
       m_core_stats->gpgpu_n_shmem_bank_access;  // Shared memory access
-  shmem_read_access[PREV_STAT_IDX] =
+  shmem_access[PREV_STAT_IDX] =
       (unsigned *)calloc(m_core_config->num_shader(), sizeof(unsigned));
 
   for (unsigned i = 0; i < NUM_STAT_IDX; ++i) {
@@ -71,6 +127,7 @@ void power_mem_stat_t::init() {
     n_pre[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned));
     n_rd[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned));
     n_wr[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned));
+    n_wr_WB[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned));
     n_req[i] = (unsigned *)calloc(m_config->m_n_mem, sizeof(unsigned));
 
     // Interconnect stats
@@ -86,8 +143,8 @@ void power_mem_stat_t::save_stats() {
   l2_cache_stats[PREV_STAT_IDX] = l2_cache_stats[CURRENT_STAT_IDX];
 
   for (unsigned i = 0; i < m_core_config->num_shader(); ++i) {
-    shmem_read_access[PREV_STAT_IDX][i] =
-        shmem_read_access[CURRENT_STAT_IDX][i];  // Shared memory access
+    shmem_access[PREV_STAT_IDX][i] =
+        shmem_access[CURRENT_STAT_IDX][i];  // Shared memory access
   }
 
   for (unsigned i = 0; i < m_config->m_n_mem; ++i) {
@@ -98,6 +155,7 @@ void power_mem_stat_t::save_stats() {
     n_pre[PREV_STAT_IDX][i] = n_pre[CURRENT_STAT_IDX][i];
     n_rd[PREV_STAT_IDX][i] = n_rd[CURRENT_STAT_IDX][i];
     n_wr[PREV_STAT_IDX][i] = n_wr[CURRENT_STAT_IDX][i];
+    n_wr_WB[PREV_STAT_IDX][i] = n_wr_WB[CURRENT_STAT_IDX][i];
     n_req[PREV_STAT_IDX][i] = n_req[CURRENT_STAT_IDX][i];
   }
 
@@ -117,7 +175,7 @@ void power_mem_stat_t::print(FILE *fout) const {
   unsigned total_mem_writes = 0;
   for (unsigned i = 0; i < m_config->m_n_mem; ++i) {
     total_mem_reads += n_rd[CURRENT_STAT_IDX][i];
-    total_mem_writes += n_wr[CURRENT_STAT_IDX][i];
+    total_mem_writes += n_wr[CURRENT_STAT_IDX][i] + n_wr_WB[CURRENT_STAT_IDX][i];
   }
   fprintf(fout, "Total memory controller accesses: %u\n",
           total_mem_reads + total_mem_writes);
@@ -147,198 +205,165 @@ void power_core_stat_t::print(FILE *fout) {
   // per core statistics
   fprintf(fout, "Power Metrics: \n");
   for (unsigned i = 0; i < m_config->num_shader(); i++) {
-    fprintf(fout, "core %u:\n", i);
-    fprintf(fout, "\tpipeline duty cycle =%f\n",
-            m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal Deocded Instructions=%u\n",
-            m_num_decoded_insn[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal FP Deocded Instructions=%u\n",
-            m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal INT Deocded Instructions=%u\n",
-            m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal LOAD Queued Instructions=%u\n",
-            m_num_loadqueued_insn[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal STORE Queued Instructions=%u\n",
-            m_num_storequeued_insn[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal IALU Acesses=%u\n",
-            m_num_ialu_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal FP Acesses=%u\n",
-            m_num_fp_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal IMUL Acesses=%u\n",
-            m_num_imul_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal IMUL24 Acesses=%u\n",
-            m_num_imul24_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal IMUL32 Acesses=%u\n",
-            m_num_imul32_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal IDIV Acesses=%u\n",
-            m_num_idiv_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal FPMUL Acesses=%u\n",
-            m_num_fpmul_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal SFU Acesses=%u\n",
-            m_num_trans_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal FPDIV Acesses=%u\n",
-            m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal SFU Acesses=%u\n",
-            m_num_sfu_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal SP Acesses=%u\n",
-            m_num_sp_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal MEM Acesses=%u\n",
-            m_num_mem_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal SFU Commissions=%u\n",
-            m_num_sfu_committed[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal SP Commissions=%u\n",
-            m_num_sp_committed[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal MEM Commissions=%u\n",
-            m_num_mem_committed[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal REG Reads=%u\n",
-            m_read_regfile_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal REG Writes=%u\n",
-            m_write_regfile_acesses[CURRENT_STAT_IDX][i]);
-    fprintf(fout, "\tTotal NON REG=%u\n",
-            m_non_rf_operands[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"core %u:\n",i);
+        fprintf(fout,"\tpipeline duty cycle =%f\n",m_pipeline_duty_cycle[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal Deocded Instructions=%u\n",m_num_decoded_insn[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal FP Deocded Instructions=%u\n",m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal INT Deocded Instructions=%u\n",m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal LOAD Queued Instructions=%u\n",m_num_loadqueued_insn[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal STORE Queued Instructions=%u\n",m_num_storequeued_insn[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal IALU Acesses=%f\n",m_num_ialu_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal FP Acesses=%f\n",m_num_fp_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal DP Acesses=%f\n",m_num_dp_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal IMUL Acesses=%f\n",m_num_imul_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal IMUL24 Acesses=%f\n",m_num_imul24_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal IMUL32 Acesses=%f\n",m_num_imul32_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal IDIV Acesses=%f\n",m_num_idiv_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal FPMUL Acesses=%f\n",m_num_fpmul_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal DPMUL Acesses=%f\n",m_num_dpmul_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SQRT Acesses=%f\n",m_num_sqrt_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal LOG Acesses=%f\n",m_num_log_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SIN Acesses=%f\n",m_num_sin_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal EXP Acesses=%f\n",m_num_exp_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal FPDIV Acesses=%f\n",m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal DPDIV Acesses=%f\n",m_num_dpdiv_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal TENSOR Acesses=%f\n",m_num_tensor_core_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal CONST Acesses=%f\n",m_num_const_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal TEX Acesses=%f\n",m_num_tex_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SFU Acesses=%f\n",m_num_sfu_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SP Acesses=%f\n",m_num_sp_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal MEM Acesses=%f\n",m_num_mem_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SFU Commissions=%u\n",m_num_sfu_committed[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal SP Commissions=%u\n",m_num_sp_committed[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal MEM Commissions=%u\n",m_num_mem_committed[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal REG Reads=%u\n",m_read_regfile_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal REG Writes=%u\n",m_write_regfile_acesses[CURRENT_STAT_IDX][i]);
+        fprintf(fout,"\tTotal NON REG=%u\n",m_non_rf_operands[CURRENT_STAT_IDX][i]);
   }
 }
 void power_core_stat_t::init() {
-  m_pipeline_duty_cycle[CURRENT_STAT_IDX] = m_core_stats->m_pipeline_duty_cycle;
-  m_num_decoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_decoded_insn;
-  m_num_FPdecoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_FPdecoded_insn;
-  m_num_INTdecoded_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_INTdecoded_insn;
-  m_num_storequeued_insn[CURRENT_STAT_IDX] =
-      m_core_stats->m_num_storequeued_insn;
-  m_num_loadqueued_insn[CURRENT_STAT_IDX] = m_core_stats->m_num_loadqueued_insn;
-  m_num_ialu_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_ialu_acesses;
-  m_num_fp_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fp_acesses;
-  m_num_imul_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul_acesses;
-  m_num_imul24_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul24_acesses;
-  m_num_imul32_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_imul32_acesses;
-  m_num_fpmul_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fpmul_acesses;
-  m_num_idiv_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_idiv_acesses;
-  m_num_fpdiv_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_fpdiv_acesses;
-  m_num_sp_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_sp_acesses;
-  m_num_sfu_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_sfu_acesses;
-  m_num_trans_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_trans_acesses;
-  m_num_mem_acesses[CURRENT_STAT_IDX] = m_core_stats->m_num_mem_acesses;
-  m_num_sp_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_sp_committed;
-  m_num_sfu_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_sfu_committed;
-  m_num_mem_committed[CURRENT_STAT_IDX] = m_core_stats->m_num_mem_committed;
-  m_read_regfile_acesses[CURRENT_STAT_IDX] =
-      m_core_stats->m_read_regfile_acesses;
-  m_write_regfile_acesses[CURRENT_STAT_IDX] =
-      m_core_stats->m_write_regfile_acesses;
-  m_non_rf_operands[CURRENT_STAT_IDX] = m_core_stats->m_non_rf_operands;
-  m_active_sp_lanes[CURRENT_STAT_IDX] = m_core_stats->m_active_sp_lanes;
-  m_active_sfu_lanes[CURRENT_STAT_IDX] = m_core_stats->m_active_sfu_lanes;
-  m_num_tex_inst[CURRENT_STAT_IDX] = m_core_stats->m_num_tex_inst;
+    m_pipeline_duty_cycle[CURRENT_STAT_IDX]=m_core_stats->m_pipeline_duty_cycle;
+    m_num_decoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_decoded_insn;
+    m_num_FPdecoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_FPdecoded_insn;
+    m_num_INTdecoded_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_INTdecoded_insn;
+    m_num_storequeued_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_storequeued_insn;
+    m_num_loadqueued_insn[CURRENT_STAT_IDX]=m_core_stats->m_num_loadqueued_insn;
+    m_num_ialu_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_ialu_acesses;
+    m_num_fp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fp_acesses;
+    m_num_imul_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul_acesses;
+    m_num_imul24_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul24_acesses;
+    m_num_imul32_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_imul32_acesses;
+    m_num_fpmul_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fpmul_acesses;
+    m_num_idiv_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_idiv_acesses;
+    m_num_fpdiv_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_fpdiv_acesses;
+    m_num_dp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_dp_acesses;
+    m_num_dpmul_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_dpmul_acesses;
+    m_num_dpdiv_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_dpdiv_acesses;
+    m_num_sp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sp_acesses;
+    m_num_sfu_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sfu_acesses;
+    m_num_sqrt_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sqrt_acesses;
+    m_num_log_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_log_acesses;
+    m_num_sin_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_sin_acesses;
+    m_num_exp_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_exp_acesses;
+    m_num_tensor_core_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_tensor_core_acesses;
+    m_num_const_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_const_acesses;
+    m_num_tex_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_tex_acesses;
+    m_num_mem_acesses[CURRENT_STAT_IDX]=m_core_stats->m_num_mem_acesses;
+    m_num_sp_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_sp_committed;
+    m_num_sfu_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_sfu_committed;
+    m_num_mem_committed[CURRENT_STAT_IDX]=m_core_stats->m_num_mem_committed;
+    m_read_regfile_acesses[CURRENT_STAT_IDX]=m_core_stats->m_read_regfile_acesses;
+    m_write_regfile_acesses[CURRENT_STAT_IDX]=m_core_stats->m_write_regfile_acesses;
+    m_non_rf_operands[CURRENT_STAT_IDX]=m_core_stats->m_non_rf_operands;
+    m_active_sp_lanes[CURRENT_STAT_IDX]=m_core_stats->m_active_sp_lanes;
+    m_active_sfu_lanes[CURRENT_STAT_IDX]=m_core_stats->m_active_sfu_lanes;
+    m_active_exu_threads[CURRENT_STAT_IDX]=m_core_stats->m_active_exu_threads;
+    m_active_exu_warps[CURRENT_STAT_IDX]=m_core_stats->m_active_exu_warps;
+    m_num_tex_inst[CURRENT_STAT_IDX]=m_core_stats->m_num_tex_inst;
+
+    m_pipeline_duty_cycle[PREV_STAT_IDX]=(float*)calloc(m_config->num_shader(),sizeof(float));
+    m_num_decoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_FPdecoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_INTdecoded_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_storequeued_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_loadqueued_insn[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_tex_inst[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+
+    m_num_ialu_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_fp_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_imul_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_imul24_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_imul32_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_fpmul_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_idiv_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_fpdiv_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_dp_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_dpmul_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_dpdiv_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_tensor_core_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_const_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_tex_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_sp_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_sfu_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_sqrt_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_log_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_sin_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_exp_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_mem_acesses[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_num_sp_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_sfu_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_num_mem_committed[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_read_regfile_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_write_regfile_acesses[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_non_rf_operands[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_active_sp_lanes[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_active_sfu_lanes[PREV_STAT_IDX]=(unsigned *)calloc(m_config->num_shader(),sizeof(unsigned));
+    m_active_exu_threads[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+    m_active_exu_warps[PREV_STAT_IDX]=(double *)calloc(m_config->num_shader(),sizeof(double));
+
 
-  m_pipeline_duty_cycle[PREV_STAT_IDX] =
-      (float *)calloc(m_config->num_shader(), sizeof(float));
-  m_num_decoded_insn[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_FPdecoded_insn[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_INTdecoded_insn[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_storequeued_insn[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_loadqueued_insn[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_ialu_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_fp_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_tex_inst[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_imul_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_imul24_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_imul32_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_fpmul_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_idiv_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_fpdiv_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_sp_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_sfu_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_trans_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_mem_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_sp_committed[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_sfu_committed[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_num_mem_committed[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_read_regfile_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_write_regfile_acesses[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_non_rf_operands[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_active_sp_lanes[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
-  m_active_sfu_lanes[PREV_STAT_IDX] =
-      (unsigned *)calloc(m_config->num_shader(), sizeof(unsigned));
 }
 
 void power_core_stat_t::save_stats() {
   for (unsigned i = 0; i < m_config->num_shader(); ++i) {
-    m_pipeline_duty_cycle[PREV_STAT_IDX][i] =
-        m_pipeline_duty_cycle[CURRENT_STAT_IDX][i];
-    m_num_decoded_insn[PREV_STAT_IDX][i] =
-        m_num_decoded_insn[CURRENT_STAT_IDX][i];
-    m_num_FPdecoded_insn[PREV_STAT_IDX][i] =
-        m_num_FPdecoded_insn[CURRENT_STAT_IDX][i];
-    m_num_INTdecoded_insn[PREV_STAT_IDX][i] =
-        m_num_INTdecoded_insn[CURRENT_STAT_IDX][i];
-    m_num_storequeued_insn[PREV_STAT_IDX][i] =
-        m_num_storequeued_insn[CURRENT_STAT_IDX][i];
-    m_num_loadqueued_insn[PREV_STAT_IDX][i] =
-        m_num_loadqueued_insn[CURRENT_STAT_IDX][i];
-    m_num_ialu_acesses[PREV_STAT_IDX][i] =
-        m_num_ialu_acesses[CURRENT_STAT_IDX][i];
-    m_num_fp_acesses[PREV_STAT_IDX][i] = m_num_fp_acesses[CURRENT_STAT_IDX][i];
-    m_num_tex_inst[PREV_STAT_IDX][i] = m_num_tex_inst[CURRENT_STAT_IDX][i];
-    m_num_imul_acesses[PREV_STAT_IDX][i] =
-        m_num_imul_acesses[CURRENT_STAT_IDX][i];
-    m_num_imul24_acesses[PREV_STAT_IDX][i] =
-        m_num_imul24_acesses[CURRENT_STAT_IDX][i];
-    m_num_imul32_acesses[PREV_STAT_IDX][i] =
-        m_num_imul32_acesses[CURRENT_STAT_IDX][i];
-    m_num_fpmul_acesses[PREV_STAT_IDX][i] =
-        m_num_fpmul_acesses[CURRENT_STAT_IDX][i];
-    m_num_idiv_acesses[PREV_STAT_IDX][i] =
-        m_num_idiv_acesses[CURRENT_STAT_IDX][i];
-    m_num_fpdiv_acesses[PREV_STAT_IDX][i] =
-        m_num_fpdiv_acesses[CURRENT_STAT_IDX][i];
-    m_num_sp_acesses[PREV_STAT_IDX][i] = m_num_sp_acesses[CURRENT_STAT_IDX][i];
-    m_num_sfu_acesses[PREV_STAT_IDX][i] =
-        m_num_sfu_acesses[CURRENT_STAT_IDX][i];
-    m_num_trans_acesses[PREV_STAT_IDX][i] =
-        m_num_trans_acesses[CURRENT_STAT_IDX][i];
-    m_num_mem_acesses[PREV_STAT_IDX][i] =
-        m_num_mem_acesses[CURRENT_STAT_IDX][i];
-    m_num_sp_committed[PREV_STAT_IDX][i] =
-        m_num_sp_committed[CURRENT_STAT_IDX][i];
-    m_num_sfu_committed[PREV_STAT_IDX][i] =
-        m_num_sfu_committed[CURRENT_STAT_IDX][i];
-    m_num_mem_committed[PREV_STAT_IDX][i] =
-        m_num_mem_committed[CURRENT_STAT_IDX][i];
-    m_read_regfile_acesses[PREV_STAT_IDX][i] =
-        m_read_regfile_acesses[CURRENT_STAT_IDX][i];
-    m_write_regfile_acesses[PREV_STAT_IDX][i] =
-        m_write_regfile_acesses[CURRENT_STAT_IDX][i];
-    m_non_rf_operands[PREV_STAT_IDX][i] =
-        m_non_rf_operands[CURRENT_STAT_IDX][i];
-    m_active_sp_lanes[PREV_STAT_IDX][i] =
-        m_active_sp_lanes[CURRENT_STAT_IDX][i];
-    m_active_sfu_lanes[PREV_STAT_IDX][i] =
-        m_active_sfu_lanes[CURRENT_STAT_IDX][i];
+    m_pipeline_duty_cycle[PREV_STAT_IDX][i]=m_pipeline_duty_cycle[CURRENT_STAT_IDX][i];
+    m_num_decoded_insn[PREV_STAT_IDX][i]= m_num_decoded_insn[CURRENT_STAT_IDX][i];
+    m_num_FPdecoded_insn[PREV_STAT_IDX][i]=m_num_FPdecoded_insn[CURRENT_STAT_IDX][i];
+    m_num_INTdecoded_insn[PREV_STAT_IDX][i]=m_num_INTdecoded_insn[CURRENT_STAT_IDX][i];
+    m_num_storequeued_insn[PREV_STAT_IDX][i]=m_num_storequeued_insn[CURRENT_STAT_IDX][i];
+    m_num_loadqueued_insn[PREV_STAT_IDX][i]=m_num_loadqueued_insn[CURRENT_STAT_IDX][i];
+    m_num_ialu_acesses[PREV_STAT_IDX][i]=m_num_ialu_acesses[CURRENT_STAT_IDX][i];
+    m_num_fp_acesses[PREV_STAT_IDX][i]=m_num_fp_acesses[CURRENT_STAT_IDX][i];
+    m_num_tex_inst[PREV_STAT_IDX][i]=m_num_tex_inst[CURRENT_STAT_IDX][i];
+    m_num_imul_acesses[PREV_STAT_IDX][i]=m_num_imul_acesses[CURRENT_STAT_IDX][i];
+    m_num_imul24_acesses[PREV_STAT_IDX][i]=m_num_imul24_acesses[CURRENT_STAT_IDX][i];
+    m_num_imul32_acesses[PREV_STAT_IDX][i]=m_num_imul32_acesses[CURRENT_STAT_IDX][i];
+    m_num_fpmul_acesses[PREV_STAT_IDX][i]=m_num_fpmul_acesses[CURRENT_STAT_IDX][i];
+    m_num_idiv_acesses[PREV_STAT_IDX][i]=m_num_idiv_acesses[CURRENT_STAT_IDX][i];
+    m_num_fpdiv_acesses[PREV_STAT_IDX][i]=m_num_fpdiv_acesses[CURRENT_STAT_IDX][i];
+    m_num_sp_acesses[PREV_STAT_IDX][i]=m_num_sp_acesses[CURRENT_STAT_IDX][i];
+    m_num_sfu_acesses[PREV_STAT_IDX][i]=m_num_sfu_acesses[CURRENT_STAT_IDX][i];
+    m_num_sqrt_acesses[PREV_STAT_IDX][i]=m_num_sqrt_acesses[CURRENT_STAT_IDX][i];
+    m_num_log_acesses[PREV_STAT_IDX][i]=m_num_log_acesses[CURRENT_STAT_IDX][i];
+    m_num_sin_acesses[PREV_STAT_IDX][i]=m_num_sin_acesses[CURRENT_STAT_IDX][i];
+    m_num_exp_acesses[PREV_STAT_IDX][i]=m_num_exp_acesses[CURRENT_STAT_IDX][i];
+    m_num_dp_acesses[PREV_STAT_IDX][i]=m_num_dp_acesses[CURRENT_STAT_IDX][i];
+    m_num_dpmul_acesses[PREV_STAT_IDX][i]=m_num_dpmul_acesses[CURRENT_STAT_IDX][i];
+    m_num_dpdiv_acesses[PREV_STAT_IDX][i]=m_num_dpdiv_acesses[CURRENT_STAT_IDX][i];
+    m_num_tensor_core_acesses[PREV_STAT_IDX][i]=m_num_tensor_core_acesses[CURRENT_STAT_IDX][i];
+    m_num_const_acesses[PREV_STAT_IDX][i]=m_num_const_acesses[CURRENT_STAT_IDX][i];
+    m_num_tex_acesses[PREV_STAT_IDX][i]=m_num_tex_acesses[CURRENT_STAT_IDX][i];
+    m_num_mem_acesses[PREV_STAT_IDX][i]=m_num_mem_acesses[CURRENT_STAT_IDX][i];
+    m_num_sp_committed[PREV_STAT_IDX][i]=m_num_sp_committed[CURRENT_STAT_IDX][i];
+    m_num_sfu_committed[PREV_STAT_IDX][i]=m_num_sfu_committed[CURRENT_STAT_IDX][i];
+    m_num_mem_committed[PREV_STAT_IDX][i]=m_num_mem_committed[CURRENT_STAT_IDX][i];
+    m_read_regfile_acesses[PREV_STAT_IDX][i]=m_read_regfile_acesses[CURRENT_STAT_IDX][i];
+    m_write_regfile_acesses[PREV_STAT_IDX][i]=m_write_regfile_acesses[CURRENT_STAT_IDX][i];
+    m_non_rf_operands[PREV_STAT_IDX][i]=m_non_rf_operands[CURRENT_STAT_IDX][i];
+    m_active_sp_lanes[PREV_STAT_IDX][i]=m_active_sp_lanes[CURRENT_STAT_IDX][i];
+    m_active_sfu_lanes[PREV_STAT_IDX][i]=m_active_sfu_lanes[CURRENT_STAT_IDX][i];
+    m_active_exu_threads[PREV_STAT_IDX][i]=m_active_exu_threads[CURRENT_STAT_IDX][i];
+    m_active_exu_warps[PREV_STAT_IDX][i]=m_active_exu_warps[CURRENT_STAT_IDX][i];
   }
 }
 
@@ -356,6 +381,51 @@ power_stat_t::power_stat_t(const shader_core_config *shader_config,
   m_active_sms = active_sms;
   m_config = shader_config;
   m_mem_config = mem_config;
+  l1r_hits_kernel = 0;
+  l1r_misses_kernel = 0;
+  l1w_hits_kernel = 0;
+  l1w_misses_kernel = 0;
+  shared_accesses_kernel = 0;
+  cc_accesses_kernel = 0;
+  dram_rd_kernel = 0;
+  dram_wr_kernel = 0;
+  dram_pre_kernel = 0;
+  l1i_hits_kernel =0;
+  l1i_misses_kernel =0;
+  l2r_hits_kernel =0;
+  l2r_misses_kernel =0;
+  l2w_hits_kernel =0;
+  l2w_misses_kernel =0;
+  noc_tr_kernel = 0;
+  noc_rc_kernel = 0;
+
+  tot_inst_execution = 0;
+  tot_int_inst_execution = 0;
+  tot_fp_inst_execution = 0;
+  commited_inst_execution = 0;
+  ialu_acc_execution = 0;
+  imul24_acc_execution = 0;
+  imul32_acc_execution = 0;
+  imul_acc_execution = 0;
+  idiv_acc_execution = 0;
+  dp_acc_execution = 0;
+  dpmul_acc_execution = 0;
+  dpdiv_acc_execution = 0;
+  fp_acc_execution = 0;
+  fpmul_acc_execution = 0;
+  fpdiv_acc_execution = 0;
+  sqrt_acc_execution = 0;
+  log_acc_execution = 0;
+  sin_acc_execution = 0;
+  exp_acc_execution = 0;
+  tensor_acc_execution = 0;
+  tex_acc_execution = 0;
+  tot_fpu_acc_execution = 0;
+  tot_sfu_acc_execution = 0;
+  tot_threads_acc_execution = 0;
+  tot_warps_acc_execution = 0;
+  sp_active_lanes_execution = 0;
+  sfu_active_lanes_execution = 0;
 }
 
 void power_stat_t::visualizer_print(gzFile visualizer_file) {
diff --git a/src/gpgpu-sim/power_stat.h b/src/gpgpu-sim/power_stat.h
index c469db3..d40f1d9 100644
--- a/src/gpgpu-sim/power_stat.h
+++ b/src/gpgpu-sim/power_stat.h
@@ -1,18 +1,20 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington
-// The University of British Columbia
+// Copyright (c) 2009-2021,  Tor M. Aamodt, Ahmed El-Shafiey, Tayler Hetherington, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -51,29 +53,40 @@ struct shader_core_power_stats_pod {
   unsigned
       *m_num_INTdecoded_insn[NUM_STAT_IDX];  // number of instructions committed
                                              // by this shader core
-  unsigned *m_num_storequeued_insn[NUM_STAT_IDX];
-  unsigned *m_num_loadqueued_insn[NUM_STAT_IDX];
-  unsigned *m_num_ialu_acesses[NUM_STAT_IDX];
-  unsigned *m_num_fp_acesses[NUM_STAT_IDX];
-  unsigned *m_num_tex_inst[NUM_STAT_IDX];
-  unsigned *m_num_imul_acesses[NUM_STAT_IDX];
-  unsigned *m_num_imul32_acesses[NUM_STAT_IDX];
-  unsigned *m_num_imul24_acesses[NUM_STAT_IDX];
-  unsigned *m_num_fpmul_acesses[NUM_STAT_IDX];
-  unsigned *m_num_idiv_acesses[NUM_STAT_IDX];
-  unsigned *m_num_fpdiv_acesses[NUM_STAT_IDX];
-  unsigned *m_num_sp_acesses[NUM_STAT_IDX];
-  unsigned *m_num_sfu_acesses[NUM_STAT_IDX];
-  unsigned *m_num_trans_acesses[NUM_STAT_IDX];
-  unsigned *m_num_mem_acesses[NUM_STAT_IDX];
-  unsigned *m_num_sp_committed[NUM_STAT_IDX];
-  unsigned *m_num_sfu_committed[NUM_STAT_IDX];
-  unsigned *m_num_mem_committed[NUM_STAT_IDX];
-  unsigned *m_active_sp_lanes[NUM_STAT_IDX];
-  unsigned *m_active_sfu_lanes[NUM_STAT_IDX];
-  unsigned *m_read_regfile_acesses[NUM_STAT_IDX];
-  unsigned *m_write_regfile_acesses[NUM_STAT_IDX];
-  unsigned *m_non_rf_operands[NUM_STAT_IDX];
+    unsigned *m_num_storequeued_insn[NUM_STAT_IDX];
+    unsigned *m_num_loadqueued_insn[NUM_STAT_IDX];
+    unsigned *m_num_tex_inst[NUM_STAT_IDX];
+    double *m_num_ialu_acesses[NUM_STAT_IDX];
+    double *m_num_fp_acesses[NUM_STAT_IDX];
+    double *m_num_imul_acesses[NUM_STAT_IDX];
+    double *m_num_imul32_acesses[NUM_STAT_IDX];
+    double *m_num_imul24_acesses[NUM_STAT_IDX];
+    double *m_num_fpmul_acesses[NUM_STAT_IDX];
+    double *m_num_idiv_acesses[NUM_STAT_IDX];
+    double *m_num_fpdiv_acesses[NUM_STAT_IDX];
+    double *m_num_dp_acesses[NUM_STAT_IDX];
+    double *m_num_dpmul_acesses[NUM_STAT_IDX];
+    double *m_num_dpdiv_acesses[NUM_STAT_IDX];
+    double *m_num_sp_acesses[NUM_STAT_IDX];
+    double *m_num_sfu_acesses[NUM_STAT_IDX];
+    double *m_num_sqrt_acesses[NUM_STAT_IDX];
+    double *m_num_log_acesses[NUM_STAT_IDX];
+    double *m_num_sin_acesses[NUM_STAT_IDX];
+    double *m_num_exp_acesses[NUM_STAT_IDX];
+    double *m_num_tensor_core_acesses[NUM_STAT_IDX];
+    double *m_num_const_acesses[NUM_STAT_IDX];
+    double *m_num_tex_acesses[NUM_STAT_IDX];
+    double *m_num_mem_acesses[NUM_STAT_IDX];
+    unsigned *m_num_sp_committed[NUM_STAT_IDX];
+    unsigned *m_num_sfu_committed[NUM_STAT_IDX];
+    unsigned *m_num_mem_committed[NUM_STAT_IDX];
+    unsigned *m_active_sp_lanes[NUM_STAT_IDX];
+    unsigned *m_active_sfu_lanes[NUM_STAT_IDX];
+    double *m_active_exu_threads[NUM_STAT_IDX];
+    double *m_active_exu_warps[NUM_STAT_IDX];    
+    unsigned *m_read_regfile_acesses[NUM_STAT_IDX];
+    unsigned *m_write_regfile_acesses[NUM_STAT_IDX];
+    unsigned *m_non_rf_operands[NUM_STAT_IDX];
 };
 
 class power_core_stat_t : public shader_core_power_stats_pod {
@@ -84,6 +97,7 @@ class power_core_stat_t : public shader_core_power_stats_pod {
   void print(FILE *fout);
   void init();
   void save_stats();
+ 
 
  private:
   shader_core_stats *m_core_stats;
@@ -96,8 +110,7 @@ struct mem_power_stats_pod {
   class cache_stats core_cache_stats[NUM_STAT_IDX];  // Total core stats
   class cache_stats l2_cache_stats[NUM_STAT_IDX];    // Total L2 partition stats
 
-  unsigned *shmem_read_access[NUM_STAT_IDX];  // Shared memory access
-
+  unsigned *shmem_access[NUM_STAT_IDX];  // Shared memory access
   // Low level DRAM stats
   unsigned *n_cmd[NUM_STAT_IDX];
   unsigned *n_activity[NUM_STAT_IDX];
@@ -106,6 +119,7 @@ struct mem_power_stats_pod {
   unsigned *n_pre[NUM_STAT_IDX];
   unsigned *n_rd[NUM_STAT_IDX];
   unsigned *n_wr[NUM_STAT_IDX];
+  unsigned *n_wr_WB[NUM_STAT_IDX];
   unsigned *n_req[NUM_STAT_IDX];
 
   // Interconnect stats
@@ -144,34 +158,88 @@ class power_stat_t {
     *m_average_pipeline_duty_cycle = 0;
     *m_active_sms = 0;
   }
-
-  unsigned get_total_inst() {
-    unsigned total_inst = 0;
+  void clear();
+  unsigned l1i_misses_kernel;
+  unsigned l1i_hits_kernel;
+  unsigned long long l1r_hits_kernel;
+  unsigned long long l1r_misses_kernel;
+  unsigned long long l1w_hits_kernel;
+  unsigned long long l1w_misses_kernel;
+  unsigned long long shared_accesses_kernel;
+  unsigned long long cc_accesses_kernel;
+  unsigned long long dram_rd_kernel;
+  unsigned long long dram_wr_kernel;
+  unsigned long long dram_pre_kernel;
+  unsigned long long l2r_hits_kernel;
+  unsigned long long l2r_misses_kernel;
+  unsigned long long l2w_hits_kernel;
+  unsigned long long l2w_misses_kernel;
+  unsigned long long noc_tr_kernel;
+  unsigned long long noc_rc_kernel;
+  unsigned long long tot_inst_execution;
+  unsigned long long tot_int_inst_execution;
+  unsigned long long tot_fp_inst_execution;
+  unsigned long long commited_inst_execution;
+  unsigned long long ialu_acc_execution;
+  unsigned long long imul24_acc_execution;
+  unsigned long long imul32_acc_execution;
+  unsigned long long imul_acc_execution;
+  unsigned long long idiv_acc_execution;
+  unsigned long long dp_acc_execution;
+  unsigned long long dpmul_acc_execution;
+  unsigned long long dpdiv_acc_execution;
+  unsigned long long fp_acc_execution;
+  unsigned long long fpmul_acc_execution;
+  unsigned long long fpdiv_acc_execution;
+  unsigned long long sqrt_acc_execution;
+  unsigned long long log_acc_execution;
+  unsigned long long sin_acc_execution;
+  unsigned long long exp_acc_execution;
+  unsigned long long tensor_acc_execution;
+  unsigned long long tex_acc_execution;
+  unsigned long long tot_fpu_acc_execution;
+  unsigned long long tot_sfu_acc_execution;
+  unsigned long long tot_threads_acc_execution;
+  unsigned long long tot_warps_acc_execution;
+  unsigned long long sp_active_lanes_execution;
+  unsigned long long sfu_active_lanes_execution;
+  double get_total_inst(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_decoded_insn[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_decoded_insn[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_core_stat->m_num_decoded_insn[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_decoded_insn[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_total_int_inst() {
-    unsigned total_inst = 0;
+  double get_total_int_inst(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst +=
+      if(aggregate_stat)
+          total_inst +=
+          (pwr_core_stat->m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]);
+      else 
+        total_inst +=
           (pwr_core_stat->m_num_INTdecoded_insn[CURRENT_STAT_IDX][i]) -
           (pwr_core_stat->m_num_INTdecoded_insn[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_total_fp_inst() {
-    unsigned total_inst = 0;
+  double get_total_fp_inst(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]);
+      else 
+        total_inst += (pwr_core_stat->m_num_FPdecoded_insn[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_FPdecoded_insn[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_total_load_inst() {
-    unsigned total_inst = 0;
+  double get_total_load_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst +=
           (pwr_core_stat->m_num_loadqueued_insn[CURRENT_STAT_IDX][i]) -
@@ -179,8 +247,8 @@ class power_stat_t {
     }
     return total_inst;
   }
-  unsigned get_total_store_inst() {
-    unsigned total_inst = 0;
+  double get_total_store_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst +=
           (pwr_core_stat->m_num_storequeued_insn[CURRENT_STAT_IDX][i]) -
@@ -188,34 +256,39 @@ class power_stat_t {
     }
     return total_inst;
   }
-  unsigned get_sp_committed_inst() {
-    unsigned total_inst = 0;
+  double get_sp_committed_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_sp_committed[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_sfu_committed_inst() {
-    unsigned total_inst = 0;
+  double get_sfu_committed_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_mem_committed_inst() {
-    unsigned total_inst = 0;
+  double get_mem_committed_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_committed_inst() {
-    unsigned total_inst = 0;
+  double get_committed_inst(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_sp_committed[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_core_stat->m_num_mem_committed[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_mem_committed[PREV_STAT_IDX][i]) +
                     (pwr_core_stat->m_num_sfu_committed[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_sfu_committed[PREV_STAT_IDX][i]) +
@@ -224,19 +297,27 @@ class power_stat_t {
     }
     return total_inst;
   }
-  unsigned get_regfile_reads() {
-    unsigned total_inst = 0;
+  double get_regfile_reads(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst +=
+      if(aggregate_stat)
+         total_inst +=
+          (pwr_core_stat->m_read_regfile_acesses[CURRENT_STAT_IDX][i]);
+      else
+        total_inst +=
           (pwr_core_stat->m_read_regfile_acesses[CURRENT_STAT_IDX][i]) -
           (pwr_core_stat->m_read_regfile_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_regfile_writes() {
-    unsigned total_inst = 0;
+  double get_regfile_writes(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst +=
+      if(aggregate_stat)
+        total_inst +=
+          (pwr_core_stat->m_write_regfile_acesses[CURRENT_STAT_IDX][i]);
+      else
+        total_inst +=
           (pwr_core_stat->m_write_regfile_acesses[CURRENT_STAT_IDX][i]) -
           (pwr_core_stat->m_write_regfile_acesses[PREV_STAT_IDX][i]);
     }
@@ -253,17 +334,20 @@ class power_stat_t {
     return total_inst;
   }
 
-  unsigned get_non_regfile_operands() {
-    unsigned total_inst = 0;
+  double get_non_regfile_operands(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_non_rf_operands[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+         total_inst += (pwr_core_stat->m_non_rf_operands[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_core_stat->m_non_rf_operands[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_non_rf_operands[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_sp_accessess() {
-    unsigned total_inst = 0;
+  double get_sp_accessess() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_sp_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_sp_acesses[PREV_STAT_IDX][i]);
@@ -271,25 +355,58 @@ class power_stat_t {
     return total_inst;
   }
 
-  unsigned get_sfu_accessess() {
-    unsigned total_inst = 0;
+  double get_sfu_accessess() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_sfu_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_sfu_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
-  unsigned get_trans_accessess() {
-    unsigned total_inst = 0;
-    for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]);
-    }
-    return total_inst;
+
+  double get_sqrt_accessess(bool aggregate_stat){
+      double total_inst=0;
+      for(unsigned i=0; i<m_config->num_shader();i++){
+          if(aggregate_stat)
+            total_inst+=(pwr_core_stat->m_num_sqrt_acesses[CURRENT_STAT_IDX][i]);
+          else
+            total_inst+=(pwr_core_stat->m_num_sqrt_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sqrt_acesses[PREV_STAT_IDX][i]);
+      }
+      return total_inst;
+  }
+  double get_log_accessess(bool aggregate_stat){
+      double total_inst=0;
+      for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst+=(pwr_core_stat->m_num_log_acesses[CURRENT_STAT_IDX][i]);
+        else 
+          total_inst+=(pwr_core_stat->m_num_log_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_log_acesses[PREV_STAT_IDX][i]);
+      }
+      return total_inst;
+  }
+  double get_sin_accessess(bool aggregate_stat){
+      double total_inst=0;
+      for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)  
+          total_inst+=(pwr_core_stat->m_num_sin_acesses[CURRENT_STAT_IDX][i]);
+        else 
+          total_inst+=(pwr_core_stat->m_num_sin_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_sin_acesses[PREV_STAT_IDX][i]);
+      }
+      return total_inst;
+  }
+  double get_exp_accessess(bool aggregate_stat){
+      double total_inst=0;
+      for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)  
+          total_inst+=(pwr_core_stat->m_num_exp_acesses[CURRENT_STAT_IDX][i]);
+        else  
+          total_inst+=(pwr_core_stat->m_num_exp_acesses[CURRENT_STAT_IDX][i]) - (pwr_core_stat->m_num_exp_acesses[PREV_STAT_IDX][i]);
+      }
+      return total_inst;
   }
 
-  unsigned get_mem_accessess() {
-    unsigned total_inst = 0;
+  double get_mem_accessess() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_mem_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_mem_acesses[PREV_STAT_IDX][i]);
@@ -297,66 +414,164 @@ class power_stat_t {
     return total_inst;
   }
 
-  unsigned get_intdiv_accessess() {
-    unsigned total_inst = 0;
+  double get_intdiv_accessess(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_fpdiv_accessess() {
-    unsigned total_inst = 0;
+  double get_fpdiv_accessess(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_intmul32_accessess() {
-    unsigned total_inst = 0;
+  double get_intmul32_accessess(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_intmul24_accessess() {
-    unsigned total_inst = 0;
+  double get_intmul24_accessess(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_intmul_accessess() {
-    unsigned total_inst = 0;
-    for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]);
+  double get_intmul_accessess(bool aggregate_stat){
+      double total_inst=0;
+      for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst+= (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]); 
+        else  
+          total_inst+= (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - 
+                       (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]);
+      }
+      return total_inst;
+  }
+
+  double get_fpmul_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst += (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]);
+        else
+          total_inst += (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_fpmul_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_fpmul_accessess() {
-    unsigned total_inst = 0;
-    for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]);
+  double get_fp_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]);
+        else  
+          total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  float get_sp_active_lanes() {
-    unsigned total_inst = 0;
+  double get_dp_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst += (pwr_core_stat->m_num_dp_acesses[CURRENT_STAT_IDX][i]);
+        else  
+          total_inst += (pwr_core_stat->m_num_dp_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_dp_acesses[PREV_STAT_IDX][i]);
+    }
+    return total_inst;
+  }
+
+  double get_dpmul_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)  
+        total_inst += (pwr_core_stat->m_num_dpmul_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_dpmul_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_dpmul_acesses[PREV_STAT_IDX][i]);
+    }
+    return total_inst;
+  }
+
+  double get_dpdiv_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)  
+        total_inst += (pwr_core_stat->m_num_dpdiv_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_dpdiv_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_dpdiv_acesses[PREV_STAT_IDX][i]);
+    }
+    return total_inst;
+  }
+
+  double get_tensor_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)  
+        total_inst += (pwr_core_stat->m_num_tensor_core_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_tensor_core_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_tensor_core_acesses[PREV_STAT_IDX][i]);
+    }
+    return total_inst;
+  }
+
+  double get_const_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+        if(aggregate_stat)
+          total_inst += pwr_core_stat->m_num_const_acesses[CURRENT_STAT_IDX][i];
+        else
+          total_inst += (pwr_core_stat->m_num_const_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_const_acesses[PREV_STAT_IDX][i]);
+    }
+    return (total_inst);
+  }
+
+  double get_tex_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)  
+        total_inst += (pwr_core_stat->m_num_tex_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_tex_acesses[CURRENT_STAT_IDX][i]) - 
+                      (pwr_core_stat->m_num_tex_acesses[PREV_STAT_IDX][i]);
+    }
+    return total_inst;
+  }
+
+  double get_sp_active_lanes() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_active_sp_lanes[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_active_sp_lanes[PREV_STAT_IDX][i]);
@@ -365,7 +580,7 @@ class power_stat_t {
   }
 
   float get_sfu_active_lanes() {
-    unsigned total_inst = 0;
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_active_sfu_lanes[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_active_sfu_lanes[PREV_STAT_IDX][i]);
@@ -375,49 +590,141 @@ class power_stat_t {
            m_config->gpgpu_num_sfu_units;
   }
 
-  unsigned get_tot_fpu_accessess() {
-    unsigned total_inst = 0;
+
+  float get_active_threads(bool aggregate_stat) {
+    unsigned total_threads = 0;
+    unsigned total_warps = 0;
+    for (unsigned i = 0; i < m_config->num_shader(); i++) {
+      if(aggregate_stat){
+        total_threads += (pwr_core_stat->m_active_exu_threads[CURRENT_STAT_IDX][i]) ;
+        total_warps += (pwr_core_stat->m_active_exu_warps[CURRENT_STAT_IDX][i]);
+      }
+      else{
+        total_threads += (pwr_core_stat->m_active_exu_threads[CURRENT_STAT_IDX][i]) -
+                    (pwr_core_stat->m_active_exu_threads[PREV_STAT_IDX][i]);
+        total_warps += (pwr_core_stat->m_active_exu_warps[CURRENT_STAT_IDX][i]) -
+                    (pwr_core_stat->m_active_exu_warps[PREV_STAT_IDX][i]);
+        }
+    }
+    if(total_warps != 0)
+      return (float)((float)total_threads / (float)total_warps);
+    else
+      return 0;
+  }
+
+  unsigned long long get_tot_threads_kernel(bool aggregate_stat) {
+    unsigned total_threads = 0;
+    for (unsigned i = 0; i < m_config->num_shader(); i++) {
+      if(aggregate_stat){
+        total_threads += (pwr_core_stat->m_active_exu_threads[CURRENT_STAT_IDX][i]) ;
+      }
+      else{
+        total_threads += (pwr_core_stat->m_active_exu_threads[CURRENT_STAT_IDX][i]) -
+                    (pwr_core_stat->m_active_exu_threads[PREV_STAT_IDX][i]);
+        }
+    }
+
+      return total_threads;
+  }
+  unsigned long long get_tot_warps_kernel(bool aggregate_stat) {
+    unsigned long long total_warps = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat){
+        total_warps += (pwr_core_stat->m_active_exu_warps[CURRENT_STAT_IDX][i]);
+      }
+      else{
+        total_warps += (pwr_core_stat->m_active_exu_warps[CURRENT_STAT_IDX][i]) -
+                    (pwr_core_stat->m_active_exu_warps[PREV_STAT_IDX][i]);
+        }
+    }
+      return total_warps;
+  }
+
+
+  double get_tot_fpu_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_dp_acesses[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_core_stat->m_num_fp_acesses[CURRENT_STAT_IDX][i]) - 
                     (pwr_core_stat->m_num_fp_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_fpmul_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]);
+                    (pwr_core_stat->m_num_dp_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_dp_acesses[PREV_STAT_IDX][i]);
     }
-    total_inst +=
-        get_total_load_inst() + get_total_store_inst() + get_tex_inst();
+    //total_inst += get_total_load_inst()+get_total_store_inst()+get_tex_inst();
     return total_inst;
   }
 
-  unsigned get_tot_sfu_accessess() {
-    unsigned total_inst = 0;
-    for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) -
+
+
+  double get_tot_sfu_accessess(bool aggregate_stat){
+    double total_inst=0;
+    for(unsigned i=0; i<m_config->num_shader();i++){
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_sqrt_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_log_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_sin_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_exp_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_dpmul_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_dpdiv_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_tensor_core_acesses[CURRENT_STAT_IDX][i])+
+                    (pwr_core_stat->m_num_tex_acesses[CURRENT_STAT_IDX][i]);
+        else
+            total_inst += (pwr_core_stat->m_num_idiv_acesses[CURRENT_STAT_IDX][i]) - 
                     (pwr_core_stat->m_num_idiv_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) -
+                    (pwr_core_stat->m_num_imul32_acesses[CURRENT_STAT_IDX][i]) - 
                     (pwr_core_stat->m_num_imul32_acesses[PREV_STAT_IDX][i]) +
-                    (pwr_core_stat->m_num_trans_acesses[CURRENT_STAT_IDX][i]) -
-                    (pwr_core_stat->m_num_trans_acesses[PREV_STAT_IDX][i]);
+                    (pwr_core_stat->m_num_sqrt_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_sqrt_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_log_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_log_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_sin_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_sin_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_exp_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_exp_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_fpdiv_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_fpdiv_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_fpmul_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_fpmul_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_dpmul_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_dpmul_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_dpdiv_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_dpdiv_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_imul24_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_imul24_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_imul_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_imul_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_tensor_core_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_tensor_core_acesses[PREV_STAT_IDX][i]) +
+                    (pwr_core_stat->m_num_tex_acesses[CURRENT_STAT_IDX][i]) - 
+                    (pwr_core_stat->m_num_tex_acesses[PREV_STAT_IDX][i]);
+
     }
     return total_inst;
   }
 
-  unsigned get_ialu_accessess() {
-    unsigned total_inst = 0;
+  double get_ialu_accessess(bool aggregate_stat) {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_core_stat->m_num_ialu_acesses[CURRENT_STAT_IDX][i]) -
+      if(aggregate_stat)
+        total_inst += (pwr_core_stat->m_num_ialu_acesses[CURRENT_STAT_IDX][i]);
+      else  
+        total_inst += (pwr_core_stat->m_num_ialu_acesses[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_ialu_acesses[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_tex_inst() {
-    unsigned total_inst = 0;
+  double get_tex_inst() {
+    double total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
       total_inst += (pwr_core_stat->m_num_tex_inst[CURRENT_STAT_IDX][i]) -
                     (pwr_core_stat->m_num_tex_inst[PREV_STAT_IDX][i]);
@@ -425,7 +732,7 @@ class power_stat_t {
     return total_inst;
   }
 
-  unsigned get_constant_c_accesses() {
+  double get_constant_c_accesses() {
     enum mem_access_type access_type[] = {CONST_ACC_R};
     enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
     unsigned num_access_type =
@@ -440,7 +747,7 @@ class power_stat_t {
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_constant_c_misses() {
+  double get_constant_c_misses() {
     enum mem_access_type access_type[] = {CONST_ACC_R};
     enum cache_request_status request_status[] = {MISS};
     unsigned num_access_type =
@@ -455,10 +762,10 @@ class power_stat_t {
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_constant_c_hits() {
+  double get_constant_c_hits() {
     return (get_constant_c_accesses() - get_constant_c_misses());
   }
-  unsigned get_texture_c_accesses() {
+  double get_texture_c_accesses() {
     enum mem_access_type access_type[] = {TEXTURE_ACC_R};
     enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
     unsigned num_access_type =
@@ -473,7 +780,7 @@ class power_stat_t {
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_texture_c_misses() {
+  double get_texture_c_misses() {
     enum mem_access_type access_type[] = {TEXTURE_ACC_R};
     enum cache_request_status request_status[] = {MISS};
     unsigned num_access_type =
@@ -488,205 +795,268 @@ class power_stat_t {
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_texture_c_hits() {
+  double get_texture_c_hits() {
     return (get_texture_c_accesses() - get_texture_c_misses());
   }
-  unsigned get_inst_c_accesses() {
+  double get_inst_c_accesses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {INST_ACC_R};
     enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat)
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    else
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_inst_c_misses() {
+  double get_inst_c_misses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {INST_ACC_R};
     enum cache_request_status request_status[] = {MISS};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat)
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    else
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
   }
-  unsigned get_inst_c_hits() {
-    return (get_inst_c_accesses() - get_inst_c_misses());
+  double get_inst_c_hits(bool aggregate_stat) {
+    return (get_inst_c_accesses(aggregate_stat) - get_inst_c_misses(aggregate_stat));
   }
 
-  unsigned get_l1d_read_accesses() {
+  double get_l1d_read_accesses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R};
-    enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
+    enum cache_request_status request_status[] = {HIT, MISS, SECTOR_MISS}; 
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
 
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+      }
   }
-  unsigned get_l1d_read_misses() {
+  double get_l1d_read_misses(bool aggregate_stat) {
+    return (get_l1d_read_accesses(aggregate_stat) - get_l1d_read_hits(aggregate_stat));
+  }
+  double get_l1d_read_hits(bool aggregate_stat) {
     enum mem_access_type access_type[] = {GLOBAL_ACC_R, LOCAL_ACC_R};
-    enum cache_request_status request_status[] = {MISS};
+    enum cache_request_status request_status[] = {HIT, MSHR_HIT};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
 
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+       return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+      }
   }
-  unsigned get_l1d_read_hits() {
-    return (get_l1d_read_accesses() - get_l1d_read_misses());
-  }
-  unsigned get_l1d_write_accesses() {
+  double get_l1d_write_accesses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W};
-    enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
+    enum cache_request_status request_status[] = {HIT, MISS, SECTOR_MISS};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
 
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+       return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+      }
   }
-  unsigned get_l1d_write_misses() {
+  double get_l1d_write_misses(bool aggregate_stat) {
+    return (get_l1d_write_accesses(aggregate_stat) - get_l1d_write_hits(aggregate_stat));
+  }
+  double get_l1d_write_hits(bool aggregate_stat) {
     enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W};
-    enum cache_request_status request_status[] = {MISS};
+    enum cache_request_status request_status[] = {HIT, MSHR_HIT};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
 
-    return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+       return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->core_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+      }
   }
-  unsigned get_l1d_write_hits() {
-    return (get_l1d_write_accesses() - get_l1d_write_misses());
-  }
-  unsigned get_cache_misses() {
-    return get_l1d_read_misses() + get_constant_c_misses() +
-           get_l1d_write_misses() + get_texture_c_misses();
+  double get_cache_misses() {
+    return get_l1d_read_misses(0) + get_constant_c_misses() +
+           get_l1d_write_misses(0) + get_texture_c_misses();
   }
 
-  unsigned get_cache_read_misses() {
-    return get_l1d_read_misses() + get_constant_c_misses() +
+  double get_cache_read_misses() {
+    return get_l1d_read_misses(0) + get_constant_c_misses() +
            get_texture_c_misses();
   }
 
-  unsigned get_cache_write_misses() { return get_l1d_write_misses(); }
+  double get_cache_write_misses() { return get_l1d_write_misses(0); }
 
-  unsigned get_shmem_read_access() {
+  double get_shmem_access(bool aggregate_stat) {
     unsigned total_inst = 0;
     for (unsigned i = 0; i < m_config->num_shader(); i++) {
-      total_inst += (pwr_mem_stat->shmem_read_access[CURRENT_STAT_IDX][i]) -
-                    (pwr_mem_stat->shmem_read_access[PREV_STAT_IDX][i]);
+      if(aggregate_stat)
+        total_inst += (pwr_mem_stat->shmem_access[CURRENT_STAT_IDX][i]);
+      else
+        total_inst += (pwr_mem_stat->shmem_access[CURRENT_STAT_IDX][i]) -
+                    (pwr_mem_stat->shmem_access[PREV_STAT_IDX][i]);
     }
     return total_inst;
   }
 
-  unsigned get_l2_read_accesses() {
+  unsigned long long  get_l2_read_accesses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {
         GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R};
-    enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
+    enum cache_request_status request_status[] = {HIT, HIT_RESERVED, MISS, SECTOR_MISS}; 
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+       return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+    }
   }
 
-  unsigned get_l2_read_misses() {
-    enum mem_access_type access_type[] = {
+  unsigned long long get_l2_read_misses(bool aggregate_stat) {
+    return (get_l2_read_accesses(aggregate_stat) - get_l2_read_hits(aggregate_stat));
+  }
+
+  unsigned long long get_l2_read_hits(bool aggregate_stat) {
+       enum mem_access_type access_type[] = {
         GLOBAL_ACC_R, LOCAL_ACC_R, CONST_ACC_R, TEXTURE_ACC_R, INST_ACC_R};
-    enum cache_request_status request_status[] = {MISS};
+    enum cache_request_status request_status[] =  {HIT, HIT_RESERVED};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+       return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+    }
   }
 
-  unsigned get_l2_read_hits() {
-    return (get_l2_read_accesses() - get_l2_read_misses());
-  }
-
-  unsigned get_l2_write_accesses() {
+  unsigned long long get_l2_write_accesses(bool aggregate_stat) {
     enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W,
                                           L1_WRBK_ACC};
-    enum cache_request_status request_status[] = {HIT, MISS, HIT_RESERVED};
+    enum cache_request_status request_status[] = {HIT, HIT_RESERVED, MISS, SECTOR_MISS}; 
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+    }
   }
 
-  unsigned get_l2_write_misses() {
-    enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W,
+  unsigned long long get_l2_write_misses(bool aggregate_stat) {
+    return (get_l2_write_accesses(aggregate_stat) - get_l2_write_hits(aggregate_stat));
+  }
+  unsigned long long get_l2_write_hits(bool aggregate_stat) {
+        enum mem_access_type access_type[] = {GLOBAL_ACC_W, LOCAL_ACC_W,
                                           L1_WRBK_ACC};
-    enum cache_request_status request_status[] = {MISS};
+    enum cache_request_status request_status[] = {HIT, HIT_RESERVED};
     unsigned num_access_type =
         sizeof(access_type) / sizeof(enum mem_access_type);
     unsigned num_request_status =
         sizeof(request_status) / sizeof(enum cache_request_status);
-
-    return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+    if(aggregate_stat){
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
+               access_type, num_access_type, request_status,
+               num_request_status));
+    }
+    else{
+      return (pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status)) -
            (pwr_mem_stat->l2_cache_stats[PREV_STAT_IDX].get_stats(
                access_type, num_access_type, request_status,
                num_request_status));
+    }
   }
-  unsigned get_l2_write_hits() {
-    return (get_l2_write_accesses() - get_l2_write_misses());
-  }
-  unsigned get_dram_cmd() {
+  double get_dram_cmd() {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
       total += (pwr_mem_stat->n_cmd[CURRENT_STAT_IDX][i] -
@@ -694,7 +1064,7 @@ class power_stat_t {
     }
     return total;
   }
-  unsigned get_dram_activity() {
+  double get_dram_activity() {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
       total += (pwr_mem_stat->n_activity[CURRENT_STAT_IDX][i] -
@@ -702,7 +1072,7 @@ class power_stat_t {
     }
     return total;
   }
-  unsigned get_dram_nop() {
+  double get_dram_nop() {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
       total += (pwr_mem_stat->n_nop[CURRENT_STAT_IDX][i] -
@@ -710,7 +1080,7 @@ class power_stat_t {
     }
     return total;
   }
-  unsigned get_dram_act() {
+  double get_dram_act() {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
       total += (pwr_mem_stat->n_act[CURRENT_STAT_IDX][i] -
@@ -718,31 +1088,49 @@ class power_stat_t {
     }
     return total;
   }
-  unsigned get_dram_pre() {
+  double get_dram_pre(bool aggregate_stat) {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
-      total += (pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i] -
+      if(aggregate_stat){
+        total += pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i];
+      }
+      else{
+        total += (pwr_mem_stat->n_pre[CURRENT_STAT_IDX][i] -
                 pwr_mem_stat->n_pre[PREV_STAT_IDX][i]);
+      }
     }
     return total;
   }
-  unsigned get_dram_rd() {
+  double get_dram_rd(bool aggregate_stat) {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
-      total += (pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i] -
+      if(aggregate_stat){
+        total += pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i];
+      }
+      else{
+        total += (pwr_mem_stat->n_rd[CURRENT_STAT_IDX][i] -
                 pwr_mem_stat->n_rd[PREV_STAT_IDX][i]);
+      }
     }
     return total;
   }
-  unsigned get_dram_wr() {
+  double get_dram_wr(bool aggregate_stat) {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
-      total += (pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i] -
-                pwr_mem_stat->n_wr[PREV_STAT_IDX][i]);
+      if(aggregate_stat){
+        total += pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i] + 
+                pwr_mem_stat->n_wr_WB[CURRENT_STAT_IDX][i];
+      }
+      else{
+        total += (pwr_mem_stat->n_wr[CURRENT_STAT_IDX][i] - 
+                pwr_mem_stat->n_wr[PREV_STAT_IDX][i]) +
+                (pwr_mem_stat->n_wr_WB[CURRENT_STAT_IDX][i] - 
+                pwr_mem_stat->n_wr_WB[PREV_STAT_IDX][i]);
+      }
     }
     return total;
   }
-  unsigned get_dram_req() {
+  double get_dram_req() {
     unsigned total = 0;
     for (unsigned i = 0; i < m_mem_config->m_n_mem; ++i) {
       total += (pwr_mem_stat->n_req[CURRENT_STAT_IDX][i] -
@@ -751,20 +1139,31 @@ class power_stat_t {
     return total;
   }
 
-  long get_icnt_simt_to_mem() {
+  unsigned long long get_icnt_simt_to_mem(bool aggregate_stat) {
     long total = 0;
-    for (unsigned i = 0; i < m_config->n_simt_clusters; ++i) {
-      total += (pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i] -
+    for (unsigned i = 0; i < m_config->n_simt_clusters; ++i){
+      if(aggregate_stat){
+        total += pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i];
+      }
+      else{
+        total += (pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i] -
                 pwr_mem_stat->n_simt_to_mem[PREV_STAT_IDX][i]);
+      }
     }
     return total;
   }
 
-  long get_icnt_mem_to_simt() {
+  unsigned long long get_icnt_mem_to_simt(bool aggregate_stat) {
     long total = 0;
     for (unsigned i = 0; i < m_config->n_simt_clusters; ++i) {
-      total += (pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i] -
+      if(aggregate_stat){
+        total += pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i];
+      }
+      
+      else{
+        total += (pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i] -
                 pwr_mem_stat->n_mem_to_simt[PREV_STAT_IDX][i]);
+      }
     }
     return total;
   }
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 90bb900..4013ae9 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
-// George L. Yuan, Andrew Turner, Inderpreet Singh
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
+// George L. Yuan, Andrew Turner, Inderpreet Singh, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -61,6 +63,20 @@ mem_fetch *shader_core_mem_fetch_allocator::alloc(
                     m_core_id, m_cluster_id, m_memory_config, cycle);
   return mf;
 }
+
+mem_fetch *shader_core_mem_fetch_allocator::alloc(
+    new_addr_type addr, mem_access_type type, const active_mask_t &active_mask,
+    const mem_access_byte_mask_t &byte_mask,
+    const mem_access_sector_mask_t &sector_mask, unsigned size, bool wr,
+    unsigned long long cycle, unsigned wid, unsigned sid, unsigned tpc,
+    mem_fetch *original_mf) const {
+  mem_access_t access(type, addr, size, wr, active_mask, byte_mask, sector_mask,
+                      m_memory_config->gpgpu_ctx);
+  mem_fetch *mf = new mem_fetch(
+      access, NULL, wr ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, wid, m_core_id,
+      m_cluster_id, m_memory_config, cycle, original_mf);
+  return mf;
+}
 /////////////////////////////////////////////////////////////////////////////
 
 std::list<unsigned> shader_core_ctx::get_regs_written(const inst_t &fvt) const {
@@ -108,7 +124,7 @@ void shader_core_ctx::create_front_pipeline() {
 
   if (m_config->sub_core_model) {
     // in subcore model, each scheduler should has its own issue register, so
-    // num scheduler = reg width
+    // ensure num scheduler = reg width
     assert(m_config->gpgpu_num_sched_per_core ==
            m_pipeline_reg[ID_OC_SP].get_size());
     assert(m_config->gpgpu_num_sched_per_core ==
@@ -124,6 +140,11 @@ void shader_core_ctx::create_front_pipeline() {
     if (m_config->gpgpu_num_int_units > 0)
       assert(m_config->gpgpu_num_sched_per_core ==
              m_pipeline_reg[ID_OC_INT].get_size());
+    for (int j = 0; j < m_config->m_specialized_unit.size(); j++) {
+      if (m_config->m_specialized_unit[j].num_units > 0)
+        assert(m_config->gpgpu_num_sched_per_core ==
+               m_config->m_specialized_unit[j].id_oc_spec_reg_width);
+    }
   }
 
   m_threadState = (thread_ctx_t *)calloc(sizeof(thread_ctx_t),
@@ -172,6 +193,8 @@ void shader_core_ctx::create_schedulers() {
                 ? CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE
                 : sched_config.find("gto") != std::string::npos
                       ? CONCRETE_SCHEDULER_GTO
+                      : sched_config.find("rrr") != std::string::npos
+                            ? CONCRETE_SCHEDULER_RRR
                       : sched_config.find("old") != std::string::npos
                             ? CONCRETE_SCHEDULER_OLDEST_FIRST
                             : sched_config.find("warp_limiting") !=
@@ -206,6 +229,14 @@ void shader_core_ctx::create_schedulers() {
             &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
             &m_pipeline_reg[ID_OC_MEM], i));
         break;
+      case CONCRETE_SCHEDULER_RRR:
+        schedulers.push_back(new rrr_scheduler(
+            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
+            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
+            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
+            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
+            &m_pipeline_reg[ID_OC_MEM], i));
+        break;
       case CONCRETE_SCHEDULER_OLDEST_FIRST:
         schedulers.push_back(new oldest_scheduler(
             m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
@@ -377,41 +408,41 @@ void shader_core_ctx::create_exec_pipeline() {
 
   // m_fu = new simd_function_unit*[m_num_function_units];
 
-  for (int k = 0; k < m_config->gpgpu_num_sp_units; k++) {
-    m_fu.push_back(new sp_unit(&m_pipeline_reg[EX_WB], m_config, this));
+  for (unsigned k = 0; k < m_config->gpgpu_num_sp_units; k++) {
+    m_fu.push_back(new sp_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
     m_dispatch_port.push_back(ID_OC_SP);
     m_issue_port.push_back(OC_EX_SP);
   }
 
-  for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) {
-    m_fu.push_back(new dp_unit(&m_pipeline_reg[EX_WB], m_config, this));
+  for (unsigned k = 0; k < m_config->gpgpu_num_dp_units; k++) {
+    m_fu.push_back(new dp_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
     m_dispatch_port.push_back(ID_OC_DP);
     m_issue_port.push_back(OC_EX_DP);
   }
-  for (int k = 0; k < m_config->gpgpu_num_int_units; k++) {
-    m_fu.push_back(new int_unit(&m_pipeline_reg[EX_WB], m_config, this));
+  for (unsigned k = 0; k < m_config->gpgpu_num_int_units; k++) {
+    m_fu.push_back(new int_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
     m_dispatch_port.push_back(ID_OC_INT);
     m_issue_port.push_back(OC_EX_INT);
   }
 
-  for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) {
-    m_fu.push_back(new sfu(&m_pipeline_reg[EX_WB], m_config, this));
+  for (unsigned k = 0; k < m_config->gpgpu_num_sfu_units; k++) {
+    m_fu.push_back(new sfu(&m_pipeline_reg[EX_WB], m_config, this, k));
     m_dispatch_port.push_back(ID_OC_SFU);
     m_issue_port.push_back(OC_EX_SFU);
   }
 
-  for (int k = 0; k < m_config->gpgpu_num_tensor_core_units; k++) {
-    m_fu.push_back(new tensor_core(&m_pipeline_reg[EX_WB], m_config, this));
+  for (unsigned k = 0; k < m_config->gpgpu_num_tensor_core_units; k++) {
+    m_fu.push_back(new tensor_core(&m_pipeline_reg[EX_WB], m_config, this, k));
     m_dispatch_port.push_back(ID_OC_TENSOR_CORE);
     m_issue_port.push_back(OC_EX_TENSOR_CORE);
   }
 
-  for (int j = 0; j < m_config->m_specialized_unit.size(); j++) {
+  for (unsigned j = 0; j < m_config->m_specialized_unit.size(); j++) {
     for (unsigned k = 0; k < m_config->m_specialized_unit[j].num_units; k++) {
       m_fu.push_back(new specialized_unit(
           &m_pipeline_reg[EX_WB], m_config, this, SPEC_UNIT_START_ID + j,
           m_config->m_specialized_unit[j].name,
-          m_config->m_specialized_unit[j].latency));
+          m_config->m_specialized_unit[j].latency, k));
       m_dispatch_port.push_back(m_config->m_specialized_unit[j].ID_OC_SPEC_ID);
       m_issue_port.push_back(m_config->m_specialized_unit[j].OC_EX_SPEC_ID);
     }
@@ -456,6 +487,10 @@ shader_core_ctx::shader_core_ctx(class gpgpu_sim *gpu,
   m_sid = shader_id;
   m_tpc = tpc_id;
 
+  if(get_gpu()->get_config().g_power_simulation_enabled){
+    scaling_coeffs =  get_gpu()->get_scaling_coeffs();
+  }
+
   m_last_inst_gpu_sim_cycle = 0;
   m_last_inst_gpu_tot_sim_cycle = 0;
 
@@ -859,7 +894,7 @@ void shader_core_ctx::decode() {
     m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline();
     if (pI1) {
       m_stats->m_num_decoded_insn[m_sid]++;
-      if (pI1->oprnd_type == INT_OP) {
+      if ((pI1->oprnd_type == INT_OP) || (pI1->oprnd_type == UN_OP))  { //these counters get added up in mcPat to compute scheduler power
         m_stats->m_num_INTdecoded_insn[m_sid]++;
       } else if (pI1->oprnd_type == FP_OP) {
         m_stats->m_num_FPdecoded_insn[m_sid]++;
@@ -870,7 +905,7 @@ void shader_core_ctx::decode() {
         m_warp[m_inst_fetch_buffer.m_warp_id]->ibuffer_fill(1, pI2);
         m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline();
         m_stats->m_num_decoded_insn[m_sid]++;
-        if (pI2->oprnd_type == INT_OP) {
+        if ((pI1->oprnd_type == INT_OP) || (pI1->oprnd_type == UN_OP))  { //these counters get added up in mcPat to compute scheduler power
           m_stats->m_num_INTdecoded_insn[m_sid]++;
         } else if (pI2->oprnd_type == FP_OP) {
           m_stats->m_num_FPdecoded_insn[m_sid]++;
@@ -916,7 +951,7 @@ void shader_core_ctx::fetch() {
               m_threadState[tid].m_active = false;
               unsigned cta_id = m_warp[warp_id]->get_cta_id();
               if (m_thread[tid] == NULL) {
-                register_cta_thread_exit(cta_id, m_kernel);
+                register_cta_thread_exit(cta_id, m_warp[warp_id]->get_kernel_info());
               } else {
                 register_cta_thread_exit(cta_id,
                                          &(m_thread[tid]->get_kernel()));
@@ -953,8 +988,10 @@ void shader_core_ctx::fetch() {
               m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
           std::list<cache_event> events;
           enum cache_request_status status;
-          if (m_config->perfect_inst_const_cache)
+          if (m_config->perfect_inst_const_cache){
             status = HIT;
+            shader_cache_access_log(m_sid, INSTRUCTION, 0);
+          }
           else
             status = m_L1I->access(
                 (new_addr_type)ppc, mf,
@@ -1082,6 +1119,33 @@ void scheduler_unit::order_lrr(
   }
 }
 
+template <class T>
+void scheduler_unit::order_rrr(
+    std::vector<T> &result_list, const typename std::vector<T> &input_list,
+    const typename std::vector<T>::const_iterator &last_issued_from_input,
+    unsigned num_warps_to_add) {
+  result_list.clear();
+
+  if (m_num_issued_last_cycle > 0 || warp(m_current_turn_warp).done_exit() ||
+      warp(m_current_turn_warp).waiting()) {
+    std::vector<shd_warp_t *>::const_iterator iter =
+      (last_issued_from_input == input_list.end()) ? 
+        input_list.begin() : last_issued_from_input + 1;
+    for (unsigned count = 0; count < num_warps_to_add; ++iter, ++count) {
+      if (iter == input_list.end()) {
+      iter = input_list.begin();
+      }
+      unsigned warp_id = (*iter)->get_warp_id();
+      if (!(*iter)->done_exit() && !(*iter)->waiting()) {
+        result_list.push_back(*iter);
+        m_current_turn_warp = warp_id;
+        break;
+      }
+    }
+  } else {
+    result_list.push_back(&warp(m_current_turn_warp));
+  }
+}
 /**
  * A general function to order things in an priority-based way.
  * The core usage of the function is similar to order_lrr.
@@ -1228,29 +1292,21 @@ void scheduler_unit::cycle() {
                 previous_issued_inst_exec_type = exec_unit_type_t::MEM;
               }
             } else {
-              bool sp_pipe_avail =
-                  (m_shader->m_config->gpgpu_num_sp_units > 0) &&
-                  m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id);
-              bool sfu_pipe_avail =
-                  (m_shader->m_config->gpgpu_num_sfu_units > 0) &&
-                  m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id);
-              bool tensor_core_pipe_avail =
-                  (m_shader->m_config->gpgpu_num_tensor_core_units > 0) &&
-                  m_tensor_core_out->has_free(
-                      m_shader->m_config->sub_core_model, m_id);
-              bool dp_pipe_avail =
-                  (m_shader->m_config->gpgpu_num_dp_units > 0) &&
-                  m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id);
-              bool int_pipe_avail =
-                  (m_shader->m_config->gpgpu_num_int_units > 0) &&
-                  m_int_out->has_free(m_shader->m_config->sub_core_model, m_id);
-
               // This code need to be refactored
               if (pI->op != TENSOR_CORE_OP && pI->op != SFU_OP &&
                   pI->op != DP_OP && !(pI->op >= SPEC_UNIT_START_ID)) {
                 bool execute_on_SP = false;
                 bool execute_on_INT = false;
 
+                bool sp_pipe_avail =
+                    (m_shader->m_config->gpgpu_num_sp_units > 0) &&
+                    m_sp_out->has_free(m_shader->m_config->sub_core_model,
+                                       m_id);
+                bool int_pipe_avail =
+                    (m_shader->m_config->gpgpu_num_int_units > 0) &&
+                    m_int_out->has_free(m_shader->m_config->sub_core_model,
+                                        m_id);
+
                 // if INT unit pipline exist, then execute ALU and INT
                 // operations on INT unit and SP-FPU on SP unit (like in Volta)
                 // if INT unit pipline does not exist, then execute all ALU, INT
@@ -1311,6 +1367,11 @@ void scheduler_unit::cycle() {
                          (pI->op == DP_OP) &&
                          !(diff_exec_units && previous_issued_inst_exec_type ==
                                                   exec_unit_type_t::DP)) {
+                bool dp_pipe_avail =
+                    (m_shader->m_config->gpgpu_num_dp_units > 0) &&
+                    m_dp_out->has_free(m_shader->m_config->sub_core_model,
+                                       m_id);
+
                 if (dp_pipe_avail) {
                   m_shader->issue_warp(*m_dp_out, pI, active_mask, warp_id,
                                        m_id);
@@ -1326,6 +1387,11 @@ void scheduler_unit::cycle() {
                         (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) &&
                        !(diff_exec_units && previous_issued_inst_exec_type ==
                                                 exec_unit_type_t::SFU)) {
+                bool sfu_pipe_avail =
+                    (m_shader->m_config->gpgpu_num_sfu_units > 0) &&
+                    m_sfu_out->has_free(m_shader->m_config->sub_core_model,
+                                        m_id);
+
                 if (sfu_pipe_avail) {
                   m_shader->issue_warp(*m_sfu_out, pI, active_mask, warp_id,
                                        m_id);
@@ -1337,6 +1403,11 @@ void scheduler_unit::cycle() {
               } else if ((pI->op == TENSOR_CORE_OP) &&
                          !(diff_exec_units && previous_issued_inst_exec_type ==
                                                   exec_unit_type_t::TENSOR)) {
+                bool tensor_core_pipe_avail =
+                    (m_shader->m_config->gpgpu_num_tensor_core_units > 0) &&
+                    m_tensor_core_out->has_free(
+                        m_shader->m_config->sub_core_model, m_id);
+
                 if (tensor_core_pipe_avail) {
                   m_shader->issue_warp(*m_tensor_core_out, pI, active_mask,
                                        warp_id, m_id);
@@ -1407,7 +1478,7 @@ void scheduler_unit::cycle() {
           m_last_supervised_issued = supervised_iter;
         }
       }
-
+      m_num_issued_last_cycle = issued;
       if (issued == 1)
         m_stats->single_issue_nums[m_id]++;
       else if (issued > 1)
@@ -1456,6 +1527,10 @@ void lrr_scheduler::order_warps() {
   order_lrr(m_next_cycle_prioritized_warps, m_supervised_warps,
             m_last_supervised_issued, m_supervised_warps.size());
 }
+void rrr_scheduler::order_warps() {
+  order_rrr(m_next_cycle_prioritized_warps, m_supervised_warps,
+            m_last_supervised_issued, m_supervised_warps.size());
+}
 
 void gto_scheduler::order_warps() {
   order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps,
@@ -1569,7 +1644,10 @@ void swl_scheduler::order_warps() {
   }
 }
 
-void shader_core_ctx::read_operands() {}
+void shader_core_ctx::read_operands() {
+  for (int i = 0; i < m_config->reg_file_port_throughput; ++i)
+    m_operand_collector.step();
+}
 
 address_type coalesced_segment(address_type addr,
                                unsigned segment_size_lg2bytes) {
@@ -1669,8 +1747,15 @@ void shader_core_ctx::execute() {
     m_fu[n]->active_lanes_in_pipeline();
     unsigned issue_port = m_issue_port[n];
     register_set &issue_inst = m_pipeline_reg[issue_port];
-    warp_inst_t **ready_reg = issue_inst.get_ready();
-    if (issue_inst.has_ready() && m_fu[n]->can_issue(**ready_reg)) {
+    unsigned reg_id;
+    bool partition_issue =
+        m_config->sub_core_model && m_fu[n]->is_issue_partitioned();
+    if (partition_issue) {
+      reg_id = m_fu[n]->get_issue_reg_id();
+    }
+    warp_inst_t **ready_reg = issue_inst.get_ready(partition_issue, reg_id);
+    if (issue_inst.has_ready(partition_issue, reg_id) &&
+        m_fu[n]->can_issue(**ready_reg)) {
       bool schedule_wb_now = !m_fu[n]->stallable();
       int resbus = -1;
       if (schedule_wb_now &&
@@ -1970,6 +2055,21 @@ void ldst_unit::L1_latency_queue_cycle() {
       } else {
         assert(status == MISS || status == HIT_RESERVED);
         l1_latency_queue[j][0] = NULL;
+        if (m_config->m_L1D_config.get_write_policy() != WRITE_THROUGH &&
+            mf_next->get_inst().is_store() &&
+            (m_config->m_L1D_config.get_write_allocate_policy() ==
+                 FETCH_ON_WRITE ||
+             m_config->m_L1D_config.get_write_allocate_policy() ==
+                 LAZY_FETCH_ON_READ) &&
+            !was_writeallocate_sent(events)) {
+          unsigned dec_ack =
+              (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
+                  ? (mf_next->get_data_size() / SECTOR_SIZE)
+                  : 1;
+          mf_next->set_reply();
+          for (unsigned i = 0; i < dec_ack; ++i) m_core->store_ack(mf_next);
+          if (!write_sent && !read_sent) delete mf_next;
+        }
       }
     }
 
@@ -1992,10 +2092,11 @@ bool ldst_unit::constant_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
   mem_stage_stall_type fail;
   if (m_config->perfect_inst_const_cache) {
     fail = NO_RC_FAIL;
+    unsigned access_count = inst.accessq_count();
     while (inst.accessq_count() > 0) inst.accessq_pop_back();
     if (inst.is_load()) {
       for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
-        if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--;
+        if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]] -= access_count;
     }
   } else {
     fail = process_memory_access_queue(m_L1C, inst);
@@ -2112,22 +2213,32 @@ simd_function_unit::simd_function_unit(const shader_core_config *config) {
   m_dispatch_reg = new warp_inst_t(config);
 }
 
+void simd_function_unit::issue(register_set &source_reg) {
+  bool partition_issue =
+      m_config->sub_core_model && this->is_issue_partitioned();
+  source_reg.move_out_to(partition_issue, this->get_issue_reg_id(),
+                         m_dispatch_reg);
+  occupied.set(m_dispatch_reg->latency);
+}
+
 sfu::sfu(register_set *result_port, const shader_core_config *config,
-         shader_core_ctx *core)
-    : pipelined_simd_unit(result_port, config, config->max_sfu_latency, core) {
+         shader_core_ctx *core, unsigned issue_reg_id)
+    : pipelined_simd_unit(result_port, config, config->max_sfu_latency, core,
+                          issue_reg_id) {
   m_name = "SFU";
 }
 
 tensor_core::tensor_core(register_set *result_port,
                          const shader_core_config *config,
-                         shader_core_ctx *core)
+                         shader_core_ctx *core, unsigned issue_reg_id)
     : pipelined_simd_unit(result_port, config, config->max_tensor_core_latency,
-                          core) {
+                          core, issue_reg_id) {
   m_name = "TENSOR_CORE";
 }
 
 void sfu::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
 
   (*ready_reg)->op_pipe = SFU__OP;
@@ -2136,7 +2247,8 @@ void sfu::issue(register_set &source_reg) {
 }
 
 void tensor_core::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
 
   (*ready_reg)->op_pipe = TENSOR_CORE__OP;
@@ -2172,7 +2284,7 @@ void sp_unit::active_lanes_in_pipeline() {
 void dp_unit::active_lanes_in_pipeline() {
   unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
   assert(active_count <= m_core->get_config()->warp_size);
-  m_core->incspactivelanes_stat(active_count);
+  //m_core->incspactivelanes_stat(active_count);
   m_core->incfuactivelanes_stat(active_count);
   m_core->incfumemactivelanes_stat(active_count);
 }
@@ -2208,34 +2320,39 @@ void tensor_core::active_lanes_in_pipeline() {
 }
 
 sp_unit::sp_unit(register_set *result_port, const shader_core_config *config,
-                 shader_core_ctx *core)
-    : pipelined_simd_unit(result_port, config, config->max_sp_latency, core) {
+                 shader_core_ctx *core, unsigned issue_reg_id)
+    : pipelined_simd_unit(result_port, config, config->max_sp_latency, core,
+                          issue_reg_id) {
   m_name = "SP ";
 }
 
 specialized_unit::specialized_unit(register_set *result_port,
                                    const shader_core_config *config,
                                    shader_core_ctx *core, unsigned supported_op,
-                                   char *unit_name, unsigned latency)
-    : pipelined_simd_unit(result_port, config, latency, core) {
+                                   char *unit_name, unsigned latency,
+                                   unsigned issue_reg_id)
+    : pipelined_simd_unit(result_port, config, latency, core, issue_reg_id) {
   m_name = unit_name;
   m_supported_op = supported_op;
 }
 
 dp_unit::dp_unit(register_set *result_port, const shader_core_config *config,
-                 shader_core_ctx *core)
-    : pipelined_simd_unit(result_port, config, config->max_dp_latency, core) {
+                 shader_core_ctx *core, unsigned issue_reg_id)
+    : pipelined_simd_unit(result_port, config, config->max_dp_latency, core,
+                          issue_reg_id) {
   m_name = "DP ";
 }
 
 int_unit::int_unit(register_set *result_port, const shader_core_config *config,
-                   shader_core_ctx *core)
-    : pipelined_simd_unit(result_port, config, config->max_int_latency, core) {
+                   shader_core_ctx *core, unsigned issue_reg_id)
+    : pipelined_simd_unit(result_port, config, config->max_int_latency, core,
+                          issue_reg_id) {
   m_name = "INT ";
 }
 
 void sp_unit ::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
   (*ready_reg)->op_pipe = SP__OP;
   m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
@@ -2243,7 +2360,8 @@ void sp_unit ::issue(register_set &source_reg) {
 }
 
 void dp_unit ::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
   (*ready_reg)->op_pipe = DP__OP;
   m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
@@ -2251,7 +2369,8 @@ void dp_unit ::issue(register_set &source_reg) {
 }
 
 void specialized_unit ::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
   (*ready_reg)->op_pipe = SPECIALIZED__OP;
   m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
@@ -2259,7 +2378,8 @@ void specialized_unit ::issue(register_set &source_reg) {
 }
 
 void int_unit ::issue(register_set &source_reg) {
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
   // m_core->incexecstat((*ready_reg));
   (*ready_reg)->op_pipe = INTP__OP;
   m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
@@ -2269,7 +2389,8 @@ void int_unit ::issue(register_set &source_reg) {
 pipelined_simd_unit::pipelined_simd_unit(register_set *result_port,
                                          const shader_core_config *config,
                                          unsigned max_latency,
-                                         shader_core_ctx *core)
+                                         shader_core_ctx *core,
+                                         unsigned issue_reg_id)
     : simd_function_unit(config) {
   m_result_port = result_port;
   m_pipeline_depth = max_latency;
@@ -2277,6 +2398,7 @@ pipelined_simd_unit::pipelined_simd_unit(register_set *result_port,
   for (unsigned i = 0; i < m_pipeline_depth; i++)
     m_pipeline_reg[i] = new warp_inst_t(config);
   m_core = core;
+  m_issue_reg_id = issue_reg_id;
   active_insts_in_pipeline = 0;
 }
 
@@ -2305,7 +2427,10 @@ void pipelined_simd_unit::cycle() {
 
 void pipelined_simd_unit::issue(register_set &source_reg) {
   // move_warp(m_dispatch_reg,source_reg);
-  warp_inst_t **ready_reg = source_reg.get_ready();
+  bool partition_issue =
+      m_config->sub_core_model && this->is_issue_partitioned();
+  warp_inst_t **ready_reg =
+      source_reg.get_ready(partition_issue, m_issue_reg_id);
   m_core->incexecstat((*ready_reg));
   // source_reg.move_out_to(m_dispatch_reg);
   simd_function_unit::issue(source_reg);
@@ -2362,7 +2487,7 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt,
                      Scoreboard *scoreboard, const shader_core_config *config,
                      const memory_config *mem_config, shader_core_stats *stats,
                      unsigned sid, unsigned tpc)
-    : pipelined_simd_unit(NULL, config, config->smem_latency, core),
+    : pipelined_simd_unit(NULL, config, config->smem_latency, core, 0),
       m_next_wb(config) {
   assert(config->smem_latency > 1);
   init(icnt, mf_allocator, core, operand_collector, scoreboard, config,
@@ -2390,7 +2515,7 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt,
                      Scoreboard *scoreboard, const shader_core_config *config,
                      const memory_config *mem_config, shader_core_stats *stats,
                      unsigned sid, unsigned tpc, l1_cache *new_l1d_cache)
-    : pipelined_simd_unit(NULL, config, 3, core),
+    : pipelined_simd_unit(NULL, config, 3, core, 0),
       m_L1D(new_l1d_cache),
       m_next_wb(config) {
   init(icnt, mf_allocator, core, operand_collector, scoreboard, config,
@@ -2552,8 +2677,7 @@ inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id();
 */
 void ldst_unit::cycle() {
   writeback();
-  for (int i = 0; i < m_config->reg_file_port_throughput; ++i)
-    m_operand_collector->step();
+
   for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++)
     if (m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage + 1]->empty())
       move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]);
@@ -2966,52 +3090,69 @@ void warp_inst_t::print(FILE *fout) const {
   m_config->gpgpu_ctx->func_sim->ptx_print_insn(pc, fout);
   fprintf(fout, "\n");
 }
-void shader_core_ctx::incexecstat(warp_inst_t *&inst) {
-  if (inst->mem_op == TEX) inctex_stat(inst->active_count(), 1);
-
-  // Latency numbers for next operations are used to scale the power values
-  // for special operations, according observations from microbenchmarking
-  // TODO: put these numbers in the xml configuration
-
-  switch (inst->sp_op) {
+void shader_core_ctx::incexecstat(warp_inst_t *&inst)
+{
+    // Latency numbers for next operations are used to scale the power values
+    // for special operations, according observations from microbenchmarking
+    // TODO: put these numbers in the xml configuration
+  if(get_gpu()->get_config().g_power_simulation_enabled){
+    switch(inst->sp_op){
     case INT__OP:
-      incialu_stat(inst->active_count(), 32);
+      incialu_stat(inst->active_count(), scaling_coeffs->int_coeff);
       break;
     case INT_MUL_OP:
-      incimul_stat(inst->active_count(), 7.2);
+      incimul_stat(inst->active_count(), scaling_coeffs->int_mul_coeff);
       break;
     case INT_MUL24_OP:
-      incimul24_stat(inst->active_count(), 4.2);
+      incimul24_stat(inst->active_count(), scaling_coeffs->int_mul24_coeff);
       break;
     case INT_MUL32_OP:
-      incimul32_stat(inst->active_count(), 4);
+      incimul32_stat(inst->active_count(), scaling_coeffs->int_mul32_coeff);
       break;
     case INT_DIV_OP:
-      incidiv_stat(inst->active_count(), 40);
+      incidiv_stat(inst->active_count(), scaling_coeffs->int_div_coeff);
       break;
     case FP__OP:
-      incfpalu_stat(inst->active_count(), 1);
+      incfpalu_stat(inst->active_count(),scaling_coeffs->fp_coeff);
       break;
     case FP_MUL_OP:
-      incfpmul_stat(inst->active_count(), 1.8);
+      incfpmul_stat(inst->active_count(), scaling_coeffs->fp_mul_coeff);
       break;
     case FP_DIV_OP:
-      incfpdiv_stat(inst->active_count(), 48);
+      incfpdiv_stat(inst->active_count(), scaling_coeffs->fp_div_coeff);
+      break;
+    case DP___OP:
+      incdpalu_stat(inst->active_count(), scaling_coeffs->dp_coeff);
+      break;
+    case DP_MUL_OP:
+      incdpmul_stat(inst->active_count(), scaling_coeffs->dp_mul_coeff);
+      break;
+    case DP_DIV_OP:
+      incdpdiv_stat(inst->active_count(), scaling_coeffs->dp_div_coeff);
       break;
     case FP_SQRT_OP:
-      inctrans_stat(inst->active_count(), 25);
+      incsqrt_stat(inst->active_count(), scaling_coeffs->sqrt_coeff);
       break;
     case FP_LG_OP:
-      inctrans_stat(inst->active_count(), 35);
+      inclog_stat(inst->active_count(), scaling_coeffs->log_coeff);
       break;
     case FP_SIN_OP:
-      inctrans_stat(inst->active_count(), 12);
+      incsin_stat(inst->active_count(), scaling_coeffs->sin_coeff);
       break;
     case FP_EXP_OP:
-      inctrans_stat(inst->active_count(), 35);
+      incexp_stat(inst->active_count(), scaling_coeffs->exp_coeff);
+      break;
+    case TENSOR__OP:
+      inctensor_stat(inst->active_count(), scaling_coeffs->tensor_coeff);
+      break;
+    case TEX__OP:
+      inctex_stat(inst->active_count(), scaling_coeffs->tex_coeff);
       break;
     default:
       break;
+    }
+    if(inst->const_cache_operand) //warp has const address space load as one operand
+      inc_const_accesses(1);
   }
 }
 void shader_core_ctx::print_stage(unsigned int stage, FILE *fout) const {
@@ -3266,49 +3407,46 @@ unsigned int shader_core_config::max_cta(const kernel_info_t &k) const {
   if (adaptive_cache_config && !k.cache_config_set) {
     // For more info about adaptive cache, see
     // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
-    unsigned total_shmed = kernel_info->smem * result;
-    assert(total_shmed >= 0 && total_shmed <= gpgpu_shmem_size);
-    // assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared
-    // assert(m_L1D_config.get_nset() == 4);  //Volta L1 has four sets
-    if (total_shmed < gpgpu_shmem_size) {
-      switch (adaptive_cache_config) {
-        case FIXED:
-          break;
-        case ADAPTIVE_VOLTA: {
-          // For Volta, we assign the remaining shared memory to L1 cache
-          // For more info about adaptive cache, see
-          // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
-          // assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared
+    unsigned total_shmem = kernel_info->smem * result;
+    assert(total_shmem >= 0 && total_shmem <= shmem_opt_list.back());
 
-          // To Do: make it flexible and not tuned to 9KB share memory
-          unsigned max_assoc = m_L1D_config.get_max_assoc();
-          if (total_shmed == 0)
-            m_L1D_config.set_assoc(max_assoc);  // L1 is 128KB and shd=0
-          else if (total_shmed > 0 && total_shmed <= 8192)
-            m_L1D_config.set_assoc(0.9375 *
-                                   max_assoc);  // L1 is 120KB and shd=8KB
-          else if (total_shmed > 8192 && total_shmed <= 16384)
-            m_L1D_config.set_assoc(0.875 *
-                                   max_assoc);  // L1 is 112KB and shd=16KB
-          else if (total_shmed > 16384 && total_shmed <= 32768)
-            m_L1D_config.set_assoc(0.75 * max_assoc);  // L1 is 96KB and
-                                                       // shd=32KB
-          else if (total_shmed > 32768 && total_shmed <= 65536)
-            m_L1D_config.set_assoc(0.5 * max_assoc);  // L1 is 64KB and shd=64KB
-          else if (total_shmed > 65536 && total_shmed <= gpgpu_shmem_size)
-            m_L1D_config.set_assoc(0.25 * max_assoc);  // L1 is 32KB and
-                                                       // shd=96KB
-          else
-            assert(0);
-          break;
-        }
-        default:
-          assert(0);
+    // Unified cache config is in KB. Converting to B
+    unsigned total_unified = m_L1D_config.m_unified_cache_size * 1024;
+
+    bool l1d_configured = false;
+    unsigned max_assoc = m_L1D_config.get_max_assoc();
+
+    for (std::vector<unsigned>::const_iterator it = shmem_opt_list.begin();
+         it < shmem_opt_list.end(); it++) {
+      if (total_shmem <= *it) {
+        float l1_ratio = 1 - ((float)*(it) / total_unified);
+        // make sure the ratio is between 0 and 1
+        assert(0 <= l1_ratio && l1_ratio <= 1);
+        // round to nearest instead of round down
+        m_L1D_config.set_assoc(max_assoc * l1_ratio + 0.5f);
+        l1d_configured = true;
+        break;
       }
+    }
 
-      printf("GPGPU-Sim: Reconfigure L1 cache to %uKB\n",
-             m_L1D_config.get_total_size_inKB());
+    assert(l1d_configured && "no shared memory option found");
+
+    if (m_L1D_config.is_streaming()) {
+      // for streaming cache, if the whole memory is allocated
+      // to the L1 cache, then make the allocation to be on_MISS
+      // otherwise, make it ON_FILL to eliminate line allocation fails
+      // i.e. MSHR throughput is the same, independent on the L1 cache
+      // size/associativity
+      if (total_shmem == 0) {
+        m_L1D_config.set_allocation_policy(ON_MISS);
+        printf("GPGPU-Sim: Reconfigure L1 allocation to ON_MISS\n");
+      } else {
+        m_L1D_config.set_allocation_policy(ON_FILL);
+        printf("GPGPU-Sim: Reconfigure L1 allocation to ON_FILL\n");
+      }
     }
+    printf("GPGPU-Sim: Reconfigure L1 cache to %uKB\n",
+           m_L1D_config.get_total_size_inKB());
 
     k.cache_config_set = true;
   }
@@ -3763,6 +3901,8 @@ void shader_core_ctx::get_icnt_power_stats(long &n_simt_to_mem,
   n_mem_to_simt += m_stats->n_mem_to_simt[m_sid];
 }
 
+kernel_info_t* shd_warp_t::get_kernel_info() const { return m_shader->get_kernel_info(); }
+
 bool shd_warp_t::functional_done() const {
   return get_n_completed() == m_warp_size;
 }
@@ -3869,15 +4009,26 @@ void opndcoll_rfu_t::init(unsigned num_banks, shader_core_ctx *shader) {
   assert((m_bank_warp_shift == 5) || (m_warp_size != 32));
 
   sub_core_model = shader->get_config()->sub_core_model;
-  m_num_warp_sceds = shader->get_config()->gpgpu_num_sched_per_core;
-  if (sub_core_model)
+  m_num_warp_scheds = shader->get_config()->gpgpu_num_sched_per_core;
+  unsigned reg_id;
+  if (sub_core_model) {
     assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0);
+    assert(m_num_warp_scheds <= m_cu.size() &&
+           m_cu.size() % m_num_warp_scheds == 0);
+  }
   m_num_banks_per_sched =
       num_banks / shader->get_config()->gpgpu_num_sched_per_core;
 
   for (unsigned j = 0; j < m_cu.size(); j++) {
+    if (sub_core_model) {
+      unsigned cusPerSched = m_cu.size() / m_num_warp_scheds;
+      reg_id = j / cusPerSched;
+    }
     m_cu[j]->init(j, num_banks, m_bank_warp_shift, shader->get_config(), this,
-                  sub_core_model, m_num_banks_per_sched);
+                  sub_core_model, reg_id, m_num_banks_per_sched);
+  }
+  for (unsigned j = 0; j < m_dispatch_units.size(); j++) {
+    m_dispatch_units[j].init(sub_core_model,m_num_warp_scheds);
   }
   m_initialized = true;
 }
@@ -3976,7 +4127,22 @@ void opndcoll_rfu_t::allocate_cu(unsigned port_num) {
       for (unsigned j = 0; j < inp.m_cu_sets.size(); j++) {
         std::vector<collector_unit_t> &cu_set = m_cus[inp.m_cu_sets[j]];
         bool allocated = false;
-        for (unsigned k = 0; k < cu_set.size(); k++) {
+        unsigned cuLowerBound = 0;
+        unsigned cuUpperBound = cu_set.size();
+        unsigned schd_id;
+        if (sub_core_model) {
+          // Sub core model only allocates on the subset of CUs assigned to the
+          // scheduler that issued
+          unsigned reg_id = (*inp.m_in[i]).get_ready_reg_id();
+          schd_id = (*inp.m_in[i]).get_schd_id(reg_id);
+          assert(cu_set.size() % m_num_warp_scheds == 0 &&
+                 cu_set.size() >= m_num_warp_scheds);
+          unsigned cusPerSched = cu_set.size() / m_num_warp_scheds;
+          cuLowerBound = schd_id * cusPerSched;
+          cuUpperBound = cuLowerBound + cusPerSched;
+          assert(0 <= cuLowerBound && cuUpperBound <= cu_set.size());
+        }
+        for (unsigned k = cuLowerBound; k < cuUpperBound; k++) {
           if (cu_set[k].is_free()) {
             collector_unit_t *cu = &cu_set[k];
             allocated = cu->allocate(inp.m_in[i], inp.m_out[i]);
@@ -3986,8 +4152,9 @@ void opndcoll_rfu_t::allocate_cu(unsigned port_num) {
         }
         if (allocated) break;  // cu has been allocated, no need to search more.
       }
-      break;  // can only service a single input, if it failed it will fail for
-              // others.
+      // break;  // can only service a single input, if it failed it will fail
+      // for
+      // others.
     }
   }
 }
@@ -4034,7 +4201,8 @@ void opndcoll_rfu_t::allocate_reads() {
 }
 
 bool opndcoll_rfu_t::collector_unit_t::ready() const {
-  return (!m_free) && m_not_ready.none() && (*m_output_register).has_free();
+  return (!m_free) && m_not_ready.none() &&
+         (*m_output_register).has_free(m_sub_core_model, m_reg_id);
 }
 
 void opndcoll_rfu_t::collector_unit_t::dump(
@@ -4052,12 +4220,10 @@ void opndcoll_rfu_t::collector_unit_t::dump(
   }
 }
 
-void opndcoll_rfu_t::collector_unit_t::init(unsigned n, unsigned num_banks,
-                                            unsigned log2_warp_size,
-                                            const core_config *config,
-                                            opndcoll_rfu_t *rfu,
-                                            bool sub_core_model,
-                                            unsigned banks_per_sched) {
+void opndcoll_rfu_t::collector_unit_t::init(
+    unsigned n, unsigned num_banks, unsigned log2_warp_size,
+    const core_config *config, opndcoll_rfu_t *rfu, bool sub_core_model,
+    unsigned reg_id, unsigned banks_per_sched) {
   m_rfu = rfu;
   m_cuid = n;
   m_num_banks = num_banks;
@@ -4065,6 +4231,7 @@ void opndcoll_rfu_t::collector_unit_t::init(unsigned n, unsigned num_banks,
   m_warp = new warp_inst_t(config);
   m_bank_warp_shift = log2_warp_size;
   m_sub_core_model = sub_core_model;
+  m_reg_id = reg_id;
   m_num_banks_per_sched = banks_per_sched;
 }
 
@@ -4077,12 +4244,19 @@ bool opndcoll_rfu_t::collector_unit_t::allocate(register_set *pipeline_reg_set,
   warp_inst_t **pipeline_reg = pipeline_reg_set->get_ready();
   if ((pipeline_reg) and !((*pipeline_reg)->empty())) {
     m_warp_id = (*pipeline_reg)->warp_id();
+    std::vector<int> prev_regs; // remove duplicate regs within same instr
     for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) {
       int reg_num =
           (*pipeline_reg)
               ->arch_reg.src[op];  // this math needs to match that used in
                                    // function_info::ptx_decode_inst
-      if (reg_num >= 0) {          // valid register
+      bool new_reg = true;
+      for (auto r : prev_regs) {
+        if (r == reg_num)
+          new_reg = false;
+      }
+      if (reg_num >= 0 && new_reg) {          // valid register
+        prev_regs.push_back(reg_num);
         m_src_op[op] = op_t(this, op, reg_num, m_num_banks, m_bank_warp_shift,
                             m_sub_core_model, m_num_banks_per_sched,
                             (*pipeline_reg)->get_schd_id());
@@ -4099,8 +4273,7 @@ bool opndcoll_rfu_t::collector_unit_t::allocate(register_set *pipeline_reg_set,
 
 void opndcoll_rfu_t::collector_unit_t::dispatch() {
   assert(m_not_ready.none());
-  // move_warp(*m_output_register,m_warp);
-  m_output_register->move_in(m_warp);
+  m_output_register->move_in(m_sub_core_model, m_reg_id, m_warp);
   m_free = true;
   m_output_register = NULL;
   for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) m_src_op[i].reset();
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 6481790..deea1c9 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1,19 +1,21 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung, Andrew Turner,
-// Ali Bakhoda
-// The University of British Columbia
+// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Andrew Turner,
+// Ali Bakhoda, Vijay Kandiah, Nikos Hardavellas, 
+// Mahmoud Khairy, Junrui Pan, Timothy G. Rogers
+// The University of British Columbia, Northwestern University, Purdue University
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer;
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution;
+// 3. Neither the names of The University of British Columbia, Northwestern 
+//    University nor the names of their contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -169,6 +171,7 @@ class shd_warp_t {
   void clear_membar() { m_membar = false; }
   bool get_membar() const { return m_membar; }
   virtual address_type get_pc() const { return m_next_pc; }
+  virtual kernel_info_t* get_kernel_info() const;
   void set_next_pc(address_type pc) { m_next_pc = pc; }
 
   void store_info_of_last_inst_at_barrier(const warp_inst_t *pI) {
@@ -238,7 +241,10 @@ class shd_warp_t {
   unsigned get_dynamic_warp_id() const { return m_dynamic_warp_id; }
   unsigned get_warp_id() const { return m_warp_id; }
 
-  class shader_core_ctx * get_shader() { return m_shader; }
+  class shader_core_ctx *get_shader() {
+    return m_shader;
+  }
+
  private:
   static const unsigned IBUFFER_SIZE = 2;
   class shader_core_ctx *m_shader;
@@ -318,6 +324,7 @@ enum concrete_scheduler {
   CONCRETE_SCHEDULER_LRR = 0,
   CONCRETE_SCHEDULER_GTO,
   CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE,
+  CONCRETE_SCHEDULER_RRR,
   CONCRETE_SCHEDULER_WARP_LIMITING,
   CONCRETE_SCHEDULER_OLDEST_FIRST,
   NUM_CONCRETE_SCHEDULERS
@@ -369,6 +376,12 @@ class scheduler_unit {  // this can be copied freely, so can be used in std
       const typename std::vector<T> &input_list,
       const typename std::vector<T>::const_iterator &last_issued_from_input,
       unsigned num_warps_to_add);
+  template <typename T>
+  void order_rrr(
+      typename std::vector<T> &result_list,
+      const typename std::vector<T> &input_list,
+      const typename std::vector<T>::const_iterator &last_issued_from_input,
+      unsigned num_warps_to_add);
 
   enum OrderingType {
     // The item that issued last is prioritized first then the sorted result
@@ -427,6 +440,8 @@ class scheduler_unit {  // this can be copied freely, so can be used in std
   register_set *m_tensor_core_out;
   register_set *m_mem_out;
   std::vector<register_set *> &m_spec_cores_out;
+  unsigned m_num_issued_last_cycle;
+  unsigned m_current_turn_warp;
 
   int m_id;
 };
@@ -450,6 +465,25 @@ class lrr_scheduler : public scheduler_unit {
   }
 };
 
+class rrr_scheduler : public scheduler_unit {
+ public:
+  rrr_scheduler(shader_core_stats *stats, shader_core_ctx *shader,
+                Scoreboard *scoreboard, simt_stack **simt,
+                std::vector<shd_warp_t *> *warp, register_set *sp_out,
+                register_set *dp_out, register_set *sfu_out,
+                register_set *int_out, register_set *tensor_core_out,
+                std::vector<register_set *> &spec_cores_out,
+                register_set *mem_out, int id)
+      : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out,
+                       sfu_out, int_out, tensor_core_out, spec_cores_out,
+                       mem_out, id) {}
+  virtual ~rrr_scheduler() {}
+  virtual void order_warps();
+  virtual void done_adding_supervised_warps() {
+    m_last_supervised_issued = m_supervised_warps.end();
+  }
+};
+
 class gto_scheduler : public scheduler_unit {
  public:
   gto_scheduler(shader_core_stats *stats, shader_core_ctx *shader,
@@ -878,11 +912,13 @@ class opndcoll_rfu_t {  // operand collector based register file unit
     }
     unsigned get_sp_op() const { return m_warp->sp_op; }
     unsigned get_id() const { return m_cuid; }  // returns CU hw id
+    unsigned get_reg_id() const { return m_reg_id; }
 
     // modifiers
     void init(unsigned n, unsigned num_banks, unsigned log2_warp_size,
               const core_config *config, opndcoll_rfu_t *rfu,
-              bool m_sub_core_model, unsigned num_banks_per_sched);
+              bool m_sub_core_model, unsigned reg_id,
+              unsigned num_banks_per_sched);
     bool allocate(register_set *pipeline_reg, register_set *output_reg);
 
     void collect_operand(unsigned op) { m_not_ready.reset(op); }
@@ -906,6 +942,7 @@ class opndcoll_rfu_t {  // operand collector based register file unit
 
     unsigned m_num_banks_per_sched;
     bool m_sub_core_model;
+    unsigned m_reg_id;  // if sub_core_model enabled, limit regs this cu can r/w
   };
 
   class dispatch_unit_t {
@@ -916,10 +953,19 @@ class opndcoll_rfu_t {  // operand collector based register file unit
       m_num_collectors = (*cus).size();
       m_next_cu = 0;
     }
+    void init(bool sub_core_model, unsigned num_warp_scheds) {
+      m_sub_core_model = sub_core_model;
+      m_num_warp_scheds = num_warp_scheds;
+    }
 
     collector_unit_t *find_ready() {
+      // With sub-core enabled round robin starts with the next cu assigned to a
+      // different sub-core than the one that dispatched last
+      unsigned cusPerSched = m_num_collectors / m_num_warp_scheds;
+      unsigned rr_increment = m_sub_core_model ?
+                              cusPerSched - (m_last_cu % cusPerSched) : 1;
       for (unsigned n = 0; n < m_num_collectors; n++) {
-        unsigned c = (m_last_cu + n + 1) % m_num_collectors;
+        unsigned c = (m_last_cu + n + rr_increment) % m_num_collectors;
         if ((*m_collector_units)[c].ready()) {
           m_last_cu = c;
           return &((*m_collector_units)[c]);
@@ -933,6 +979,8 @@ class opndcoll_rfu_t {  // operand collector based register file unit
     std::vector<collector_unit_t> *m_collector_units;
     unsigned m_last_cu;  // dispatch ready cu's rr
     unsigned m_next_cu;  // for initialization
+    bool m_sub_core_model;
+    unsigned m_num_warp_scheds;
   };
 
   // opndcoll_rfu_t data members
@@ -947,7 +995,7 @@ class opndcoll_rfu_t {  // operand collector based register file unit
   arbiter_t m_arbiter;
 
   unsigned m_num_banks_per_sched;
-  unsigned m_num_warp_sceds;
+  unsigned m_num_warp_scheds;
   bool sub_core_model;
 
   // unsigned m_num_ports;
@@ -1039,10 +1087,7 @@ class simd_function_unit {
   ~simd_function_unit() { delete m_dispatch_reg; }
 
   // modifiers
-  virtual void issue(register_set &source_reg) {
-    source_reg.move_out_to(m_dispatch_reg);
-    occupied.set(m_dispatch_reg->latency);
-  }
+  virtual void issue(register_set &source_reg);
   virtual void cycle() = 0;
   virtual void active_lanes_in_pipeline() = 0;
 
@@ -1051,6 +1096,8 @@ class simd_function_unit {
   virtual bool can_issue(const warp_inst_t &inst) const {
     return m_dispatch_reg->empty() && !occupied.test(inst.latency);
   }
+  virtual bool is_issue_partitioned() = 0;
+  virtual unsigned get_issue_reg_id() = 0;
   virtual bool stallable() const = 0;
   virtual void print(FILE *fp) const {
     fprintf(fp, "%s dispatch= ", m_name.c_str());
@@ -1070,7 +1117,7 @@ class pipelined_simd_unit : public simd_function_unit {
  public:
   pipelined_simd_unit(register_set *result_port,
                       const shader_core_config *config, unsigned max_latency,
-                      shader_core_ctx *core);
+                      shader_core_ctx *core, unsigned issue_reg_id);
 
   // modifiers
   virtual void cycle();
@@ -1091,6 +1138,8 @@ class pipelined_simd_unit : public simd_function_unit {
   virtual bool can_issue(const warp_inst_t &inst) const {
     return simd_function_unit::can_issue(inst);
   }
+  virtual bool is_issue_partitioned() = 0;
+  unsigned get_issue_reg_id() { return m_issue_reg_id; }
   virtual void print(FILE *fp) const {
     simd_function_unit::print(fp);
     for (int s = m_pipeline_depth - 1; s >= 0; s--) {
@@ -1106,6 +1155,8 @@ class pipelined_simd_unit : public simd_function_unit {
   warp_inst_t **m_pipeline_reg;
   register_set *m_result_port;
   class shader_core_ctx *m_core;
+  unsigned m_issue_reg_id;  // if sub_core_model is enabled we can only issue
+                            // from a subset of operand collectors
 
   unsigned active_insts_in_pipeline;
 };
@@ -1113,7 +1164,7 @@ class pipelined_simd_unit : public simd_function_unit {
 class sfu : public pipelined_simd_unit {
  public:
   sfu(register_set *result_port, const shader_core_config *config,
-      shader_core_ctx *core);
+      shader_core_ctx *core, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     switch (inst.op) {
       case SFU_OP:
@@ -1129,12 +1180,13 @@ class sfu : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 };
 
 class dp_unit : public pipelined_simd_unit {
  public:
   dp_unit(register_set *result_port, const shader_core_config *config,
-          shader_core_ctx *core);
+          shader_core_ctx *core, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     switch (inst.op) {
       case DP_OP:
@@ -1146,12 +1198,13 @@ class dp_unit : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 };
 
 class tensor_core : public pipelined_simd_unit {
  public:
   tensor_core(register_set *result_port, const shader_core_config *config,
-              shader_core_ctx *core);
+              shader_core_ctx *core, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     switch (inst.op) {
       case TENSOR_CORE_OP:
@@ -1163,12 +1216,13 @@ class tensor_core : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 };
 
 class int_unit : public pipelined_simd_unit {
  public:
   int_unit(register_set *result_port, const shader_core_config *config,
-           shader_core_ctx *core);
+           shader_core_ctx *core, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     switch (inst.op) {
       case SFU_OP:
@@ -1194,12 +1248,13 @@ class int_unit : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 };
 
 class sp_unit : public pipelined_simd_unit {
  public:
   sp_unit(register_set *result_port, const shader_core_config *config,
-          shader_core_ctx *core);
+          shader_core_ctx *core, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     switch (inst.op) {
       case SFU_OP:
@@ -1223,13 +1278,14 @@ class sp_unit : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 };
 
 class specialized_unit : public pipelined_simd_unit {
  public:
   specialized_unit(register_set *result_port, const shader_core_config *config,
                    shader_core_ctx *core, unsigned supported_op,
-                   char *unit_name, unsigned latency);
+                   char *unit_name, unsigned latency, unsigned issue_reg_id);
   virtual bool can_issue(const warp_inst_t &inst) const {
     if (inst.op != m_supported_op) {
       return false;
@@ -1238,6 +1294,7 @@ class specialized_unit : public pipelined_simd_unit {
   }
   virtual void active_lanes_in_pipeline();
   virtual void issue(register_set &source_reg);
+  bool is_issue_partitioned() { return true; }
 
  private:
   unsigned m_supported_op;
@@ -1259,6 +1316,7 @@ class ldst_unit : public pipelined_simd_unit {
 
   // modifiers
   virtual void issue(register_set &inst);
+  bool is_issue_partitioned() { return false; }
   virtual void cycle();
 
   void fill(mem_fetch *mf);
@@ -1479,6 +1537,17 @@ class shader_core_config : public core_config {
       } else
         break;  // we only accept continuous specialized_units, i.e., 1,2,3,4
     }
+
+    // parse gpgpu_shmem_option for adpative cache config
+    if (adaptive_cache_config) {
+      std::stringstream ss(gpgpu_shmem_option);
+      while (ss.good()) {
+        std::string option;
+        std::getline(ss, option, ',');
+        shmem_opt_list.push_back((unsigned)std::stoi(option) * 1024);
+      }
+      std::sort(shmem_opt_list.begin(), shmem_opt_list.end());
+    }
   }
   void reg_options(class OptionParser *opp);
   unsigned max_cta(const kernel_info_t &k) const;
@@ -1619,18 +1688,26 @@ struct shader_core_stats_pod {
   unsigned *m_num_INTdecoded_insn;
   unsigned *m_num_storequeued_insn;
   unsigned *m_num_loadqueued_insn;
-  unsigned *m_num_ialu_acesses;
-  unsigned *m_num_fp_acesses;
-  unsigned *m_num_imul_acesses;
   unsigned *m_num_tex_inst;
-  unsigned *m_num_fpmul_acesses;
-  unsigned *m_num_idiv_acesses;
-  unsigned *m_num_fpdiv_acesses;
-  unsigned *m_num_sp_acesses;
-  unsigned *m_num_sfu_acesses;
-  unsigned *m_num_tensor_core_acesses;
-  unsigned *m_num_trans_acesses;
-  unsigned *m_num_mem_acesses;
+  double *m_num_ialu_acesses;
+  double *m_num_fp_acesses;
+  double *m_num_imul_acesses;
+  double *m_num_fpmul_acesses;
+  double *m_num_idiv_acesses;
+  double *m_num_fpdiv_acesses;
+  double *m_num_sp_acesses;
+  double *m_num_sfu_acesses;
+  double *m_num_tensor_core_acesses;
+  double *m_num_tex_acesses;
+  double *m_num_const_acesses;
+  double *m_num_dp_acesses;
+  double *m_num_dpmul_acesses;
+  double *m_num_dpdiv_acesses;
+  double *m_num_sqrt_acesses;
+  double *m_num_log_acesses;
+  double *m_num_sin_acesses;
+  double *m_num_exp_acesses;
+  double *m_num_mem_acesses;
   unsigned *m_num_sp_committed;
   unsigned *m_num_tlb_hits;
   unsigned *m_num_tlb_accesses;
@@ -1640,13 +1717,15 @@ struct shader_core_stats_pod {
   unsigned *m_read_regfile_acesses;
   unsigned *m_write_regfile_acesses;
   unsigned *m_non_rf_operands;
-  unsigned *m_num_imul24_acesses;
-  unsigned *m_num_imul32_acesses;
+  double *m_num_imul24_acesses;
+  double *m_num_imul32_acesses;
   unsigned *m_active_sp_lanes;
   unsigned *m_active_sfu_lanes;
   unsigned *m_active_tensor_core_lanes;
   unsigned *m_active_fu_lanes;
   unsigned *m_active_fu_mem_lanes;
+  double *m_active_exu_threads; //For power model
+  double *m_active_exu_warps; //For power model
   unsigned *m_n_diverge;  // number of divergence occurring in this shader
   unsigned gpgpu_n_load_insn;
   unsigned gpgpu_n_store_insn;
@@ -1717,38 +1796,56 @@ class shader_core_stats : public shader_core_stats_pod {
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_num_loadqueued_insn =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+    m_num_tex_inst = 
+        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_num_INTdecoded_insn =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_num_ialu_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_fp_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
-    m_num_tex_inst = (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_imul_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_imul24_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_imul32_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_fpmul_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_idiv_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_fpdiv_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_num_dp_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
+    m_num_dpmul_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
+    m_num_dpdiv_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
     m_num_sp_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_sfu_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
-    m_num_tensor_core_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
-    m_num_trans_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_num_tensor_core_acesses = 
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_num_const_acesses =
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_num_tex_acesses =
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_num_sqrt_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
+    m_num_log_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
+    m_num_sin_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
+    m_num_exp_acesses = 
+        (double*) calloc(config->num_shader(),sizeof(double));
     m_num_mem_acesses =
-        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_num_sp_committed =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
-    m_num_tlb_hits = (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+    m_num_tlb_hits = 
+        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_num_tlb_accesses =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_active_sp_lanes =
@@ -1759,6 +1856,10 @@ class shader_core_stats : public shader_core_stats_pod {
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_active_fu_lanes =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+    m_active_exu_threads =
+        (double *)calloc(config->num_shader(), sizeof(double));
+    m_active_exu_warps =
+        (double *)calloc(config->num_shader(), sizeof(double));
     m_active_fu_mem_lanes =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_num_sfu_committed =
@@ -1773,7 +1874,8 @@ class shader_core_stats : public shader_core_stats_pod {
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     m_non_rf_operands =
         (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
-    m_n_diverge = (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
+    m_n_diverge = 
+        (unsigned *)calloc(config->num_shader(), sizeof(unsigned));
     shader_cycle_distro =
         (unsigned *)calloc(config->warp_size + 3, sizeof(unsigned));
     last_shader_cycle_distro =
@@ -1802,6 +1904,48 @@ class shader_core_stats : public shader_core_stats_pod {
     delete m_incoming_traffic_stats;
     free(m_num_sim_insn);
     free(m_num_sim_winsn);
+    free(m_num_FPdecoded_insn);
+    free(m_num_INTdecoded_insn);
+    free(m_num_storequeued_insn);
+    free(m_num_loadqueued_insn);
+    free(m_num_ialu_acesses);
+    free(m_num_fp_acesses);
+    free(m_num_imul_acesses);
+    free(m_num_tex_inst);
+    free(m_num_fpmul_acesses);
+    free(m_num_idiv_acesses);
+    free(m_num_fpdiv_acesses);
+    free(m_num_sp_acesses);
+    free(m_num_sfu_acesses);
+    free(m_num_tensor_core_acesses);
+    free(m_num_tex_acesses);
+    free(m_num_const_acesses);
+    free(m_num_dp_acesses);
+    free(m_num_dpmul_acesses);
+    free(m_num_dpdiv_acesses);
+    free(m_num_sqrt_acesses);
+    free(m_num_log_acesses);
+    free(m_num_sin_acesses);
+    free(m_num_exp_acesses);
+    free(m_num_mem_acesses);
+    free(m_num_sp_committed);
+    free(m_num_tlb_hits);
+    free(m_num_tlb_accesses);
+    free(m_num_sfu_committed);
+    free(m_num_tensor_core_committed);
+    free(m_num_mem_committed);
+    free(m_read_regfile_acesses);
+    free(m_write_regfile_acesses);
+    free(m_non_rf_operands);
+    free(m_num_imul24_acesses);
+    free(m_num_imul32_acesses);
+    free(m_active_sp_lanes);
+    free(m_active_sfu_lanes);
+    free(m_active_tensor_core_lanes);
+    free(m_active_fu_lanes);
+    free(m_active_exu_threads);
+    free(m_active_exu_warps);
+    free(m_active_fu_mem_lanes);
     free(m_n_diverge);
     free(shader_cycle_distro);
     free(last_shader_cycle_distro);
@@ -1856,6 +2000,12 @@ class shader_core_mem_fetch_allocator : public mem_fetch_allocator {
   }
   mem_fetch *alloc(new_addr_type addr, mem_access_type type, unsigned size,
                    bool wr, unsigned long long cycle) const;
+  mem_fetch *alloc(new_addr_type addr, mem_access_type type,
+                   const active_mask_t &active_mask,
+                   const mem_access_byte_mask_t &byte_mask,
+                   const mem_access_sector_mask_t &sector_mask, unsigned size,
+                   bool wr, unsigned long long cycle, unsigned wid,
+                   unsigned sid, unsigned tpc, mem_fetch *original_mf) const;
   mem_fetch *alloc(const warp_inst_t &inst, const mem_access_t &access,
                    unsigned long long cycle) const {
     warp_inst_t inst_copy = inst;
@@ -1900,7 +2050,7 @@ class shader_core_ctx : public core_t {
     printf("GPGPU-Sim uArch: Shader %d bind to kernel %u \'%s\'\n", m_sid,
            m_kernel->get_uid(), m_kernel->name().c_str());
   }
-
+  PowerscalingCoefficients *scaling_coeffs;
   // accessors
   bool fetch_unit_response_buffer_full() const;
   bool ldst_unit_response_buffer_full() const;
@@ -1958,119 +2108,206 @@ class shader_core_ctx : public core_t {
 
   void incload_stat() { m_stats->m_num_loadqueued_insn[m_sid]++; }
   void incstore_stat() { m_stats->m_num_storequeued_insn[m_sid]++; }
-  void incialu_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_ialu_acesses[m_sid] =
-          m_stats->m_num_ialu_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_nonsfu(active_count, latency);
-    } else {
-      m_stats->m_num_ialu_acesses[m_sid] =
-          m_stats->m_num_ialu_acesses[m_sid] + active_count * latency;
+  void incialu_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_ialu_acesses[m_sid]=m_stats->m_num_ialu_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+      m_stats->m_num_ialu_acesses[m_sid]=m_stats->m_num_ialu_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void inctex_stat(unsigned active_count, double latency) {
-    m_stats->m_num_tex_inst[m_sid] =
-        m_stats->m_num_tex_inst[m_sid] + active_count * latency;
-  }
-  void incimul_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_imul_acesses[m_sid] =
-          m_stats->m_num_imul_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_nonsfu(active_count, latency);
-    } else {
-      m_stats->m_num_imul_acesses[m_sid] =
-          m_stats->m_num_imul_acesses[m_sid] + active_count * latency;
+  void incimul_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_imul_acesses[m_sid]=m_stats->m_num_imul_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+      m_stats->m_num_imul_acesses[m_sid]=m_stats->m_num_imul_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void incimul24_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_imul24_acesses[m_sid] =
-          m_stats->m_num_imul24_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_nonsfu(active_count, latency);
-    } else {
-      m_stats->m_num_imul24_acesses[m_sid] =
-          m_stats->m_num_imul24_acesses[m_sid] + active_count * latency;
+  void incimul24_stat(unsigned active_count,double latency) {
+  if(m_config->gpgpu_clock_gated_lanes==false){
+    m_stats->m_num_imul24_acesses[m_sid]=m_stats->m_num_imul24_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+      m_stats->m_num_imul24_acesses[m_sid]=m_stats->m_num_imul24_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;    
+   }
+   void incimul32_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_imul32_acesses[m_sid]=m_stats->m_num_imul32_acesses[m_sid]+(double)active_count*latency
+         + inactive_lanes_accesses_sfu(active_count, latency);          
+    }else{
+      m_stats->m_num_imul32_acesses[m_sid]=m_stats->m_num_imul32_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void incimul32_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_imul32_acesses[m_sid] =
-          m_stats->m_num_imul32_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_sfu(active_count, latency);
-    } else {
-      m_stats->m_num_imul32_acesses[m_sid] =
-          m_stats->m_num_imul32_acesses[m_sid] + active_count * latency;
+   void incidiv_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_idiv_acesses[m_sid]=m_stats->m_num_idiv_acesses[m_sid]+(double)active_count*latency
+         + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else {
+      m_stats->m_num_idiv_acesses[m_sid]=m_stats->m_num_idiv_acesses[m_sid]+(double)active_count*latency;
     }
-    // printf("Int_Mul -- Active_count: %d\n",active_count);
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;    
   }
-  void incidiv_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_idiv_acesses[m_sid] =
-          m_stats->m_num_idiv_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_sfu(active_count, latency);
-    } else {
-      m_stats->m_num_idiv_acesses[m_sid] =
-          m_stats->m_num_idiv_acesses[m_sid] + active_count * latency;
+   void incfpalu_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_fp_acesses[m_sid]=m_stats->m_num_fp_acesses[m_sid]+(double)active_count*latency
+         + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+    m_stats->m_num_fp_acesses[m_sid]=m_stats->m_num_fp_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;     
   }
-  void incfpalu_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_fp_acesses[m_sid] =
-          m_stats->m_num_fp_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_nonsfu(active_count, latency);
-    } else {
-      m_stats->m_num_fp_acesses[m_sid] =
-          m_stats->m_num_fp_acesses[m_sid] + active_count * latency;
+   void incfpmul_stat(unsigned active_count,double latency) {
+              // printf("FP MUL stat increament\n");
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_fpmul_acesses[m_sid]=m_stats->m_num_fpmul_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+    m_stats->m_num_fpmul_acesses[m_sid]=m_stats->m_num_fpmul_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+   void incfpdiv_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_fpdiv_acesses[m_sid]=m_stats->m_num_fpdiv_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else {
+      m_stats->m_num_fpdiv_acesses[m_sid]=m_stats->m_num_fpdiv_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+   void incdpalu_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_dp_acesses[m_sid]=m_stats->m_num_dp_acesses[m_sid]+(double)active_count*latency
+         + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+    m_stats->m_num_dp_acesses[m_sid]=m_stats->m_num_dp_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++; 
+   }
+   void incdpmul_stat(unsigned active_count,double latency) {
+              // printf("FP MUL stat increament\n");
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_dpmul_acesses[m_sid]=m_stats->m_num_dpmul_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_nonsfu(active_count, latency);
+    }else {
+    m_stats->m_num_dpmul_acesses[m_sid]=m_stats->m_num_dpmul_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+   void incdpdiv_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_dpdiv_acesses[m_sid]=m_stats->m_num_dpdiv_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else {
+      m_stats->m_num_dpdiv_acesses[m_sid]=m_stats->m_num_dpdiv_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+
+   void incsqrt_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_sqrt_acesses[m_sid]=m_stats->m_num_sqrt_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_sqrt_acesses[m_sid]=m_stats->m_num_sqrt_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+
+   void inclog_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_log_acesses[m_sid]=m_stats->m_num_log_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_log_acesses[m_sid]=m_stats->m_num_log_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+   }
+
+   void incexp_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_exp_acesses[m_sid]=m_stats->m_num_exp_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_exp_acesses[m_sid]=m_stats->m_num_exp_acesses[m_sid]+(double)active_count*latency;
+    }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void incfpmul_stat(unsigned active_count, double latency) {
-    // printf("FP MUL stat increament\n");
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_fpmul_acesses[m_sid] =
-          m_stats->m_num_fpmul_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_nonsfu(active_count, latency);
-    } else {
-      m_stats->m_num_fpmul_acesses[m_sid] =
-          m_stats->m_num_fpmul_acesses[m_sid] + active_count * latency;
+
+   void incsin_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_sin_acesses[m_sid]=m_stats->m_num_sin_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_sin_acesses[m_sid]=m_stats->m_num_sin_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void incfpdiv_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_fpdiv_acesses[m_sid] =
-          m_stats->m_num_fpdiv_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_sfu(active_count, latency);
-    } else {
-      m_stats->m_num_fpdiv_acesses[m_sid] =
-          m_stats->m_num_fpdiv_acesses[m_sid] + active_count * latency;
+
+
+   void inctensor_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_tensor_core_acesses[m_sid]=m_stats->m_num_tensor_core_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_tensor_core_acesses[m_sid]=m_stats->m_num_tensor_core_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
   }
-  void inctrans_stat(unsigned active_count, double latency) {
-    if (m_config->gpgpu_clock_gated_lanes == false) {
-      m_stats->m_num_trans_acesses[m_sid] =
-          m_stats->m_num_trans_acesses[m_sid] + active_count * latency +
-          inactive_lanes_accesses_sfu(active_count, latency);
-    } else {
-      m_stats->m_num_trans_acesses[m_sid] =
-          m_stats->m_num_trans_acesses[m_sid] + active_count * latency;
+
+  void inctex_stat(unsigned active_count,double latency) {
+    if(m_config->gpgpu_clock_gated_lanes==false){
+      m_stats->m_num_tex_acesses[m_sid]=m_stats->m_num_tex_acesses[m_sid]+(double)active_count*latency
+        + inactive_lanes_accesses_sfu(active_count, latency); 
+    }else{
+      m_stats->m_num_tex_acesses[m_sid]=m_stats->m_num_tex_acesses[m_sid]+(double)active_count*latency;
     }
+    m_stats->m_active_exu_threads[m_sid]+=active_count;
+    m_stats->m_active_exu_warps[m_sid]++;
+  }
+
+  void inc_const_accesses(unsigned active_count) {
+    m_stats->m_num_const_acesses[m_sid]=m_stats->m_num_const_acesses[m_sid]+active_count;
   }
 
   void incsfu_stat(unsigned active_count, double latency) {
     m_stats->m_num_sfu_acesses[m_sid] =
-        m_stats->m_num_sfu_acesses[m_sid] + active_count * latency;
+        m_stats->m_num_sfu_acesses[m_sid] + (double)active_count*latency;
   }
   void incsp_stat(unsigned active_count, double latency) {
     m_stats->m_num_sp_acesses[m_sid] =
-        m_stats->m_num_sp_acesses[m_sid] + active_count * latency;
+        m_stats->m_num_sp_acesses[m_sid] + (double)active_count*latency;
   }
   void incmem_stat(unsigned active_count, double latency) {
     if (m_config->gpgpu_clock_gated_lanes == false) {
       m_stats->m_num_mem_acesses[m_sid] =
-          m_stats->m_num_mem_acesses[m_sid] + active_count * latency +
+          m_stats->m_num_mem_acesses[m_sid] + (double)active_count*latency +
           inactive_lanes_accesses_nonsfu(active_count, latency);
     } else {
       m_stats->m_num_mem_acesses[m_sid] =
-          m_stats->m_num_mem_acesses[m_sid] + active_count * latency;
+          m_stats->m_num_mem_acesses[m_sid] + (double)active_count*latency;
     }
   }
   void incexecstat(warp_inst_t *&inst);
@@ -2133,8 +2370,8 @@ class shader_core_ctx : public core_t {
   friend class TwoLevelScheduler;
   friend class LooseRoundRobbinScheduler;
   virtual void issue_warp(register_set &warp, const warp_inst_t *pI,
-                  const active_mask_t &active_mask, unsigned warp_id,
-                  unsigned sch_id);
+                          const active_mask_t &active_mask, unsigned warp_id,
+                          unsigned sch_id);
 
   void create_front_pipeline();
   void create_schedulers();
diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc
index 6fafaa6..0513d17 100644
--- a/src/gpgpu-sim/stat-tool.cc
+++ b/src/gpgpu-sim/stat-tool.cc
@@ -369,8 +369,6 @@ void shader_mem_lat_print(FILE *fout) {
 static int s_cache_access_logger_n_types = 0;
 static std::vector<linear_histogram_logger> s_cache_access_logger;
 
-enum cache_access_logger_types { NORMALS, TEXTURE, CONSTANT, INSTRUCTION };
-
 int get_shader_normal_cache_id() { return NORMALS; }
 int get_shader_texture_cache_id() { return TEXTURE; }
 int get_shader_constant_cache_id() { return CONSTANT; }
diff --git a/src/gpgpu-sim/stat-tool.h b/src/gpgpu-sim/stat-tool.h
index 3a291be..fdf8756 100644
--- a/src/gpgpu-sim/stat-tool.h
+++ b/src/gpgpu-sim/stat-tool.h
@@ -268,6 +268,8 @@ class linear_histogram_logger : public snap_shot_trigger,
   static int s_ids;
 };
 
+enum cache_access_logger_types { NORMALS, TEXTURE, CONSTANT, INSTRUCTION };
+
 void try_snap_shot(unsigned long long current_cycle);
 void set_spill_interval(unsigned long long interval);
 void spill_log_to_file(FILE *fout, int final, unsigned long long current_cycle);
diff --git a/src/gpuwattch/gpgpu_sim_wrapper.cc b/src/gpuwattch/gpgpu_sim_wrapper.cc
deleted file mode 100644
index f2989f6..0000000
--- a/src/gpuwattch/gpgpu_sim_wrapper.cc
+++ /dev/null
@@ -1,863 +0,0 @@
-// Copyright (c) 2009-2011, Tor M. Aamodt, Tayler Hetherington, Ahmed ElTantawy,
-// The University of British Columbia
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// Redistributions of source code must retain the above copyright notice, this
-// list of conditions and the following disclaimer.
-// Redistributions in binary form must reproduce the above copyright notice,
-// this list of conditions and the following disclaimer in the documentation
-// and/or other materials provided with the distribution. Neither the name of
-// The University of British Columbia nor the names of its contributors may be
-// used to endorse or promote products derived from this software without
-// specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "gpgpu_sim_wrapper.h"
-#include <sys/stat.h>
-#define SP_BASE_POWER 0
-#define SFU_BASE_POWER 0
-
-static const char* pwr_cmp_label[] = {
-    "IBP,", "ICP,",  "DCP,",   "TCP,",   "CCP,",        "SHRDP,",
-    "RFP,", "SPP,",  "SFUP,",  "FPUP,",  "SCHEDP,",     "L2CP,",
-    "MCP,", "NOCP,", "DRAMP,", "PIPEP,", "IDLE_COREP,", "CONST_DYNAMICP"};
-
-enum pwr_cmp_t {
-  IBP = 0,
-  ICP,
-  DCP,
-  TCP,
-  CCP,
-  SHRDP,
-  RFP,
-  SPP,
-  SFUP,
-  FPUP,
-  SCHEDP,
-  L2CP,
-  MCP,
-  NOCP,
-  DRAMP,
-  PIPEP,
-  IDLE_COREP,
-  CONST_DYNAMICP,
-  NUM_COMPONENTS_MODELLED
-};
-
-gpgpu_sim_wrapper::gpgpu_sim_wrapper(bool power_simulation_enabled,
-                                     char* xmlfile) {
-  kernel_sample_count = 0;
-  total_sample_count = 0;
-
-  kernel_tot_power = 0;
-
-  num_pwr_cmps = NUM_COMPONENTS_MODELLED;
-  num_perf_counters = NUM_PERFORMANCE_COUNTERS;
-
-  // Initialize per-component counter/power vectors
-  avg_max_min_counters<double> init;
-  kernel_cmp_pwr.resize(NUM_COMPONENTS_MODELLED, init);
-  kernel_cmp_perf_counters.resize(NUM_PERFORMANCE_COUNTERS, init);
-
-  kernel_power = init;   // Per-kernel powers
-  gpu_tot_power = init;  // Global powers
-
-  sample_cmp_pwr.resize(NUM_COMPONENTS_MODELLED, 0);
-
-  sample_perf_counters.resize(NUM_PERFORMANCE_COUNTERS, 0);
-  initpower_coeff.resize(NUM_PERFORMANCE_COUNTERS, 0);
-  effpower_coeff.resize(NUM_PERFORMANCE_COUNTERS, 0);
-
-  const_dynamic_power = 0;
-  proc_power = 0;
-
-  g_power_filename = NULL;
-  g_power_trace_filename = NULL;
-  g_metric_trace_filename = NULL;
-  g_steady_state_tracking_filename = NULL;
-  xml_filename = xmlfile;
-  g_power_simulation_enabled = power_simulation_enabled;
-  g_power_trace_enabled = false;
-  g_steady_power_levels_enabled = false;
-  g_power_trace_zlevel = 0;
-  g_power_per_cycle_dump = false;
-  gpu_steady_power_deviation = 0;
-  gpu_steady_min_period = 0;
-
-  gpu_stat_sample_freq = 0;
-  p = new ParseXML();
-  if (g_power_simulation_enabled) {
-    p->parse(xml_filename);
-  }
-  proc = new Processor(p);
-  power_trace_file = NULL;
-  metric_trace_file = NULL;
-  steady_state_tacking_file = NULL;
-  has_written_avg = false;
-  init_inst_val = false;
-}
-
-gpgpu_sim_wrapper::~gpgpu_sim_wrapper() {}
-
-bool gpgpu_sim_wrapper::sanity_check(double a, double b) {
-  if (b == 0)
-    return (abs(a - b) < 0.00001);
-  else
-    return (abs(a - b) / abs(b) < 0.00001);
-
-  return false;
-}
-void gpgpu_sim_wrapper::init_mcpat(
-    char* xmlfile, char* powerfilename, char* power_trace_filename,
-    char* metric_trace_filename, char* steady_state_filename,
-    bool power_sim_enabled, bool trace_enabled, bool steady_state_enabled,
-    bool power_per_cycle_dump, double steady_power_deviation,
-    double steady_min_period, int zlevel, double init_val,
-    int stat_sample_freq) {
-  // Write File Headers for (-metrics trace, -power trace)
-
-  reset_counters();
-  static bool mcpat_init = true;
-
-  // initialize file name if it is not set
-  time_t curr_time;
-  time(&curr_time);
-  char* date = ctime(&curr_time);
-  char* s = date;
-  while (*s) {
-    if (*s == ' ' || *s == '\t' || *s == ':') *s = '-';
-    if (*s == '\n' || *s == '\r') *s = 0;
-    s++;
-  }
-
-  if (mcpat_init) {
-    g_power_filename = powerfilename;
-    g_power_trace_filename = power_trace_filename;
-    g_metric_trace_filename = metric_trace_filename;
-    g_steady_state_tracking_filename = steady_state_filename;
-    xml_filename = xmlfile;
-    g_power_simulation_enabled = power_sim_enabled;
-    g_power_trace_enabled = trace_enabled;
-    g_steady_power_levels_enabled = steady_state_enabled;
-    g_power_trace_zlevel = zlevel;
-    g_power_per_cycle_dump = power_per_cycle_dump;
-    gpu_steady_power_deviation = steady_power_deviation;
-    gpu_steady_min_period = steady_min_period;
-
-    gpu_stat_sample_freq = stat_sample_freq;
-
-    // p->sys.total_cycles=gpu_stat_sample_freq*4;
-    p->sys.total_cycles = gpu_stat_sample_freq;
-    power_trace_file = NULL;
-    metric_trace_file = NULL;
-    steady_state_tacking_file = NULL;
-
-    if (g_power_trace_enabled) {
-      power_trace_file = gzopen(g_power_trace_filename, "w");
-      metric_trace_file = gzopen(g_metric_trace_filename, "w");
-      if ((power_trace_file == NULL) || (metric_trace_file == NULL)) {
-        printf("error - could not open trace files \n");
-        exit(1);
-      }
-      gzsetparams(power_trace_file, g_power_trace_zlevel, Z_DEFAULT_STRATEGY);
-
-      gzprintf(power_trace_file, "power,");
-      for (unsigned i = 0; i < num_pwr_cmps; i++) {
-        gzprintf(power_trace_file, pwr_cmp_label[i]);
-      }
-      gzprintf(power_trace_file, "\n");
-
-      gzsetparams(metric_trace_file, g_power_trace_zlevel, Z_DEFAULT_STRATEGY);
-      for (unsigned i = 0; i < num_perf_counters; i++) {
-        gzprintf(metric_trace_file, perf_count_label[i]);
-      }
-      gzprintf(metric_trace_file, "\n");
-
-      gzclose(power_trace_file);
-      gzclose(metric_trace_file);
-    }
-    if (g_steady_power_levels_enabled) {
-      steady_state_tacking_file = gzopen(g_steady_state_tracking_filename, "w");
-      if ((steady_state_tacking_file == NULL)) {
-        printf("error - could not open trace files \n");
-        exit(1);
-      }
-      gzsetparams(steady_state_tacking_file, g_power_trace_zlevel,
-                  Z_DEFAULT_STRATEGY);
-      gzprintf(steady_state_tacking_file, "start,end,power,IPC,");
-      for (unsigned i = 0; i < num_perf_counters; i++) {
-        gzprintf(steady_state_tacking_file, perf_count_label[i]);
-      }
-      gzprintf(steady_state_tacking_file, "\n");
-
-      gzclose(steady_state_tacking_file);
-    }
-
-    mcpat_init = false;
-    has_written_avg = false;
-    powerfile.open(g_power_filename);
-    int flg = chmod(g_power_filename, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-    assert(flg == 0);
-  }
-  sample_val = 0;
-  init_inst_val = init_val;  // gpu_tot_sim_insn+gpu_sim_insn;
-}
-
-void gpgpu_sim_wrapper::reset_counters() {
-  avg_max_min_counters<double> init;
-  for (unsigned i = 0; i < num_perf_counters; ++i) {
-    sample_perf_counters[i] = 0;
-    kernel_cmp_perf_counters[i] = init;
-  }
-  for (unsigned i = 0; i < num_pwr_cmps; ++i) {
-    sample_cmp_pwr[i] = 0;
-    kernel_cmp_pwr[i] = init;
-  }
-
-  // Reset per-kernel counters
-  kernel_sample_count = 0;
-  kernel_tot_power = 0;
-  kernel_power = init;
-
-  return;
-}
-
-void gpgpu_sim_wrapper::set_inst_power(bool clk_gated_lanes, double tot_cycles,
-                                       double busy_cycles, double tot_inst,
-                                       double int_inst, double fp_inst,
-                                       double load_inst, double store_inst,
-                                       double committed_inst) {
-  p->sys.core[0].gpgpu_clock_gated_lanes = clk_gated_lanes;
-  p->sys.core[0].total_cycles = tot_cycles;
-  p->sys.core[0].busy_cycles = busy_cycles;
-  p->sys.core[0].total_instructions =
-      tot_inst * p->sys.scaling_coefficients[TOT_INST];
-  p->sys.core[0].int_instructions =
-      int_inst * p->sys.scaling_coefficients[FP_INT];
-  p->sys.core[0].fp_instructions =
-      fp_inst * p->sys.scaling_coefficients[FP_INT];
-  p->sys.core[0].load_instructions = load_inst;
-  p->sys.core[0].store_instructions = store_inst;
-  p->sys.core[0].committed_instructions = committed_inst;
-  sample_perf_counters[FP_INT] = int_inst + fp_inst;
-  sample_perf_counters[TOT_INST] = tot_inst;
-}
-
-void gpgpu_sim_wrapper::set_regfile_power(double reads, double writes,
-                                          double ops) {
-  p->sys.core[0].int_regfile_reads =
-      reads * p->sys.scaling_coefficients[REG_RD];
-  p->sys.core[0].int_regfile_writes =
-      writes * p->sys.scaling_coefficients[REG_WR];
-  p->sys.core[0].non_rf_operands =
-      ops * p->sys.scaling_coefficients[NON_REG_OPs];
-  sample_perf_counters[REG_RD] = reads;
-  sample_perf_counters[REG_WR] = writes;
-  sample_perf_counters[NON_REG_OPs] = ops;
-}
-
-void gpgpu_sim_wrapper::set_icache_power(double hits, double misses) {
-  p->sys.core[0].icache.read_accesses =
-      hits * p->sys.scaling_coefficients[IC_H] +
-      misses * p->sys.scaling_coefficients[IC_M];
-  p->sys.core[0].icache.read_misses =
-      misses * p->sys.scaling_coefficients[IC_M];
-  sample_perf_counters[IC_H] = hits;
-  sample_perf_counters[IC_M] = misses;
-}
-
-void gpgpu_sim_wrapper::set_ccache_power(double hits, double misses) {
-  p->sys.core[0].ccache.read_accesses =
-      hits * p->sys.scaling_coefficients[CC_H] +
-      misses * p->sys.scaling_coefficients[CC_M];
-  p->sys.core[0].ccache.read_misses =
-      misses * p->sys.scaling_coefficients[CC_M];
-  sample_perf_counters[CC_H] = hits;
-  sample_perf_counters[CC_M] = misses;
-  // TODO: coalescing logic is counted as part of the caches power (this is not
-  // valid for no-caches architectures)
-}
-
-void gpgpu_sim_wrapper::set_tcache_power(double hits, double misses) {
-  p->sys.core[0].tcache.read_accesses =
-      hits * p->sys.scaling_coefficients[TC_H] +
-      misses * p->sys.scaling_coefficients[TC_M];
-  p->sys.core[0].tcache.read_misses =
-      misses * p->sys.scaling_coefficients[TC_M];
-  sample_perf_counters[TC_H] = hits;
-  sample_perf_counters[TC_M] = misses;
-  // TODO: coalescing logic is counted as part of the caches power (this is not
-  // valid for no-caches architectures)
-}
-
-void gpgpu_sim_wrapper::set_shrd_mem_power(double accesses) {
-  p->sys.core[0].sharedmemory.read_accesses =
-      accesses * p->sys.scaling_coefficients[SHRD_ACC];
-  sample_perf_counters[SHRD_ACC] = accesses;
-}
-
-void gpgpu_sim_wrapper::set_l1cache_power(double read_hits, double read_misses,
-                                          double write_hits,
-                                          double write_misses) {
-  p->sys.core[0].dcache.read_accesses =
-      read_hits * p->sys.scaling_coefficients[DC_RH] +
-      read_misses * p->sys.scaling_coefficients[DC_RM];
-  p->sys.core[0].dcache.read_misses =
-      read_misses * p->sys.scaling_coefficients[DC_RM];
-  p->sys.core[0].dcache.write_accesses =
-      write_hits * p->sys.scaling_coefficients[DC_WH] +
-      write_misses * p->sys.scaling_coefficients[DC_WM];
-  p->sys.core[0].dcache.write_misses =
-      write_misses * p->sys.scaling_coefficients[DC_WM];
-  sample_perf_counters[DC_RH] = read_hits;
-  sample_perf_counters[DC_RM] = read_misses;
-  sample_perf_counters[DC_WH] = write_hits;
-  sample_perf_counters[DC_WM] = write_misses;
-  // TODO: coalescing logic is counted as part of the caches power (this is not
-  // valid for no-caches architectures)
-}
-
-void gpgpu_sim_wrapper::set_l2cache_power(double read_hits, double read_misses,
-                                          double write_hits,
-                                          double write_misses) {
-  p->sys.l2.total_accesses = read_hits * p->sys.scaling_coefficients[L2_RH] +
-                             read_misses * p->sys.scaling_coefficients[L2_RM] +
-                             write_hits * p->sys.scaling_coefficients[L2_WH] +
-                             write_misses * p->sys.scaling_coefficients[L2_WM];
-  p->sys.l2.read_accesses = read_hits * p->sys.scaling_coefficients[L2_RH] +
-                            read_misses * p->sys.scaling_coefficients[L2_RM];
-  p->sys.l2.write_accesses = write_hits * p->sys.scaling_coefficients[L2_WH] +
-                             write_misses * p->sys.scaling_coefficients[L2_WM];
-  p->sys.l2.read_hits = read_hits * p->sys.scaling_coefficients[L2_RH];
-  p->sys.l2.read_misses = read_misses * p->sys.scaling_coefficients[L2_RM];
-  p->sys.l2.write_hits = write_hits * p->sys.scaling_coefficients[L2_WH];
-  p->sys.l2.write_misses = write_misses * p->sys.scaling_coefficients[L2_WM];
-  sample_perf_counters[L2_RH] = read_hits;
-  sample_perf_counters[L2_RM] = read_misses;
-  sample_perf_counters[L2_WH] = write_hits;
-  sample_perf_counters[L2_WM] = write_misses;
-}
-
-void gpgpu_sim_wrapper::set_idle_core_power(double num_idle_core) {
-  p->sys.num_idle_cores = num_idle_core;
-  sample_perf_counters[IDLE_CORE_N] = num_idle_core;
-}
-
-void gpgpu_sim_wrapper::set_duty_cycle_power(double duty_cycle) {
-  p->sys.core[0].pipeline_duty_cycle =
-      duty_cycle * p->sys.scaling_coefficients[PIPE_A];
-  sample_perf_counters[PIPE_A] = duty_cycle;
-}
-
-void gpgpu_sim_wrapper::set_mem_ctrl_power(double reads, double writes,
-                                           double dram_precharge) {
-  p->sys.mc.memory_accesses = reads * p->sys.scaling_coefficients[MEM_RD] +
-                              writes * p->sys.scaling_coefficients[MEM_WR];
-  p->sys.mc.memory_reads = reads * p->sys.scaling_coefficients[MEM_RD];
-  p->sys.mc.memory_writes = writes * p->sys.scaling_coefficients[MEM_WR];
-  p->sys.mc.dram_pre = dram_precharge * p->sys.scaling_coefficients[MEM_PRE];
-  sample_perf_counters[MEM_RD] = reads;
-  sample_perf_counters[MEM_WR] = writes;
-  sample_perf_counters[MEM_PRE] = dram_precharge;
-}
-
-void gpgpu_sim_wrapper::set_exec_unit_power(double fpu_accesses,
-                                            double ialu_accesses,
-                                            double sfu_accesses) {
-  p->sys.core[0].fpu_accesses =
-      fpu_accesses * p->sys.scaling_coefficients[FPU_ACC];
-  // Integer ALU (not present in Tesla)
-  p->sys.core[0].ialu_accesses =
-      ialu_accesses * p->sys.scaling_coefficients[SP_ACC];
-  // Sfu accesses
-  p->sys.core[0].mul_accesses =
-      sfu_accesses * p->sys.scaling_coefficients[SFU_ACC];
-
-  sample_perf_counters[SP_ACC] = ialu_accesses;
-  sample_perf_counters[SFU_ACC] = sfu_accesses;
-  sample_perf_counters[FPU_ACC] = fpu_accesses;
-}
-
-void gpgpu_sim_wrapper::set_active_lanes_power(double sp_avg_active_lane,
-                                               double sfu_avg_active_lane) {
-  p->sys.core[0].sp_average_active_lanes = sp_avg_active_lane;
-  p->sys.core[0].sfu_average_active_lanes = sfu_avg_active_lane;
-}
-
-void gpgpu_sim_wrapper::set_NoC_power(double noc_tot_reads,
-                                      double noc_tot_writes) {
-  p->sys.NoC[0].total_accesses =
-      noc_tot_reads * p->sys.scaling_coefficients[NOC_A] +
-      noc_tot_writes * p->sys.scaling_coefficients[NOC_A];
-  sample_perf_counters[NOC_A] = noc_tot_reads + noc_tot_writes;
-}
-
-void gpgpu_sim_wrapper::power_metrics_calculations() {
-  total_sample_count++;
-  kernel_sample_count++;
-
-  // Current sample power
-  double sample_power =
-      proc->rt_power.readOp.dynamic + sample_cmp_pwr[CONST_DYNAMICP];
-
-  // Average power
-  // Previous + new + constant dynamic power (e.g., dynamic clocking power)
-  kernel_tot_power += sample_power;
-  kernel_power.avg = kernel_tot_power / kernel_sample_count;
-  for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
-    kernel_cmp_pwr[ind].avg += (double)sample_cmp_pwr[ind];
-  }
-
-  for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
-    kernel_cmp_perf_counters[ind].avg += (double)sample_perf_counters[ind];
-  }
-
-  // Max Power
-  if (sample_power > kernel_power.max) {
-    kernel_power.max = sample_power;
-    for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
-      kernel_cmp_pwr[ind].max = (double)sample_cmp_pwr[ind];
-    }
-    for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
-      kernel_cmp_perf_counters[ind].max = sample_perf_counters[ind];
-    }
-  }
-
-  // Min Power
-  if (sample_power < kernel_power.min || (kernel_power.min == 0)) {
-    kernel_power.min = sample_power;
-    for (unsigned ind = 0; ind < num_pwr_cmps; ++ind) {
-      kernel_cmp_pwr[ind].min = (double)sample_cmp_pwr[ind];
-    }
-    for (unsigned ind = 0; ind < num_perf_counters; ++ind) {
-      kernel_cmp_perf_counters[ind].min = sample_perf_counters[ind];
-    }
-  }
-
-  gpu_tot_power.avg = (gpu_tot_power.avg + sample_power);
-  gpu_tot_power.max =
-      (sample_power > gpu_tot_power.max) ? sample_power : gpu_tot_power.max;
-  gpu_tot_power.min =
-      ((sample_power < gpu_tot_power.min) || (gpu_tot_power.min == 0))
-          ? sample_power
-          : gpu_tot_power.min;
-}
-
-void gpgpu_sim_wrapper::print_trace_files() {
-  open_files();
-
-  for (unsigned i = 0; i < num_perf_counters; ++i) {
-    gzprintf(metric_trace_file, "%f,", sample_perf_counters[i]);
-  }
-  gzprintf(metric_trace_file, "\n");
-
-  gzprintf(power_trace_file, "%f,", proc_power);
-  for (unsigned i = 0; i < num_pwr_cmps; ++i) {
-    gzprintf(power_trace_file, "%f,", sample_cmp_pwr[i]);
-  }
-  gzprintf(power_trace_file, "\n");
-
-  close_files();
-}
-
-void gpgpu_sim_wrapper::update_coefficients() {
-  initpower_coeff[FP_INT] = proc->cores[0]->get_coefficient_fpint_insts();
-  effpower_coeff[FP_INT] =
-      initpower_coeff[FP_INT] * p->sys.scaling_coefficients[FP_INT];
-
-  initpower_coeff[TOT_INST] = proc->cores[0]->get_coefficient_tot_insts();
-  effpower_coeff[TOT_INST] =
-      initpower_coeff[TOT_INST] * p->sys.scaling_coefficients[TOT_INST];
-
-  initpower_coeff[REG_RD] =
-      proc->cores[0]->get_coefficient_regreads_accesses() *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-  initpower_coeff[REG_WR] =
-      proc->cores[0]->get_coefficient_regwrites_accesses() *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-  initpower_coeff[NON_REG_OPs] =
-      proc->cores[0]->get_coefficient_noregfileops_accesses() *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-  effpower_coeff[REG_RD] =
-      initpower_coeff[REG_RD] * p->sys.scaling_coefficients[REG_RD];
-  effpower_coeff[REG_WR] =
-      initpower_coeff[REG_WR] * p->sys.scaling_coefficients[REG_WR];
-  effpower_coeff[NON_REG_OPs] =
-      initpower_coeff[NON_REG_OPs] * p->sys.scaling_coefficients[NON_REG_OPs];
-
-  initpower_coeff[IC_H] = proc->cores[0]->get_coefficient_icache_hits();
-  initpower_coeff[IC_M] = proc->cores[0]->get_coefficient_icache_misses();
-  effpower_coeff[IC_H] =
-      initpower_coeff[IC_H] * p->sys.scaling_coefficients[IC_H];
-  effpower_coeff[IC_M] =
-      initpower_coeff[IC_M] * p->sys.scaling_coefficients[IC_M];
-
-  initpower_coeff[CC_H] = (proc->cores[0]->get_coefficient_ccache_readhits() +
-                           proc->get_coefficient_readcoalescing());
-  initpower_coeff[CC_M] = (proc->cores[0]->get_coefficient_ccache_readmisses() +
-                           proc->get_coefficient_readcoalescing());
-  effpower_coeff[CC_H] =
-      initpower_coeff[CC_H] * p->sys.scaling_coefficients[CC_H];
-  effpower_coeff[CC_M] =
-      initpower_coeff[CC_M] * p->sys.scaling_coefficients[CC_M];
-
-  initpower_coeff[TC_H] = (proc->cores[0]->get_coefficient_tcache_readhits() +
-                           proc->get_coefficient_readcoalescing());
-  initpower_coeff[TC_M] = (proc->cores[0]->get_coefficient_tcache_readmisses() +
-                           proc->get_coefficient_readcoalescing());
-  effpower_coeff[TC_H] =
-      initpower_coeff[TC_H] * p->sys.scaling_coefficients[TC_H];
-  effpower_coeff[TC_M] =
-      initpower_coeff[TC_M] * p->sys.scaling_coefficients[TC_M];
-
-  initpower_coeff[SHRD_ACC] =
-      proc->cores[0]->get_coefficient_sharedmemory_readhits();
-  effpower_coeff[SHRD_ACC] =
-      initpower_coeff[SHRD_ACC] * p->sys.scaling_coefficients[SHRD_ACC];
-
-  initpower_coeff[DC_RH] = (proc->cores[0]->get_coefficient_dcache_readhits() +
-                            proc->get_coefficient_readcoalescing());
-  initpower_coeff[DC_RM] =
-      (proc->cores[0]->get_coefficient_dcache_readmisses() +
-       proc->get_coefficient_readcoalescing());
-  initpower_coeff[DC_WH] = (proc->cores[0]->get_coefficient_dcache_writehits() +
-                            proc->get_coefficient_writecoalescing());
-  initpower_coeff[DC_WM] =
-      (proc->cores[0]->get_coefficient_dcache_writemisses() +
-       proc->get_coefficient_writecoalescing());
-  effpower_coeff[DC_RH] =
-      initpower_coeff[DC_RH] * p->sys.scaling_coefficients[DC_RH];
-  effpower_coeff[DC_RM] =
-      initpower_coeff[DC_RM] * p->sys.scaling_coefficients[DC_RM];
-  effpower_coeff[DC_WH] =
-      initpower_coeff[DC_WH] * p->sys.scaling_coefficients[DC_WH];
-  effpower_coeff[DC_WM] =
-      initpower_coeff[DC_WM] * p->sys.scaling_coefficients[DC_WM];
-
-  initpower_coeff[L2_RH] = proc->get_coefficient_l2_read_hits();
-  initpower_coeff[L2_RM] = proc->get_coefficient_l2_read_misses();
-  initpower_coeff[L2_WH] = proc->get_coefficient_l2_write_hits();
-  initpower_coeff[L2_WM] = proc->get_coefficient_l2_write_misses();
-  effpower_coeff[L2_RH] =
-      initpower_coeff[L2_RH] * p->sys.scaling_coefficients[L2_RH];
-  effpower_coeff[L2_RM] =
-      initpower_coeff[L2_RM] * p->sys.scaling_coefficients[L2_RM];
-  effpower_coeff[L2_WH] =
-      initpower_coeff[L2_WH] * p->sys.scaling_coefficients[L2_WH];
-  effpower_coeff[L2_WM] =
-      initpower_coeff[L2_WM] * p->sys.scaling_coefficients[L2_WM];
-
-  initpower_coeff[IDLE_CORE_N] =
-      p->sys.idle_core_power * proc->cores[0]->executionTime;
-  effpower_coeff[IDLE_CORE_N] =
-      initpower_coeff[IDLE_CORE_N] * p->sys.scaling_coefficients[IDLE_CORE_N];
-
-  initpower_coeff[PIPE_A] = proc->cores[0]->get_coefficient_duty_cycle();
-  effpower_coeff[PIPE_A] =
-      initpower_coeff[PIPE_A] * p->sys.scaling_coefficients[PIPE_A];
-
-  initpower_coeff[MEM_RD] = proc->get_coefficient_mem_reads();
-  initpower_coeff[MEM_WR] = proc->get_coefficient_mem_writes();
-  initpower_coeff[MEM_PRE] = proc->get_coefficient_mem_pre();
-  effpower_coeff[MEM_RD] =
-      initpower_coeff[MEM_RD] * p->sys.scaling_coefficients[MEM_RD];
-  effpower_coeff[MEM_WR] =
-      initpower_coeff[MEM_WR] * p->sys.scaling_coefficients[MEM_WR];
-  effpower_coeff[MEM_PRE] =
-      initpower_coeff[MEM_PRE] * p->sys.scaling_coefficients[MEM_PRE];
-
-  initpower_coeff[SP_ACC] =
-      proc->cores[0]->get_coefficient_ialu_accesses() *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-  ;
-  initpower_coeff[SFU_ACC] = proc->cores[0]->get_coefficient_sfu_accesses();
-  initpower_coeff[FPU_ACC] = proc->cores[0]->get_coefficient_fpu_accesses();
-
-  effpower_coeff[SP_ACC] =
-      initpower_coeff[SP_ACC] * p->sys.scaling_coefficients[SP_ACC];
-  effpower_coeff[SFU_ACC] =
-      initpower_coeff[SFU_ACC] * p->sys.scaling_coefficients[SFU_ACC];
-  effpower_coeff[FPU_ACC] =
-      initpower_coeff[FPU_ACC] * p->sys.scaling_coefficients[FPU_ACC];
-
-  initpower_coeff[NOC_A] = proc->get_coefficient_noc_accesses();
-  effpower_coeff[NOC_A] =
-      initpower_coeff[NOC_A] * p->sys.scaling_coefficients[NOC_A];
-
-  const_dynamic_power =
-      proc->get_const_dynamic_power() / (proc->cores[0]->executionTime);
-
-  for (unsigned i = 0; i < num_perf_counters; i++) {
-    initpower_coeff[i] /= (proc->cores[0]->executionTime);
-    effpower_coeff[i] /= (proc->cores[0]->executionTime);
-  }
-}
-
-void gpgpu_sim_wrapper::update_components_power() {
-  update_coefficients();
-
-  proc_power = proc->rt_power.readOp.dynamic;
-
-  sample_cmp_pwr[IBP] =
-      (proc->cores[0]->ifu->IB->rt_power.readOp.dynamic +
-       proc->cores[0]->ifu->IB->rt_power.writeOp.dynamic +
-       proc->cores[0]->ifu->ID_misc->rt_power.readOp.dynamic +
-       proc->cores[0]->ifu->ID_operand->rt_power.readOp.dynamic +
-       proc->cores[0]->ifu->ID_inst->rt_power.readOp.dynamic) /
-      (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[ICP] = proc->cores[0]->ifu->icache.rt_power.readOp.dynamic /
-                        (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[DCP] = proc->cores[0]->lsu->dcache.rt_power.readOp.dynamic /
-                        (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[TCP] = proc->cores[0]->lsu->tcache.rt_power.readOp.dynamic /
-                        (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[CCP] = proc->cores[0]->lsu->ccache.rt_power.readOp.dynamic /
-                        (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[SHRDP] =
-      proc->cores[0]->lsu->sharedmemory.rt_power.readOp.dynamic /
-      (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[RFP] =
-      (proc->cores[0]->exu->rfu->rt_power.readOp.dynamic /
-       (proc->cores[0]->executionTime)) *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-
-  sample_cmp_pwr[SPP] =
-      (proc->cores[0]->exu->exeu->rt_power.readOp.dynamic /
-       (proc->cores[0]->executionTime)) *
-      (proc->cores[0]->exu->rf_fu_clockRate / proc->cores[0]->exu->clockRate);
-
-  sample_cmp_pwr[SFUP] = (proc->cores[0]->exu->mul->rt_power.readOp.dynamic /
-                          (proc->cores[0]->executionTime));
-
-  sample_cmp_pwr[FPUP] = (proc->cores[0]->exu->fp_u->rt_power.readOp.dynamic /
-                          (proc->cores[0]->executionTime));
-
-  sample_cmp_pwr[SCHEDP] = proc->cores[0]->exu->scheu->rt_power.readOp.dynamic /
-                           (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[L2CP] = (proc->XML->sys.number_of_L2s > 0)
-                             ? proc->l2array[0]->rt_power.readOp.dynamic /
-                                   (proc->cores[0]->executionTime)
-                             : 0;
-
-  sample_cmp_pwr[MCP] = (proc->mc->rt_power.readOp.dynamic -
-                         proc->mc->dram->rt_power.readOp.dynamic) /
-                        (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[NOCP] =
-      proc->nocs[0]->rt_power.readOp.dynamic / (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[DRAMP] =
-      proc->mc->dram->rt_power.readOp.dynamic / (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[PIPEP] =
-      proc->cores[0]->Pipeline_energy / (proc->cores[0]->executionTime);
-
-  sample_cmp_pwr[IDLE_COREP] =
-      proc->cores[0]->IdleCoreEnergy / (proc->cores[0]->executionTime);
-
-  // This constant dynamic power (e.g., clock power) part is estimated via
-  // regression model.
-  sample_cmp_pwr[CONST_DYNAMICP] = 0;
-  double cnst_dyn =
-      proc->get_const_dynamic_power() / (proc->cores[0]->executionTime);
-  // If the regression scaling term is greater than the recorded constant
-  // dynamic power then use the difference (other portion already added to
-  // dynamic power). Else, all the constant dynamic power is accounted for, add
-  // nothing.
-  if (p->sys.scaling_coefficients[CONST_DYNAMICN] > cnst_dyn)
-    sample_cmp_pwr[CONST_DYNAMICP] =
-        (p->sys.scaling_coefficients[CONST_DYNAMICN] - cnst_dyn);
-
-  proc_power += sample_cmp_pwr[CONST_DYNAMICP];
-
-  double sum_pwr_cmp = 0;
-  for (unsigned i = 0; i < num_pwr_cmps; i++) {
-    sum_pwr_cmp += sample_cmp_pwr[i];
-  }
-  bool check = false;
-  check = sanity_check(sum_pwr_cmp, proc_power);
-  assert("Total Power does not equal the sum of the components\n" && (check));
-}
-
-void gpgpu_sim_wrapper::compute() { proc->compute(); }
-void gpgpu_sim_wrapper::print_power_kernel_stats(
-    double gpu_sim_cycle, double gpu_tot_sim_cycle, double init_value,
-    const std::string& kernel_info_string, bool print_trace) {
-  detect_print_steady_state(1, init_value);
-  if (g_power_simulation_enabled) {
-    powerfile << kernel_info_string << std::endl;
-
-    sanity_check((kernel_power.avg * kernel_sample_count), kernel_tot_power);
-    powerfile << "Kernel Average Power Data:" << std::endl;
-    powerfile << "kernel_avg_power = " << kernel_power.avg << std::endl;
-
-    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
-      powerfile << "gpu_avg_" << pwr_cmp_label[i] << " = "
-                << kernel_cmp_pwr[i].avg / kernel_sample_count << std::endl;
-    }
-    for (unsigned i = 0; i < num_perf_counters; ++i) {
-      powerfile << "gpu_avg_" << perf_count_label[i] << " = "
-                << kernel_cmp_perf_counters[i].avg / kernel_sample_count
-                << std::endl;
-    }
-
-    powerfile << std::endl << "Kernel Maximum Power Data:" << std::endl;
-    powerfile << "kernel_max_power = " << kernel_power.max << std::endl;
-    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
-      powerfile << "gpu_max_" << pwr_cmp_label[i] << " = "
-                << kernel_cmp_pwr[i].max << std::endl;
-    }
-    for (unsigned i = 0; i < num_perf_counters; ++i) {
-      powerfile << "gpu_max_" << perf_count_label[i] << " = "
-                << kernel_cmp_perf_counters[i].max << std::endl;
-    }
-
-    powerfile << std::endl << "Kernel Minimum Power Data:" << std::endl;
-    powerfile << "kernel_min_power = " << kernel_power.min << std::endl;
-    for (unsigned i = 0; i < num_pwr_cmps; ++i) {
-      powerfile << "gpu_min_" << pwr_cmp_label[i] << " = "
-                << kernel_cmp_pwr[i].min << std::endl;
-    }
-    for (unsigned i = 0; i < num_perf_counters; ++i) {
-      powerfile << "gpu_min_" << perf_count_label[i] << " = "
-                << kernel_cmp_perf_counters[i].min << std::endl;
-    }
-
-    powerfile << std::endl
-              << "Accumulative Power Statistics Over Previous Kernels:"
-              << std::endl;
-    powerfile << "gpu_tot_avg_power = "
-              << gpu_tot_power.avg / total_sample_count << std::endl;
-    powerfile << "gpu_tot_max_power = " << gpu_tot_power.max << std::endl;
-    powerfile << "gpu_tot_min_power = " << gpu_tot_power.min << std::endl;
-    powerfile << std::endl << std::endl;
-    powerfile.flush();
-
-    if (print_trace) {
-      print_trace_files();
-    }
-  }
-}
-void gpgpu_sim_wrapper::dump() {
-  if (g_power_per_cycle_dump) proc->displayEnergy(2, 5);
-}
-
-void gpgpu_sim_wrapper::print_steady_state(int position, double init_val) {
-  double temp_avg = sample_val / (double)samples.size();
-  double temp_ipc = (init_val - init_inst_val) /
-                    (double)(samples.size() * gpu_stat_sample_freq);
-
-  if ((samples.size() >
-       gpu_steady_min_period)) {  // If steady state occurred for some time,
-                                  // print to file
-    has_written_avg = true;
-    gzprintf(steady_state_tacking_file, "%u,%d,%f,%f,", sample_start,
-             total_sample_count, temp_avg, temp_ipc);
-    for (unsigned i = 0; i < num_perf_counters; ++i) {
-      gzprintf(steady_state_tacking_file, "%f,",
-               samples_counter.at(i) / ((double)samples.size()));
-    }
-    gzprintf(steady_state_tacking_file, "\n");
-  } else {
-    if (!has_written_avg && position)
-      gzprintf(steady_state_tacking_file,
-               "ERROR! Not enough steady state points to generate average\n");
-  }
-
-  sample_start = 0;
-  sample_val = 0;
-  init_inst_val = init_val;
-  samples.clear();
-  samples_counter.clear();
-  pwr_counter.clear();
-  assert(samples.size() == 0);
-}
-
-void gpgpu_sim_wrapper::detect_print_steady_state(int position,
-                                                  double init_val) {
-  // Calculating Average
-  if (g_power_simulation_enabled && g_steady_power_levels_enabled) {
-    steady_state_tacking_file = gzopen(g_steady_state_tracking_filename, "a");
-    if (position == 0) {
-      if (samples.size() == 0) {
-        // First sample
-        sample_start = total_sample_count;
-        sample_val = proc->rt_power.readOp.dynamic;
-        init_inst_val = init_val;
-        samples.push_back(proc->rt_power.readOp.dynamic);
-        assert(samples_counter.size() == 0);
-        assert(pwr_counter.size() == 0);
-
-        for (unsigned i = 0; i < (num_perf_counters); ++i) {
-          samples_counter.push_back(sample_perf_counters[i]);
-        }
-
-        for (unsigned i = 0; i < (num_pwr_cmps); ++i) {
-          pwr_counter.push_back(sample_cmp_pwr[i]);
-        }
-        assert(pwr_counter.size() == (double)num_pwr_cmps);
-        assert(samples_counter.size() == (double)num_perf_counters);
-      } else {
-        // Get current average
-        double temp_avg = sample_val / (double)samples.size();
-
-        if (abs(proc->rt_power.readOp.dynamic - temp_avg) <
-            gpu_steady_power_deviation) {  // Value is within threshold
-          sample_val += proc->rt_power.readOp.dynamic;
-          samples.push_back(proc->rt_power.readOp.dynamic);
-          for (unsigned i = 0; i < (num_perf_counters); ++i) {
-            samples_counter.at(i) += sample_perf_counters[i];
-          }
-
-          for (unsigned i = 0; i < (num_pwr_cmps); ++i) {
-            pwr_counter.at(i) += sample_cmp_pwr[i];
-          }
-
-        } else {  // Value exceeds threshold, not considered steady state
-          print_steady_state(position, init_val);
-        }
-      }
-    } else {
-      print_steady_state(position, init_val);
-    }
-    gzclose(steady_state_tacking_file);
-  }
-}
-
-void gpgpu_sim_wrapper::open_files() {
-  if (g_power_simulation_enabled) {
-    if (g_power_trace_enabled) {
-      power_trace_file = gzopen(g_power_trace_filename, "a");
-      metric_trace_file = gzopen(g_metric_trace_filename, "a");
-    }
-  }
-}
-void gpgpu_sim_wrapper::close_files() {
-  if (g_power_simulation_enabled) {
-    if (g_power_trace_enabled) {
-      gzclose(power_trace_file);
-      gzclose(metric_trace_file);
-    }
-  }
-}