creating branch for adding support for CUDA 3.x and Fermi

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 6829]

33 files changed, 17010 insertions, 0 deletions

diff --git a/src/gpgpu-sim/Makefile b/src/gpgpu-sim/Makefile
new file mode 100644
index 0000000..2b1412a
--- /dev/null
+++ b/src/gpgpu-sim/Makefile
@@ -0,0 +1,116 @@
+# Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+# George L. Yuan, Ivan Sham and the 
+# University of British Columbia
+# Vancouver, BC  V6T 1Z4
+# All Rights Reserved.
+# 
+# THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+# TERMS AND CONDITIONS.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# 
+# NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+# are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+# (property of NVIDIA).  The files benchmarks/BlackScholes/* and 
+# benchmarks/template/* are derived from the CUDA SDK available from 
+# http://www.nvidia.com/cuda (also property of NVIDIA).  The files 
+# src/gpgpusim_entrypoint.c and src/simplesim-3.0/* are derived from the 
+# SimpleScalar Toolset available from http://www.simplescalar.com/ 
+# (property of SimpleScalar LLC) and the files src/intersim/* are derived 
+# from Booksim (Simulator provided with the textbook "Principles and 
+# Practices of Interconnection Networks" available from 
+# http://cva.stanford.edu/books/ppin/).  As such, those files are bound by 
+# the corresponding legal terms and conditions set forth separately (original 
+# copyright notices are left in files from these sources and where we have 
+# modified a file our copyright notice appears before the original copyright 
+# notice).  
+# 
+# Using this version of GPGPU-Sim requires a complete installation of CUDA 
+# version 1.1, which is distributed seperately by NVIDIA under separate terms 
+# and conditions.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 
+# 3. Neither the name of the University of British Columbia nor the names of
+# its contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+# 
+# 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+#  
+# 5. No nonprofit user may place any restrictions on the use of this software,
+# including as modified by the user, by any other authorized user.
+# 
+# 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+# Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+# Vancouver, BC V6T 1Z4
+
+# GPGPU-Sim Makefile
+
+DEBUG=0
+
+CXXFLAGS = -Wall -DDEBUG
+CXXFLAGS_L2CACHE = -Wall -DDEBUG
+
+GNUC_CPP0X := $(shell gcc --version | perl -ne 'if (/gcc\s+\(.*\)\s+([0-9.]+)/){ if($$1 >= 4.3) {$$n=1} else {$$n=0;} } END { print $$n; }')
+ifeq ($(GNUC_CPP0X), 1)
+    CXXFLAGS += -std=c++0x
+    CXXFLAGS_L2CACHE += -std=c++0x
+endif
+
+ifneq ($(DEBUG),1)
+	OPTFLAGS += -O3
+else
+	CXXFLAGS_L2CACHE += -DGLIBCXX_DEBUG_PEDANTIC
+	CXXFLAGS += -D_GLIBCXX_DEBUG -DGLIBCXX_DEBUG_PEDANTIC
+endif
+
+OPTFLAGS += -g3 -fPIC
+
+CPP = g++ $(SNOW)
+OEXT = o
+
+SRCS = $(shell ls *.cc)
+OBJS = $(SRCS:.cc=.$(OEXT))
+ 
+libgpu_uarch_sim.a:$(OBJS)
+	ar rcs  libgpu_uarch_sim.a $(OBJS)
+
+depend:
+	makedepend $(SRCS) 2> /dev/null
+
+l2cache.$(OEXT): l2cache.cc
+	$(CPP) $(OPTFLAGS) $(CXXFLAGS_L2CACHE) -o $*.$(OEXT) -c $*.cc
+
+.cc.$(OEXT):
+	$(CPP) $(OPTFLAGS) $(CXXFLAGS) -o $*.$(OEXT) -c $*.cc
+
+clean:
+	rm -f *.o core *~ *.a 
+
+option_parser.$(OEXT): option_parser.h
+
+dram_sched.$(OEXT): ../cuda-sim/ptx.tab.h
+
+../cuda-sim/ptx.tab.h:
+	make -C ../cuda-sim/ ptx.tab.c
+
+# DO NOT DELETE
+
diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc
new file mode 100644
index 0000000..333d6de
--- /dev/null
+++ b/src/gpgpu-sim/addrdec.cc
@@ -0,0 +1,471 @@
+/* 
+ * addrdec.c 
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <string.h>
+#include "addrdec.h"
+//#include "gpu-sim.h"
+#include "../option_parser.h"
+
+int ADDR_CHIP_S = 10;
+extern int gpgpu_mem_address_mask;
+
+long int powli( long int x, long int y ) // compute x to the y
+{
+   long int r = 1;
+   int i; 
+   for (i = 0; i < y; ++i ) {
+      r *= x;
+   }
+   return r;
+}
+
+void addrdec_display(addrdec_t *a) {
+   //printf("DRAM:  unused:%x chip:%x row:%x col:%x bk:%x\n",
+   //       a.dram.unused, a.dram.chip, GET_ROW(a), GET_COL(a), a.dram.bk);
+
+   if (a->chip)   printf("\tchip:%x ",  a->chip);
+   if (a->row)    printf("\trow:%x ",   a->row);
+   if (a->col)    printf("\tcol:%x ",   a->col);
+   if (a->bk)     printf("\tbk:%x ",    a->bk);
+   if (a->burst)  printf("\tburst:%x ", a->burst);
+}  
+
+unsigned long long int addrdec_packbits(unsigned long long int mask, 
+                                        unsigned long long int val,
+                                        unsigned char high, unsigned char low) 
+{
+   int i, pos;
+   unsigned long long int out;
+   out = 0;
+   pos = 0;
+   for (i=low;i<high;i++) {
+      if ((mask & ((unsigned long long int)1<<i)) != 0) {
+         out |= ((val & ((unsigned long long int)1<<i)) >> i) << pos;
+         pos++;
+      }
+      // printf("%02d: %016llx %d\n",i,out,pos);
+   }
+
+   return out;
+}
+
+unsigned long long int addrdec_mask[5] = {
+   0x0000000000001C00,
+   0x0000000000000300,
+   0x000000000FFF0000,
+   0x000000000000E0FF,
+   0x000000000000000F
+};
+
+void addrdec_getmasklimit(unsigned long long int mask, unsigned char *high, unsigned char *low) 
+{
+   *high = 64;
+   *low = 0;
+   int i;
+   int low_found = 0;
+
+   for (i=0;i<64;i++) {
+      if ((mask & ((unsigned long long int)1<<i)) != 0) {
+         if (low_found) {
+            *high = i + 1;
+         } else {
+            *high = i + 1;
+            *low = i;
+            low_found = 1;
+         }
+      }
+      // printf("%02d: %016llx %d\n",i,out,pos);
+   }
+}
+
+unsigned char addrdec_mklow[5] = { 0, 0, 0, 0, 0};
+unsigned char addrdec_mkhigh[5] = { 64, 64, 64, 64, 64};
+
+static unsigned int gap;
+static int Nchips;
+
+void addrdec_tlx(unsigned long long int addr, addrdec_t *tlx) 
+{  
+   unsigned long long int addr_for_chip,rest_of_addr;
+   if (!gap) {
+      tlx->chip = addrdec_packbits(addrdec_mask[CHIP], addr, addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]);
+      tlx->bk   = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], addrdec_mklow[BK]);
+      tlx->row  = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]);
+      tlx->col  = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], addrdec_mklow[COL]);
+      tlx->burst= addrdec_packbits(addrdec_mask[BURST], addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]);
+   } else {
+      addr_for_chip= ( (addr>>ADDR_CHIP_S) % Nchips) << ADDR_CHIP_S;
+      rest_of_addr= ( (addr>>ADDR_CHIP_S) / Nchips) << ADDR_CHIP_S;                
+
+      tlx->chip = addrdec_packbits(addrdec_mask[CHIP], addr_for_chip, addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]);
+      if (addrdec_mask[BK] > addrdec_mask[CHIP]) {
+         tlx->bk   = addrdec_packbits(addrdec_mask[BK], rest_of_addr, addrdec_mkhigh[BK], addrdec_mklow[BK]);
+      } else {
+         tlx->bk   = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], addrdec_mklow[BK]);
+      }
+      if (addrdec_mask[ROW] > addrdec_mask[CHIP]) {
+         tlx->row  = addrdec_packbits(addrdec_mask[ROW], rest_of_addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]);
+      } else {
+         tlx->row  = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]);
+      }
+      tlx->col  = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], addrdec_mklow[COL]);
+      tlx->burst= addrdec_packbits(addrdec_mask[BURST], addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]);
+   }
+}
+
+unsigned int LOGB2_32( unsigned int v ) {
+   unsigned int shift;
+   unsigned int r;
+
+   r = 0;
+
+   shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift;
+   shift = (( v & 0xFF00    ) != 0 ) << 3; v >>= shift; r |= shift;
+   shift = (( v & 0xF0      ) != 0 ) << 2; v >>= shift; r |= shift;
+   shift = (( v & 0xC       ) != 0 ) << 1; v >>= shift; r |= shift;
+   shift = (( v & 0x2       ) != 0 ) << 0; v >>= shift; r |= shift;
+
+   return r;
+}
+
+
+static char *addrdec_option = NULL;
+void addrdec_setoption(option_parser_t opp)
+{
+   option_parser_register(opp, "-gpgpu_mem_addr_mapping", OPT_CSTR, &addrdec_option,
+      "mapping memory address to dram model {dramid@<start bit>;<memory address map>}",
+      NULL);
+}
+
+void addrdec_parseoption(const char *option)
+{
+   unsigned int dramid_start = 0;
+   int dramid_parsed = sscanf(option, "dramid@%d", &dramid_start);
+   if (dramid_parsed == 1) {
+      ADDR_CHIP_S = dramid_start;
+   } else {
+      ADDR_CHIP_S = -1;
+   }
+   
+   const char *cmapping = strchr(option, ';');
+   if (cmapping == NULL) {
+      cmapping = option;
+   } else {
+      cmapping += 1;
+   }
+
+   addrdec_mask[CHIP] = 0x0;
+   addrdec_mask[BK]   = 0x0;
+   addrdec_mask[ROW]  = 0x0;
+   addrdec_mask[COL]  = 0x0;
+   addrdec_mask[BURST]= 0x0;
+   
+   int ofs = 63;
+   while ((*cmapping) != '\0') {
+      switch (*cmapping) {
+         case 'D': case 'd':  
+            assert(dramid_parsed != 1); addrdec_mask[CHIP]  |= (1ULL << ofs); ofs--; break;
+         case 'B': case 'b':   addrdec_mask[BK]    |= (1ULL << ofs); ofs--; break;
+         case 'R': case 'r':   addrdec_mask[ROW]   |= (1ULL << ofs); ofs--; break;
+         case 'C': case 'c':   addrdec_mask[COL]   |= (1ULL << ofs); ofs--; break;
+         case 'S': case 's':   addrdec_mask[BURST] |= (1ULL << ofs); addrdec_mask[COL]   |= (1ULL << ofs); ofs--; break;
+         // ignore bit
+         case '0': ofs--; break;
+         // ignore character
+         case '|':
+         case ' ':
+         case '.': break;
+         default:
+            fprintf(stderr, "ERROR: Invalid address mapping character '%c' in option '%s'\n", *cmapping, option);
+      }
+      cmapping += 1;
+   }
+
+   if (ofs != -1) {
+      fprintf(stderr, "ERROR: Invalid address mapping length (%d) in option '%s'\n", 63 - ofs, option);
+      assert(ofs == -1);
+   }
+}
+
+
+void addrdec_setnchip(unsigned int nchips) 
+{
+   unsigned i;
+   unsigned long long int mask;
+   unsigned int nchipbits = LOGB2_32(nchips);
+   Nchips = nchips;
+
+   gap = (nchips - powli(2,nchipbits));
+   if (gap) {
+      nchipbits++;
+   }
+   switch (gpgpu_mem_address_mask) {
+   case 0: 
+      //old, added 2row bits, use #define ADDR_CHIP_S 10
+      ADDR_CHIP_S = 10;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000000300;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x0000000000001CFF;
+      break;
+   case 1:
+      ADDR_CHIP_S = 13;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 2:
+      ADDR_CHIP_S = 11;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 3:
+      ADDR_CHIP_S = 11;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x000000000FFFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+
+   case 14:
+      ADDR_CHIP_S = 14;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 15:
+      ADDR_CHIP_S = 15;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 16:
+      ADDR_CHIP_S = 16;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 6:
+      ADDR_CHIP_S = 6;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 5:
+      ADDR_CHIP_S = 5;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;                         
+   case 100:
+      ADDR_CHIP_S = 1;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000000003;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x0000000000001FFC;
+      break;
+   case 103:
+      ADDR_CHIP_S = 3;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000000003;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x0000000000001FFC;
+      break;
+   case 106:
+      ADDR_CHIP_S = 6;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000001800;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x00000000000007FF;
+      break;
+   case 160: 
+      //old, added 2row bits, use #define ADDR_CHIP_S 10
+      ADDR_CHIP_S = 6;
+      addrdec_mask[CHIP] = 0x0000000000000000;
+      addrdec_mask[BK]   = 0x0000000000000300;
+      addrdec_mask[ROW]  = 0x0000000007FFE000;
+      addrdec_mask[COL]  = 0x0000000000001CFF;
+
+   default:
+      break;
+   }
+
+   if (addrdec_option != NULL) {
+      addrdec_parseoption(addrdec_option);
+   }
+
+   if (ADDR_CHIP_S != -1) {
+      mask = ((unsigned long long int)1 << ADDR_CHIP_S) - 1;
+      addrdec_mask[BK]   = ((addrdec_mask[BK] & ~mask) << nchipbits) | (addrdec_mask[BK] & mask);
+      addrdec_mask[ROW]  = ((addrdec_mask[ROW] & ~mask) << nchipbits) | (addrdec_mask[ROW] & mask);
+      addrdec_mask[COL]  = ((addrdec_mask[COL] & ~mask) << nchipbits) | (addrdec_mask[COL] & mask);
+
+      for (i=ADDR_CHIP_S;i<(ADDR_CHIP_S+nchipbits);i++) {
+         mask = (unsigned long long int)1 << i;
+         addrdec_mask[CHIP] |= mask;
+      }
+   } else {
+      // make sure nchips is power of two when explicit dram id mask is used
+      assert((nchips & (nchips - 1)) == 0); 
+   }
+
+   addrdec_getmasklimit(addrdec_mask[CHIP],  &addrdec_mkhigh[CHIP],  &addrdec_mklow[CHIP] );
+   addrdec_getmasklimit(addrdec_mask[BK],    &addrdec_mkhigh[BK],    &addrdec_mklow[BK]   );
+   addrdec_getmasklimit(addrdec_mask[ROW],   &addrdec_mkhigh[ROW],   &addrdec_mklow[ROW]  );
+   addrdec_getmasklimit(addrdec_mask[COL],   &addrdec_mkhigh[COL],   &addrdec_mklow[COL]  );
+   addrdec_getmasklimit(addrdec_mask[BURST], &addrdec_mkhigh[BURST], &addrdec_mklow[BURST]);
+
+   printf("addr_dec_mask[CHIP]  = %016llx \thigh:%d low:%d\n", addrdec_mask[CHIP],  addrdec_mkhigh[CHIP],  addrdec_mklow[CHIP] );
+   printf("addr_dec_mask[BK]    = %016llx \thigh:%d low:%d\n", addrdec_mask[BK],    addrdec_mkhigh[BK],    addrdec_mklow[BK]   );
+   printf("addr_dec_mask[ROW]   = %016llx \thigh:%d low:%d\n", addrdec_mask[ROW],   addrdec_mkhigh[ROW],   addrdec_mklow[ROW]  );
+   printf("addr_dec_mask[COL]   = %016llx \thigh:%d low:%d\n", addrdec_mask[COL],   addrdec_mkhigh[COL],   addrdec_mklow[COL]  );
+   printf("addr_dec_mask[BURST] = %016llx \thigh:%d low:%d\n", addrdec_mask[BURST], addrdec_mkhigh[BURST], addrdec_mklow[BURST]);
+}
+
+#ifdef UNIT_TEST
+
+int main () {
+   unsigned int tb = 1;
+   unsigned pos;
+   addrdec_t_o tlx;
+
+   printf("DRAM: %d %d %d %d %d %d %d\n", 
+          D_COLL, D_BK, D_COLU, D_ROWL, D_CHIP, D_ROWU, D_UNUSED);
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      tlx.plain = tb;
+      addrdec_fetch(tlx);
+      addrdec_dram(tlx);
+      printf("\n");
+   }
+
+   unsigned long long int packed;
+   packed = addrdec_packbits(0xFFFF0000FFFF0000, 0x2244113322441133);
+   assert (packed == 0x22442244);
+   printf("%016llx\n", packed);
+
+   packed = addrdec_packbits(0x5555555555555555, 0x3333333333333333);
+   assert (packed == 0x55555555);
+   printf("%016llx\n", packed);
+
+   packed = addrdec_packbits(0x5555555555555555, 0x6363636363636363);
+   assert (packed == 0x99999999);
+   printf("%016llx\n", packed);
+
+   addrdec_t tls;
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }
+
+   addrdec_setnchip(32);
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }
+   addrdec_setnchip(16);
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }
+   addrdec_setnchip(8);
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }
+   addrdec_setnchip(7);
+   for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }
+/*   addrdec_setnchip(6);
+   for(tb=1, pos=0; tb!=0; tb ++, pos++) {
+      printf("%08lx|%02d =>", tb,pos);
+      addrdec_tlx(tb, &tls);
+      addrdec_display(&tls);
+      printf("\n");
+   }*/
+   return 0;
+}
+
+#endif
diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h
new file mode 100644
index 0000000..ba28c05
--- /dev/null
+++ b/src/gpgpu-sim/addrdec.h
@@ -0,0 +1,96 @@
+/*
+ * addrdec.h
+ * 
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung and the University of 
+ * British Columbia 
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "../option_parser.h"
+
+#ifndef ADDRDEC_H
+#define ADDRDEC_H
+
+enum {
+   CHIP  = 0,
+   BK    = 1,
+   ROW   = 2,
+   COL   = 3,
+   BURST = 4,
+   N_ADDRDEC
+};
+
+typedef struct {
+   unsigned chip;
+   unsigned bk;
+   unsigned row;
+   unsigned col;
+   unsigned burst;
+} addrdec_t;
+
+void addrdec_tlx(unsigned long long int addr, addrdec_t *tlx);
+void addrdec_setnchip(unsigned int nchips);
+void addrdec_setoption(option_parser_t opp);
+void addrdec_parseoption(const char *option);
+
+#endif
diff --git a/src/gpgpu-sim/cflogger.h b/src/gpgpu-sim/cflogger.h
new file mode 100644
index 0000000..bdf57e5
--- /dev/null
+++ b/src/gpgpu-sim/cflogger.h
@@ -0,0 +1,121 @@
+/* 
+ * cflogger.h 
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef CFLOGGER_H
+#define CFLOGGER_H
+
+void try_snap_shot (unsigned long long  current_cycle);
+void set_spill_interval (unsigned long long  interval);
+void spill_log_to_file (FILE *fout, int final, unsigned long long  current_cycle);
+
+void create_thread_CFlogger( int n_loggers, int n_threads, int n_insn, address_type start_pc, unsigned long long  logging_interval);
+void destroy_thread_CFlogger( );
+void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc );
+void cflog_snapshot( int logger_id, unsigned long long  cycle );
+void cflog_print(FILE *fout);
+void cflog_print_path_expression(FILE *fout);
+void cflog_visualizer_print(FILE *fout);
+
+
+void insn_warp_occ_create( int n_loggers, int simd_width, int n_insn );
+void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ );
+void insn_warp_occ_print( FILE *fout );
+
+
+void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long  logging_interval );
+void shader_warp_occ_log( int logger_id, int warp_occ );
+void shader_warp_occ_snapshot( int logger_id, unsigned long long  current_cycle );
+void shader_warp_occ_print( FILE *fout );
+
+
+void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long  logging_interval );
+void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw );
+void shader_mem_acc_snapshot( int logger_id, unsigned long long  current_cycle );
+void shader_mem_acc_print( FILE *fout );
+
+
+void shader_mem_lat_create( int n_loggers, unsigned long long  logging_interval );
+void shader_mem_lat_log( int logger_id, int latency );
+void shader_mem_lat_snapshot( int logger_id, unsigned long long  current_cycle );
+void shader_mem_lat_print( FILE *fout );
+
+
+int get_shader_normal_cache_id();
+int get_shader_texture_cache_id();
+int get_shader_constant_cache_id();
+void shader_cache_access_create( int n_loggers, int n_types, unsigned long long  logging_interval );
+void shader_cache_access_log( int logger_id, int type, int miss);
+void shader_cache_access_unlog( int logger_id, int type, int miss);
+void shader_cache_access_print( FILE *fout );
+
+
+void shader_CTA_count_create( int n_shaders, unsigned long long  logging_interval);
+void shader_CTA_count_log( int shader_id, int nCTAadded );
+void shader_CTA_count_unlog( int shader_id, int nCTAdone );
+void shader_CTA_count_resetnow( );
+void shader_CTA_count_print( FILE *fout );
+void shader_CTA_count_visualizer_print( FILE *fout );
+
+#endif /* CFLOGGER_H */
diff --git a/src/gpgpu-sim/delayqueue.cc b/src/gpgpu-sim/delayqueue.cc
new file mode 100644
index 0000000..dc870d0
--- /dev/null
+++ b/src/gpgpu-sim/delayqueue.cc
@@ -0,0 +1,468 @@
+/* 
+ * delayqueue.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda,
+ * Ivan Sham, Henry Tran and the University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "delayqueue.h"
+#include "gpu-misc.h"
+#include "../intersim/statwraper.h"
+
+extern unsigned long long  gpu_sim_cycle; //for stat collection
+
+unsigned char dq_full( delay_queue* dq ) 
+{
+   if (dq->max_len && dq->length >= dq->max_len)
+      return 1;
+   return 0;
+}
+
+unsigned char dq_empty(delay_queue* dq )
+{
+   return(dq->head == NULL)?1:0;
+}
+
+unsigned int dq_n_element(delay_queue* dq )
+{
+   return(dq->n_element);
+}
+
+unsigned char dq_push(delay_queue* dq, void* data) {
+   if (dq->max_len) assert(dq->length < dq->max_len);
+   if (dq->head) {
+      if (dq->tail->data || dq->length < dq->min_len) {
+         dq->tail->next = (delay_data*) malloc(sizeof(delay_data)); 
+         dq->tail = dq->tail->next;
+         dq->length++;
+         dq->n_element++;
+      }
+   } else {
+      dq->head = dq->tail = (delay_data*) malloc(sizeof(delay_data)); 
+      dq->length++;
+      dq->n_element++;
+   }
+   dq->tail->next = NULL;
+   dq->tail->time_elapsed = dq->latency;
+   dq->tail->data = (void*)data;
+   dq->tail->push_time = gpu_sim_cycle;
+   return 1;
+}
+
+void* dq_top(delay_queue* dq) {
+   if (dq->head) {
+      return dq->head->data;
+   } else {
+      return NULL;
+   }
+}
+
+void* dq_pop(delay_queue* dq) {
+   delay_data* next;
+   void* data;
+   if (dq->head) {
+      if (dq->head->time_elapsed) {
+         dq->head->time_elapsed--;
+         data = NULL;
+      } else {
+         next = dq->head->next;
+         data = dq->head->data;
+         StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - dq->head->push_time));
+         if ( dq->head == dq->tail ) {
+            assert( next == NULL );
+            dq->tail = NULL;     
+         }
+         free(dq->head);
+         dq->head = next;
+         dq->length--;
+         if (dq->length == 0) {
+            assert( dq->head == NULL );
+            dq->tail = dq->head;
+         }
+         dq->n_element--; 
+      }
+      if (dq->min_len && dq->length < dq->min_len) {
+         dq_push(dq,NULL);
+         dq->n_element--; // uncount NULL elements inserted to create delays
+      }
+   } else {
+      data = NULL;
+   }
+   return data;
+}
+
+void dq_set_min_length(delay_queue* dq, unsigned int new_min_len) {
+   if (new_min_len == dq->min_len) return;
+
+   if (new_min_len > dq->min_len) {
+      dq->min_len = new_min_len;
+      while (dq->length < dq->min_len) {
+         dq_push(dq,NULL);
+         dq->n_element--; // uncount NULL elements inserted to create delays
+      }
+   } else {
+      // in this branch imply that the original min_len is larger then 0
+      // ie. dq->head != 0
+      assert(dq->head);
+      dq->min_len = new_min_len;
+      while ((dq->length > dq->min_len) && (dq->tail->data == 0)) {
+         delay_data *iter;
+         iter = dq->head;
+         while (iter && (iter->next != dq->tail))
+            iter = iter->next;
+         if (!iter) {
+            // there is only one node, and that node is empty
+            assert(dq->head->data == 0);
+            dq_pop(dq);
+         } else {
+            // there are more than one node, and tail node is empty
+            assert(iter->next == dq->tail);
+            free(dq->tail);
+            dq->tail = iter;
+            dq->tail->next = 0;
+            dq->length--;
+         }
+      }
+   }
+}
+
+void dq_remove(void* data, delay_queue* dq)
+{
+   // removes an item from the queue without deallocating the memory
+   delay_data* ptr = NULL;
+   delay_data* temp = NULL;
+
+   assert(dq);
+   assert(data);
+
+   ptr = dq->head;
+   if (ptr) {
+      if (ptr->data == data) {
+         StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - ptr->push_time));
+         dq->head = ptr->next;
+         if ( dq->head == NULL )
+            dq->tail = NULL;
+         dq->length--;
+         return;      
+      }
+      while (ptr->next) {
+         if (ptr->next->data == data) {
+            temp = ptr->next;
+            StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - temp->push_time));
+            if ( ptr->next == dq->tail ) {
+               dq->tail = ptr;
+            }
+            ptr->next = ptr->next->next;
+            dq->length--;
+            return;
+         }
+         ptr = ptr->next;
+      }
+   }
+}
+
+void removeEntry(void* data, delay_queue** dqq, int size_dq)
+{
+   int i;
+   delay_data* ptr = NULL;
+   delay_queue* dq = NULL;
+   delay_data* temp = NULL;
+
+   assert(dqq);
+   assert(data);
+
+
+   for (i = 0; i<size_dq; i++) {
+      dq = dqq[i];
+      ptr = dq->head;
+      if (ptr) {
+         if (ptr->data == data) {
+            dq->head = ptr->next;
+            if ( dq->head == NULL )
+               dq->tail = NULL;
+            StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - ptr->push_time));
+            free(ptr);  
+            dq->length--;
+            return;     
+         }
+         while (ptr->next) {
+            if (ptr->next->data == data) {
+               temp = ptr->next;
+               if ( ptr->next == dq->tail ) {
+                  dq->tail = ptr;
+               }
+               ptr->next = ptr->next->next;
+               StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - temp->push_time));
+               free(temp);
+               dq->length--;
+               return;
+            }
+            ptr = ptr->next;
+         }
+      }
+
+   }
+}
+
+static int dq_uid_counter = 0;
+
+delay_queue* dq_create(const char* name, unsigned int latency, unsigned int min_len, unsigned int max_len) {
+   unsigned i;
+   delay_queue* dq;
+   dq = (delay_queue*) malloc(sizeof(delay_queue));
+   dq->name = name;
+   dq->latency = latency;
+   dq->min_len = min_len;
+   dq->max_len = max_len;
+   dq->length = 0;
+   dq->n_element = 0;
+   dq->head = NULL;
+   dq->tail = NULL;
+   for (i=0;i<min_len;i++) dq_push(dq,NULL);
+   dq->uid = dq_uid_counter;
+   dq_uid_counter++;
+   if (1) {
+      dq->lat_stat = StatCreate(dq->name,1,32);  
+   }
+   dq->max_size_stat = 0;
+   dq->avg_size_stat =0.0 ;
+   return dq;
+}
+
+void dq_print(delay_queue* dq) {
+   delay_data* ddp = dq->head;
+   printf("%s(%d): ", dq->name, dq->length);
+   while (ddp) {
+      printf("%p ", ddp->data);
+      ddp = ddp->next;
+   }
+   printf("\n");
+}
+
+void dq_free(delay_queue* dq) {
+   while (dq->head) {
+      dq->tail = dq->head;
+      dq->head = dq->head->next;
+      free(dq->tail);
+   }
+   free(dq);
+   dq = NULL;
+}
+
+void dq_update_stat(delay_queue* dq) {
+   if (dq->n_element > dq->max_size_stat) {
+      dq->max_size_stat = dq->n_element;
+   }
+   dq->avg_size_stat = (dq->avg_size_stat*dq->n_stat_samples + dq->n_element)/(++dq->n_stat_samples); 
+}
+void dq_print_stat(delay_queue* dq) {
+   printf("Max Length: %d, Average Length: %f\n",dq->max_size_stat,dq->avg_size_stat );
+}
+
+
+#ifdef TEST_DQ
+
+void regresstion_test01() {
+   delay_queue *dqa, *dqb;
+   int i;
+   int a[7];
+   for (i=0;i<7;i++) a[i]=i;
+
+   dqa = dq_create("dqa", 0, 7, 0);
+   for (i=0;i<3;i++) dq_push(dqa, &a[i]);
+
+   for (i=0;i<6;i++) {
+      dq_print(dqa);
+      assert(dq_pop(dqa) == 0);
+   }
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[0]);
+
+   // shortening queue 
+   dq_print(dqa);
+   dq_set_min_length(dqa, 4);
+   // see if data in the queue still persist
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[1]);
+   // see if the queue behave with min length = 4
+   dq_push(dqa, &a[3]);
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[2]);
+   for (i=0;i<2;i++) {
+      dq_print(dqa);
+      assert(dq_pop(dqa) == 0);
+   }
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[3]);
+
+   // lengthening queue 
+   dq_set_min_length(dqa, 6);
+   dq_push(dqa, &a[4]);
+   dq_push(dqa, &a[5]);
+   for (i=0;i<5;i++) {
+      dq_print(dqa);
+      assert(dq_pop(dqa) == 0);
+   }
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[4]);
+
+   // queue with no min length
+   dq_set_min_length(dqa, 0);
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[5]);
+   dq_print(dqa);
+   dq_push(dqa, &a[6]);
+   dq_print(dqa);
+   assert(dq_pop(dqa) == &a[6]);
+
+   // lengthening the queue, then shorten it again, 
+   // but with some data exceeding the new min length
+   // the data should retain.
+   dq_print(dqa);
+   dq_set_min_length(dqa, 7);
+   dq_print(dqa);
+   dq_push(dqa, &a[0]);
+   assert(dq_pop(dqa) == 0);
+   dq_print(dqa);
+   dq_set_min_length(dqa, 4);
+   dq_print(dqa);
+   assert(dq_pop(dqa) == 0);
+   assert(dq_pop(dqa) == 0);
+   assert(dq_pop(dqa) == 0);
+   assert(dq_pop(dqa) == 0);
+   assert(dq_pop(dqa) == 0);
+   // This is the 7th pop: min-length is obeyed
+   assert(dq_pop(dqa) == &a[0]); 
+   dq_print(dqa);
+
+   // Shortening a queue with null entry only
+   dq_set_min_length(dqa, 0);
+   assert(dqa->length == 0);
+   dq_print(dqa);
+
+   // Lengthening
+   dq_set_min_length(dqa, 6);
+   assert(dqa->length == 6);
+   dq_print(dqa);
+
+   // Shortening a queue with null entry only
+   dq_set_min_length(dqa, 3);
+   assert(dqa->length == 3);
+   dq_print(dqa);
+
+   dq_free(dqa);
+   printf("regression test 01 passed!\n");
+}
+
+int regresstion_test00() {
+   delay_queue *dqa, *dqb, *dqc, *dqd;
+   int i;
+   int a[4];
+   int *b;
+   for (i=0;i<4;i++) a[i]=i;
+   dqa = dq_create("dqa", 0, 4, 0);
+   dqb = dq_create("dqb", 0, 10, 0);
+   dq_print(dqa);
+   dq_print(dqb);
+   dq_push(dqa,a);
+   dq_print(dqa);
+   dq_pop(dqa);
+   dq_print(dqa);
+   dq_push(dqa,a);
+   dq_print(dqa);
+   dq_pop(dqa);
+   dq_print(dqa);
+   dq_pop(dqa);
+   dq_print(dqa);
+   b = dq_pop(dqa);
+   dq_print(dqa);
+   for (i=0;i<4;i++) printf("%d\n",b[i]);
+   dqc = dq_create("dqc", 0, 0, 3);
+   for (i=0;i<4;i++) {
+      if (!dq_push(dqc,&a[i])) printf("cannot push.\n");
+      dq_print(dqc);
+   }
+   dqd = dq_create("dqd", 0, 2, 3);
+   if (!dq_push(dqd,&a[0])) printf("cannot push.\n");
+   dq_print(dqd);
+   if (!dq_push(dqd,&a[1])) printf("cannot push.\n");
+   dq_print(dqd);
+   if (!dq_push(dqd,&a[2])) printf("cannot push.\n");
+   dq_print(dqd);
+   dq_pop(dqd);
+   if (!dq_push(dqd,&a[3])) printf("cannot push.\n");
+   dq_print(dqd);
+
+   dq_free(dqa);
+   dq_free(dqb);
+   dq_free(dqc);
+   dq_free(dqd);
+
+   return 0;
+}
+
+int main() {
+   regresstion_test01();
+   return 0;
+}
+
+
+#endif
diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h
new file mode 100644
index 0000000..baa3892
--- /dev/null
+++ b/src/gpgpu-sim/delayqueue.h
@@ -0,0 +1,122 @@
+/* 
+ * delayqueue.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda,
+ * Ivan Sham, Henry Tran and the University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+
+#ifndef DELAYQUEUE_H
+#define DELAYQUEUE_H
+
+#include "../util.h"
+
+typedef struct delay_data_t delay_data;
+struct delay_data_t {
+   void *data;
+   unsigned int time_elapsed;
+   delay_data *next;
+   unsigned long long  push_time; //for stat collection
+};
+
+typedef struct {
+   const char* name;
+   int uid;
+
+   unsigned int latency;
+   unsigned int min_len;
+   unsigned int max_len;
+   unsigned int length;
+   unsigned int n_element;
+
+   delay_data *head;
+   delay_data *tail;
+
+   void* lat_stat; //a pointer to latency stats distribution structure
+   //occupancy stat
+   unsigned int max_size_stat;
+   unsigned int n_stat_samples;
+   float  avg_size_stat;
+} delay_queue;
+
+unsigned char dq_full(delay_queue* dq );
+unsigned char dq_empty(delay_queue* dq );
+unsigned int dq_n_element(delay_queue* dq );
+unsigned char dq_push(delay_queue* dq, void* data);
+void* dq_pop(delay_queue* dq);
+void dq_set_min_length(delay_queue* dq, unsigned int new_min_len);
+void removeEntry(void* data, delay_queue** dq, int size_dq);
+delay_queue* dq_create( const char* name, 
+		   unsigned int latency, 
+		   unsigned int min_len, 
+		   unsigned int max_len);
+void dq_remove(void* data, delay_queue* dq);
+void dq_print(delay_queue* dq);
+void dq_free(delay_queue* dq);
+void* dq_top(delay_queue* dq);//return the data in the head without poping the queue
+
+void dq_update_stat(delay_queue* dq);
+void dq_print_stat(delay_queue* dq);
+
+#endif
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
new file mode 100644
index 0000000..9f76b32
--- /dev/null
+++ b/src/gpgpu-sim/dram.cc
@@ -0,0 +1,604 @@
+/* 
+ * dram.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, George L. Yuan,
+ * Ivan Sham, Justin Kwong, Dan O'Connor and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+//#include "gpu-sim.h"
+#include "gpu-misc.h"
+//#include "shader.h"
+#include "dram.h"
+
+extern unsigned long long  gpu_sim_cycle;
+extern signed long long gpu_tot_sim_cycle;
+extern int gpgpu_memlatency_stat;
+extern unsigned max_dq_latency;
+extern unsigned dq_lat_table[24];
+extern unsigned max_dq_latency;
+extern unsigned dq_lat_table[24];
+unsigned int gpu_n_warps;
+extern unsigned int gpu_n_mem_per_ctrlr;
+extern unsigned int recent_dram_util;
+
+#ifdef DRAM_VERIFY
+int PRINT_CYCLE = 0;
+#endif
+
+dram_t* dram_create( unsigned int id, unsigned int nbk, 
+                     unsigned int tCCD, unsigned int tRRD,
+                     unsigned int tRCD, unsigned int tRAS,
+                     unsigned int tRP, unsigned int tRC,
+                     unsigned int CL, unsigned int WL,
+                     unsigned int BL, unsigned int tWTR,
+                     unsigned int busW, unsigned int queue_limit,
+                     unsigned char scheduler_type ) 
+{
+   dram_t *dm;
+   unsigned i;
+
+   dm = (dram_t*)calloc(1,sizeof(dram_t));
+
+   dm->id = id;
+
+   dm->nbk = nbk;
+   dm->tCCD = tCCD;
+   dm->tRRD = tRRD;
+   dm->tRCD = tRCD;
+   dm->tRCDWR = tRCD - (WL + 1); //formula given in datasheet
+   dm->tRAS = tRAS;
+   dm->tRP  = tRP;
+   dm->tRC  = tRC;
+   dm->CL = CL;
+   dm->WL = WL;
+   dm->BL = BL;
+
+   dm->tRTW = (CL+(BL/2)+2-WL); //read to write time according to datasheet
+   dm->tWTR = tWTR;
+
+   dm->busW = busW;
+
+   dm->CCDc = 0;
+   dm->RRDc = 0;
+   dm->RTWc = 0;
+   dm->WTRc = 0;
+
+   dm->rw = READ; //read mode is default
+
+   dm->bk = (bank_t**) calloc(sizeof(bank_t*),dm->nbk);
+   dm->bk[0] = (bank_t*) calloc(sizeof(bank_t),dm->nbk);
+   for (i=1;i<dm->nbk;i++) {
+      dm->bk[i] = dm->bk[0] + i;
+   }
+   for (i=0;i<dm->nbk;i++) {
+      dm->bk[i]->state = BANK_IDLE;
+   }
+   dm->prio = 0;  
+   dm->rwq = dq_create("rwq",0,dm->CL,dm->CL+1);
+   dm->mrqq = dq_create("mrqq",0,0,0);
+   dm->queue_limit = queue_limit;
+
+   dm->returnq = dq_create("dramreturnq",0,0, queue_limit); 
+
+   dm->m_fast_ideal_scheduler = NULL;
+   if ( scheduler_type == DRAM_IDEAL_FAST )
+      dm->m_fast_ideal_scheduler = alloc_fast_ideal_scheduler(dm);
+
+
+   dm->n_cmd = 0;
+   dm->n_activity = 0;
+   dm->n_nop = 0; 
+   dm->n_act = 0; 
+   dm->n_pre = 0; 
+   dm->n_rd = 0;
+   dm->n_wr = 0;
+   dm->n_req = 0;
+   dm->max_mrqs_temp = 0;
+
+   dm->bwutil = 0;
+
+   dm->max_mrqs = 0;
+
+   dm->scheduler_type = scheduler_type;
+
+   dm->realistic_scheduler_mode = READ; //realistic scheduler defaults to read
+   for (i=0;i<10;i++) {
+      dm->dram_util_bins[i]=0;
+      dm->dram_eff_bins[i]=0;
+   }
+   dm->last_n_cmd = dm->last_n_activity = dm->last_bwutil = 0;
+
+   dm->n_cmd_partial = 0;
+   dm->n_activity_partial = 0;
+   dm->n_nop_partial = 0;  
+   dm->n_act_partial = 0;  
+   dm->n_pre_partial = 0;  
+   dm->n_req_partial = 0;
+   dm->ave_mrqs_partial = 0;
+   dm->bwutil_partial = 0;
+   return dm;
+}
+
+void dram_free( dram_t *dm ) 
+{
+   dq_free(dm->mrqq);
+   dq_free(dm->rwq);
+   dq_free( dm->returnq );
+
+   free(dm->bk[0]);
+   free(dm->bk);
+   free(dm);
+}
+
+int dram_full( dram_t *dm ) 
+{
+   int full = 0;
+   if ( dm->queue_limit == 0 ) return 0;
+   if ( dm->scheduler_type == DRAM_IDEAL_FAST ) {
+      unsigned nreqs = fast_scheduler_queue_length(dm) + dq_n_element(dm->mrqq);
+      full = (nreqs >= dm->queue_limit);
+   } else {
+      full = (dm->mrqq->length >= dm->queue_limit);
+   }
+
+   return full;
+}
+
+unsigned int dram_que_length( dram_t *dm ) 
+{
+   unsigned nreqs = 0;
+   if (dm->scheduler_type == DRAM_IDEAL_FAST ) {
+      nreqs = fast_scheduler_queue_length(dm);
+   } else {
+      nreqs = dm->mrqq->length ;
+   }
+   return nreqs;
+}
+
+void dram_push( dram_t *dm, unsigned int bank,
+                unsigned int row, unsigned int col,
+                unsigned int nbytes, unsigned int write,
+                unsigned int wid, 
+                unsigned int sid, int cache_hits_waiting, unsigned long long addr,
+                void *data ) 
+{
+   dram_req_t *mrq;
+
+   if (bank>=dm->nbk) printf("ERROR: no such bank exist in DRAM %d\n", bank);
+
+   mrq = (dram_req_t *) malloc(sizeof(dram_req_t));
+
+   mrq->bk = bank;
+   mrq->row = row;
+   mrq->col = col;
+   mrq->nbytes = nbytes;
+   mrq->txbytes = 0;
+   mrq->dqbytes = 0;
+   mrq->data = data;
+   mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle;
+   mrq->cache_hits_waiting = cache_hits_waiting;
+   mrq->addr = addr;
+   mrq->insertion_time = (unsigned) gpu_sim_cycle;
+
+   if (!write) {
+      mrq->rw = READ;   //request is a read
+   } else {
+      mrq->rw = WRITE;  //request is a write
+   }
+
+   dq_push(dm->mrqq,mrq);
+   dm->n_req += 1;
+   dm->n_req_partial += 1;
+
+   if ( dm->scheduler_type == DRAM_IDEAL_FAST ) {
+      unsigned nreqs = fast_scheduler_queue_length(dm);
+      if ( nreqs > dm->max_mrqs_temp)
+         dm->max_mrqs_temp = nreqs;
+   } else {
+      dm->max_mrqs_temp = (dm->max_mrqs_temp > dm->mrqq->length)? dm->max_mrqs_temp : dm->mrqq->length;
+   }
+}
+
+void scheduler_fifo(dram_t* dm) 
+{
+   if (dm->mrqq->head) {
+      dram_req_t *head_mrqq;
+      unsigned int bkn;
+      head_mrqq = (dram_req_t *)dm->mrqq->head->data;
+      bkn = head_mrqq->bk;
+      if (!dm->bk[bkn]->mrq) {
+         dm->bk[bkn]->mrq = (dram_req_t*) dq_pop(dm->mrqq);
+      }
+   }
+}
+
+
+#define DEC2ZERO(x) x = (x)? (x-1) : 0;
+#define SWAP(a,b) a ^= b; b ^= a; a ^= b;
+
+void dram_issueCMD (dram_t* dm) 
+{
+   unsigned i,j,k;
+   unsigned char issued;
+   issued = 0;
+
+   /* check if the upcoming request is on an idle bank */
+   /* Should we modify this so that multiple requests are checked? */
+
+   switch (dm->scheduler_type) {
+   case DRAM_FIFO:
+      scheduler_fifo(dm);
+      break;
+   case DRAM_IDEAL_FAST:
+      fast_scheduler_ideal(dm);
+      break;
+	default:
+		printf("Error: Unknown DRAM scheduler type\n");
+		assert(0);
+   }
+   if ( dm->scheduler_type == DRAM_IDEAL_FAST ) {
+      unsigned nreqs = fast_scheduler_queue_length(dm);
+      if ( nreqs > dm->max_mrqs) {
+         dm->max_mrqs = nreqs;
+      }
+      dm->ave_mrqs += nreqs;
+      dm->ave_mrqs_partial += nreqs;
+   } else {
+      if (dm->mrqq->length > dm->max_mrqs) {
+         dm->max_mrqs = dm->mrqq->length;
+      }
+      dm->ave_mrqs += dm->mrqq->length;
+      dm->ave_mrqs_partial +=  dm->mrqq->length;
+   }
+   k=dm->nbk;
+   // check if any bank is ready to issue a new read
+   for (i=0;i<dm->nbk;i++) {
+      j = (i + dm->prio) % dm->nbk;
+      if (dm->bk[j]->mrq) { //if currently servicing a memory request
+         // correct row activated for a READ
+         if ( !issued && !dm->CCDc && !dm->bk[j]->RCDc &&
+              (dm->bk[j]->curr_row == dm->bk[j]->mrq->row) && 
+              (dm->bk[j]->mrq->rw == READ) && (dm->WTRc == 0 )  &&
+              (dm->bk[j]->state == BANK_ACTIVE) &&
+              !dq_full(dm->rwq) ) {
+            if (dm->rw==WRITE) {
+               dm->rw=READ;
+               dq_set_min_length(dm->rwq, dm->CL);
+            }
+            dq_push(dm->rwq,(void*)dm->bk[j]->mrq); //only push when rwq empty?
+            dm->bk[j]->mrq->txbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; //16 bytes
+            dm->CCDc = dm->tCCD;
+            dm->RTWc = dm->tRTW;
+            issued = 1;
+            dm->n_rd++;
+            //printf("\tn_rd++ Bank: %d Row: %d Col: %d\n", j, dm->bk[j]->mrq->row, dm->bk[j]->mrq->col);
+            dm->bwutil+= dm->BL/2;
+            dm->bwutil_partial += dm->BL/2;
+            dm->bk[j]->n_access++;
+#ifdef DRAM_VERIFY
+            PRINT_CYCLE=1;
+            printf("\tRD  Bk:%d Row:%03x Col:%03x \n",
+                   j, dm->bk[j]->curr_row,
+                   dm->bk[j]->mrq->col+dm->bk[j]->mrq->txbytes-dm->BL*dm->busW);
+#endif            
+            // transfer done
+            if ( !(dm->bk[j]->mrq->txbytes < dm->bk[j]->mrq->nbytes) ) {
+               dm->bk[j]->mrq = NULL;
+            }
+         } else
+            // correct row activated for a WRITE
+            if ( !issued && !dm->CCDc && !dm->bk[j]->RCDWRc &&
+                 (dm->bk[j]->curr_row == dm->bk[j]->mrq->row)  && 
+                 (dm->bk[j]->mrq->rw == WRITE) && (dm->RTWc == 0 )  &&
+                 (dm->bk[j]->state == BANK_ACTIVE) &&
+                 !dq_full(dm->rwq) ) {
+            if (dm->rw==READ) {
+               dm->rw=WRITE;
+               dq_set_min_length(dm->rwq, dm->WL);
+            }
+            dq_push(dm->rwq,(void*)dm->bk[j]->mrq);
+
+            dm->bk[j]->mrq->txbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; /*16 bytes*/
+            dm->CCDc = dm->tCCD;
+            issued = 1;
+            dm->n_wr++;
+            dm->bwutil+=2;
+            dm->bwutil_partial += dm->BL/2;
+#ifdef DRAM_VERIFY
+            PRINT_CYCLE=1;
+            printf("\tWR  Bk:%d Row:%03x Col:%03x \n",
+                   j, dm->bk[j]->curr_row, 
+                   dm->bk[j]->mrq->col+dm->bk[j]->mrq->txbytes-dm->BL*dm->busW);
+#endif  
+            // transfer done 
+            if ( !(dm->bk[j]->mrq->txbytes < dm->bk[j]->mrq->nbytes) ) {
+               dm->bk[j]->mrq = NULL;
+            }
+         }
+
+         else
+            // bank is idle
+            if ( !issued && !dm->RRDc && 
+                 (dm->bk[j]->state == BANK_IDLE) &&
+                 !dm->bk[j]->RPc && !dm->bk[j]->RCc ) {
+#ifdef DRAM_VERIFY
+            PRINT_CYCLE=1;
+            printf("\tACT BK:%d NewRow:%03x From:%03x \n",
+                   j,dm->bk[j]->mrq->row,dm->bk[j]->curr_row);
+#endif
+            // activate the row with current memory request 
+            dm->bk[j]->curr_row = dm->bk[j]->mrq->row;
+            dm->bk[j]->state = BANK_ACTIVE;
+            dm->RRDc = dm->tRRD;
+            dm->bk[j]->RCDc = dm->tRCD;
+            dm->bk[j]->RCDWRc = dm->tRCDWR;
+            dm->bk[j]->RASc = dm->tRAS;
+            dm->bk[j]->RCc = dm->tRC;
+            dm->prio = (j + 1) % dm->nbk;
+            issued = 1;
+            dm->n_act_partial++;
+            dm->n_act++;
+         }
+
+         else
+            // different row activated
+            if ( (!issued) && 
+                 (dm->bk[j]->curr_row != dm->bk[j]->mrq->row) &&
+                 (dm->bk[j]->state == BANK_ACTIVE) && 
+                 (!dm->bk[j]->RASc) ) {
+            //printf("\tRASc: %d \n", dm->bk[j]->RASc);
+            // make the bank idle again
+            dm->bk[j]->state = BANK_IDLE;
+            dm->bk[j]->RPc = dm->tRP;
+            dm->prio = (j + 1) % dm->nbk;
+            issued = 1;
+            dm->n_pre++;
+            dm->n_pre_partial++;
+#ifdef DRAM_VERIFY
+            PRINT_CYCLE=1;
+            printf("\tPRE BK:%d Row:%03x \n", j,dm->bk[j]->curr_row);
+            //printf("\tRASc: %d \n", dm->bk[j]->RASc);
+#endif
+         }
+      } else {
+         if (!dm->CCDc && !dm->RRDc && !dm->RTWc && !dm->WTRc && !dm->bk[j]->RCDc && !dm->bk[j]->RASc
+             && !dm->bk[j]->RCc && !dm->bk[j]->RPc  && !dm->bk[j]->RCDWRc) k--;
+         dm->bk[i]->n_idle++;
+      }
+   }
+   if (!issued) {
+      dm->n_nop++;
+      dm->n_nop_partial++;
+#ifdef DRAM_VIEWCMD
+      printf("\tNOP                        ");
+#endif
+   }
+   if (k) {
+      dm->n_activity++;
+      dm->n_activity_partial++;
+   }
+   dm->n_cmd++;
+   dm->n_cmd_partial++;
+
+   // decrements counters once for each time dram_issueCMD is called
+   DEC2ZERO(dm->RRDc);
+   DEC2ZERO(dm->CCDc);
+   DEC2ZERO(dm->RTWc);
+   DEC2ZERO(dm->WTRc);
+   for (j=0;j<dm->nbk;j++) {
+      DEC2ZERO(dm->bk[j]->RCDc);
+      DEC2ZERO(dm->bk[j]->RASc);
+      DEC2ZERO(dm->bk[j]->RCc);
+      DEC2ZERO(dm->bk[j]->RPc);
+      DEC2ZERO(dm->bk[j]->RCDWRc);
+   }
+
+#ifdef DRAM_VISUALIZE
+   dram_visualize(dm);
+#endif
+}
+
+//if mrq is being serviced by dram, gets popped after CL latency fulfilled
+void* dram_pop( dram_t *dm ) 
+{ 
+   dram_req_t *mrq;
+   void *data;
+   unsigned dq_latency;
+
+   data = NULL;
+   mrq = (dram_req_t*)dq_pop(dm->rwq);
+   if (mrq) {
+      // data = mrq->data; 
+#ifdef DRAM_VIEWCMD 
+      printf("\tDQ: BK%d Row:%03x Col:%03x",
+             mrq->bk, mrq->row, mrq->col + mrq->dqbytes);
+#endif
+      mrq->dqbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; /*16 bytes*/
+      if (mrq->dqbytes >= mrq->nbytes) {
+
+         if (gpgpu_memlatency_stat) {
+            dq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - mrq->timestamp;
+            dq_lat_table[LOGB2(dq_latency)]++;
+            if (dq_latency > max_dq_latency)
+               max_dq_latency = dq_latency;
+         }
+         data = mrq->data; 
+
+         free(mrq);
+      }
+   }
+#ifdef DRAM_VIEWCMD 
+   printf("\n");
+#endif
+
+   return data;
+}
+
+// a hack to allow peeking into what memory request will be serviced.
+void* dram_top( dram_t *dm )
+{
+   dram_req_t *mrq;
+   void *data;
+
+   data = NULL;
+   mrq = (dram_req_t*)dq_top(dm->rwq);
+   if (mrq) {
+      // number of bytes returned from dram if this is ever popped
+      unsigned tobe_dqbytes = mrq->dqbytes + dm->BL * dm->busW * gpu_n_mem_per_ctrlr; 
+      if (tobe_dqbytes >= mrq->nbytes) {
+         data = mrq->data; 
+      }
+   }
+
+   return data;
+}
+
+void dram_print( dram_t* dm, FILE* simFile) 
+{
+   unsigned i;
+   fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", 
+           dm->id, dm->nbk, dm->busW, dm->BL, dm->CL );
+   fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n",
+           dm->tCCD, dm->tRRD, dm->tRCD, dm->tRAS, dm->tRP, dm->tRC );
+   fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g\n",
+           dm->n_cmd, dm->n_nop, dm->n_act, dm->n_pre, dm->n_req, dm->n_rd, dm->n_wr,
+           (float)dm->bwutil/dm->n_cmd);
+   fprintf(simFile,"n_activity=%d dram_eff=%.4g\n",
+           dm->n_activity, (float)dm->bwutil/dm->n_activity);
+   for (i=0;i<dm->nbk;i++) {
+      fprintf(simFile, "bk%d: %da %di ",i,dm->bk[i]->n_access,dm->bk[i]->n_idle);
+   }
+   fprintf(simFile, "\n");
+   fprintf(simFile, "dram_util_bins:");
+   for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_util_bins[i]);
+   fprintf(simFile, "\ndram_eff_bins:");
+   for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_eff_bins[i]);
+   fprintf(simFile, "\n");
+   /*
+   {
+   delay_data* mrq;
+   mrq = dm->mrqq->head;
+   while (mrq) {
+      printf("%d",((dram_req_t*)mrq->data)->bk);
+      mrq = mrq->next;
+   }
+   printf("\n");
+   }
+   */
+   fprintf(simFile, "mrqq: max=%d avg=%g\n", dm->max_mrqs, (float)dm->ave_mrqs/dm->n_cmd);
+}
+
+void dram_visualize( dram_t* dm ) 
+{
+   unsigned i;
+
+   printf("RRDc=%d CCDc=%d mrqq.Length=%d rwq.Length=%d\n", 
+          dm->RRDc, dm->CCDc, dm->mrqq->length,dm->rwq->length);
+   for (i=0;i<dm->nbk;i++) {
+      printf("BK%d: state=%c curr_row=%03x, %2d %2d %2d %2d %p ", 
+             i, dm->bk[i]->state, dm->bk[i]->curr_row,
+             dm->bk[i]->RCDc, dm->bk[i]->RASc,
+             dm->bk[i]->RPc, dm->bk[i]->RCc,
+             dm->bk[i]->mrq );
+      if (dm->bk[i]->mrq)
+         printf("txf: %d %d", dm->bk[i]->mrq->nbytes, dm->bk[i]->mrq->txbytes);
+      printf("\n");
+   }
+   if ( dm->m_fast_ideal_scheduler ) {
+      dump_fast_ideal_scheduler( dm );
+   }
+
+}
+
+void dram_print_stat( dram_t* dm, FILE* simFile ) 
+{
+   int i;
+   fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ",
+           dm->id, dm->n_cmd, dm->n_nop, dm->n_act, dm->n_pre, dm->n_req, dm->n_rd, dm->n_wr,
+           (float)dm->bwutil/dm->n_cmd);
+   fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", dm->max_mrqs, (float)dm->ave_mrqs/dm->n_cmd, dm->max_mrqs_temp);
+   fprintf(simFile, "\n");
+   fprintf(simFile, "dram_util_bins:");
+   for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_util_bins[i]);
+   fprintf(simFile, "\ndram_eff_bins:");
+   for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_eff_bins[i]);
+   fprintf(simFile, "\n");
+   dm->max_mrqs_temp = 0;
+}
+
+
+unsigned dram_busy( dram_t* dm) 
+{
+   unsigned busy = 0;
+
+   switch (dm->scheduler_type) {
+   case DRAM_FIFO:
+      busy = (dm->mrqq->length > 0);
+      break;
+   case DRAM_IDEAL_FAST:
+      busy = (fast_scheduler_queue_length(dm) > 0) || (dm->mrqq->length > 0);
+      break;
+   }
+
+   return busy;
+}
+
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
new file mode 100644
index 0000000..fff23f6
--- /dev/null
+++ b/src/gpgpu-sim/dram.h
@@ -0,0 +1,237 @@
+/* 
+ * dram.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, George L. Yuan,
+ * Ivan Sham, Justin Kwong, Dan O'Connor and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files 
+ * src/gpgpusim_entrypoint.c and src/simplesim-3.0/ are derived from the 
+ * SimpleScalar Toolset available from http://www.simplescalar.com/ 
+ * (property of SimpleScalar LLC) and the files src/intersim/ are derived 
+ * from Booksim (Simulator provided with the textbook "Principles and 
+ * Practices of Interconnection Networks" available from 
+ * http://cva.stanford.edu/books/ppin/).  As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA
+ * which is distributed seperately by NVIDIA under separate terms and
+ * conditions.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "delayqueue.h"
+#include "../cuda-sim/dram_callback.h"
+
+#ifndef DRAM_H
+#define DRAM_H
+
+#define FIFO_AGE_LIMIT 50     //used for both BANK_CONF and REALISTIC schedulers
+#define FIFO_NUM_WRITE_LIMIT 3   //used for both BANK_CONF and REALISTIC schedulers
+#define LOOKAHEAD_VALUE 10    //used for REALISTIC scheduler ONLY
+
+enum {
+   DRAM_FIFO,
+   DRAM_IDEAL_FAST,
+   DRAM_NUM_HANDLES
+};
+
+
+#define READ 'R'  //define read and write states
+#define WRITE 'W'
+typedef struct {
+   unsigned int row;
+   unsigned int col;
+   unsigned int bk;
+   unsigned int nbytes;
+   unsigned int txbytes;
+   unsigned int dqbytes;
+   unsigned int age;
+   unsigned int timestamp;
+   unsigned char rw;    //is the request a read or a write?
+   unsigned long long int addr;
+   unsigned int insertion_time;
+   void* data;
+
+   int cache_hits_waiting; 
+} dram_req_t;
+
+#define BANK_IDLE 'I'
+#define BANK_ACTIVE 'A'
+
+typedef struct {
+   unsigned int RCDc;
+   unsigned int RCDWRc;
+   unsigned int RASc;
+   unsigned int RPc;
+   unsigned int RCc;
+
+   unsigned char rw;    //is the bank reading or writing?
+   unsigned char state; //is the bank active or idle?
+   unsigned int curr_row;
+
+   dram_req_t *mrq;
+
+   unsigned int n_access;
+   unsigned int n_writes;
+   unsigned int n_idle;
+} bank_t;
+
+typedef struct {
+   unsigned int id;
+
+   unsigned int tCCD;   //column to column delay
+   unsigned int tRRD;   //minimal time required between activation of rows in different banks
+   unsigned int tRCD;   //row to column delay - time required to activate a row before a read
+   unsigned int tRCDWR;//row to column delay for a write command
+   unsigned int tRAS;   //time needed to activate row
+   unsigned int tRP; //row precharge ie. deactivate row
+   unsigned int tRC; //row cycle time ie. precharge current, then activate different row
+
+   unsigned int CL;  //CAS latency
+   unsigned int WL;  //WRITE latency
+   unsigned int BL;  //Burst Length in bytes (we're using 4? could be 8)
+   unsigned int tRTW;   //time to switch from read to write
+   unsigned int tWTR;   //time to switch from write to read 5? look in datasheet
+   unsigned int busW;
+
+   unsigned int nbk;
+   bank_t **bk;
+   unsigned int prio;
+
+   unsigned int RRDc;
+   unsigned int CCDc;
+   unsigned int RTWc;   //read to write penalty applies across banks
+   unsigned int WTRc;   //write to read penalty applies across banks
+
+   unsigned char rw; //was last request a read or write? (important for RTW, WTR)
+
+   unsigned int pending_writes;
+   unsigned char realistic_scheduler_mode;
+
+   delay_queue *rwq;
+   delay_queue *mrqq;
+   //buffer to hold packets when DRAM processing is over
+   //should be filled with dram clock and popped with l2or icnt clock 
+   delay_queue *returnq;      
+
+
+   unsigned int dram_util_bins[10];
+   unsigned int dram_eff_bins[10];
+   unsigned int last_n_cmd, last_n_activity, last_bwutil;
+
+   unsigned int queue_limit;
+
+   unsigned int n_cmd;
+   unsigned int n_activity;
+   unsigned int n_nop;
+   unsigned int n_act;
+   unsigned int n_pre;
+   unsigned int n_rd;
+   unsigned int n_wr;
+   unsigned int n_req;
+   unsigned int max_mrqs_temp;
+
+   unsigned int bwutil;
+   unsigned int max_mrqs;
+   unsigned int ave_mrqs;
+   unsigned char scheduler_type;
+
+   void *m_fast_ideal_scheduler;
+
+   void *m_L2cache;
+
+   unsigned int n_cmd_partial;
+   unsigned int n_activity_partial;
+   unsigned int n_nop_partial; 
+   unsigned int n_act_partial; 
+   unsigned int n_pre_partial; 
+   unsigned int n_req_partial;
+   unsigned int ave_mrqs_partial;
+   unsigned int bwutil_partial;
+
+   void * req_hist;
+} dram_t;
+
+
+dram_t* dram_create( unsigned int id, unsigned int nbk, 
+		unsigned int tCCD, unsigned int tRRD,
+		unsigned int tRCD, unsigned int tRAS,
+		unsigned int tRP, unsigned int tRC,
+		unsigned int CL, unsigned int WL, 
+		unsigned int BL, unsigned int tWTR,
+		unsigned int busW, unsigned int queue_limit,
+		unsigned char scheduler_type );
+void dram_free( dram_t *dm );
+int dram_full( dram_t *dm );
+void dram_push( dram_t *dm, unsigned int bank,
+	   unsigned int row, unsigned int col,
+	   unsigned int nbytes, unsigned int write,
+	   unsigned int wid, unsigned int sid, int cache_hits_waiting, unsigned long long addr,
+	   void *data );
+void scheduler_fifo(dram_t* dm);
+void dram_issueCMD (dram_t* dm);
+void* dram_pop( dram_t *dm );
+void* dram_top( dram_t *dm );
+unsigned dram_busy( dram_t *dm);
+void dram_print( dram_t* dm, FILE* simFile );
+void dram_visualize( dram_t* dm );
+void dram_print_stat( dram_t* dm, FILE* simFile );
+void fast_scheduler_ideal(dram_t* dm);
+void* alloc_fast_ideal_scheduler(dram_t *dm);
+void dump_fast_ideal_scheduler(dram_t *dm);
+unsigned fast_scheduler_queue_length(dram_t *dm);
+
+//supposed to return the current queue length for all memory scheduler types.
+unsigned int dram_que_length( dram_t *dm ); 
+
+#endif /*DRAM_H*/
diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc
new file mode 100644
index 0000000..24b518f
--- /dev/null
+++ b/src/gpgpu-sim/dram_sched.cc
@@ -0,0 +1,247 @@
+/* 
+ * dram_sched.cc
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "dram_sched.h"
+#include "gpu-misc.h"
+#include "gpu-sim.h"
+#include "../util.h"
+
+extern unsigned long long  gpu_sim_cycle;
+extern signed long long gpu_tot_sim_cycle;
+extern unsigned max_mrq_latency;
+extern unsigned mrq_lat_table[24];
+extern int gpgpu_memlatency_stat;
+extern int gpgpu_dram_sched_queue_size;
+extern unsigned int **concurrent_row_access; //concurrent_row_access[dram chip id][bank id]
+extern unsigned int **row_access; //concurrent_row_access[dram chip id][bank id]
+extern unsigned int **num_activates; //num_activates[dram chip id][bank id]
+extern unsigned int **max_conc_access2samerow; //max_conc_access2samerow[dram chip id][bank id]
+extern unsigned int **max_servicetime2samerow;
+
+ideal_dram_scheduler::ideal_dram_scheduler( dram_t *dm )
+{
+
+   m_num_pending = 0;
+   m_dram = dm;
+   m_queue = new std::list<dram_req_t*>[dm->nbk];
+   m_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ dm->nbk ];
+   m_last_row = new std::list<std::list<dram_req_t*>::iterator>*[ dm->nbk ];
+   curr_row_service_time = new unsigned[dm->nbk];
+   row_service_timestamp = new unsigned[dm->nbk];
+   for ( unsigned i=0; i < dm->nbk; i++ ) {
+      m_queue[i].clear();
+      m_bins[i].clear();
+      m_last_row[i] = NULL;
+      curr_row_service_time[i] = 0;
+      row_service_timestamp[i] = 0;
+   }
+
+}
+
+void ideal_dram_scheduler::add_req( dram_req_t *req )
+{
+   m_num_pending++;
+
+   m_queue[req->bk].push_front(req);
+   std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin();
+
+   m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+
+
+}
+
+
+inline void ideal_dram_scheduler::data_collection(unsigned int bank)
+{
+   if (gpu_sim_cycle > row_service_timestamp[bank]) {
+      curr_row_service_time[bank] = gpu_sim_cycle - row_service_timestamp[bank];
+      if (curr_row_service_time[bank] > max_servicetime2samerow[m_dram->id][bank])
+         max_servicetime2samerow[m_dram->id][bank] = curr_row_service_time[bank];
+   }
+   curr_row_service_time[bank] = 0;
+   row_service_timestamp[bank] = gpu_sim_cycle;
+   if (concurrent_row_access[m_dram->id][bank] > max_conc_access2samerow[m_dram->id][bank]) {
+      max_conc_access2samerow[m_dram->id][bank] = concurrent_row_access[m_dram->id][bank];
+   }
+   concurrent_row_access[m_dram->id][bank] = 0;
+   num_activates[m_dram->id][bank]++;
+}
+
+dram_req_t *ideal_dram_scheduler::schedule( unsigned bank, unsigned curr_row )
+{
+   int row_hit = 0;
+   if ( m_last_row[bank] == NULL ) {
+      if ( m_queue[bank].empty() )
+         return NULL;
+
+      std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row );
+      if ( bin_ptr == m_bins[bank].end()) {
+         dram_req_t *req = m_queue[bank].back();
+         bin_ptr = m_bins[bank].find( req->row );
+         assert( bin_ptr != m_bins[bank].end() ); // where did the request go???
+         m_last_row[bank] = &(bin_ptr->second);
+         data_collection(bank);
+      } else {
+         m_last_row[bank] = &(bin_ptr->second);
+
+      }
+   }
+   row_hit=1;
+   std::list<dram_req_t*>::iterator next = m_last_row[bank]->back();
+   dram_req_t *req = (*next);
+
+   concurrent_row_access[m_dram->id][bank]++;
+   row_access[m_dram->id][bank]++;
+   m_last_row[bank]->pop_back();
+
+   m_queue[bank].erase(next);
+   if ( m_last_row[bank]->empty() ) {
+      m_bins[bank].erase( req->row );
+      m_last_row[bank] = NULL;
+   }
+#ifdef DEBUG_FAST_IDEAL_SCHED
+   if ( req )
+      printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", 
+             (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row );
+#endif
+   assert( req != NULL && m_num_pending != 0 ); 
+   m_num_pending--;
+
+   return req;
+}
+
+
+void ideal_dram_scheduler::print( FILE *fp )
+{
+   for ( unsigned b=0; b < m_dram->nbk; b++ ) {
+      printf(" %u: queue length = %u\n", b, (unsigned)m_queue[b].size() );
+   }
+}
+
+void* alloc_fast_ideal_scheduler(dram_t *dm)
+{
+   return new ideal_dram_scheduler(dm);
+}
+
+void fast_scheduler_ideal(dram_t* dm)
+{
+
+
+   unsigned mrq_latency;
+   // replacement for scheduler_ideal()
+
+   ideal_dram_scheduler *sched = reinterpret_cast<ideal_dram_scheduler*>( dm->m_fast_ideal_scheduler );
+   while ( !dq_empty(dm->mrqq) && (!gpgpu_dram_sched_queue_size || sched->num_pending() < (unsigned) gpgpu_dram_sched_queue_size)) {
+      dram_req_t *req = (dram_req_t*)dq_pop(dm->mrqq);
+      sched->add_req(req);
+   }
+
+   dram_req_t *req;
+   unsigned i;
+   for ( i=0; i < dm->nbk; i++ ) {
+      unsigned b = (i+dm->prio)%dm->nbk;
+      if ( !dm->bk[b]->mrq ) {
+
+         req = sched->schedule(b, dm->bk[b]->curr_row);
+
+         if ( req ) {
+            dm->prio = (dm->prio+1)%dm->nbk;
+            dm->bk[b]->mrq = req;
+            if (gpgpu_memlatency_stat) {
+               mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - dm->bk[b]->mrq->timestamp;
+               dm->bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle;
+               mrq_lat_table[LOGB2(mrq_latency)]++;
+               if (mrq_latency > max_mrq_latency) {
+                  max_mrq_latency = mrq_latency;
+               }
+            }
+
+            break;
+         }
+      }
+   }
+}
+
+
+
+void dump_fast_ideal_scheduler( dram_t *dm )
+{
+   ideal_dram_scheduler *sched = reinterpret_cast<ideal_dram_scheduler*>( dm->m_fast_ideal_scheduler );
+   sched->print(stdout);
+}
+
+unsigned fast_scheduler_queue_length(dram_t *dm)
+{
+   if (dm->m_fast_ideal_scheduler ) {
+      ideal_dram_scheduler *sched = reinterpret_cast<ideal_dram_scheduler*>( dm->m_fast_ideal_scheduler );
+      return sched->num_pending();
+   } else {
+      printf("fast_scheduler_queue_length(): Where did the scheduler go?\n");
+      exit(1);
+   }
+}
+
+
diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h
new file mode 100644
index 0000000..49cae22
--- /dev/null
+++ b/src/gpgpu-sim/dram_sched.h
@@ -0,0 +1,99 @@
+/* 
+ * dram_sched.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef dram_sched_h_INCLUDED
+#define dram_sched_h_INCLUDED
+
+#include "dram.h"
+#include "shader.h"
+#include "gpu-sim.h"
+#include "gpu-misc.h"
+#include <list>
+#include <map>
+
+class ideal_dram_scheduler {
+public:
+   ideal_dram_scheduler( dram_t *dm );
+   void add_req( dram_req_t *req );
+   std::list<dram_req_t*>::iterator binarysort_VFTF(dram_req_t *req);
+   std::list<dram_req_t*>::iterator sort_VFTF(dram_req_t *req);
+   inline void data_collection(unsigned bank);
+   dram_req_t *schedule( unsigned bank, unsigned curr_row );
+   void print( FILE *fp );
+   unsigned num_pending() const { return m_num_pending;}
+
+private:
+
+   dram_t *m_dram;
+   unsigned m_num_pending;
+   std::list<dram_req_t*>                                    *m_queue;
+   std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >    *m_bins;
+   std::list<std::list<dram_req_t*>::iterator>                 **m_last_row;
+   unsigned *curr_row_service_time; //one set of variables for each bank.
+   unsigned *row_service_timestamp; //tracks when scheduler began servicing current row
+};
+
+#endif
diff --git a/src/gpgpu-sim/dwf.cc b/src/gpgpu-sim/dwf.cc
new file mode 100644
index 0000000..cd66084
--- /dev/null
+++ b/src/gpgpu-sim/dwf.cc
@@ -0,0 +1,2609 @@
+/* 
+ * dwf.cc
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+
+#include "dwf.h"
+#include "histogram.h"
+#include <map>
+#include <set>
+#include <deque>
+#include <queue>
+#include <string.h>
+
+using namespace std;
+
+unsigned int gpgpu_dwf_regbk = 1;
+unsigned int gpgpu_dwf_heuristic = 0;
+enum {
+   MAJORITY  = 0,
+   MINORITY  = 1,
+   FIFO      = 2,
+   PDOMPRIO  = 3,
+   PC        = 4,
+   MAJORITY_MAXHEAP = 5,
+   N_DWFMODE
+};
+
+typedef struct warp_entry {
+   address_type pc;
+   int* tid; // thread id's
+   int occ; // occupancy vector
+   int pdom_prio; // pdom_priority
+   int pdom_occ; // pdom_priority's aux data
+   int next_warp; // index to next warp in an implicit queue
+   void* lut_ptr; // pointer to the warp lut entry that last update this warp (a hack), done to decouple warp lut and warp pool
+   int uid; // unique id of a warp
+} warp_entry_t;
+
+class issue_warp_majority {
+public:
+
+   virtual void add_threads( address_type pc, int *tid) = 0;
+   virtual void push_warp( address_type pc, int idx) = 0;
+   virtual int pop_warp( ) = 0;
+   virtual void print( FILE *fout ) = 0;
+   virtual ~issue_warp_majority( ) {}
+};
+
+typedef struct maxheap_lut_entry {
+   address_type pc;    // pc of the warps
+   int maxheap_idx; // index to the max heap
+} maxheap_lut_entry_t;
+
+typedef struct maxheap_entry {
+   address_type pc;    // pc of the warps
+   int n_thds;  // number of threads with this pc (from lut)
+   int wpool_head;  // the first warp with this pc
+   int wpool_tail;  // the last warp with this pc
+   int lut_idx; // reverse index to the lut (for update in entry movement)
+} maxheap_entry_t;
+
+class mh_lut_class {
+private:
+
+   maxheap_lut_entry_t *lut_data;
+   list<int> *lru_stack; // front = LRU
+   int n_set;
+   int insn_size_lgb2;
+
+public:
+
+   int size;
+   int assoc;
+   int n_read;
+   int n_write;
+   int n_read_per_cycle;
+   int n_write_per_cycle;
+
+   int n_aliased;
+   static maxheap_lut_entry_t clean_entry;
+
+   mh_lut_class (int size, int assoc, int n_read_per_cycle, int n_write_per_cycle ) {
+      int i;
+
+      this->size = size;
+      this->assoc = assoc;
+      lut_data = new maxheap_lut_entry_t[size];
+
+      for (i=0; i<size; i++) {
+         lut_data[i] = clean_entry;
+      }
+
+      n_set = size/assoc;
+      assert(n_set && !((n_set - 1) & n_set)); // make sure n_set is a power of 2
+
+      insn_size_lgb2 = 0;
+
+      lru_stack = new list<int>[n_set];
+      for (i=0; i<n_set; i++) {
+         int j;
+         for (j=0; j<assoc; j++) {
+            lru_stack[i].push_back(i * assoc + j);
+         }
+      }
+
+      this->n_read_per_cycle = n_read_per_cycle;
+      this->n_write_per_cycle = n_write_per_cycle;
+      this->n_read = 0;
+      this->n_write = 0;
+      this->n_aliased = 0;
+   }
+
+   ~mh_lut_class ( ) {
+      delete[] lut_data;
+   }
+
+   // obtain entry at a known location
+   maxheap_lut_entry_t get( int lut_idx ) {
+      assert(lut_idx < size);
+      n_read++;
+      return lut_data[lut_idx];
+   }
+
+   // modify an entry at a known location
+   void set( int lut_idx, maxheap_lut_entry_t lut_entry ) {
+      n_write++;
+      lut_data[lut_idx] = lut_entry;
+   }
+
+   // update a lut entry with a new index
+   void update_mh_idx( int lut_idx, int mh_idx ) {
+      n_write++;
+      lut_data[lut_idx].maxheap_idx = mh_idx;
+   }
+
+   // lookup an entry with a pc
+   int lookup( address_type pc ) {
+      int i;
+      int lut_idx = -1;
+      int set_start_idx = get_set(pc) * assoc;
+
+      // look for the matched entry within the set
+      for (i = set_start_idx; i < (set_start_idx + assoc); i++) {
+         if (lut_data[i].pc == pc) {
+            lut_idx = i;
+         }
+      }
+
+      // update lru stack if hit
+      if (lut_idx != -1) {
+         int set_idx = set_start_idx / assoc;
+         list<int>::iterator it;
+         it = find(lru_stack[set_idx].begin(), lru_stack[set_idx].end(), lut_idx);
+
+         if (it != lru_stack[set_idx].end()) {
+            lru_stack[set_idx].erase(it);
+            lru_stack[set_idx].push_back(lut_idx);
+         }
+      }
+
+      return lut_idx;
+   }
+
+   void free(int lut_idx) {
+      set(lut_idx, clean_entry);
+
+      int set_idx = lut_idx / assoc;
+      list<int>::iterator it;
+      it = find(lru_stack[set_idx].begin(), lru_stack[set_idx].end(), lut_idx);
+
+      if (it != lru_stack[set_idx].end()) {
+         lru_stack[set_idx].erase(it);
+         lru_stack[set_idx].push_front(lut_idx);
+      }
+   }
+
+   // find the LRU entry to be replaced
+   int find_lru( maxheap_lut_entry_t lut_entry ) {
+      int set_idx = get_set(lut_entry.pc);
+      int lru_idx = lru_stack[set_idx].front();
+
+      return lru_idx;
+   }
+
+   // actually replacing the LRU entry
+   int replace_lru( maxheap_lut_entry_t lut_entry ) {
+      int set_idx = get_set(lut_entry.pc);
+      int lru_idx = lru_stack[set_idx].front();
+      lru_stack[set_idx].pop_front();
+
+      // counting the number of overwritten entries
+      if (lut_data[lru_idx].maxheap_idx != 0) n_aliased++;
+
+      set(lru_idx, lut_entry);
+      lru_stack[set_idx].push_back(lru_idx);
+
+      return lru_idx;
+   }
+
+   // reset the number of accesses to zero
+   void reset_access( ) {
+      n_read = 0;
+      n_write = 0;
+   }
+
+   // clear the number of accesses - done at the end of scheduler cycle
+   void clear_access( ) {
+      n_read -= n_read_per_cycle;
+      n_read = (n_read >= 0)? n_read : 0;
+      n_write -= n_write_per_cycle;
+      n_write = (n_write >= 0)? n_write : 0;
+   }
+
+   // test if the structure is done with all the required accesses
+   int all_access_done( ) {
+      return(n_read == 0 && n_write == 0);
+   }
+
+   void print_lut_e(FILE *fout, maxheap_lut_entry_t lut_e) {
+      fprintf(fout, "[%08x]mh%02d", 
+              lut_e.pc, lut_e.maxheap_idx);
+   }
+
+   void print(FILE *fout) {
+      int i, j;
+      for (i=0; i<n_set; i++) {
+         fprintf(fout, "S%02d", i);
+         for (j=0; j<assoc; j++) {
+            fprintf(fout, " |%02d:", i * assoc + j);
+            print_lut_e(fout, lut_data[i * assoc + j]);
+         }
+         fprintf(fout, "  ");
+         list<int>::iterator it = lru_stack[i].begin();
+         for (; it != lru_stack[i].end(); it++) {
+            fprintf(fout, "%02d-", *it);
+         }
+         fprintf(fout, "\n");
+      }
+   }
+
+private:
+
+   inline int get_set(address_type pc) {
+      return((pc >> insn_size_lgb2) & (n_set - 1));
+   }
+};
+
+maxheap_lut_entry_t mh_lut_class::clean_entry = {0xDEADBEEF, 0};
+
+// A class tracking the number of accesses done to the maxheap structure
+// and the index ranges from 1..n_entries with 1 being the root
+class maxheap_class {
+private:
+
+   maxheap_entry_t *maxheap_data;
+   mh_lut_class *lut;
+
+public:
+
+   int n_read;
+   int n_write;
+   int n_entries;
+   int size;
+   int n_read_per_cycle;
+   int n_write_per_cycle;
+
+   int max_n_entries;
+   static maxheap_entry_t clean_entry;
+
+   maxheap_class( int size, mh_lut_class *lut, int n_read_per_cycle, int n_write_per_cycle ) {
+      n_read = 0;
+      n_write = 0;
+      n_entries = 0; // index to the last element
+      this->size = size;
+      maxheap_data = new maxheap_entry_t[size];
+
+      for (int i=0; i<size; i++) {
+         maxheap_data[i] = clean_entry;
+      }
+
+      this->lut = lut;
+
+      this->n_read_per_cycle = n_read_per_cycle;
+      this->n_write_per_cycle = n_write_per_cycle;
+      this->n_read = 0;
+      this->n_write = 0;
+      this->max_n_entries = 0;
+   }
+
+   ~maxheap_class( ) {
+      delete[] maxheap_data;
+   }
+
+   // insert a new entry into the maxheap
+   // return: the index to the new entry
+   int insert( maxheap_entry_t mh_entry ) {
+      assert(n_entries + 1 < size); 
+      n_write++;
+      n_entries++;
+      maxheap_data[n_entries] = mh_entry;
+      max_n_entries = (max_n_entries >= n_entries)? max_n_entries : n_entries;
+      return n_entries;
+   }
+
+   // retrieve the max heap entry at index [mh_idx]
+   maxheap_entry_t get( int mh_idx ) {
+      assert(mh_idx > 0);
+      assert(mh_idx <= n_entries);
+      n_read++;
+      return maxheap_data[mh_idx];
+   }
+
+   // replace the max heap entry at index [mh_idx]
+   void set( int mh_idx, maxheap_entry_t mh_entry ) {
+      assert(mh_idx > 0);
+      assert(mh_idx <= n_entries);
+      n_write++;
+      maxheap_data[mh_idx] = mh_entry;
+   }
+
+   // a special version of set that only reset the lut_idx
+   void remove_lut_idx( int mh_idx ) {
+      assert(mh_idx > 0);
+      assert(mh_idx <= n_entries);
+      n_write++;
+      maxheap_data[mh_idx].lut_idx = -1;
+   }
+
+   // read both childrens of a given node, count as one read
+   // return the number of child read
+   int get_childof(int mh_idx, maxheap_entry_t *child) {
+      int child_idx = childof(mh_idx);
+      int child_read = 0;
+
+      if (child_idx <= n_entries) {
+         n_read++;
+         child[0] = maxheap_data[child_idx]; 
+         child_read++;
+      }
+      if (child_idx + 1 <= n_entries) {
+         child[1] = maxheap_data[child_idx + 1]; 
+         child_read++;
+      }
+
+      return child_read;
+   }
+
+   // pop the root entry of max heap
+   maxheap_entry_t pop_root( ) {
+      maxheap_entry_t old_root = get(1);
+      maxheap_entry_t curr_mhe[3];
+      curr_mhe[0] = get(n_entries);
+
+      set(1, curr_mhe[0]);
+      if (curr_mhe[0].lut_idx >= 0)
+         lut->update_mh_idx(curr_mhe[0].lut_idx, 1);
+
+      n_entries--;
+
+      int curr_node = 1;
+      int n_child = 0;
+
+      n_child = get_childof(curr_node, curr_mhe + 1);
+      while (n_child > 0) {
+         int max_child = 0;
+         int i;
+         for (i = 1; i < n_child + 1; i++) {
+            if (cmp_mh(curr_mhe[i], curr_mhe[max_child])) {
+               max_child = i;
+            }
+         }
+
+         n_child = 0;
+         if (max_child > 0) {
+            int max_child_node = childof(curr_node) + max_child - 1;
+            set(curr_node, curr_mhe[max_child]);
+            set(max_child_node, curr_mhe[0]);
+
+            // update the lut for this swap
+            if (curr_mhe[max_child].lut_idx >= 0)
+               lut->update_mh_idx(curr_mhe[max_child].lut_idx, curr_node);
+            if (curr_mhe[0].lut_idx >= 0)
+               lut->update_mh_idx(curr_mhe[0].lut_idx, max_child_node);
+
+            // get the next child
+            curr_node = max_child_node;
+            n_child = get_childof(curr_node, curr_mhe + 1);
+         }
+      }
+
+      return old_root;
+   }
+
+   // probe if the maxheap is empty
+   int empty( ) {
+      return(n_entries == 0);
+   }
+
+   // reset the number of accesses to zero
+   void reset_access( ) {
+      n_read = 0;
+      n_write = 0;
+   }
+
+   // clear the number of accesses - done at the end of scheduler cycle
+   void clear_access( ) {
+      n_read -= n_read_per_cycle;
+      n_read = (n_read >= 0)? n_read : 0;
+      n_write -= n_write_per_cycle;
+      n_write = (n_write >= 0)? n_write : 0;
+   }
+
+   // test if the structure is done with all the required accesses
+   int all_access_done( ) {
+      return(n_read == 0 && n_write == 0);
+   }
+
+   // sort the max heap again starting from start_idx 
+   // (this entry can only go up in the tree to the root)
+   void sort_bottomup(int start_idx) {
+      maxheap_entry_t mh_entry;
+      maxheap_entry_t mh_parent;
+
+      if (start_idx == 1) return; // no need to resort if the root is incremented
+
+      int curr_idx = start_idx;
+      int parent_idx = parentof(start_idx);
+
+      int continue_sort = 1;
+      while (curr_idx > 1 && continue_sort) {
+         mh_entry = get(curr_idx);
+         mh_parent = get(parent_idx);
+
+         // swap the entries if it is now larger than it's parent
+         if (cmp_mh(mh_entry, mh_parent)) {
+            set(parent_idx, mh_entry);
+            set(curr_idx, mh_parent);
+
+            // update the lut for this swap
+            if (mh_entry.lut_idx >= 0)
+               lut->update_mh_idx(mh_entry.lut_idx, parent_idx);
+            if (mh_parent.lut_idx >= 0)
+               lut->update_mh_idx(mh_parent.lut_idx, curr_idx);
+
+            // update index for next iteration
+            curr_idx = parent_idx;
+            parent_idx = parentof(curr_idx);
+         } else {
+            // swap did not happen, no need to sort anymore 
+            continue_sort = 0;
+         }
+      }
+   }
+
+   void print_mh_e(FILE *fout, maxheap_entry_t mh_e) {
+      fprintf(fout, "[%08x]%03d(H%03dT%03d)p%02d | ", 
+              mh_e.pc, mh_e.n_thds, mh_e.wpool_head, mh_e.wpool_tail, mh_e.lut_idx);
+   }
+
+   void print(FILE *fout) {
+      fprintf(fout, "MaxHeap: ");
+      fprintf(fout, "N_entries = %d\n", n_entries);
+      for (int i=0; i<n_entries; i++) {
+         print_mh_e(fout, maxheap_data[i + 1]);
+         if (!((i + 2) & (i + 1))) fprintf(fout, "\n");
+      }
+      fprintf(fout, "\n");
+   }
+
+private:
+
+   static inline int parentof(int mh_idx) {
+      assert(mh_idx > 0);
+      return(mh_idx / 2);
+   }
+
+   static inline int childof(int mh_idx) {
+      return(mh_idx * 2);
+   }
+
+   static inline int cmp_mh(maxheap_entry_t &a, maxheap_entry_t &b) {
+      if (a.n_thds > b.n_thds) return 1;
+      if (a.n_thds == b.n_thds) {
+         if (a.pc < b.pc) return 1;
+      }
+      return 0;
+   }
+
+};
+
+maxheap_entry_t maxheap_class::clean_entry = {0, 0, -1, -1, -1};
+
+typedef struct mh_update_struct {
+   int n_maxheap_read;
+   int n_maxheap_write;
+   int n_mhlut_read;
+   int n_mhlut_write;
+} mh_update;
+
+// heap implementation of majority policy
+class issue_warp_majority_heap : public issue_warp_majority {
+public:
+
+   mh_lut_class mh_lut;
+   maxheap_class maxheap;
+
+   maxheap_lut_entry_t major_lut_e;
+   maxheap_entry_t major_mh_e;
+
+   vector<warp_entry_t> *warp_pool;
+   int simd_width;
+
+   int n_stall_on_maxheap;
+
+   queue<mh_update> update_queue;
+   static pow2_histogram n_pending_updates_histo;
+
+   issue_warp_majority_heap (int simd_width = 0, vector<warp_entry_t> *bp = NULL,
+                             int lut_size = 32, int lut_assoc = 4, int maxheap_size = 128,
+                             int n_read_lut = 4, int n_write_lut = 4, 
+                             int n_read_mh = 4, int n_write_mh = 4) 
+   : mh_lut(lut_size, lut_assoc, n_read_lut, n_write_lut), 
+   maxheap(maxheap_size, &mh_lut, n_read_mh, n_write_mh)
+   {
+      this->simd_width = simd_width;
+      this->warp_pool = bp;
+
+      this->major_lut_e = mh_lut_class::clean_entry;
+      this->major_mh_e  = maxheap_class::clean_entry;
+
+      this->n_stall_on_maxheap = 0;
+   }
+
+   // adding more threads to a specify pc
+   // these threads may end up in different warpes
+   void add_threads( address_type pc, int *tid) {
+      int i;
+      int n_thds = 0;
+      for (i=0; i<simd_width; i++) {
+         if (tid[i] >= 0) n_thds++;
+      }
+
+      // handle special case with adding threads to current majority pc
+      if (major_lut_e.pc == pc) {
+         assert(major_mh_e.pc == pc);
+         major_mh_e.n_thds += n_thds;
+         return;
+      }
+
+      maxheap_lut_entry_t lut_e;
+      maxheap_entry_t mh_entry;
+
+      // snapshot the current maxheap read/write demand
+      mh_update new_mh_update;
+      new_mh_update.n_maxheap_read  = maxheap.n_read;
+      new_mh_update.n_maxheap_write = maxheap.n_write;
+      new_mh_update.n_mhlut_read  = mh_lut.n_read;
+      new_mh_update.n_mhlut_write = mh_lut.n_write;
+
+      int lut_idx = mh_lut.lookup(pc);
+
+      int sort_from_idx = 0;
+
+      if (lut_idx >= 0) {
+         // obtain the entry
+         lut_e = mh_lut.get(lut_idx);
+
+         // get the maxheap entry and update its number of threads
+         mh_entry = maxheap.get(lut_e.maxheap_idx);
+         mh_entry.n_thds += n_thds;
+         maxheap.set(lut_e.maxheap_idx, mh_entry);
+
+         // sort from this specific entry
+         sort_from_idx = lut_e.maxheap_idx;
+      } else {
+         // create a new lut entry 
+         lut_e = mh_lut_class::clean_entry;
+         lut_e.pc = pc;
+
+         // get index to the LRU lut entry in this set
+         lut_idx = mh_lut.find_lru(lut_e);
+
+         // get the replaced lut entry and remove its link with the maxheap entry
+         maxheap_lut_entry_t lut_old = mh_lut.get(lut_idx);
+         if (lut_old.maxheap_idx > 0) maxheap.remove_lut_idx(lut_old.maxheap_idx);
+
+         // create a new maxheap entry
+         mh_entry = maxheap_class::clean_entry;
+         mh_entry.pc = pc;
+         mh_entry.n_thds = n_thds;
+         mh_entry.lut_idx = lut_idx;
+
+         // push the new entry into the maxheap and lut respectively
+         lut_e.maxheap_idx = maxheap.insert(mh_entry);
+         mh_lut.replace_lru(lut_e);
+
+         // start sorting from the bottom?
+         sort_from_idx = lut_e.maxheap_idx;
+      }
+
+      maxheap.sort_bottomup(sort_from_idx);
+
+      // record the newly generated maxheap read/write demand from this update
+      new_mh_update.n_maxheap_read  = maxheap.n_read - new_mh_update.n_maxheap_read;
+      new_mh_update.n_maxheap_write = maxheap.n_write - new_mh_update.n_maxheap_write;
+      new_mh_update.n_mhlut_read  = mh_lut.n_read - new_mh_update.n_mhlut_read;
+      new_mh_update.n_mhlut_write = mh_lut.n_write - new_mh_update.n_mhlut_write;
+
+      update_queue.push(new_mh_update);
+   }
+
+   // call this when a new warp allocated for a specific pc
+   void push_warp( address_type pc, int idx) {
+      maxheap_entry_t *p_mh_e = NULL;
+      maxheap_entry_t mh_e;
+      maxheap_lut_entry_t lut_e = mh_lut_class::clean_entry;
+      int lut_idx = -1;
+
+      if (major_mh_e.pc == pc) {
+         p_mh_e = &major_mh_e;
+      } else {
+         lut_idx = mh_lut.lookup(pc);
+         assert(lut_idx >= 0); // if it is a miss, a new entry should have been created already
+         lut_e = mh_lut.get(lut_idx);
+         mh_e = maxheap.get(lut_e.maxheap_idx);
+         p_mh_e = &mh_e;
+
+         // discounting these 'gets'
+         // because they should be combined with the 'gets' in add_threads()
+         mh_lut.n_read--;
+         maxheap.n_read--;
+      }
+
+      if (p_mh_e->wpool_head == -1) {
+         p_mh_e->wpool_head = idx;
+         p_mh_e->wpool_tail = idx;
+      } else {
+         (*warp_pool)[p_mh_e->wpool_tail].next_warp = idx;
+         p_mh_e->wpool_tail = idx;
+      }
+
+      if (major_mh_e.pc == pc) {
+      } else {
+         maxheap.set(lut_e.maxheap_idx, mh_e);
+         // discounting this 'set'
+         // because it should be combined with the 'set' in add_threads()
+         maxheap.n_write--;
+      }
+   }
+
+   // obtain a warp index from this issue logic
+   int pop_warp( ) {
+      int bidx = -1;
+      if (major_mh_e.wpool_head == -1 && !maxheap.empty()) {
+         if (this->all_access_done( )) {
+            // pop the majority PC from max heap
+            major_mh_e = maxheap.pop_root();
+
+            // pop its corresponding entry from the lut as well (if it exists)
+            if (major_mh_e.lut_idx >= 0) {
+               major_lut_e = mh_lut.get(major_mh_e.lut_idx);
+               mh_lut.free(major_mh_e.lut_idx);
+            } else {
+               major_lut_e = mh_lut_class::clean_entry;
+            }
+         } else {
+            n_stall_on_maxheap += 1;
+            bidx = -1;
+            return bidx;
+         }
+      }
+
+      // just pop and entry to from the virtual queue (and set the head pointer to next warp)
+      bidx = major_mh_e.wpool_head;
+      if (bidx >= 0) {
+         major_mh_e.wpool_head = (*warp_pool)[major_mh_e.wpool_head].next_warp;
+      }
+
+      return bidx;
+   }
+
+   void reset_access( ) {
+      maxheap.reset_access();
+      mh_lut.reset_access();
+
+      while (!update_queue.empty()) {
+         update_queue.pop();
+      }
+   }
+
+   inline void consume_access( int &req_acc, int &avl_acc) {
+      if (req_acc > avl_acc) {
+         req_acc -= avl_acc;
+         avl_acc = 0;
+      } else {
+         avl_acc -= req_acc;
+         req_acc = 0;
+      }
+   }
+
+   void clear_access( ) {
+      maxheap.clear_access();
+      mh_lut.clear_access();
+
+      int n_maxheap_read_bw  = maxheap.n_read_per_cycle;
+      int n_maxheap_write_bw = maxheap.n_write_per_cycle;
+      int n_mhlut_read_bw  = mh_lut.n_read_per_cycle;
+      int n_mhlut_write_bw = mh_lut.n_write_per_cycle;
+
+      while ((n_maxheap_read_bw > 0 || n_maxheap_read_bw > 0 || 
+              n_mhlut_read_bw > 0 || n_mhlut_write_bw > 0) && !update_queue.empty()) {
+         mh_update &c_update = update_queue.front();
+
+         consume_access (c_update.n_maxheap_read,  n_maxheap_read_bw);
+         consume_access (c_update.n_maxheap_write, n_maxheap_write_bw);
+         consume_access (c_update.n_mhlut_read,   n_mhlut_read_bw);
+         consume_access (c_update.n_mhlut_write,  n_mhlut_write_bw);
+
+         if (c_update.n_maxheap_read == 0 && c_update.n_maxheap_write == 0 && 
+             c_update.n_mhlut_read == 0 && c_update.n_mhlut_write == 0) {
+            update_queue.pop();
+         } else {
+            break;
+         }
+      }
+
+      n_pending_updates_histo.add2bin(update_queue.size());
+   }
+
+   void print( FILE *fout ) {
+      fprintf(fout, "LUT: ");
+      mh_lut.print_lut_e(fout, major_lut_e);
+      fprintf(fout, " \tMH: ");
+      maxheap.print_mh_e(fout, major_mh_e);
+      fprintf(fout, "\n");
+      mh_lut.print(fout);
+      maxheap.print(fout);
+   }
+
+   static void print_stat( FILE *fout) {
+      fprintf(fout, "n_pending_maxheap_updates = ");
+      n_pending_updates_histo.fprint(fout);
+      fprintf(fout, "\n");
+   }
+
+private:
+
+   int all_access_done( ) {
+      return(maxheap.all_access_done() && mh_lut.all_access_done());
+
+   }
+};
+pow2_histogram issue_warp_majority_heap::n_pending_updates_histo;
+
+class warp_queue {
+public:
+   int m_pc;
+   int n_thds;
+   int simd_width;
+   deque<int> idx_queue;
+
+   warp_queue( address_type pc, int simd_width) {
+      this->m_pc = pc;
+      this->n_thds = 0;
+      this->simd_width = simd_width;
+   }
+
+   // called right after a lut_entry is looked up
+   void add_threads( int *tid ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] >= 0) this->n_thds++;
+      }
+   }
+
+   // called right after a warp is issued
+   void sub_threads( int *tid ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] >= 0) this->n_thds--;
+      }
+   }
+
+   // if other warp queue should be ahead
+   bool operator<(const warp_queue& other) const {
+      if (n_thds == other.n_thds) {
+         return(m_pc > other.m_pc); // smaller pc first
+      } else {
+         return(n_thds < other.n_thds);
+      }
+   }
+   bool operator>(const warp_queue& other) const {
+      if (n_thds == other.n_thds) {
+         return(m_pc > other.m_pc); // smaller pc first
+      } else {
+         return(n_thds > other.n_thds);
+      }
+   }
+
+   void print( FILE *fout ) {
+      fprintf(fout, "0x%08x(%03d)=[", m_pc, n_thds);
+      deque<int>::iterator dit = idx_queue.begin();
+      for (; dit != idx_queue.end(); dit++) {
+         fprintf(fout, "%03d ", *dit);
+      }
+      fprintf(fout, "]\n");
+   }
+};
+
+bool minor_warp( const warp_queue* a, const warp_queue* b ) {
+   return(*a<*b);
+}
+
+// queue implementation of majority scheduling policy
+class issue_warp_majority_queue : public issue_warp_majority {
+public:
+   map<address_type, warp_queue* > majority_map;
+   set<warp_queue*> warpq_set;
+   warp_queue* maj_warp;
+
+   vector<warp_entry_t> *warp_pool;
+   int simd_width;
+
+   issue_warp_majority_queue(int simd_width = 0, vector<warp_entry_t> *bp = NULL) {
+      this->maj_warp = NULL;
+      this->simd_width = simd_width;
+      this->warp_pool = bp;
+   }
+
+   // adding more threads to a specify pc
+   // these threads may end up in different warps
+   void add_threads( address_type pc, int *tid) {
+      warp_queue* bq = majority_map[pc];
+      if (bq == NULL) {
+         bq = new warp_queue(pc,simd_width);
+         warpq_set.insert(bq);
+         majority_map[pc] = bq;
+      }
+      bq->add_threads(tid);
+   }
+
+   // call this when a new warp allocated for a specific pc
+   void push_warp( address_type pc, int idx) {
+      warp_queue* bq = majority_map[pc];
+      assert(bq != NULL);
+      bool check_redundant_idx = false;
+      if (check_redundant_idx) {
+         deque<int>::iterator dit = find(bq->idx_queue.begin(), bq->idx_queue.end(), idx);
+         assert(dit == bq->idx_queue.end());
+      }
+      bq->idx_queue.push_back(idx);
+   }
+
+   // obtain a warp index from this issue logic
+   int pop_warp( ) {
+      int bidx = -1;
+
+      // find the new majority pc if it didn't exist
+      if (maj_warp == NULL && warpq_set.size()) {
+         maj_warp = *max_element(warpq_set.begin(), warpq_set.end(), minor_warp);
+      }
+
+      // if a majority pc indeed exist
+      if (maj_warp) {
+         assert(!maj_warp->idx_queue.empty());
+         bidx = maj_warp->idx_queue.front();
+         maj_warp->idx_queue.pop_front();
+         maj_warp->sub_threads((*warp_pool)[bidx].tid);
+
+         // when the majority pc runs out of thread
+         if (maj_warp->n_thds == 0) {
+            // remove that warp queue
+            warpq_set.erase(maj_warp);
+            majority_map.erase(maj_warp->m_pc);
+            delete maj_warp;
+            maj_warp = NULL;
+         }
+      }
+
+      return bidx;
+   }
+
+   void print( FILE *fout ) {
+      fprintf(fout, "issue_warp_majority:\n");
+      set<warp_queue*>::iterator dit = warpq_set.begin();
+      for (; dit != warpq_set.end(); dit++) {
+         fprintf(fout, " %c ", ((*dit)==maj_warp)? 'M':' ');
+         (*dit)->print(fout);
+      }
+   }
+
+   void check_consistency( ) {
+      set<warp_queue*>::iterator set_it = warpq_set.begin();
+      for (; set_it != warpq_set.end(); set_it++) {
+         warp_queue* bq = (*set_it);
+
+         int real_nthds = 0;
+         deque<int>::iterator dit = bq->idx_queue.begin();
+         for (; dit != bq->idx_queue.end(); dit++) {
+            int *tid = (*warp_pool)[*dit].tid;
+            for (int i = 0; i < simd_width; i++) {
+               real_nthds += (tid[i] >= 0)? 1 : 0;
+            }
+         }
+
+         assert(real_nthds == bq->n_thds);
+      }
+   }
+};
+
+// pdom priority 
+class lesspdom_first {
+public:
+   vector<warp_entry_t> *warp_pool;
+   lesspdom_first( vector<warp_entry_t> *bp=NULL ) {
+      this->warp_pool = bp;
+   }
+   bool operator() (const int &idx_a, const int &idx_b) const {
+      if ((*warp_pool)[idx_a].pdom_prio != (*warp_pool)[idx_b].pdom_prio) {
+         return((*warp_pool)[idx_a].pdom_prio < (*warp_pool)[idx_b].pdom_prio);
+      } else {
+         return((*warp_pool)[idx_a].occ > (*warp_pool)[idx_b].occ);
+      }
+   }
+};
+
+
+class issue_warp_pdom_prio {
+public:
+   vector<warp_entry_t> *warp_pool;
+   int* thd_pdom_prio; 
+   int simd_width;
+   int n_threads;
+
+   int resort_needed;
+   list<int> pdom_pqueue; //the queue holding all index
+
+   lesspdom_first lesspdom_cmp;
+
+   static set<address_type> reconvgence_pt; //table holding all recvg pt
+
+   issue_warp_pdom_prio (int simd_width = 0, vector<warp_entry_t> *bp = NULL, 
+                         int n_threads = 0) 
+   : lesspdom_cmp(bp)
+   {
+      this->simd_width = simd_width;
+      this->warp_pool = bp;
+      this->n_threads = n_threads;
+      this->thd_pdom_prio = new int[n_threads];
+      memset(this->thd_pdom_prio, 0, sizeof(int)*n_threads);
+      this->resort_needed = 0;
+   }
+
+   ~issue_warp_pdom_prio( ) {
+      delete[] this->thd_pdom_prio;
+   }
+
+   void reinit( ) {
+      memset(this->thd_pdom_prio, 0, sizeof(int)*n_threads);
+   }
+
+   // adding more threads to a warp
+   void add_threads( int idx, address_type pc) {
+      assert((*warp_pool)[idx].pc == pc);
+
+      // check to see if this is a newly allocated warp
+      bool check_pdom = false;
+      if ((*warp_pool)[idx].pdom_prio == -1) {
+         check_pdom = true;
+      }
+
+      // check for newly assigned threads to the warp
+      int pdom_occ = (*warp_pool)[idx].pdom_occ;
+      int *tid = (*warp_pool)[idx].tid;
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] >= 0 && !(pdom_occ & (1<<i))) {
+            if ((*warp_pool)[idx].pdom_prio < thd_pdom_prio[tid[i]]) {
+               (*warp_pool)[idx].pdom_prio = thd_pdom_prio[tid[i]];
+               resort_needed = 1;
+            }
+            pdom_occ |= (1<<i);
+         }
+      }
+      if (check_pdom) {
+         if (reconvgence_pt.find(pc) != reconvgence_pt.end()) {
+            (*warp_pool)[idx].pdom_prio += 1;
+         }
+      }
+   }
+
+   // call this when a new warp allocated for a specific pc
+   void push_warp( address_type pc, int idx ) {
+      assert((*warp_pool)[idx].pc == pc);
+      // initialize the pdom_prio for this newly allocated warp
+      (*warp_pool)[idx].pdom_prio = -1;
+      (*warp_pool)[idx].pdom_occ = 0;
+      pdom_pqueue.push_back(idx);
+   }
+
+   // obtain a warp index from this issue logic
+   int front_warp( ) {
+      int bidx = -1;
+
+      if (!pdom_pqueue.empty()) {
+         if (resort_needed) {
+            pdom_pqueue.sort(lesspdom_cmp);
+            resort_needed = 0;
+         }
+
+         bidx = pdom_pqueue.front();
+      }
+
+      return bidx;
+   }
+
+   int size( ) {
+      return pdom_pqueue.size();
+   }
+
+   void enforce_resort( ) {
+      resort_needed = 1;
+   }
+
+   int pop_warp( ) {
+      int bidx = -1;
+
+      if (!pdom_pqueue.empty()) {
+         if (resort_needed) {
+            pdom_pqueue.sort(lesspdom_cmp);
+            resort_needed = 0;
+         }
+
+         bidx = pdom_pqueue.front();
+         pdom_pqueue.pop_front();
+
+         // update the pdom prio of each thread inside a warp
+         for (int i=0; i<simd_width; i++) {
+            if ((*warp_pool)[bidx].tid[i] >= 0) {
+               thd_pdom_prio[(*warp_pool)[bidx].tid[i]] = (*warp_pool)[bidx].pdom_prio;
+            }
+         }
+      }
+
+      return bidx;
+   }
+
+};
+
+set<address_type> issue_warp_pdom_prio::reconvgence_pt = set<address_type>(); 
+//*/
+
+
+class npc_tracker_class {
+public:
+   map<address_type, unsigned> pc_count;
+   unsigned* acc_pc_count;
+   int simd_width;
+   static map<unsigned, unsigned> histogram;
+
+   npc_tracker_class( ) {
+      this->acc_pc_count = NULL;
+      this->simd_width = 0;
+   }
+
+   npc_tracker_class(unsigned* acc_pc_count, int simd_width) {
+      this->acc_pc_count = acc_pc_count;
+      this->simd_width = simd_width;
+   }
+
+   void add_threads( int *tid, address_type pc ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] != -1) pc_count[pc] += 1; // automatically create a new entry if not exist
+      }
+   }
+
+   void sub_threads( int *tid, address_type pc ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] != -1) {
+            pc_count[pc] -= 1;
+            assert((int)pc_count[pc] >= 0);
+            if (pc_count[pc] == 0) pc_count.erase(pc); // manually erasing entries with 0 count
+         }
+      }
+   }
+
+   void update_acc_count( ) { 
+      (*acc_pc_count) += pc_count.size(); 
+      histogram[pc_count.size()] += 1;
+   }
+
+   unsigned count( ) { return pc_count.size();}
+
+   static void histo_print( FILE* fout ) {
+      map<unsigned, unsigned>::iterator i;
+      fprintf(fout, "DYHW nPC Histogram: ");
+      for (i = histogram.begin(); i != histogram.end(); i++) {
+         fprintf(fout, "%d:%d ", i->first, i->second);
+      }
+      fprintf(fout, "\n");
+   }
+};
+
+map<unsigned, unsigned> npc_tracker_class::histogram;
+
+class pc_tag {
+private:
+
+   address_type m_pc;
+
+public:
+
+   pc_tag () {
+      this->reset();
+   }
+
+   pc_tag (const pc_tag& p) { this->m_pc = p.m_pc;}
+   pc_tag (const address_type& other_pc) { this->m_pc = other_pc;}
+
+   pc_tag& operator=(const pc_tag& p) { m_pc = p.m_pc; return *this;}
+   pc_tag& operator=(const address_type& other_pc) { m_pc = other_pc; return *this;}
+
+   inline bool operator==(const pc_tag& p) const { return(m_pc == p.m_pc);}
+   inline bool operator==(const address_type& other_pc) const { return(m_pc == other_pc);}
+
+   inline bool operator!=(const pc_tag& p) const { return(m_pc != p.m_pc);}
+   inline bool operator!=(const address_type& other_pc) const { return(m_pc != other_pc);}
+
+   inline bool operator<(const pc_tag& p) const { return(m_pc < p.m_pc);}
+
+   inline void reset() {
+      m_pc = -1;
+   }
+
+   inline address_type get_pc() const { return m_pc;}
+
+   // the hash function to warp LUT
+   inline unsigned lut_hash( int insn_size_lgb2, int lut_nsets ) const {
+      return(m_pc >> insn_size_lgb2) & (lut_nsets - 1);
+   }
+
+   inline void to_print(char *buffer, unsigned length) {
+      snprintf(buffer, length, "0x%08x", m_pc);
+   }
+};
+
+template <class Tag>
+class tag2warp_entry_t {
+public: 
+
+   Tag tag;
+   int idx; // pointing to warp pool
+   int occ; // occupancy vector
+   int accessed; // is the entry accessed this cycle
+
+   tag2warp_entry_t () {
+      this->reset();
+   }
+
+   ~tag2warp_entry_t () {}
+
+   tag2warp_entry_t (const tag2warp_entry_t& p) {
+      this->tag = p.tag;
+      this->idx = p.idx;
+      this->occ = p.occ;
+      this->accessed = p.accessed;
+   }
+
+   tag2warp_entry_t& operator=(const tag2warp_entry_t& p) {
+      if (this != &p) {
+         tag = p.tag;
+         idx = p.idx;
+         occ = p.occ;
+         accessed = p.accessed;
+      }
+      return *this;
+   }
+
+   inline bool operator==(const tag2warp_entry_t& p) const {
+      return(tag == p.tag);
+   }
+
+   inline bool operator==(const Tag& test_tag) const {
+      return(tag == test_tag);
+   }
+
+   inline bool operator()(const tag2warp_entry_t& p) const {
+      return(tag == p.tag);
+   }
+
+   inline void reset() {
+      tag.reset();
+      idx = 0; 
+      occ = 0; 
+      accessed = 0; 
+   }
+
+   void print( FILE *fout ) {
+      static char buffer[20];
+      tag.to_print(buffer,20);
+      fprintf(fout, "\t%s->%03d (%02x)\n", buffer, idx, occ);
+   }
+
+};
+
+template <class Tag>
+class tag2warp_set {
+public:
+   vector< tag2warp_entry_t<Tag> > entry;
+   list< tag2warp_entry_t<Tag>* > lru_stack;
+
+   tag2warp_set(int assoc = 0) : entry(assoc) {
+      for (unsigned j=0; j<this->entry.size(); j++) {
+         this->lru_stack.push_back(&(this->entry[j]));
+      }
+   }
+
+   tag2warp_set(const tag2warp_set& other) : entry(other.entry.size()) {
+      for (unsigned j=0; j<this->entry.size(); j++) {
+         this->lru_stack.push_back(&(this->entry[j]));
+      }
+   }
+
+   tag2warp_set& operator=(const tag2warp_set& p) {
+      printf("tag2warp_set assignment operator called!\n");
+      return *this;
+   }
+
+   ~tag2warp_set() {}
+};
+
+template <class Tag>
+class warp_lut {
+public:
+   virtual ~warp_lut() {}
+   virtual tag2warp_entry_t<Tag>* lookup_pc2warp( const Tag& tag, bool& lut_missed ) = 0;
+   virtual void invalidate_entry( tag2warp_entry_t<Tag>* lut_entry, int warp_idx ) = 0;
+   virtual void clear_accessed( ) = 0;
+   virtual void print( FILE* fout) = 0;
+};
+
+template <class Tag>
+class warp_lut_sa : public warp_lut<Tag> {
+private: 
+   int lut_size;
+   int lut_assoc;
+   vector< tag2warp_set<Tag> > tag2warp_lut;
+   int insn_size_lgb2;
+
+   queue< tag2warp_entry_t<Tag>* > lut_accessed_q; // store accessed lut entry for clear
+
+   struct same_tag {
+      Tag tag;
+      bool operator()(tag2warp_entry_t<Tag>* a) {
+         return(a->tag == tag);
+      }
+   };
+
+   static unsigned int lut_aliased;
+
+public:
+   warp_lut_sa(int lut_size, int lut_assoc, int insn_size) {
+      this->lut_size = lut_size;
+      this->lut_assoc = lut_assoc;
+
+      // optimize for LUT hash function
+      insn_size_lgb2 = 0;
+      while ( (1 << insn_size_lgb2) < insn_size ) insn_size_lgb2++;
+
+      // initialize the pc2warp LUT
+      // note: lut_size is the absolute size of LUT regardless of assoc.
+      this->tag2warp_lut.assign(lut_size/lut_assoc, tag2warp_set<Tag>(lut_assoc));
+
+      // assert on #set in LUT to be power of 2
+      int lut_nset_pow2 = 1;
+      while ( lut_nset_pow2 < (int)tag2warp_lut.size() ) lut_nset_pow2 <<= 1;
+      assert((int)tag2warp_lut.size() == lut_nset_pow2);
+   }
+
+   tag2warp_entry_t<Tag>* lookup_pc2warp( const Tag& tag, bool& lut_missed );
+   void invalidate_entry( tag2warp_entry_t<Tag>* lut_entry, int warp_idx ) {
+      if (lut_entry != NULL) { // check for warp lut entry invalidation
+         if (lut_entry->idx == warp_idx) {
+            lut_entry->reset();
+         }
+      }
+   }
+
+   void clear_accessed( );
+
+   void print( FILE* fout) {
+      for (unsigned i=0; i< tag2warp_lut.size(); i++) {
+         for (unsigned j=0; j< tag2warp_lut[i].entry.size(); j++) {
+            fprintf(fout, "lut%03d-%02d:", i, j);
+            tag2warp_lut[i].entry[j].print(fout);
+         }
+      }
+   }
+
+   static void print_stats ( FILE* fout ) {
+      fprintf( fout, "lut_aliased = %d\n", lut_aliased);
+   }
+};
+template <class Tag> unsigned int warp_lut_sa<Tag>::lut_aliased = 0;
+
+
+// lookup function in LUT
+// may return an entry that has different PC for replacement
+// or return a NULL pointer to indicate that the entry is accessed by another port
+template <class Tag>
+tag2warp_entry_t<Tag>* warp_lut_sa<Tag>::lookup_pc2warp( const Tag &tag, bool &lut_missed )
+{
+   tag2warp_entry_t<Tag>* lut_entry = NULL;
+   unsigned hashed_pc = tag.lut_hash(insn_size_lgb2, tag2warp_lut.size());
+   list< tag2warp_entry_t<Tag>* > &hashed_lru_stack = tag2warp_lut.at(hashed_pc).lru_stack;
+   struct same_tag same_tag_f;
+
+   same_tag_f.tag = tag;
+   typename list< tag2warp_entry_t<Tag>* >::iterator lut_it;
+   lut_it = find_if(hashed_lru_stack.begin(), 
+                    hashed_lru_stack.end(),
+                    same_tag_f);
+   if (lut_it != hashed_lru_stack.end()) {
+      lut_entry = *lut_it;
+      lut_entry->accessed = 1;
+      lut_accessed_q.push(lut_entry);
+      hashed_lru_stack.splice(hashed_lru_stack.end(), hashed_lru_stack, lut_it);
+      assert(lut_entry == hashed_lru_stack.back());
+      lut_missed = false;
+   } else {
+      assert(!hashed_lru_stack.empty());
+      lut_entry = hashed_lru_stack.front();
+      if (lut_entry->accessed) {
+         lut_entry = NULL;
+      } else {
+         lut_entry->accessed = 1;
+         lut_accessed_q.push(lut_entry);
+         hashed_lru_stack.splice(hashed_lru_stack.end(), hashed_lru_stack, hashed_lru_stack.begin());
+         assert(lut_entry == hashed_lru_stack.back());
+         lut_aliased++;
+      }
+      lut_missed = true;
+   }
+   assert(hashed_lru_stack.size() == tag2warp_lut[hashed_pc].entry.size());
+
+   return lut_entry;
+}
+
+template <class Tag>
+void warp_lut_sa<Tag>::clear_accessed( ) {
+   while ( !lut_accessed_q.empty() ) {
+      lut_accessed_q.front()->accessed = 0;
+      lut_accessed_q.pop();
+   }
+}
+
+// a perfect warp lut that never misses.
+template <class Tag>
+class warp_lut_perfect : public warp_lut<Tag> {
+private:
+   typedef map< Tag, tag2warp_entry_t<Tag>* > warp_map_t;
+   warp_map_t m_tag2entry_map;
+
+   static unsigned int lut_max_size;
+public:
+   warp_lut_perfect() {}
+   ~warp_lut_perfect() {
+      typename warp_map_t::iterator mit = m_tag2entry_map.begin();
+      for (; mit != m_tag2entry_map.end(); mit++) {
+         delete mit->second;
+      }
+   }
+
+   // idealistic implementation of lookup: the entry is never aliased, 
+   // and a new one is created automatically if it does not exist
+   tag2warp_entry_t<Tag>* lookup_pc2warp( const Tag& tag, bool& lut_missed ) {
+      typename warp_map_t::iterator mit = m_tag2entry_map.find(tag);
+
+      tag2warp_entry_t<Tag>* lut_entry = NULL;
+      if (mit != m_tag2entry_map.end()) {
+         lut_entry = mit->second;
+         assert(lut_entry->tag == tag);
+      } else {
+         lut_entry = new tag2warp_entry_t<Tag>();
+         m_tag2entry_map.insert(make_pair(tag, lut_entry));
+      }
+
+      lut_missed = false;
+      lut_max_size = (lut_max_size < m_tag2entry_map.size())? m_tag2entry_map.size() : lut_max_size;
+
+      return lut_entry;
+   }
+
+   void invalidate_entry( tag2warp_entry_t<Tag>* lut_entry, int warp_idx ) {
+      if (lut_entry == NULL) return;
+      if (lut_entry->idx != warp_idx) return;
+
+      typename warp_map_t::iterator mit = m_tag2entry_map.find(lut_entry->tag);
+      if (mit != m_tag2entry_map.end()) {
+         assert(mit->second == lut_entry);
+         mit->second->reset();
+         delete mit->second;
+         m_tag2entry_map.erase(mit);
+      }
+   }
+
+   void clear_accessed( ) {}
+
+   void print( FILE* fout) {
+      typename warp_map_t::iterator mit = m_tag2entry_map.begin();
+      for (; mit != m_tag2entry_map.end(); mit++) {
+         mit->second->print(fout);
+      }
+   }
+
+   static void print_stats ( FILE* fout ) {
+      fprintf( fout, "lut_max_size = %d\n", lut_max_size);
+   }
+};
+template <class Tag> unsigned int warp_lut_perfect<Tag>::lut_max_size = 0;
+
+
+typedef tag2warp_entry_t<pc_tag> warplut_entry_t;
+typedef pc_tag warp_tag_t;
+
+class dwf_hw_sche_class {
+public:
+   int m_id;
+   warp_lut<pc_tag> *warp_lut_pc;
+   vector<warp_entry_t> warp_pool;
+   deque<int> free_warp_q; // the warp allocator
+   int simd_width;
+   int regf_width;
+   int insn_size_lgb2;
+   bool just_resume;
+
+   vector<char> m_req;  // request vector from incoming warp
+   vector<char> m_occ_new; // occupancy vector of the new warp, double as conflict vector
+   vector<char> m_occ_upd; // occupancy vector of the updated existing warp
+   vector<char> m_occ_ext; // occupancy vector of the existing warp
+
+   dwf_hw_sche_class( int lut_size, int lut_assoc, 
+                      int simd_width, int regf_width, 
+                      int n_threads, int insn_size, 
+                      int heuristic, int id, 
+                      char *policy_opt = NULL );
+   ~dwf_hw_sche_class();
+
+   warplut_entry_t* lookup_pc2warp( const warp_tag_t& lookup_tag );
+   int update_warp( int* tid, address_type pc );
+
+   // barrier handling
+   int m_nbarriers; 
+   class dwf_barrier {
+   public:
+      bool m_release; // see if a barrier is to be released (ie. all warp in cta hit already)
+      deque<int> m_queue; // queue storing warps currently hitting a barrier, skipping warplut and scheduler
+
+      dwf_barrier() : m_release(false) {}
+      dwf_barrier(const dwf_barrier& that) 
+      : m_release(that.m_release), m_queue(that.m_queue) {}
+      bool ready_to_issue() {
+         return(m_release && !m_queue.empty());
+      }
+   };
+   set< int > m_cta_released_barrier; // set of cta with released barrier 
+   map< int, dwf_barrier > m_barrier; // map <barrier id == cta id, barrier>
+   int update_warp_at_barrier( int* tid, address_type pc, int cta_id, int barrier_num = 0 );
+   void hit_barrier( int cta_id, int barrier_num = 0 );
+   void release_barrier( int cta_id, int barrier_num = 0 );
+
+   int allocate_warp( address_type pc, bool update_scheduler = true );
+   void free_warp( int idx, bool update_warplut = true );
+
+   void issue_warp( int *tid, address_type *pc );
+
+   void clear_accessed( ) {
+      warp_lut_pc->clear_accessed();
+   }
+
+   void init_cta(int start_thread, int cta_size, address_type start_pc);
+
+   void print_pc2warp_lut( FILE *fout );
+   void print_warp_pool( FILE *fout );
+   void print_free_warp_q( FILE *fout );
+
+   int heuristic;
+
+   // FIFO warp issue logic
+   queue<int> issue_warp_FIFO_q; 
+
+   // PC warp issue logic
+   class pc_first {
+   public:
+      vector<warp_entry_t> &warp_pool;
+      pc_first( vector<warp_entry_t> &bp ) : warp_pool(bp) {}
+      bool operator() (const int &idx_a, const int &idx_b) const {
+         if (warp_pool[idx_a].pc != warp_pool[idx_b].pc) {
+            return(warp_pool[idx_a].pc > warp_pool[idx_b].pc);
+         } else {
+            return(warp_pool[idx_a].occ < warp_pool[idx_b].occ);
+         }
+      }
+   };
+   pc_first mypc_first;
+   priority_queue<int, vector<int>, pc_first > issue_warp_PC_q;
+
+   // Majority warp issue logic
+   issue_warp_majority *issue_warp_MAJ;
+   void clear_policy_access( );
+   void reset_policy_access( );
+
+   // PDOM Priority issue logic
+   issue_warp_pdom_prio issue_warp_pdom;
+
+   // statistics
+   npc_tracker_class npc_tracker;
+   int max_warppool_occ;
+   int *warppool_occ_histo; // histogram of warppool occupancy
+   static unsigned int lut_realmiss;
+   static unsigned int uid_cnt;
+   static unsigned int warp_fragmentation;
+   static unsigned int warp_merge_conflict;
+   static void print_stats ( FILE* fout ) {
+      warp_lut_perfect<warp_tag_t>::print_stats( fout );
+      warp_lut_sa<warp_tag_t>::print_stats( fout );
+      fprintf( fout, "lut_realmiss = %d\n", lut_realmiss);
+      fprintf( fout, "warp_fragmentation = %d\n", warp_fragmentation);
+      fprintf( fout, "warp_merge_conflict = %d\n", warp_merge_conflict);
+   }
+};
+
+unsigned int dwf_hw_sche_class::lut_realmiss = 0;
+unsigned int dwf_hw_sche_class::uid_cnt = 0;
+unsigned int dwf_hw_sche_class::warp_fragmentation = 0;
+unsigned int dwf_hw_sche_class::warp_merge_conflict = 0;
+
+
+dwf_hw_sche_class::dwf_hw_sche_class( int lut_size, int lut_assoc, 
+                                      int simd_width, int regf_width,
+                                      int n_threads, int insn_size, 
+                                      int heuristic, int id,
+                                      char *policy_opt ) 
+: m_id(id), 
+// WarpLUT w/ pc tag
+warp_lut_pc( (lut_size == 0)? (warp_lut<pc_tag> *) new warp_lut_perfect<pc_tag>() : 
+             (warp_lut<pc_tag> *) new warp_lut_sa<pc_tag>(lut_size, lut_assoc, insn_size) ), 
+m_nbarriers(1), // for barrier
+mypc_first( warp_pool ), issue_warp_PC_q( mypc_first ),   // DPC
+issue_warp_pdom(simd_width, &warp_pool, n_threads),       // DPdPri
+npc_tracker( NULL, simd_width )
+{
+   unsigned i;
+
+   this->simd_width = simd_width;
+   this->regf_width = regf_width;
+   this->m_req.resize(regf_width);
+   this->m_occ_new.resize(regf_width);
+   this->m_occ_upd.resize(regf_width);
+   this->m_occ_ext.resize(regf_width);
+
+   // initialize the warp pool 
+   // (make sure the thread id's are init to -1)
+   this->warp_pool.resize(n_threads);
+   for (i=0; i<warp_pool.size(); i++) {
+      warp_pool[i].pc = -1;
+      warp_pool[i].tid = new int[simd_width];
+      memset(warp_pool[i].tid, -1, sizeof(int)*simd_width);
+      warp_pool[i].occ = 0;
+      warp_pool[i].next_warp = -1;
+
+      // push the index to the warp allocator
+      free_warp_q.push_back(i);
+   }
+
+   // setup for various heuristics
+   this->heuristic = heuristic;
+   switch (heuristic) {
+   case MAJORITY: 
+      issue_warp_MAJ = new issue_warp_majority_queue(simd_width, &warp_pool);
+      break;
+   case MAJORITY_MAXHEAP: {
+         int mh_lut_size = 32;
+         int mh_lut_assoc = 4;
+         int n_reads_per_cycle_lut = 4;
+         int n_writes_per_cycle_lut = 4;
+         int mh_size = 128;
+         int n_reads_per_cycle_mh = 4;
+         int n_writes_per_cycle_mh = 4;
+         if (policy_opt != NULL) {
+            sscanf(policy_opt, ";LUT=%d:%dr%dw%d;MH=%dr%dw%d", 
+                   &mh_lut_size, &mh_lut_assoc, &n_reads_per_cycle_lut, &n_writes_per_cycle_lut,
+                   &mh_size, &n_reads_per_cycle_mh, &n_writes_per_cycle_mh);
+         }
+         issue_warp_MAJ = new issue_warp_majority_heap(simd_width, &warp_pool, 
+                                                       mh_lut_size, mh_lut_assoc, mh_size,
+                                                       n_reads_per_cycle_lut, n_writes_per_cycle_lut,
+                                                       n_reads_per_cycle_mh, n_writes_per_cycle_mh);
+      }
+      break;
+   }
+
+   this->just_resume = false;
+
+   this->max_warppool_occ = 0;
+   this->warppool_occ_histo = new int[n_threads];
+   memset(this->warppool_occ_histo, 0, n_threads*sizeof(int));
+}
+
+// should never be called (only at exit?)
+dwf_hw_sche_class::~dwf_hw_sche_class( )
+{
+   unsigned i;
+
+   for (i=0; i<warp_pool.size(); i++) {
+      free(warp_pool[i].tid);
+   }
+
+   delete[] this->warppool_occ_histo;
+
+   delete warp_lut_pc;
+}
+
+// allocate a new warp in warp pool
+int dwf_hw_sche_class::allocate_warp( address_type pc, bool update_scheduler )
+{
+   int idx;
+   assert(!free_warp_q.empty());
+   idx = free_warp_q.front();
+   free_warp_q.pop_front();
+   warp_pool[idx].uid = uid_cnt;
+   uid_cnt++;
+   warp_pool[idx].pc = pc;
+   warp_pool[idx].next_warp = -1;
+   warp_pool[idx].lut_ptr = NULL;
+
+   if (update_scheduler) {
+      if (heuristic == FIFO) issue_warp_FIFO_q.push(idx);
+      if (heuristic == PC) issue_warp_PC_q.push(idx);
+      if (heuristic == MAJORITY || heuristic == MAJORITY_MAXHEAP)
+         issue_warp_MAJ->push_warp(pc, idx);
+      if (heuristic == PDOMPRIO) issue_warp_pdom.push_warp(pc, idx);
+   }
+
+   return idx;
+}
+
+// free a warp in warp pool
+// it will reset the content of the warp entry as well
+void dwf_hw_sche_class::free_warp( int idx, bool update_warplut ) 
+{
+   bool redundant_idx_check = false;
+   if (redundant_idx_check) {
+      deque<int>::iterator dit = find(free_warp_q.begin(), free_warp_q.end(), idx);
+      assert(dit == free_warp_q.end());
+   }
+
+   warp_pool[idx].pc = -1;
+   memset(warp_pool[idx].tid, -1, sizeof(int)*simd_width);
+   warp_pool[idx].occ = 0;
+   warp_pool[idx].next_warp = -1;
+   if (update_warplut) {
+      warp_lut_pc->invalidate_entry( (warplut_entry_t*)warp_pool[idx].lut_ptr, idx );
+   }
+
+   free_warp_q.push_back(idx);
+   assert(free_warp_q.size() <= warp_pool.size());
+}
+
+warplut_entry_t* dwf_hw_sche_class::lookup_pc2warp( const warp_tag_t& lookup_tag )
+{
+   bool lut_missed = false;
+
+   warplut_entry_t* lut_entry;
+   lut_entry = warp_lut_pc->lookup_pc2warp( lookup_tag, lut_missed );
+
+   if (!lut_missed) {
+      if (lut_entry->tag != warp_pool[lut_entry->idx].pc) lut_missed = true;
+   }
+
+   if (lut_missed) {
+      if (npc_tracker.pc_count.find(lookup_tag.get_pc()) != npc_tracker.pc_count.end()) {
+         lut_realmiss++; // ie. the incoming warp lost an opportunity to merge
+      }
+   }
+
+   return lut_entry;
+}   
+
+
+void fill_all (vector<char>& container, const char& value)
+{
+   fill(container.begin(), container.end(), value);
+}
+
+int regfile_hash(signed istream_number, unsigned simd_size, unsigned n_banks);
+int dwf_hw_sche_class::update_warp( int *tid, address_type pc )
+{
+   int i;
+   bool newwarp = false;
+   bool newwarp_alloc = false;
+   warplut_entry_t* lut_entry;
+   warp_tag_t warp_tag(pc);
+   lut_entry = lookup_pc2warp(warp_tag);
+
+   // no LUT entry returned, stall
+   if (!lut_entry) {
+      assert(0);
+   }
+
+   if (heuristic == MAJORITY || heuristic == MAJORITY_MAXHEAP) {
+      issue_warp_MAJ->add_threads(pc, tid);
+   }
+
+   npc_tracker.add_threads( tid, pc );
+
+   // if the pc of the LUT entry does not match,
+   // allocate a new entry
+   if (lut_entry->tag != warp_tag) {
+      lut_entry->idx = allocate_warp(pc);
+      lut_entry->tag = warp_tag;
+      lut_entry->occ = 0;
+      assert(warp_pool[lut_entry->idx].pc == pc);
+      newwarp = true;
+      newwarp_alloc = true;
+   }
+
+   // create the request vector
+   bool tid_has_valid_entry = false;
+   fill_all(m_req, 0);
+   for (i = 0; i<simd_width; i++) {
+      if (tid[i] != -1) {
+         int lane = regfile_hash(tid[i],simd_width,regf_width);
+         // make sure we are not having two threads going to same lane
+         assert(lane < regf_width);
+         m_req[lane] += 1;
+         tid_has_valid_entry = true;
+      }
+   }
+   assert(tid_has_valid_entry);
+
+   // read the old idx pointing to an existing warp
+   int old_idx = lut_entry->idx;
+
+   // create the conflict vector
+   fill_all(m_occ_ext, 0);
+   int regf_mask = regf_width - 1;
+   for (i = 0; i<simd_width; i++) {
+      m_occ_ext[i & regf_mask] += ((lut_entry->occ & (1 << i)) == 0)? 0 : 1;
+   }
+   fill_all(m_occ_upd, 0);
+   fill_all(m_occ_new, 0);
+   int n_regf_slot = simd_width / regf_width;
+   bool conflict = false;
+   for (i = 0; i<regf_width; i++) {
+      if (m_occ_ext[i] + m_req[i] > n_regf_slot) {
+         m_occ_new[i] = m_occ_ext[i] + m_req[i] - n_regf_slot;
+         m_occ_upd[i] = n_regf_slot - m_occ_ext[i];
+         conflict = true;
+      } else {
+         m_occ_upd[i] = m_req[i];
+      }
+   }
+
+   // if the pc of the warp mismatch with lut, 
+   // set conflict vector to all one. 
+   // that force all threads to the newly allocated warp
+   if (warp_pool[old_idx].pc != pc) {
+      conflict = true;
+      for (i = 0; i<regf_width; i++) {
+         m_occ_new[i] = m_req[i];
+         m_occ_upd[i] = 0;
+         m_occ_ext[i] = n_regf_slot;
+      }
+   }
+
+   // if there are conflicted entries, get a new warp
+   int new_idx = -1;
+   if (conflict) {
+      new_idx = allocate_warp(pc);
+      lut_entry->idx = new_idx;
+      lut_entry->occ = 0; //update the lut_entry
+      assert(warp_pool[new_idx].pc == pc);
+
+      int total_occ = 0;
+      for (i = 0; i < regf_width; i++)
+         total_occ += m_occ_ext[i] + m_req[i];
+      if (total_occ <= simd_width) warp_fragmentation += 1;
+      warp_merge_conflict += 1;
+
+      newwarp_alloc = true;
+   }
+
+   // update the warp as indicated by the LUT
+   // if the lane is conflicted, or the old warp is just not 
+   // write to the new warp 
+   int new_occ = 0;
+   fill_all(m_occ_new, 0);
+   for (i = 0; i<simd_width; i++) {
+      if (tid[i] != -1) {
+         int rfbank = regfile_hash(tid[i],simd_width,regf_width);
+         int lane = -1;
+         if ((m_occ_ext[rfbank] < n_regf_slot) || newwarp) {
+            lane = rfbank + m_occ_ext[rfbank] * regf_width;
+            assert(lane < simd_width);
+            warp_pool[old_idx].tid[lane] = tid[i];
+            warp_pool[old_idx].occ++;
+            lut_entry->occ |= (1<<lane);
+            m_occ_ext[rfbank]++;
+         } else {
+            lane = rfbank + m_occ_new[rfbank] * regf_width;
+            assert(lane < simd_width);
+            warp_pool[new_idx].tid[lane] = tid[i];
+            warp_pool[new_idx].occ++;
+            new_occ |= (1<<lane);
+            m_occ_new[rfbank]++;
+            assert(m_occ_new[rfbank] <= n_regf_slot);
+         }
+      }
+   }
+
+   // to cover the case where the pc of the warp mismatch with lut 
+   // (because the warp is issued) 
+   if (warp_pool[old_idx].pc == pc) {
+      issue_warp_pdom.add_threads(old_idx, pc);
+   }
+   if (conflict) {
+      lut_entry->occ = new_occ;
+      issue_warp_pdom.add_threads(new_idx, pc);
+   }
+
+   warp_pool[lut_entry->idx].lut_ptr = lut_entry; // link up the lut entry and warp 
+
+   bool scheduler_consistency_check = false;
+   if (scheduler_consistency_check && heuristic == MAJORITY) {
+      ((issue_warp_majority_queue*)issue_warp_MAJ)->check_consistency();
+   }
+
+   return 1;
+}
+
+// called AFTER threads hit a barrier to insert them into the barrier queue
+// ASSUME: threads from released barrier are not hitting second barrier right away
+int dwf_hw_sche_class::update_warp_at_barrier( int* tid, address_type pc, int cta_id, int barrier_num )
+{
+   assert(barrier_num < m_nbarriers);
+   assert(cta_id >= 0);
+
+   int i;
+   int warp_index = 0xDEADBEEF;
+
+   npc_tracker.add_threads( tid, pc );
+
+   // always allocate new warp
+   warp_index = allocate_warp(pc, false);
+   assert(warp_pool[warp_index].pc == pc);
+
+   // no need to create the request vector
+   // no need to create the conflict vector
+
+   // assign threads into the new warp
+   fill_all(m_occ_ext, 0);
+   int max_nthreads_per_rfbank = simd_width / regf_width;
+   for (i = 0; i<simd_width; i++) {
+      if (tid[i] != -1) {
+         int rfbank = regfile_hash(tid[i],simd_width,regf_width);
+         int lane = -1;
+
+         assert(m_occ_ext[rfbank] < max_nthreads_per_rfbank);
+         lane = rfbank + m_occ_ext[rfbank] * regf_width;
+         assert(lane < simd_width);
+         warp_pool[warp_index].tid[lane] = tid[i];
+         warp_pool[warp_index].occ++;
+         m_occ_ext[rfbank]++;
+      }
+   }
+
+   warp_pool[warp_index].lut_ptr = NULL; // no link to any lut entry 
+
+   // put the warp id into barrier queue
+   m_barrier[cta_id].m_queue.push_back(warp_index);
+
+   // notify issue module to check this barrier at issue
+   if ( m_barrier[cta_id].ready_to_issue() ) {
+      m_cta_released_barrier.insert(cta_id);
+   }
+
+   return 1;
+}
+
+// called at decode stage when thread hit a barrier
+// ASSUME: threads from released barrier are not hitting second barrier right away
+void dwf_hw_sche_class::hit_barrier( int cta_id, int barrier_num )
+{
+   assert(barrier_num < m_nbarriers);
+   assert(cta_id >= 0);
+
+   m_barrier[cta_id].m_release = false;
+}
+
+// called at decode stage when all thread in cta hit the barrier
+// ASSUME: threads from released barrier are not hitting second barrier right away
+void dwf_hw_sche_class::release_barrier( int cta_id, int barrier_num )
+{
+   assert(barrier_num < m_nbarriers);
+   assert(cta_id >= 0);
+
+   map<int, dwf_barrier>::iterator i_barrier = m_barrier.find(cta_id);
+   assert(i_barrier != m_barrier.end()); // barrier has to exists in the first place!
+   i_barrier->second.m_release = true;
+}
+
+void dwf_hw_sche_class::issue_warp( int *tid, address_type *pc )
+{
+   int i;
+   bool warp_issued = false;
+
+   // scan the released barriers for ready warp
+   // TODO: arbitrate between different queues?
+   set<int>::iterator i_ctabar = m_cta_released_barrier.begin();
+   for (; i_ctabar != m_cta_released_barrier.end(); ++i_ctabar) {
+      int cta_id = *i_ctabar;
+      map<int, dwf_barrier>::iterator i_barrier = m_barrier.find(cta_id);
+
+      if ( i_barrier->second.ready_to_issue() ) {
+         int warp_idx = i_barrier->second.m_queue.front();
+
+         for (i = 0; i < simd_width; i++) {
+            tid[i] =  warp_pool[warp_idx].tid[i];
+         }
+         *pc = warp_pool[warp_idx].pc;
+
+         i_barrier->second.m_queue.pop_front();
+         free_warp(warp_idx, false); // don't update warplut as the warp is not linked to it
+
+         // remove cta from checking list if the queue is emptied 
+         // (if the last threads haven't made it back to scheduler in time, 
+         //  update_warp_at_barrier will insert the cta id again)
+         if (i_barrier->second.m_queue.empty()) {
+            m_cta_released_barrier.erase(i_ctabar); 
+         }
+
+         warp_issued = true;
+
+         break;
+      }
+   }
+
+   if (!warp_issued) {
+      switch (heuristic) {
+      case FIFO:
+         // Oldest warp are issued first
+         if (!issue_warp_FIFO_q.empty()) {
+            int idx = issue_warp_FIFO_q.front();
+            for (i = 0; i < simd_width; i++) {
+               tid[i] =  warp_pool[idx].tid[i];
+            }
+            *pc = warp_pool[idx].pc;
+
+            issue_warp_FIFO_q.pop();
+            free_warp(idx);
+         } else {
+            memset(tid, -1, sizeof(int)*simd_width);
+            *pc = -1;
+         }
+         break;
+      case PC:
+         // lowest PC warp are issued first
+         if (!issue_warp_PC_q.empty()) {
+            int idx = issue_warp_PC_q.top();
+            for (i = 0; i < simd_width; i++) {
+               tid[i] =  warp_pool[idx].tid[i];
+            }
+            *pc = warp_pool[idx].pc;
+
+            issue_warp_PC_q.pop();
+            free_warp(idx);
+         } else {
+            memset(tid, -1, sizeof(int)*simd_width);
+            *pc = -1;
+         }
+         break;
+      case MAJORITY:
+      case MAJORITY_MAXHEAP:
+         // issue the most common PC first
+         {
+            int idx = issue_warp_MAJ->pop_warp();
+            if (idx >= 0) {
+               for (i = 0; i < simd_width; i++) {
+                  tid[i] =  warp_pool[idx].tid[i];
+               }
+               *pc = warp_pool[idx].pc;
+               free_warp(idx);
+            } else {
+               memset(tid, -1, sizeof(int)*simd_width);
+               *pc = -1;
+            }
+         }
+         break;
+      case PDOMPRIO:
+         // issue the warp with lowest PDOM count
+         {
+            int idx = issue_warp_pdom.front_warp();
+            if (idx >= 0) {
+               issue_warp_pdom.pop_warp();
+
+               for (i = 0; i < simd_width; i++) {
+                  tid[i] =  warp_pool[idx].tid[i];
+               }
+               *pc = warp_pool[idx].pc;
+               free_warp(idx);
+
+               just_resume = false;
+            } else {
+               memset(tid, -1, sizeof(int)*simd_width);
+               *pc = -1;
+            }
+         }
+         break;
+      default:
+         printf("Unsupported Heuristics!\n");
+         abort();
+         break;
+      }
+   }
+
+   npc_tracker.sub_threads( tid, *pc );
+
+   int warppool_occ = warp_pool.size() - free_warp_q.size();
+   if (max_warppool_occ < warppool_occ) {
+      max_warppool_occ = warppool_occ;
+   }
+   warppool_occ_histo[warppool_occ] += 1;
+}
+
+void dwf_hw_sche_class::init_cta(int start_thread, int cta_size, address_type start_pc)
+{
+   assert((start_thread % simd_width) == 0); // thread id starting at a warp
+
+   int n_warp_2assign = cta_size / simd_width;
+   n_warp_2assign += (cta_size % simd_width)? 1 : 0; // round up
+
+   static int *thd_id = NULL;
+   if (thd_id == NULL) thd_id = new int[simd_width];
+
+   for (int w = 0; w < n_warp_2assign; w++) {
+      // generate the warp update register for each warp
+      fill_n(thd_id, simd_width, -1);
+      int warp_start_tid = start_thread + w * simd_width;
+      for (int i = 0; (i < simd_width) && (warp_start_tid + i) < (start_thread + cta_size); i++) {
+         thd_id[i] = warp_start_tid + i;
+      }
+
+      // push these warps into DWF scheduler
+      update_warp( thd_id, start_pc );
+   }
+}
+
+void dwf_hw_sche_class::print_free_warp_q( FILE *fout )
+{
+   fprintf(fout, "free_node_q (%zd)= ", free_warp_q.size() );
+   deque<int>::iterator dit = free_warp_q.begin();
+   for (; dit != free_warp_q.end(); dit++) {
+      fprintf(fout, "%03d ", *dit);
+   }
+   fprintf(fout, "\n");
+}
+
+void print_warp( FILE *fout, warp_entry_t warp_e, int simd_width ) 
+{
+   fprintf(fout, "\t%02d 0x%08x: (", warp_e.pdom_prio, warp_e.pc );
+   for (int i=0;i<simd_width;i++) {
+      fprintf(fout, "%03d ", warp_e.tid[i]);
+   }
+   fprintf(fout, ")\n");
+}
+
+void dwf_hw_sche_class::print_warp_pool( FILE *fout )
+{
+   for (unsigned i=0; i< warp_pool.size(); i++) {
+      if (warp_pool[i].pc != (address_type)-1) {
+         fprintf(fout, "bp%03d:", i);
+         print_warp(fout, warp_pool[i], simd_width);
+      }
+   }
+}
+
+void dwf_hw_sche_class::clear_policy_access( ) {
+   if (heuristic == MAJORITY_MAXHEAP) {
+      ((issue_warp_majority_heap*)issue_warp_MAJ)->clear_access( );
+   }
+}
+
+void dwf_hw_sche_class::reset_policy_access( ) {
+   if (heuristic == MAJORITY_MAXHEAP) {
+      ((issue_warp_majority_heap*)issue_warp_MAJ)->reset_access( );
+   }
+}
+
+///////////////////////////////////////////////////////////////////////////
+// c-wrapper interface
+///////////////////////////////////////////////////////////////////////////
+
+int dwf_hw_n_sche = 0;
+dwf_hw_sche_class **dwf_hw_sche;
+unsigned *acc_dyn_pcs = NULL;
+
+void create_dwf_schedulers( int n_shaders, 
+                                       int lut_size, int lut_assoc, 
+                                       int simd_width, int regf_width,
+                                       int n_threads, int insn_size, 
+                                       int heuristic,
+                                       char *policy_opt )
+{
+   dwf_hw_n_sche = n_shaders;
+   dwf_hw_sche = new dwf_hw_sche_class*[n_shaders];
+   for (int i=0; i<n_shaders; i++) {
+      dwf_hw_sche[i] = new dwf_hw_sche_class( lut_size, lut_assoc, 
+                                              simd_width, regf_width,
+                                              n_threads, insn_size, 
+                                              heuristic, i, 
+                                              policy_opt );
+   }
+
+   if (acc_dyn_pcs == NULL) {
+      acc_dyn_pcs = new unsigned[n_shaders];
+      std::fill_n(acc_dyn_pcs, n_shaders, 0);
+   }
+   for (int i=0; i<n_shaders; i++) {
+      dwf_hw_sche[i]->npc_tracker.acc_pc_count = &acc_dyn_pcs[i];
+   }   
+}
+
+int dwf_update_warp( int shd_id, int* tid, address_type pc )
+{
+   return dwf_hw_sche[shd_id]->update_warp( tid, pc );
+}
+
+int dwf_update_warp_at_barrier( int shd_id, int* tid, address_type pc, int cta_id )
+{
+   return dwf_hw_sche[shd_id]->update_warp_at_barrier( tid, pc, cta_id);
+}
+
+void dwf_hit_barrier( int shd_id, int cta_id )
+{
+   dwf_hw_sche[shd_id]->hit_barrier( cta_id );
+}
+
+void dwf_release_barrier( int shd_id, int cta_id )
+{
+   dwf_hw_sche[shd_id]->release_barrier( cta_id );
+}
+
+void dwf_issue_warp( int shd_id, int *tid, address_type *pc )
+{
+   dwf_hw_sche[shd_id]->issue_warp( tid, pc );
+}
+
+void dwf_clear_accessed( int shd_id )
+{
+   dwf_hw_sche[shd_id]->clear_accessed( );
+}
+
+void dwf_clear_policy_access( int shd_id )
+{
+   dwf_hw_sche[shd_id]->clear_policy_access( );
+}
+
+void dwf_reset_policy_access( int shd_id )
+{
+   dwf_hw_sche[shd_id]->reset_policy_access( );
+}
+
+void dwf_init_CTA(int shd_id, int start_thread, int cta_size, address_type start_pc)
+{
+   dwf_hw_sche[shd_id]->init_cta(start_thread, cta_size, start_pc);
+   dwf_hw_sche[shd_id]->clear_accessed( );
+   dwf_hw_sche[shd_id]->clear_policy_access( );
+}
+
+void dwf_print_stat( FILE* fout )
+{
+   dwf_hw_sche_class::print_stats( fout );
+   npc_tracker_class::histo_print( fout );
+   fprintf(fout, "max_warppool_occ = ");
+   for (int i=0; i<dwf_hw_n_sche; i++) {
+      fprintf(fout, "%d ", dwf_hw_sche[i]->max_warppool_occ);
+   }
+   fprintf(fout, "\n");
+   for (int i=0; i<dwf_hw_n_sche; i++) {
+      fprintf(fout, "warppool_occ[%d] = ", i);
+      for (int j=0; j<dwf_hw_sche[i]->max_warppool_occ; j++) {
+         fprintf(fout, "%d ", dwf_hw_sche[i]->warppool_occ_histo[j]);
+      }
+      fprintf(fout, "\n");
+   }
+   if (dwf_hw_sche[0]->heuristic == MAJORITY_MAXHEAP) {
+      fprintf(fout, "n_stall_on_maxheap = ");
+      for (int i=0; i<dwf_hw_n_sche; i++) {
+         fprintf(fout, "%d ", 
+                 ((issue_warp_majority_heap*)dwf_hw_sche[i]->issue_warp_MAJ)->n_stall_on_maxheap);
+      }
+      fprintf(fout, "\n");
+      fprintf(fout, "maxheap_n_entries = ");
+      for (int i=0; i<dwf_hw_n_sche; i++) {
+         fprintf(fout, "%d ", 
+                 ((issue_warp_majority_heap*)dwf_hw_sche[i]->issue_warp_MAJ)->maxheap.max_n_entries);
+      }
+      fprintf(fout, "\n");
+      fprintf(fout, "maxheap_lut_n_aliased = ");
+      for (int i=0; i<dwf_hw_n_sche; i++) {
+         fprintf(fout, "%d ", 
+                 ((issue_warp_majority_heap*)dwf_hw_sche[i]->issue_warp_MAJ)->mh_lut.n_aliased);
+      }
+      fprintf(fout, "\n");
+      issue_warp_majority_heap::print_stat(fout);
+   }
+}
+
+void dwf_reset_reconv_pt() 
+{
+   issue_warp_pdom_prio::reconvgence_pt.clear();
+}
+
+void dwf_insert_reconv_pt(address_type pc) 
+{
+   issue_warp_pdom_prio::reconvgence_pt.insert(pc);
+}
+
+void dwf_reinit_schedulers( int n_shaders )
+{
+   for (int i=0; i<n_shaders; i++) {
+      dwf_hw_sche[i]->issue_warp_pdom.reinit();
+   }
+}
+
+void dwf_update_statistics( int shader_id )
+{
+   dwf_hw_sche[shader_id]->npc_tracker.update_acc_count();
+}
+
+void g_print_dmaj_scheduler(int sid) {
+   dwf_hw_sche[sid]->issue_warp_MAJ->print(stdout);
+}
+
+void g_print_warp_lut(int sid) {
+   dwf_hw_sche[sid]->warp_lut_pc->print(stdout);
+}
+
+void g_print_free_warp_q(int sid) {
+   dwf_hw_sche[sid]->print_free_warp_q(stdout);
+}
+
+void g_print_warp_pool(int sid) {
+   dwf_hw_sche[sid]->print_warp_pool(stdout);
+}
+
+void g_print_max_heap(int sid) {
+   dwf_hw_sche[sid]->issue_warp_MAJ->print(stdout);
+}
+
+#ifdef UNIT_TEST
+
+   #undef UNIT_TEST
+   #include "stat-tool.cc"
+
+unsigned gpgpu_thread_swizzling = 0;
+unsigned long long  gpu_sim_cycle = 0;
+
+int regfile_hash(signed istream_number, unsigned simd_size, unsigned n_banks) {
+   if (gpgpu_thread_swizzling) {
+      signed warp_ID = istream_number / simd_size;
+      return((istream_number + warp_ID) % n_banks);
+   } else {
+      return(istream_number % n_banks);
+   }
+}
+
+int log2i(int n) {
+   int lg;
+   lg = -1;
+   while (n) {
+      n>>=1;lg++;
+   }
+   return lg;
+}
+
+int test_FIFO() 
+{
+   dwf_hw_sche_class *dwf_sche;
+   int i;
+   int tid[6][4] = {
+      { 0, 1, 2, 3},
+      { 4, 5, 6, 7},
+      { 8,-1,10,-1},
+      {-1, 1,-1, 3},
+      { 4, 9,-1,11},
+      {-1,13,14,-1}
+   };
+
+   int expect_out[12][4] = {
+      { 0, 1, 2, 3},
+      { 0, 1, 2, 3},
+      { 0, 1, 2, 3},
+      { 4, 5, 6, 7},
+      { 8, 1,10, 3},
+      { 4, 9,14,11},
+      {-1,13,-1,-1},
+      { 4, 9,-1,11},
+      {-1,13,14,-1},
+      { 8,-1,10,-1},
+      { 4, 9,14,11},
+      { 8,13,10,-1}
+   };
+
+   int tid_out[4];
+   address_type pc_out;
+
+   dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, FIFO);
+
+   // same threads - different pc
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[0], 0x409a80);
+   dwf_sche->update_warp(tid[0], 0x409a88);
+
+   // different threads - different pc
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[0], 0x409a90);
+   dwf_sche->update_warp(tid[1], 0x409a80);
+
+   // different threads - same pc 
+   // expect two warp to merge into one as there is no lane conflict
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[2], 0x409a90);
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[3], 0x409a90);
+
+   // same as above, but with lane conflict
+   // expect a new warp allocated, 
+   // but only the conflicting threads goes to new warp
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[4], 0x409a80);
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[5], 0x409a80);
+
+   // different threads - different pc
+   // purposely try to alias an existing mapping 
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[4], 0x410a80);
+   dwf_sche->update_warp(tid[5], 0x411a80);
+
+   // going back to that mapping 
+   // a new warp should be allocated (despite lack of conflict)
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[2], 0x409a80);
+
+   // testing the occupancy vector
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[4], 0x409aa0);
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[5], 0x409aa0);
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[2], 0x409aa0);
+
+   // fill the warp pool up
+   for (i=12; i<64; ) {
+      dwf_sche->clear_accessed();
+      dwf_sche->update_warp(tid[1], 0x409a80 + 8 * i++);
+      dwf_sche->update_warp(tid[4], 0x409a80 + 8 * i++);
+   }
+   // issue all the warp (do some auto checking on the way)
+   for (i=0; i<64; i++) {
+      dwf_sche->issue_warp(tid_out, &pc_out);
+      printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]);
+      if (i<12) {
+         if ( memcmp(tid_out, expect_out[i], 4*sizeof(int) ) ) {
+            printf("%d warp mismatches\n", i);
+            assert(0);
+         }
+      }
+   }
+
+   // now that all warpes are issue, no entries in the lut is valid
+   // updating warp with an old address that remains in the lut
+   // to see if detects the invalid lut entry
+   dwf_sche->clear_accessed();
+   dwf_sche->update_warp(tid[2], 0x409a80 + 8 * 63);
+   dwf_sche->update_warp(tid[3], 0x409a80 + 8 * 62);
+   dwf_sche->issue_warp(tid_out, &pc_out);
+   assert(!memcmp(tid_out, tid[2], 4*sizeof(int) ));
+   dwf_sche->issue_warp(tid_out, &pc_out);
+   assert(!memcmp(tid_out, tid[3], 4*sizeof(int) ));
+
+   dwf_sche->print_warp_pool(stdout);
+   dwf_sche->warp_lut_pc->print(stdout);
+   dwf_hw_sche_class::print_stats(stdout);
+
+   delete dwf_sche;
+
+   return 0;
+}
+
+int test_PC ()
+{
+   dwf_hw_sche_class *dwf_sche;
+   int i;
+   int tid[4][4] = {
+      { 0, 1, 2, 3},
+      { 4, 5, 6, 7},
+      { 8,-1,10,-1},
+      {-1,13,14,-1}
+   };
+
+   int tid_out[4];
+   address_type pc_out;
+
+   dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, PC);
+
+   // fill the warp pool up in reverse PC order
+   for (i=0; i<4; i++) {
+      for (int j=0; j<4; j++) {
+         dwf_sche->clear_accessed();
+         dwf_sche->update_warp(tid[j], 0x409a80 - 8 * i);
+      }
+   }
+
+   // issue the warps, expect them to be in PC order, with higher occ warp issued first
+   printf("PC Issue Logic:\n");
+   for (i=0; i<4; i++) {
+      for (int j=0; j<4; j++) {
+         dwf_sche->issue_warp(tid_out, &pc_out);
+         printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]);
+      }
+   }  
+
+}
+
+int test_MAJ ()
+{
+   dwf_hw_sche_class *dwf_sche;
+   int i;
+   int tid[4][4] = {
+      { 0, 1, 2, 3},
+      { 4, 5, 6, 7},
+      { 8,-1,10,-1},
+      {-1,13,14,-1}
+   };
+
+   int tid_out[4];
+   address_type pc_out;
+
+   dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, MAJORITY);
+
+   // fill the warp pool up in reverse PC order
+   for (i=0; i<4; i++) {
+      for (int j=0; j<(4-i); j++) {
+         dwf_sche->clear_accessed();
+         dwf_sche->update_warp(tid[j], 0x409a80 - 8 * i);
+      }
+   }
+
+   // issue the warps, expect them to be in PC order, with higher occ warp issued first
+   printf("Majority Issue Logic:\n");
+   for (i=0; i<4; i++) {
+      for (int j=0; j<4; j++) {
+         dwf_sche->issue_warp(tid_out, &pc_out);
+         printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]);
+      }
+   }  
+}
+
+int test_MAJ_HEAP ()
+{
+   printf("\ntest_MAJ_HEAP:\n");
+   dwf_hw_sche_class *dwf_sche;
+   int i;
+   int tid[4][4] = {
+      { 0, 1, 2, 3},
+      { 4, 5, 6, 7},
+      { 8,-1,10,-1},
+      {-1,13,14,-1}
+   };
+
+   int tid_out[4];
+   address_type pc_out;
+
+   dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, MAJORITY_MAXHEAP);
+
+   // fill the warp pool up in reverse PC order
+   for (i=0; i<4; i++) {
+      for (int j=0; j<(i+1); j++) {
+         dwf_sche->clear_accessed();
+         dwf_sche->update_warp(tid[j], 0x409a80 + 8 * i);
+      }
+   }
+
+   dwf_sche->reset_policy_access();
+   dwf_sche->issue_warp_MAJ->print(stdout);
+
+   // issue the warps, expect them to be in PC order, with higher occ warp issued first
+   printf("Majority (Max Heap) Issue Logic:\n");
+   for (i=0; i<4; i++) {
+      for (int j=0; j<4; j++) {
+         dwf_sche->issue_warp(tid_out, &pc_out);
+         printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]);
+      }
+      dwf_sche->reset_policy_access();
+   }  
+}
+
+void test_warp_lut_pc ()
+{
+   printf("\ntest_warp_lut_pc:\n");
+   warp_lut_sa<pc_tag> warp_lut_pc(16,   // size
+                                   4,   // assoc
+                                   1);  // insn_size
+
+   address_type pc_value[] = {0, 4, 0, 8, 12, 16, 20, 8, 8, 0};
+   int n_entry = sizeof(pc_value) / sizeof(address_type);
+   vector<pc_tag> pc_stream(pc_value, pc_value + n_entry);
+
+   int misses = 0;
+   for (int n = 0; n < n_entry * 100; n++) {
+      int i = n % n_entry;
+      tag2warp_entry_t<pc_tag> *lut_entry = NULL;
+      bool lut_miss = false;
+
+      lut_entry = warp_lut_pc.lookup_pc2warp(pc_stream[i], lut_miss);
+
+      if (lut_entry->tag != pc_stream[i]) {
+         lut_entry->tag = pc_stream[i];
+         lut_entry->occ = 1;
+         misses += 1;
+      }
+      warp_lut_pc.clear_accessed();
+      lut_entry->accessed = 0;
+   }
+
+   printf("Number of Miss = %d\n", misses);
+}
+
+int main () {
+   //test_FIFO();
+   //test_PC();
+   //test_MAJ();
+   test_MAJ_HEAP();
+   test_warp_lut_pc();
+   return 0;
+}
+
+#endif
diff --git a/src/gpgpu-sim/dwf.h b/src/gpgpu-sim/dwf.h
new file mode 100644
index 0000000..6328f1a
--- /dev/null
+++ b/src/gpgpu-sim/dwf.h
@@ -0,0 +1,118 @@
+/* 
+ * dwf.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef dwf_h_INCLUDED
+#define dwf_h_INCLUDED
+
+#ifdef __cplusplus
+
+   #include <cstdio>
+   #include <cstdlib>
+   #include <cassert>
+   #include <vector>
+   #include <queue>
+   #include <list>
+   #include <algorithm>
+
+#endif
+
+#include "../util.h"
+
+extern unsigned *acc_dyn_pcs;
+
+void create_dwf_schedulers( int n_shaders, 
+		       int lut_size, int lut_assoc, 
+		       int simd_width, int regf_width,
+		       int n_threads, int insn_size, 
+		       int heuristic,
+		       char *policy_opt );
+
+int dwf_update_warp( int shd_id, int* tid, address_type pc );
+
+void dwf_issue_warp( int shd_id, int *tid, address_type *pc );
+
+void dwf_clear_accessed( int shd_id );
+
+void dwf_clear_policy_access( int shd_id );
+void dwf_reset_policy_access( int shd_id );
+
+int dwf_update_warp_at_barrier( int shd_id, int* tid, address_type pc, int cta_id );
+void dwf_hit_barrier( int shd_id, int cta_id );
+void dwf_release_barrier( int shd_id, int cta_id );
+
+void dwf_init_CTA(int shd_id, int start_thread, int cta_size, address_type start_pc);
+
+void dwf_print_stat( FILE* fout );
+
+void dwf_reset_reconv_pt();
+void dwf_insert_reconv_pt(address_type pc); 
+
+void dwf_reinit_schedulers( int n_shaders );
+
+void dwf_set_accPC( int n_shaders, unsigned *acc_pc_count );
+
+void dwf_update_statistics( int shader_id );
+
+#endif
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
new file mode 100644
index 0000000..679d1d2
--- /dev/null
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -0,0 +1,609 @@
+/* 
+ * gpu-cache.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "gpu-cache.h"
+#include "gpu-misc.h"
+#include "addrdec.h"
+#include <assert.h>
+#include <string.h>
+
+// both shd_cache_access and shd_cache_probe functions use 
+// shd_cache_access_internal to access/probe cache
+shd_cache_line_t* shd_cache_access_internal( shd_cache_t *cp, 
+                                             unsigned long long int addr, 
+                                             unsigned int nbytes, 
+                                             unsigned char write,
+                                             unsigned int sim_cycle,
+                                             unsigned int real_access); 
+
+shd_cache_t * shd_cache_create( char *name,
+                                unsigned int nset,
+                                unsigned int assoc,
+                                unsigned int line_sz,
+                                unsigned char policy,
+                                unsigned int hit_latency,
+                                unsigned long long int bank_mask,
+                                enum cache_write_policy wp) {
+
+   shd_cache_t *cp;
+   unsigned int nlines;
+
+   unsigned int i;
+
+   if (!nset || !assoc) {
+      printf("Creating non-existing cache!\n");
+      return 0;
+   }
+
+   nlines = nset * assoc;
+   cp = (shd_cache_t*) calloc(1, sizeof(shd_cache_t));
+
+   cp->bank_mask = bank_mask;
+   cp->name = (char*) malloc(sizeof(char) * (strlen(name) + 1));
+   strcpy(cp->name, name);
+   cp->nset = nset;
+   cp->nset_log2 = LOGB2(nset);
+   cp->assoc = assoc;
+   cp->line_sz = line_sz;
+   cp->line_sz_log2 = LOGB2(line_sz);
+   cp->policy = policy;
+   cp->hit_latency = hit_latency; 
+   cp->lines = (shd_cache_line_t*) calloc(nlines, sizeof(shd_cache_line_t));
+   cp->write_policy = wp;
+
+   for (i=0; i<nlines; i++) {
+      cp->lines[i].line_sz = line_sz;
+      cp->lines[i].status = 0;
+   }
+
+   // don't hook up with any logger
+   cp->core_id = -1; 
+   cp->type_id = -1;
+
+   // initialize snapshot counters for visualizer
+   cp->prev_snapshot_access = 0;
+   cp->prev_snapshot_miss = 0;
+   cp->prev_snapshot_merge_hit = 0;
+
+//   printf("%s: %d(%x) x %d x %d(%x) %c, %d\n",
+//          cp->name, cp->nset, cp->nset_log2, cp->assoc, cp->line_sz,
+//          cp->line_sz_log2, cp->policy, nlines);
+
+   return cp;
+}
+
+void shd_cache_destroy( shd_cache_t* cp ) {
+
+   free(cp->lines);
+   free(cp);
+}
+
+extern void shader_cache_miss_log( int logger_id, int type );
+// hook up with shader core logger
+void shd_cache_bind_logger(shd_cache_t* cp, int core_id, int type_id) {
+   cp->core_id = core_id; 
+   cp->type_id = type_id;
+}
+
+extern unsigned long long int addrdec_packbits(unsigned long long int mask, 
+                                               unsigned long long int val,
+                                               unsigned char high, unsigned char low);
+extern void shader_cache_access_log( int logger_id, int type, int miss);
+extern void shader_cache_access_unlog( int logger_id, int type, int miss);
+
+shd_cache_line_t* shd_cache_access_internal( shd_cache_t *cp, 
+                                             unsigned long long int addr, 
+                                             unsigned int nbytes, 
+                                             unsigned char write,
+                                             unsigned int sim_cycle,
+                                             unsigned int real_access) {
+   //if real_access==0 then its only a cache probe stats and LRU tags should not be updated 
+   unsigned int i;
+   unsigned int set;
+   unsigned long long int tag; 
+   unsigned long long int packed_addr;
+   shd_cache_line_t *pline;
+
+   if (cp->bank_mask)
+      packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0);
+   else
+      packed_addr = addr;
+
+   set = (packed_addr >> cp->line_sz_log2) & ( (1<<cp->nset_log2) - 1 );
+   tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2);
+
+   if (real_access) {
+      cp->access++;
+      shader_cache_access_log(cp->core_id, cp->type_id, 0);
+   }
+
+   for (i=0; i<cp->assoc; i++) {
+      pline = &(cp->lines[set*cp->assoc+i] );
+      if (pline->status & VALID) {
+         if (pline->tag == tag) {
+            //printf("Cache Hit! Addr=%08x Set=%x Way=%x Tag=%x\n", packed_addr, set, i, tag);
+            if (real_access) {
+               pline->last_used = sim_cycle;
+               if (write) {
+                  pline->status |= DIRTY;
+               }
+            }
+            return pline;
+         }
+      }
+   }
+   if (real_access) {
+      cp->miss++;
+      shader_cache_access_log(cp->core_id, cp->type_id, 1);
+   }
+   return 0;
+}
+
+shd_cache_line_t* shd_cache_access( shd_cache_t *cp, 
+                                    unsigned long long int addr, 
+                                    unsigned int nbytes, 
+                                    unsigned char write,
+                                    unsigned int sim_cycle ) 
+{
+   return shd_cache_access_internal(cp,addr,nbytes,write,sim_cycle,1/*this is a real access*/);
+}
+
+extern int gpgpu_cache_wt_through;
+shd_cache_t *test = NULL;
+enum cache_request_status shd_cache_access_wb( shd_cache_t *cp, 
+                                    unsigned long long int addr, 
+                                    unsigned int nbytes, 
+                                    unsigned char write,
+                                    unsigned int sim_cycle, address_type *wb_address) 
+{
+   unsigned int i;
+   unsigned int set;
+   unsigned long long int tag; 
+   unsigned long long int packed_addr;
+   shd_cache_line_t *pline;
+
+   unsigned already_reserved = 0;
+   unsigned all_reserved = 1;
+   shd_cache_line_t *free_line = NULL;
+
+   if (cp->bank_mask)
+      packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0);
+   else
+      packed_addr = addr;
+
+   set = (packed_addr >> cp->line_sz_log2) & ( (1<<cp->nset_log2) - 1 );
+   tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2);
+
+   cp->access++;
+   shader_cache_access_log(cp->core_id, cp->type_id, 0);
+
+   for (i=0; i<cp->assoc; i++) {
+      pline = &(cp->lines[set*cp->assoc+i] );
+      if (pline->tag == tag) {
+         if (pline->status & RESERVED) {
+            already_reserved = 1;
+            break;
+         } else if (pline->status & VALID) {
+            //printf("Cache Hit! Addr=%08x Set=%x Way=%x Tag=%x\n", packed_addr, set, i, tag);
+            pline->last_used = sim_cycle;
+            if (write) {
+               pline->status |= DIRTY;
+            }            
+            //return pline;
+            if (cp->write_policy == write_through) return HIT_W_WT;
+            return HIT;
+         }
+      } 
+      if (!(pline->status & RESERVED)) {
+         all_reserved = 0;
+         if (!(pline->status & VALID)) { 
+             free_line = pline;
+         }
+      }
+   }
+   cp->miss++;
+   shader_cache_access_log(cp->core_id, cp->type_id, 1);
+
+   if (already_reserved || cp->write_policy != write_back || write) {
+      //not in cache yet, but no wb as place is reserved.
+      //or wt caches never nead to worry about it (as do no_write caches) 
+      //or is a write
+      if (already_reserved && write) {
+         //write the data into the cache line, make it dirty
+         //up to mshrs to save write mask to not overwrite this data when the read returns
+         pline->status |= DIRTY;
+         return WB_HIT_ON_MISS;
+      } else if (already_reserved) {
+         return WB_HIT_ON_MISS;
+      }
+      return MISS_NO_WB;
+   }
+
+   //if not in cache, and a write back cache, and not already allocated, need to allocate a place for this request
+
+   if (all_reserved) {
+      //cannot service this request, because we can't garantee that we have room for the line when it comes back
+      return RESERVATION_FAIL;
+   }
+
+   //printf("RESRV %d\n",tag);
+
+   if (free_line) {
+      //reserve fo this request
+      free_line->status |= RESERVED;
+      free_line->tag = tag;
+      //no writeback
+      return MISS_NO_WB;
+   }
+
+   // need to kick a line out to reserve a spot
+   shd_cache_line_t *rline = NULL;
+           
+   for (i=0; i<cp->assoc; i++) {
+      pline = &(cp->lines[set*cp->assoc+i] );
+      if (pline->status & VALID && !(pline->status & RESERVED)) {
+         if (!rline) {
+            rline = pline; //select first available for ejection for later comparison
+            continue;
+         }
+         switch (cp->policy) {
+         case LRU: 
+            if (pline->last_used < rline->last_used)
+               rline = pline;
+            break;
+         case FIFO:
+            if (pline->fetch_time < rline->fetch_time)
+               rline = pline;
+            break;
+         default:
+            rline = pline; //pick one, ie. the last valied one.
+         }   
+      }
+   }
+   assert(rline); //ensure we actually found one.
+
+   unsigned needs_wb = (rline->status & (DIRTY|VALID)) == (DIRTY|VALID);
+   /* Set the replaced cache line address */
+   if (needs_wb) {   
+      *wb_address = rline->addr; 
+   }
+   
+   /* reserve this new line */
+   rline->status |= RESERVED;
+   rline->status &= ~VALID;
+   rline->status &= ~DIRTY;
+   rline->tag = tag; 
+
+   /* printf("Fetching! Addr=%08x ReplAddr=%08x(%d) Set=%x Tag=%x\n",
+             packed_addr, repl_addr, nofreeslot, set, tag);
+   */
+   if (needs_wb) {
+      return MISS_W_WB;
+   } else {
+      return MISS_NO_WB;
+   }
+}
+
+shd_cache_line_t* shd_cache_probe( shd_cache_t *cp, 
+                                   unsigned long long int addr)
+{
+   return shd_cache_access_internal(cp,addr,
+                                    1,0,0, /*do not matter*/
+                                    0/*this is just a probe*/);  
+}
+
+void shd_cache_undo_stats( shd_cache_t *cp, int miss )
+{
+   if (miss) {
+      cp->miss--;
+      shader_cache_access_unlog(cp->core_id, cp->type_id, 1);
+   }
+   cp->access--;
+   shader_cache_access_unlog(cp->core_id, cp->type_id, 0);
+}
+
+// Obtain the windowed cache miss rate for visualizer
+float shd_cache_windowed_cache_miss_rate( shd_cache_t *cp, int minus_merge_hit )
+{
+   unsigned int n_access = cp->access - cp->prev_snapshot_access;
+   unsigned int n_miss = cp->miss - cp->prev_snapshot_miss;
+   unsigned int n_merge_hit = cp->merge_hit - cp->prev_snapshot_merge_hit;
+   
+   if (minus_merge_hit) {
+      n_miss -= n_merge_hit;
+   }
+   float missrate = 0.0f;
+   if (n_access != 0) {
+      missrate = (float) n_miss / n_access;
+   }
+   
+   return missrate;
+}
+
+// start a new sampling window
+void shd_cache_new_window( shd_cache_t *cp )
+{
+   cp->prev_snapshot_access = cp->access;
+   cp->prev_snapshot_miss = cp->miss;
+   cp->prev_snapshot_merge_hit = cp->merge_hit;
+}
+
+unsigned long long int L2_shd_cache_fill( shd_cache_t *cp, 
+                                          unsigned long long int addr,
+                                          unsigned int sim_cycle ) {
+   unsigned long long int result = shd_cache_fill(cp, addr, sim_cycle);
+   return result;
+}
+
+static unsigned int _n_line_existed = 0; // debug counter
+
+// Fetch requested data into cache line. 
+// Returning address on the replaced line if it is dirty, or -1 if it is clean
+// Assume the line is filled all at once. 
+unsigned long long int shd_cache_fill( shd_cache_t *cp, 
+                                       unsigned long long int addr,
+                                       unsigned int sim_cycle ) {
+
+   unsigned int i;
+   unsigned int set;
+   unsigned long long int tag;
+   unsigned long long int packed_addr;
+   unsigned long long int repl_addr;
+
+   unsigned char nofreeslot; 
+   unsigned char line_exists;
+   unsigned int base = 0 ; 
+   unsigned int maxway = cp->assoc ; 
+
+   shd_cache_line_t *pline, *cline;
+
+   if (cp->bank_mask)
+      packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0);
+   else
+      packed_addr = addr;
+   set = (packed_addr >> cp->line_sz_log2) & ( (1<<cp->nset_log2) - 1 );
+   tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2);
+
+   if (cp->write_policy == write_back) {
+      //this request must have a reserved spot
+      cline = NULL;
+      for (i=base; i<maxway; i++) {
+         pline = &(cp->lines[set*cp->assoc+i] );
+         if ((pline->tag == tag) && (pline->status & RESERVED)) { 
+            cline = pline;
+            break;
+         }
+         if ((pline->tag == tag) && (pline->status & VALID)) {
+            //A second fill has returned to a line in the cache
+            //discard it as line in cache may have been modified, or is the same
+            _n_line_existed++;
+
+            return -1;
+         }
+      }
+
+      //if (cline) printf("FOUND %d\n",tag);
+      //else printf("UNFOUND!!! %d\n", tag);
+
+      if (!cline) printf("----!!! about to abort - this probably happened because global memory msrh merging is not enabled with a writeback cache !!!----\n");
+
+      assert(cline); //error if it doesn't have a reserved space
+      
+      /* Fetch data into block */
+      cline->status &= ~RESERVED;
+      cline->status |= VALID;
+      //cline->status &= ~DIRTY; Don't clear dirty bit, as might be dirty from write.
+      cline->tag = tag; 
+      cline->addr = addr;
+      cline->last_used = sim_cycle;
+      cline->fetch_time = sim_cycle;
+
+      // no wb, already handled.
+      return -1;
+   }  
+
+   //behavior unchanged for write through cache... probably not all necessary.
+   
+   // Look for any free slots and the possibility that the line is in the cache already
+   nofreeslot = 1;
+   line_exists = 0;
+   for (i=base; i<maxway; i++) {
+      pline = &(cp->lines[set*cp->assoc+i] );
+      if (!(pline->status & VALID)) {
+         cline = pline;
+         nofreeslot = 0;
+         break;
+      } else if (pline->tag == tag) {
+         cline = pline;
+         line_exists = 1;
+         break;
+      }
+   }
+
+   if (line_exists) {
+      _n_line_existed += 1;
+      return -1; // don't need to spill any line, nor it needs to be filled
+   }
+
+   if (nofreeslot) {
+      cline = &(cp->lines[set*cp->assoc+base] );
+      for (i=1+base; i<maxway; i++) {
+         pline = &(cp->lines[set*cp->assoc+i] );
+         if (pline->status & VALID) {
+            switch (cp->policy) {
+            case LRU: 
+               if (pline->last_used < cline->last_used)
+                  cline = pline;
+               break;
+            case FIFO:
+               if (pline->fetch_time < cline->fetch_time)
+                  cline = pline;
+               break;
+            default:
+               break;
+            }   
+         }
+      }
+   }
+
+   /* Set the replaced cache line address */
+   if ((cline->status & (DIRTY|VALID)) == (DIRTY|VALID)) {
+      repl_addr = cline->addr; 
+   } else {
+      repl_addr = -1;
+   }
+
+   /* Fetch data into block */
+   cline->status |= VALID;
+   cline->status &= ~DIRTY;
+   cline->tag = tag; 
+   cline->addr = addr;
+   cline->last_used = sim_cycle;
+   cline->fetch_time = sim_cycle;
+
+/*    printf("Fetching! Addr=%08x ReplAddr=%08x(%d) Set=%x Tag=%x\n",
+          packed_addr, repl_addr, nofreeslot, set, tag);
+ */
+   return repl_addr;
+}
+
+void shd_cache_mergehit( shd_cache_t *cp, unsigned long long int addr )
+{
+   cp->merge_hit += 1;
+}
+
+void shd_cache_print( shd_cache_t *cp,  FILE *stream) {
+   fprintf( stream, "Cache %s:\t", cp->name);
+   fprintf( stream, "Size = %d B (%d Set x %d-way x %d byte line)\n", 
+            cp->line_sz * cp->nset * cp->assoc,
+            cp->nset, cp->assoc, cp->line_sz );
+   fprintf( stream, "\t\tAccess = %d, Miss = %d (%.3g), -MgHts = %d (%.3g)\n", 
+            cp->access, cp->miss, (float) cp->miss / cp->access, 
+            cp->miss - cp->merge_hit, (float) (cp->miss - cp->merge_hit) / cp->access);
+}
+
+#ifdef UNIT_TEST
+
+int main() {
+   shd_cache_t *cp[3];
+   unsigned int addr, i;
+   unsigned int cachenum;
+   unsigned int sim_cycle;
+
+   unsigned int test_addrs[8] = { 0x100, 0x200, 0x300, 0x400, 
+      0x104, 0x204, 0x500, 0x100}; 
+   unsigned int repl_addr[8] = {0,0,0,0,0,0,0,0};
+   unsigned int rdwr[8] = {0,1,0,0,0,0,0,0};
+
+   sim_cycle = 0;
+   cp[0] = shd_cache_create ("cp1", 16, 4, 16, LRU, 1);
+   cp[1] = shd_cache_create ("cp2", 16, 4, 16, FIFO, 1);
+
+   for (cachenum = 0; cachenum<2; cachenum++)
+      for (i=0; i<8; i++) {
+         if ( !shd_cache_access(cp[cachenum], test_addrs[i], 4, rdwr[i], sim_cycle) ) {
+            repl_addr[i] = shd_cache_fill(cp[cachenum], test_addrs[i], sim_cycle);
+            shd_cache_access(cp[cachenum], test_addrs[i], 4, rdwr[i], sim_cycle);
+         }
+         sim_cycle++;
+      }
+
+   printf("replaced address:");
+   for (i=0; i<8; i++) {
+      printf("0x%x ", repl_addr[i]);
+   }
+   printf("\n");
+   shd_cache_print(cp[0],stdout);
+   shd_cache_print(cp[1],stdout);
+
+   shd_cache_fill(cp[0], 0x104b3ecb0, sim_cycle);
+   printf("Accessing 64-bit address tag: %d\n",
+          shd_cache_access(cp[0], 0x104b3ecb2, 4, 0, sim_cycle));
+   printf("Accessing 64-bit address tag: %d\n",
+          shd_cache_access(cp[0], 0x103433330, 4, 0, sim_cycle));
+
+
+   shd_set_coherency_policy(2);
+   cp[2] = shd_cache_create("cp2", 16, 4, 16, LRU, 1);
+   shd_cache_fill(cp[2], 0x12345000, 0);
+   shd_cache_access(cp[2], 0x12345000, 4, 1, 0);
+   shd_cache_access(cp[2], 0x12345004, 4, 0, 0);
+   shd_cache_access(cp[2], 0x12345008, 4, 0, 0);
+   shd_cache_access(cp[2], 0x1234500C, 4, 1, 0);
+   printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0xf,
+          shd_cache_linedirty(cp[2], 0x12345000, 0xf), 1 );
+   printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x6,
+          shd_cache_linedirty(cp[2], 0x12345000, 0x6), 0 );
+   printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x1,
+          shd_cache_linedirty(cp[2], 0x12345000, 0x1), 1 );
+   printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x8,
+          shd_cache_linedirty(cp[2], 0x12345000, 0x8), 1 );
+   printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x9,
+          shd_cache_linedirty(cp[2], 0x12345000, 0x9), 1 );
+
+}
+
+#endif
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
new file mode 100644
index 0000000..e82ad05
--- /dev/null
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -0,0 +1,191 @@
+/* 
+ * gpu-cache.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "../util.h"
+
+#ifndef GPU_CACHE_H
+#define GPU_CACHE_H
+
+#define VALID 0x01
+#define DIRTY 0x02
+#define RESERVED 0x04
+
+enum cache_request_status {
+    HIT,
+    HIT_W_WT, /* Hit, but write through cache, still needs to send to memory */
+    MISS_NO_WB, /* miss, but witeback not necessary*/
+    MISS_W_WB, /* miss, must do writeback */
+    WB_HIT_ON_MISS, /* request hit on a reservation in wb cache*/
+    RESERVATION_FAIL,
+    NUM_CACHE_REQUEST_STATUS
+};
+
+
+typedef struct {
+   unsigned long long int tag;
+   unsigned long long int addr;
+   unsigned int set;
+   unsigned int line_sz; /* bytes */
+   unsigned int fetch_time;
+   unsigned int last_used;
+   unsigned char status; /* valid, dirty... etc */
+} shd_cache_line_t;
+
+
+#define LRU 'L'
+#define FIFO 'F'
+#define RANDOM 'R'
+
+enum cache_write_policy{
+    no_writes, //line replacement when new line arrives
+    write_back, //line replacement when new line arrives
+    write_through //reservation based, use much handle reservation full error.
+};
+
+typedef struct {
+
+   char *name;
+
+   shd_cache_line_t *lines; /* nset x assoc lines in total */
+   unsigned int nset;
+   unsigned int nset_log2;
+   unsigned int assoc;
+   unsigned int line_sz; // bytes 
+   unsigned int line_sz_log2;
+   enum cache_write_policy write_policy; 
+   unsigned char policy;
+   unsigned int hit_latency;
+
+   unsigned int access;
+   unsigned int miss;
+   unsigned int merge_hit; // number of cache miss that hit the same line (and merged as a result)
+
+   // performance counters for calculating the amount of misses within a time window
+   unsigned int prev_snapshot_access;
+   unsigned int prev_snapshot_miss;
+   unsigned int prev_snapshot_merge_hit; 
+   
+   int core_id; // which shader core is using this
+   int type_id; // what kind of cache is this (normal, texture, constant)
+
+   unsigned long long int bank_mask;
+
+} shd_cache_t;
+
+shd_cache_t * shd_cache_create( char *name,
+                                unsigned int nset,
+                                unsigned int assoc,
+                                unsigned int line_sz,
+                                unsigned char policy,
+                                unsigned int hit_latency,
+                                unsigned long long int bank_mask,
+                                enum cache_write_policy wp);
+
+void shd_cache_destroy( shd_cache_t* cp );
+
+// hook up with shader core logger
+void shd_cache_bind_logger(shd_cache_t* cp, int core_id, int type_id);
+
+//depercated, use _wb
+shd_cache_line_t* shd_cache_access( shd_cache_t *cp, 
+                                    unsigned long long int addr, 
+                                    unsigned int nbytes, 
+                                    unsigned char write,
+                                    unsigned int sim_cycle );
+
+//cache check checks for wb and forwards information over.
+enum cache_request_status shd_cache_access_wb( shd_cache_t *cp, 
+                                    unsigned long long int addr, 
+                                    unsigned int nbytes, 
+                                    unsigned char write,
+                                    unsigned int sim_cycle, 
+                                    address_type *wb_address);
+
+
+//just probe the tag array to see if addr is in the cache or not
+//does not update LRU or stats...
+shd_cache_line_t* shd_cache_probe( shd_cache_t *cp, 
+                                   unsigned long long int addr);
+
+// undo the statistic record when the memory access is stalled/squashed and will try again next cycle
+void shd_cache_undo_stats( shd_cache_t *cp, int miss );
+
+void shd_cache_mergehit( shd_cache_t *cp, unsigned long long int addr );
+
+unsigned long long int shd_cache_fill( shd_cache_t *cp, 
+                                       unsigned long long int addr,
+                                       unsigned int sim_cycle );
+
+unsigned long long int L2_shd_cache_fill( shd_cache_t *cp, 
+                                          unsigned long long int addr,
+                                          unsigned int sim_cycle );
+
+void shd_cache_print( shd_cache_t *cp,  FILE *stream);
+
+
+#endif
diff --git a/src/gpgpu-sim/gpu-misc.cc b/src/gpgpu-sim/gpu-misc.cc
new file mode 100644
index 0000000..06a0f68
--- /dev/null
+++ b/src/gpgpu-sim/gpu-misc.cc
@@ -0,0 +1,96 @@
+/* 
+ * gpu-misc.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "gpu-misc.h"
+
+unsigned int LOGB2( unsigned int v ) {
+   unsigned int shift;
+   unsigned int r;
+
+   r = 0;
+
+   shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift;
+   shift = (( v & 0xFF00    ) != 0 ) << 3; v >>= shift; r |= shift;
+   shift = (( v & 0xF0      ) != 0 ) << 2; v >>= shift; r |= shift;
+   shift = (( v & 0xC       ) != 0 ) << 1; v >>= shift; r |= shift;
+   shift = (( v & 0x2       ) != 0 ) << 0; v >>= shift; r |= shift;
+
+   return r;
+}
+
+unsigned int MAX2NUM( unsigned int a, unsigned int b ) {
+   if (a > b) {
+      return a;
+   } else
+      return b;
+}
+
+unsigned int MIN2NUM( unsigned int a, unsigned int b ) {
+   if (a < b) {
+      return a;
+   } else
+      return b;
+}
diff --git a/src/gpgpu-sim/gpu-misc.h b/src/gpgpu-sim/gpu-misc.h
new file mode 100644
index 0000000..3d07d77
--- /dev/null
+++ b/src/gpgpu-sim/gpu-misc.h
@@ -0,0 +1,106 @@
+/* 
+ * gpu-misc.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef GPU_MISC_H
+#define GPU_MISC_H
+
+#define CONSTC  100
+#define DCACHE   200
+#define TEXTC      300
+#define SHD_CACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1cache->line_sz - 1)))
+#define SHD_TEXCACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1texcache->line_sz - 1)))
+#define SHD_CONSTCACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1constcache->line_sz - 1)))
+#define CACHE_TAG_OF(x,cache) ((x) & (~((unsigned long long int)cache->line_sz - 1)))
+#define CACHE_TAG_OF_64(x) ((x) & (~((unsigned long long int)64 - 1)))
+
+#define ispowerof2(x)   ((((x) - 1) & (x)) == 0)
+#define powerof2(x)  (1 << (x))
+
+
+enum mem_space {  //used for cudasim
+   SHARED_SPACE, 
+   CONST_SPACE, 
+   GLOBAL_SPACE,
+   LOCAL_SPACE,
+   TEX_SPACE
+};
+//enables a verbose printout of all L1 cache misses and all MSHR status changes 
+//good for a single shader configuration
+#define DEBUGL1MISS 0
+
+unsigned int LOGB2( unsigned int v );
+
+unsigned int MAX2NUM( unsigned int a, unsigned int b );
+
+unsigned int MIN2NUM( unsigned int a, unsigned int b );
+
+
+#define gs_max2(a,b) (((a)>(b))?(a):(b))
+#define gs_min2(a,b) (((a)<(b))?(a):(b))
+#define min3(x,y,z) (((x)<(y) && (x)<(z))?(x):(gs_min2((y),(z))))
+
+#endif
+
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
new file mode 100644
index 0000000..6ae7430
--- /dev/null
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -0,0 +1,1752 @@
+/* 
+ * gpu-sim.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan, Ivan Sham, Henry Wong, Dan O'Connor and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "gpu-sim.h"
+
+#include <time.h>
+#include "gpu-cache.h"
+#include "gpu-misc.h"
+#include "delayqueue.h"
+#include "shader.h"
+#include "icnt_wrapper.h"
+#include "dram.h"
+#include "addrdec.h"
+#include "dwf.h"
+#include "warp_tracker.h"
+#include "cflogger.h"
+#include "l2cache.h"
+
+#include "../cuda-sim/ptx-stats.h"
+#include "../intersim/statwraper.h"
+#include "../abstract_hardware_model.h"
+
+#include <stdio.h>
+#include <string.h>
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+extern unsigned L2_write_miss;
+extern unsigned L2_write_hit;
+extern unsigned L2_read_hit;
+extern unsigned L2_read_miss;
+unsigned made_read_mfs = 0;
+unsigned made_write_mfs = 0;
+unsigned freed_read_mfs = 0;
+unsigned freed_L1write_mfs = 0;
+unsigned freed_L2write_mfs = 0;
+unsigned freed_dummy_read_mfs = 0;
+unsigned long long  gpu_sim_cycle = 0;
+unsigned long long  gpu_sim_insn = 0;
+unsigned long long  gpu_sim_insn_no_ld_const = 0;
+unsigned long long  gpu_sim_prev_insn = 0;
+unsigned long long  gpu_sim_insn_last_update = 0;
+unsigned long long  gpu_tot_sim_cycle = 0;
+unsigned long long  gpu_tot_sim_insn = 0;
+unsigned long long  gpu_last_sim_cycle = 0;
+unsigned long long  gpu_completed_thread = 0;
+unsigned long long  gpu_tot_issued_cta = 0;
+unsigned long long  gpu_tot_completed_thread = 0;
+
+unsigned int **concurrent_row_access; //concurrent_row_access[dram chip id][bank id]
+unsigned int **num_activates; //num_activates[dram chip id][bank id]
+unsigned int **row_access; //row_access[dram chip id][bank id]
+unsigned int **max_conc_access2samerow; //max_conc_access2samerow[dram chip id][bank id]
+unsigned int **max_servicetime2samerow; //max_servicetime2samerow[dram chip id][bank id]
+unsigned int mergemiss = 0;
+unsigned int L1_read_miss = 0;
+unsigned int L1_write_miss = 0;
+unsigned int L1_write_hit_on_miss = 0;
+unsigned int L1_writeback = 0;
+unsigned int L1_texture_miss = 0;
+unsigned int L1_const_miss = 0;
+unsigned int gpgpu_n_sent_writes = 0;
+unsigned int gpgpu_n_processed_writes = 0;
+unsigned int *max_return_queue_length;
+
+// performance counter for stalls due to congestion.
+unsigned int gpu_stall_shd_mem = 0;
+unsigned int gpu_stall_wr_back = 0;
+unsigned int gpu_stall_dramfull = 0; 
+unsigned int gpu_stall_icnt2sh = 0;
+unsigned int gpu_stall_by_MSHRwb = 0;
+
+//shader cannot send to icnt because icnt buffer is full
+//Note: it is accumulative for all shaders and is never reset
+//so it might increase 8 times in a cycle if we have 8 shaders
+unsigned int gpu_stall_sh2icnt = 0;        
+// performance counters to account for instruction distribution
+extern unsigned int gpgpu_n_load_insn;
+extern unsigned int gpgpu_n_store_insn;
+extern unsigned int gpgpu_n_shmem_insn;
+extern unsigned int gpgpu_n_tex_insn;
+extern unsigned int gpgpu_n_const_insn;
+extern unsigned int gpgpu_multi_unq_fetches;
+char *gpgpu_runtime_stat;
+int gpu_stat_sample_freq = 10000;
+int gpu_runtime_stat_flag = 0;
+extern int gpgpu_warpdistro_shader;
+
+// GPGPU options
+unsigned long long  gpu_max_cycle = 0;
+unsigned long long  gpu_max_insn = 0;
+int gpu_max_cycle_opt = 0;
+int gpu_max_insn_opt = 0;
+int gpu_max_cta_opt = 0;
+int gpu_deadlock_detect = 0;
+int gpu_deadlock = 0;
+static unsigned long long  last_gpu_sim_insn = 0;
+int gpgpu_dram_scheduler = DRAM_FIFO;
+int g_save_embedded_ptx = 0;
+int gpgpu_simd_model = 0;
+int gpgpu_no_dl1 = 0;
+char *gpgpu_cache_texl1_opt;
+char *gpgpu_cache_constl1_opt;
+char *gpgpu_cache_dl1_opt;
+char *gpgpu_cache_dl2_opt;
+extern int gpgpu_l2_readoverwrite;
+int gpgpu_partial_write_mask = 0;
+
+int gpgpu_perfect_mem = FALSE;
+char *gpgpu_shader_core_pipeline_opt;
+extern unsigned int *requests_by_warp;
+unsigned int gpgpu_dram_buswidth = 4;
+unsigned int gpgpu_dram_burst_length = 4;
+int gpgpu_dram_sched_queue_size = 0; 
+char * gpgpu_dram_timing_opt;
+int gpgpu_flush_cache = 0;
+int gpgpu_mem_address_mask = 0;
+unsigned int recent_dram_util = 0;
+
+int gpgpu_cflog_interval = 0;
+
+unsigned int finished_trace = 0;
+
+unsigned g_next_request_uid = 1;
+
+extern struct regs_t regs;
+
+extern long int gpu_reads;
+
+void ptx_dump_regs( void *thd );
+
+int g_nthreads_issued;
+int g_total_cta_left;
+
+
+unsigned ptx_kernel_program_size();
+void visualizer_printstat();
+void time_vector_create(int ld_size,int st_size);
+void time_vector_print(void);
+void time_vector_update(unsigned int uid,int slot ,long int cycle,int type);
+void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type);
+void node_req_hist_clear(void *p);
+void node_req_hist_dump(void *p);
+void node_req_hist_update(void * p,int node, long long cycle);
+
+/* functionally simulated memory */
+extern struct mem_t *mem;
+
+/* Defining Clock Domains
+basically just the ratio is important */
+
+#define  CORE  0x01
+#define  L2    0x02
+#define  DRAM  0x04
+#define  ICNT  0x08  
+
+double core_time=0;
+double icnt_time=0;
+double dram_time=0;
+double l2_time=0;
+
+#define MhZ *1000000
+double core_freq=2 MhZ;
+double icnt_freq=2 MhZ;
+double dram_freq=2 MhZ;
+double l2_freq=2 MhZ;
+
+double core_period  = 1 /( 2 MhZ);
+double icnt_period   = 1 /( 2 MhZ);
+double dram_period = 1 /( 2 MhZ);
+double l2_period = 1 / (2 MhZ);
+
+char * gpgpu_clock_domains;
+
+/* GPU uArch parameters */
+unsigned int gpu_n_mem = 8;
+unsigned int gpu_mem_n_bk = 4;
+unsigned int gpu_n_mem_per_ctrlr = 1;
+unsigned int gpu_n_shader = 8;
+int gpu_concentration = 1;
+int gpu_n_tpc = 8;
+unsigned int gpu_n_mshr_per_shader;
+unsigned int gpu_n_thread_per_shader = 128;
+unsigned int gpu_n_warp_per_shader;
+unsigned int gpu_n_mshr_per_thread = 1;
+
+extern int gpgpu_interwarp_mshr_merge ;
+
+extern unsigned int gpgpu_shmem_size;
+extern unsigned int gpgpu_shader_registers;
+extern unsigned int gpgpu_shader_cta;
+extern int gpgpu_shmem_bkconflict;
+extern int gpgpu_cache_bkconflict;
+extern int gpgpu_n_cache_bank;
+extern unsigned int warp_size; 
+extern int pipe_simd_width;
+extern unsigned int gpgpu_dwf_heuristic;
+extern unsigned int gpgpu_dwf_regbk;
+int gpgpu_reg_bankconflict = FALSE;
+extern int gpgpu_shmem_port_per_bank;
+extern int gpgpu_cache_port_per_bank;
+extern int gpgpu_const_port_per_bank;
+extern int gpgpu_shmem_pipe_speedup;  
+extern int gpgpu_reg_bank_conflict_model;
+extern int gpgpu_num_reg_banks;
+
+extern unsigned int gpu_max_cta_per_shader;
+extern unsigned int gpu_padded_cta_size;
+extern int gpgpu_local_mem_map;
+
+unsigned int gpgpu_pre_mem_stages = 0;
+unsigned int gpgpu_no_divg_load = 0;
+char *gpgpu_dwf_hw_opt;
+unsigned int gpgpu_thread_swizzling = 0;
+unsigned int gpgpu_strict_simd_wrbk = 0;
+
+int pdom_sched_type = 0;
+int n_pdom_sc_orig_stat = 0; //the selected pdom schedular is used 
+int n_pdom_sc_single_stat = 0; //only a single warp is ready to go in that cycle.  
+int *num_warps_issuable;
+int *num_warps_issuable_pershader;
+
+// Thread Dispatching Unit option 
+int gpgpu_cuda_sim = 1;
+int gpgpu_spread_blocks_across_cores = 1;
+
+/* GPU uArch structures */
+shader_core_ctx_t **sc;
+dram_t **dram;
+unsigned int common_clock = 0;
+unsigned int more_thread = 1;
+extern unsigned int n_regconflict_stall;
+unsigned int warp_conflict_at_writeback = 0;
+unsigned int gpgpu_commit_pc_beyond_two = 0;
+extern int g_network_mode;
+int gpgpu_cache_wt_through = 0;
+
+
+//memory access classification
+int gpgpu_n_mem_read_local = 0;
+int gpgpu_n_mem_write_local = 0;
+int gpgpu_n_mem_texture = 0;
+int gpgpu_n_mem_const = 0;
+int gpgpu_n_mem_read_global = 0;
+int gpgpu_n_mem_write_global = 0;
+
+#define MEM_LATENCY_STAT_IMPL
+#include "mem_latency_stat.h"
+
+unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid );
+unsigned char fq_push(unsigned long long int addr, int bsize, unsigned char write, partial_write_mask_t partial_write_mask, 
+                      int sid, int wid, mshr_entry* mshr, int cache_hits_waiting,
+                      enum mem_access_type mem_acc, address_type pc);
+int issue_mf_from_fq(mem_fetch_t *mf);
+unsigned char single_check_icnt_has_buffer(int chip, int sid, unsigned char is_write );
+unsigned char fq_pop(int tpc_id);
+void fill_shd_L1_with_new_line(shader_core_ctx_t * sc, mem_fetch_t * mf);
+
+void set_option_gpgpu_spread_blocks_across_cores(int option);
+void set_param_gpgpu_num_shaders(int num_shaders);
+unsigned ptx_sim_grid_size();
+void icnt_init_grid();
+void interconnect_stats();
+void icnt_overal_stat();
+unsigned ptx_sim_cta_size();
+unsigned ptx_sim_init_thread( void** thread_info, int sid, unsigned tid,unsigned threads_left,unsigned num_threads, core_t *core, unsigned hw_cta_id, unsigned hw_warp_id );
+
+void gpu_sim_loop( int grid_num );
+
+void print_shader_cycle_distro( FILE *fout ) ;
+void find_reconvergence_points();
+void dwf_process_reconv_pts();
+
+extern int gpgpu_ptx_instruction_classification ;
+extern int g_ptx_sim_mode;
+
+extern int gpgpu_coalesce_arch;
+
+#define CREATELOG 111
+#define SAMPLELOG 222
+#define DUMPLOG 333
+void L2c_log(int task);
+void dram_log(int task);
+
+void visualizer_options(option_parser_t opp);
+void gpu_reg_options(option_parser_t opp)
+{
+   option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, 
+                "saves ptx files embedded in binary as <n>.ptx",
+                "0");
+   option_parser_register(opp, "-gpgpu_simd_model", OPT_INT32, &gpgpu_simd_model, 
+               "0 = no recombination, 1 = post-dominator, 2 = MIMD, 3 = dynamic warp formation", "0");
+   option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &gpgpu_dram_scheduler, 
+               "0 = fifo (default), 1 = fast ideal", "0");
+
+   option_parser_register(opp, "-gpgpu_max_cycle", OPT_INT32, &gpu_max_cycle_opt, 
+               "terminates gpu simulation early (0 = no limit)",
+               "0");
+   option_parser_register(opp, "-gpgpu_max_insn", OPT_INT32, &gpu_max_insn_opt, 
+               "terminates gpu simulation early (0 = no limit)",
+               "0");
+   option_parser_register(opp, "-gpgpu_max_cta", OPT_INT32, &gpu_max_cta_opt, 
+               "terminates gpu simulation early (0 = no limit)",
+               "0");
+
+   option_parser_register(opp, "-gpgpu_tex_cache:l1", OPT_CSTR, &gpgpu_cache_texl1_opt, 
+                  "per-shader L1 texture cache  (READ-ONLY) config, i.e., {<nsets>:<linesize>:<assoc>:<repl>|none}",
+                  "512:64:2:L");
+
+   option_parser_register(opp, "-gpgpu_const_cache:l1", OPT_CSTR, &gpgpu_cache_constl1_opt, 
+                  "per-shader L1 constant memory cache  (READ-ONLY) config, i.e., {<nsets>:<linesize>:<assoc>:<repl>|none}",
+                  "64:64:2:L");
+
+   option_parser_register(opp, "-gpgpu_no_dl1", OPT_BOOL, &gpgpu_no_dl1, 
+                "no dl1 cache (voids -gpgpu_cache:dl1 option)",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_cache:dl1", OPT_CSTR, &gpgpu_cache_dl1_opt, 
+                  "shader L1 data cache config, i.e., {<nsets>:<bsize>:<assoc>:<repl>|none}",
+                  "256:128:1:L");
+
+   option_parser_register(opp, "-gpgpu_cache:dl2", OPT_CSTR, &gpgpu_cache_dl2_opt, 
+                  "unified banked L2 data cache config, i.e., {<nsets>:<bsize>:<assoc>:<repl>|none}; disabled by default",
+                  NULL); 
+
+   option_parser_register(opp, "-gpgpu_perfect_mem", OPT_BOOL, &gpgpu_perfect_mem, 
+                "enable perfect memory mode (no cache miss)",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_shader_core_pipeline", OPT_CSTR, &gpgpu_shader_core_pipeline_opt, 
+                  "shader core pipeline config, i.e., {<nthread>:<warpsize>:<pipe_simd_width>}",
+                  "256:32:32");
+
+   option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, 
+                "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)",
+                "8192");
+
+   option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &gpgpu_shader_cta, 
+                "Maximum number of concurrent CTAs in shader (default 8)",
+                "8");
+
+   option_parser_register(opp, "-gpgpu_n_shader", OPT_UINT32, &gpu_n_shader, 
+                "number of shaders in gpu",
+                "8");
+   option_parser_register(opp, "-gpgpu_n_mem", OPT_UINT32, &gpu_n_mem, 
+                "number of memory modules (e.g. memory controllers) in gpu",
+                "8");
+   option_parser_register(opp, "-gpgpu_n_mem_per_ctrlr", OPT_UINT32, &gpu_n_mem_per_ctrlr, 
+                "number of memory chips per memory controller",
+                "1");
+   option_parser_register(opp, "-gpgpu_runtime_stat", OPT_CSTR, &gpgpu_runtime_stat, 
+                  "display runtime statistics such as dram utilization {<freq>:<flag>}",
+                  "10000:0");
+
+   option_parser_register(opp, "-gpgpu_dwf_heuristic", OPT_UINT32, &gpgpu_dwf_heuristic, 
+                "DWF scheduling heuristic: 0 = majority, 1 = minority, 2 = timestamp, 3 = pdom priority, 4 = pc-based, 5 = max-heap",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_reg_bankconflict", OPT_BOOL, &gpgpu_reg_bankconflict, 
+                "Check for bank conflict in the pipeline",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_dwf_regbk", OPT_BOOL, (int*)&gpgpu_dwf_regbk, 
+                "Have dwf scheduler to avoid bank conflict",
+                "1");
+
+   option_parser_register(opp, "-gpgpu_memlatency_stat", OPT_INT32, &gpgpu_memlatency_stat, 
+               "track and display latency statistics 0x2 enables MC, 0x4 enables queue logs",
+               "0");
+
+   option_parser_register(opp, "-gpu_n_mshr_per_shader", OPT_UINT32, &gpu_n_mshr_per_shader, 
+                "Number of MSHRs per shader",
+                "64");
+
+   option_parser_register(opp, "-gpgpu_interwarp_mshr_merge", OPT_INT32, &gpgpu_interwarp_mshr_merge, 
+               "interwarp coalescing",
+               "0");
+
+   option_parser_register(opp, "-gpgpu_dram_sched_queue_size", OPT_INT32, &gpgpu_dram_sched_queue_size, 
+               "0 = unlimited (default); # entries per chip",
+               "0");
+
+   option_parser_register(opp, "-gpgpu_dram_buswidth", OPT_UINT32, &gpgpu_dram_buswidth, 
+                "default = 4 bytes (8 bytes per cycle at DDR)",
+                "4");
+
+   option_parser_register(opp, "-gpgpu_dram_burst_length", OPT_UINT32, &gpgpu_dram_burst_length, 
+                "Burst length of each DRAM request (default = 4 DDR cycle)",
+                "4");
+
+   option_parser_register(opp, "-gpgpu_dram_timing_opt", OPT_CSTR, &gpgpu_dram_timing_opt, 
+               "DRAM timing parameters = {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tWTR}",
+               "4:2:8:12:21:13:34:9:4:5");
+
+
+   option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, 
+               "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits",
+               "0");
+
+   option_parser_register(opp, "-gpgpu_flush_cache", OPT_BOOL, &gpgpu_flush_cache, 
+                "Flush cache at the end of each kernel call",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_pre_mem_stages", OPT_UINT32, &gpgpu_pre_mem_stages, 
+                "default = 0 pre-memory pipeline stages",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_no_divg_load", OPT_BOOL, (int*)&gpgpu_no_divg_load, 
+                "Don't allow divergence on load",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_dwf_hw", OPT_CSTR, &gpgpu_dwf_hw_opt, 
+                  "dynamic warp formation hw config, i.e., {<#LUT_entries>:<associativity>|none}",
+                  "32:2");
+
+   option_parser_register(opp, "-gpgpu_thread_swizzling", OPT_BOOL, (int*)&gpgpu_thread_swizzling, 
+                "Thread Swizzling (1=on, 0=off)",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_strict_simd_wrbk", OPT_BOOL, (int*)&gpgpu_strict_simd_wrbk, 
+                "Applying Strick SIMD WriteBack Stage (1=on, 0=off)",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, 
+                "Size of shared memory per shader core (default 16kB)",
+                "16384");
+
+   option_parser_register(opp, "-gpgpu_shmem_bkconflict", OPT_BOOL, &gpgpu_shmem_bkconflict,  
+                "Turn on bank conflict check for shared memory",
+                "0");
+
+   option_parser_register(opp, "-gpgpu_shmem_pipe_speedup", OPT_INT32, &gpgpu_shmem_pipe_speedup,  
+                "Number of groups each warp is divided for shared memory bank conflict check",
+                "2");
+
+   option_parser_register(opp, "-gpgpu_cache_wt_through", OPT_BOOL, &gpgpu_cache_wt_through, 
+                "L1 cache become write through (1=on, 0=off)", 
+                "0");
+
+   option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, 
+                "Stop the simulation at deadlock (1=on (default), 0=off)", 
+                "1");
+
+   option_parser_register(opp, "-gpgpu_cache_bkconflict", OPT_BOOL, &gpgpu_cache_bkconflict, 
+                "Turn on bank conflict check for L1 cache access", 
+                "0");
+
+   option_parser_register(opp, "-gpgpu_n_cache_bank", OPT_INT32, &gpgpu_n_cache_bank, 
+               "Number of banks in L1 cache, also for memory coalescing stall", 
+               "1");
+
+   option_parser_register(opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, 
+               "Specify which shader core to collect the warp size distribution from", 
+               "-1");
+
+
+   option_parser_register(opp, "-gpgpu_pdom_sched_type", OPT_INT32, &pdom_sched_type, 
+               "0 = first ready warp found, 1 = random, 8 = loose round robin", 
+               "8");
+
+   option_parser_register(opp, "-gpgpu_spread_blocks_across_cores", OPT_BOOL, 
+                &gpgpu_spread_blocks_across_cores, 
+                "Spread block-issuing across all cores instead of filling up core by core (do NOT disable)", 
+                "1");
+
+   option_parser_register(opp, "-gpgpu_cuda_sim", OPT_BOOL, &gpgpu_cuda_sim, 
+                "use PTX instruction set", 
+                "1");
+   option_parser_register(opp, "-gpgpu_ptx_instruction_classification", OPT_INT32, 
+               &gpgpu_ptx_instruction_classification, 
+               "if enabled will classify ptx instruction types per kernel (Max 255 kernels now)", 
+               "0");
+   option_parser_register(opp, "-gpgpu_ptx_sim_mode", OPT_INT32, &g_ptx_sim_mode, 
+               "Select between Performance (default) or Functional simulation (1)", 
+               "0");
+   option_parser_register(opp, "-gpgpu_clock_domains", OPT_CSTR, &gpgpu_clock_domains, 
+                  "Clock Domain Frequencies in MhZ {<Core Clock>:<ICNT Clock>:<L2 Clock>:<DRAM Clock>}",
+                  "500.0:2000.0:2000.0:2000.0");
+
+   option_parser_register(opp, "-gpgpu_shmem_port_per_bank", OPT_INT32, &gpgpu_shmem_port_per_bank, 
+               "Number of access processed by a shared memory bank per cycle (default = 2)", 
+               "2");
+   option_parser_register(opp, "-gpgpu_cache_port_per_bank", OPT_INT32, &gpgpu_cache_port_per_bank, 
+               "Number of access processed by a cache bank per cycle (default = 2)", 
+               "2");
+   option_parser_register(opp, "-gpgpu_const_port_per_bank", OPT_INT32, &gpgpu_const_port_per_bank,
+               "Number of access processed by a constant cache bank per cycle (default = 2)", 
+               "2");
+   option_parser_register(opp, "-gpgpu_cflog_interval", OPT_INT32, &gpgpu_cflog_interval, 
+               "Interval between each snapshot in control flow logger", 
+               "0");
+   option_parser_register(opp, "-gpgpu_partial_write_mask", OPT_INT32, &gpgpu_partial_write_mask, 
+               "use partial write mask to filter memory requests <1>No extra reads(use this!)<2>extra reads generated for partial chunks", 
+               "0");
+   option_parser_register(opp, "-gpu_concentration", OPT_INT32, &gpu_concentration, 
+               "Number of shader cores per interconnection port (default = 1)", 
+               "1");
+   option_parser_register(opp, "-gpgpu_local_mem_map", OPT_INT32, &gpgpu_local_mem_map, 
+               "Mapping from local memory space address to simulated GPU physical address space (default = 1)", 
+               "1");
+   option_parser_register(opp, "-gpgpu_reg_bank_conflict_model", OPT_BOOL, &gpgpu_reg_bank_conflict_model, 
+               "Turn on register bank conflict model (default = off)", 
+               "0");
+   option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, 
+               "Number of register banks (default = 8)", 
+               "8");
+    option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, 
+                           "Coalescing arch (default = 13, anything else is off for now)", 
+                           "13");
+   addrdec_setoption(opp);
+   L2c_options(opp);
+   visualizer_options(opp);
+   ptx_file_line_stats_options(opp);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+inline int mem2device(int memid) {
+   return memid + gpu_n_tpc;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
+
+/* Allocate memory for uArch structures */
+void init_gpu () 
+{ 
+   int i;
+
+   gpu_max_cycle = gpu_max_cycle_opt;
+   gpu_max_insn = gpu_max_insn_opt;
+
+   i = sscanf(gpgpu_shader_core_pipeline_opt,"%d:%d:%d", 
+              &gpu_n_thread_per_shader, &warp_size, &pipe_simd_width);
+   gpu_n_warp_per_shader = gpu_n_thread_per_shader / warp_size;
+   num_warps_issuable = (int*) calloc(gpu_n_warp_per_shader+1, sizeof(int));
+   num_warps_issuable_pershader = (int*) calloc(gpu_n_shader, sizeof(int));
+   if (i == 2) {
+      pipe_simd_width = warp_size;
+   } else if (i == 3) {
+      assert(warp_size % pipe_simd_width == 0);
+   }
+
+   sscanf(gpgpu_runtime_stat, "%d:%x",
+          &gpu_stat_sample_freq, &gpu_runtime_stat_flag);
+
+   sc = (shader_core_ctx_t**) calloc(gpu_n_shader, sizeof(shader_core_ctx_t*));
+   int mshr_que = gpu_n_mshr_per_thread;
+   for (i=0;(unsigned)i<gpu_n_shader;i++) {
+      sc[i] = shader_create("sh", i, /* shader id*/
+                            gpu_n_thread_per_shader, /* number of threads */
+                            mshr_que, /* number of MSHR per threads */
+                            fq_push, fq_has_buffer, gpgpu_simd_model);
+   }
+
+   ptx_file_line_stats_create_exposed_latency_tracker(gpu_n_shader);
+
+   // initialize dynamic warp formation scheduler
+   int dwf_lut_size, dwf_lut_assoc;
+   sscanf(gpgpu_dwf_hw_opt,"%d:%d", &dwf_lut_size, &dwf_lut_assoc);
+   char *dwf_hw_policy_opt = strchr(gpgpu_dwf_hw_opt, ';');
+   int insn_size = 1; // for cuda-sim
+   create_dwf_schedulers(gpu_n_shader, dwf_lut_size, dwf_lut_assoc, 
+                         warp_size, pipe_simd_width, 
+                         gpu_n_thread_per_shader, insn_size, 
+                         gpgpu_dwf_heuristic, dwf_hw_policy_opt );
+
+   gpgpu_no_divg_load = gpgpu_no_divg_load && (gpgpu_simd_model == DWF);
+   // always use no diverge on load for PDOM and NAIVE
+   gpgpu_no_divg_load = gpgpu_no_divg_load || (gpgpu_simd_model == POST_DOMINATOR || gpgpu_simd_model == NO_RECONVERGE);
+   if (gpgpu_no_divg_load)
+      init_warp_tracker();
+
+   assert(gpu_n_shader % gpu_concentration == 0);
+   gpu_n_tpc = gpu_n_shader / gpu_concentration;
+
+   dram = (dram_t**) calloc(gpu_n_mem, sizeof(dram_t*));
+   // L2request = (mem_fetch_t**) calloc(gpu_n_mem, sizeof(mem_fetch_t*));
+   addrdec_setnchip(gpu_n_mem);
+   unsigned int nbk,tCCD,tRRD,tRCD,tRAS,tRP,tRC,CL,WL,tWTR;
+   sscanf(gpgpu_dram_timing_opt,"%d:%d:%d:%d:%d:%d:%d:%d:%d:%d",&nbk,&tCCD,&tRRD,&tRCD,&tRAS,&tRP,&tRC,&CL,&WL,&tWTR);
+   gpu_mem_n_bk = nbk;
+   for (i=0;(unsigned)i<gpu_n_mem;i++) {
+      dram[i] = dram_create(i, nbk, tCCD, tRRD, tRCD, tRAS, tRP, tRC, 
+                            CL, WL, gpgpu_dram_burst_length/*BL*/, tWTR, gpgpu_dram_buswidth/*busW*/, 
+                            gpgpu_dram_sched_queue_size, gpgpu_dram_scheduler);
+      if (gpgpu_cache_dl2_opt)
+         L2c_create(dram[i], gpgpu_cache_dl2_opt);
+   }
+   dram_log(CREATELOG);
+   if (gpgpu_cache_dl2_opt && 1) {
+      L2c_log(CREATELOG);
+   }
+   concurrent_row_access = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   num_activates = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   row_access = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   max_conc_access2samerow = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   max_servicetime2samerow = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+
+   for (i=0;(unsigned)i<gpu_n_mem ;i++ ) {
+      concurrent_row_access[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      row_access[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      num_activates[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      max_conc_access2samerow[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      max_servicetime2samerow[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+   }
+
+   memlatstat_init();
+
+   L2c_init_stat();
+   max_return_queue_length = (unsigned int*) calloc(gpu_n_shader, sizeof(unsigned int));
+   icnt_init(gpu_n_tpc, gpu_n_mem);
+
+   common_clock = 0;
+
+   time_vector_create(NUM_MEM_REQ_STAT,MR_2SH_ICNT_INJECTED);
+}
+
+
+
+void gpu_print_stat();
+
+void init_clock_domains(void ) {
+   sscanf(gpgpu_clock_domains,"%lf:%lf:%lf:%lf", 
+          &core_freq, &icnt_freq, &l2_freq, &dram_freq);
+   core_freq = core_freq MhZ;
+   icnt_freq = icnt_freq MhZ;
+   l2_freq = l2_freq MhZ;
+   dram_freq = dram_freq MhZ;        
+   core_period = 1/core_freq;
+   icnt_period = 1/icnt_freq;
+   dram_period = 1/dram_freq;
+   l2_period = 1/l2_freq;
+   core_time = 0 ;
+   dram_time = 0 ;
+   icnt_time = 0;
+   l2_time = 0;
+   printf("GPGPU-Sim uArch: clock freqs: %lf:%lf:%lf:%lf\n",core_freq,icnt_freq,l2_freq,dram_freq);
+   printf("GPGPU-Sim uArch: clock periods: %.20lf:%.20lf:%.20lf:%.20lf\n",core_period,icnt_period,l2_period,dram_period);
+}
+
+void reinit_clock_domains(void)
+{
+   core_time = 0 ;
+   dram_time = 0 ;
+   icnt_time = 0;
+   l2_time = 0;
+}
+
+void init_once(void ) {
+   init_clock_domains();
+}
+
+// return the number of cycle required to run all the trace on the gpu 
+unsigned int run_gpu_sim(int grid_num) 
+{
+
+   int not_completed;
+   int mem_busy;
+   int icnt2mem_busy;
+
+   gpu_sim_cycle = 0;
+   not_completed = 1;
+   mem_busy = 1;
+   icnt2mem_busy = 1;
+   finished_trace = 0;
+   g_next_request_uid = 1;
+   more_thread = 1;
+   gpu_sim_insn = 0;
+   gpu_sim_insn_no_ld_const = 0;
+
+   gpu_completed_thread = 0;
+
+   g_nthreads_issued = 0;
+
+   static int one_time_inits_done = 0 ;
+   if (!one_time_inits_done ) {
+      init_once();
+   }
+   reinit_clock_domains();
+   assert(gpgpu_spread_blocks_across_cores); // this seems to be required, so let's make it explicit
+   set_option_gpgpu_spread_blocks_across_cores(gpgpu_spread_blocks_across_cores);
+   set_param_gpgpu_num_shaders(gpu_n_shader);
+   for (unsigned i=0;i<gpu_n_shader;i++) {
+      sc[i]->not_completed = 0;
+      shader_reinit(sc[i],0,sc[i]->n_threads);
+   }
+   if (gpu_max_cta_opt != 0) {
+      g_total_cta_left = gpu_max_cta_opt;
+   } else {
+      g_total_cta_left =  ptx_sim_grid_size();
+   }
+   if (gpu_max_cta_opt != 0) {
+      // the maximum number of CTA has been reached, stop any further simulation
+      if (gpu_tot_issued_cta >= (unsigned)gpu_max_cta_opt) {
+         return 0;
+      }
+   }
+
+   if (gpu_max_cycle && (gpu_tot_sim_cycle + gpu_sim_cycle) >= gpu_max_cycle) {
+      return gpu_sim_cycle;
+   }
+   if (gpu_max_insn && (gpu_tot_sim_insn + gpu_sim_insn) >= gpu_max_insn) {
+      return gpu_sim_cycle;
+   }
+
+   // refind the diverge/reconvergence pairs
+   dwf_reset_reconv_pt();
+   find_reconvergence_points();
+
+   dwf_process_reconv_pts();
+   dwf_reinit_schedulers(gpu_n_shader);
+
+   // initialize the control-flow, memory access, memory latency logger
+   create_thread_CFlogger( gpu_n_shader, gpu_n_thread_per_shader, ptx_kernel_program_size(), 0, gpgpu_cflog_interval );
+   shader_CTA_count_create( gpu_n_shader, gpgpu_cflog_interval);
+   if (gpgpu_cflog_interval != 0) {
+      insn_warp_occ_create( gpu_n_shader, warp_size, ptx_kernel_program_size() );
+      shader_warp_occ_create( gpu_n_shader, warp_size, gpgpu_cflog_interval);
+      shader_mem_acc_create( gpu_n_shader, gpu_n_mem, 4, gpgpu_cflog_interval);
+      shader_mem_lat_create( gpu_n_shader, gpgpu_cflog_interval);
+      shader_cache_access_create( gpu_n_shader, 3, gpgpu_cflog_interval);
+      set_spill_interval (gpgpu_cflog_interval * 40);
+   }
+
+   // calcaulte the max cta count and cta size for local memory address mapping
+   gpu_max_cta_per_shader = max_cta_per_shader(sc[0]);
+   //gpu_max_cta_per_shader is limited by number of CTAs if not enough    
+   if (ptx_sim_grid_size() < gpu_max_cta_per_shader*gpu_n_shader) { 
+      gpu_max_cta_per_shader = (ptx_sim_grid_size() / gpu_n_shader);
+      if (ptx_sim_grid_size() % gpu_n_shader)
+         gpu_max_cta_per_shader++;
+   }
+   unsigned int gpu_cta_size = ptx_sim_cta_size();
+   gpu_padded_cta_size = (gpu_cta_size%32) ? 32*((gpu_cta_size/32)+1) : gpu_cta_size;
+
+   if (g_network_mode) {
+      icnt_init_grid(); 
+   }
+   last_gpu_sim_insn = 0;
+   // add this condition as well? (gpgpu_n_processed_writes < gpgpu_n_sent_writes)
+   while (not_completed || mem_busy || icnt2mem_busy) {
+      gpu_sim_loop(grid_num);
+
+      not_completed = 0;
+      for (unsigned i=0;i<gpu_n_shader;i++) {
+         not_completed += sc[i]->not_completed;
+      }
+      // dram_busy just check the request queue length into the dram 
+      // to make sure all the memory requests (esp the writes) are done
+      mem_busy = 0; 
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         mem_busy += dram_busy(dram[i]);
+      }
+     // icnt to the memory should clean of any pending tranfers as well
+      icnt2mem_busy = icnt_busy( );
+
+      if (gpu_max_cycle && (gpu_tot_sim_cycle + gpu_sim_cycle) >= gpu_max_cycle) {
+         break;
+      }
+      if (gpu_max_insn && (gpu_tot_sim_insn + gpu_sim_insn) >= gpu_max_insn) {
+         break;
+      }
+      if (gpu_deadlock_detect && gpu_deadlock) {
+         break;
+      }
+
+   }
+   memlatstat_lat_pw();
+   gpu_tot_sim_cycle += gpu_sim_cycle;
+   gpu_tot_sim_insn += gpu_sim_insn;
+   gpu_tot_completed_thread += gpu_completed_thread;
+   
+   ptx_file_line_stats_write_file();
+
+   printf("stats for grid: %d\n", grid_num);
+   gpu_print_stat();
+   if (g_network_mode) {
+      interconnect_stats();
+      printf("----------------------------Interconnect-DETAILS---------------------------------" );
+      icnt_overal_stat();
+      printf("----------------------------END-of-Interconnect-DETAILS-------------------------" );
+   }
+   if (gpgpu_memlatency_stat & GPU_MEMLATSTAT_QUEUELOGS ) {
+      dramqueue_latency_log_dump();
+      dram_log(DUMPLOG);
+      if (gpgpu_cache_dl2_opt) {
+         L2c_log(DUMPLOG);
+         L2c_latency_log_dump();
+      }
+   }
+
+#define DEADLOCK 0
+   if (gpu_deadlock_detect && gpu_deadlock) {
+      fflush(stdout);
+      printf("ERROR ** deadlock detected: last writeback @ gpu_sim_cycle %u (+ gpu_tot_sim_cycle %u) (%u cycles ago)\n", 
+             (unsigned) gpu_sim_insn_last_update, (unsigned) (gpu_tot_sim_cycle-gpu_sim_cycle),
+             (unsigned) (gpu_sim_cycle - gpu_sim_insn_last_update )); 
+      fflush(stdout);
+      assert(DEADLOCK);
+   }
+   return gpu_sim_cycle;
+}
+
+extern void ** g_inst_classification_stat;
+extern void ** g_inst_op_classification_stat;
+extern int g_ptx_kernel_count; // used for classification stat collection purposes 
+
+extern unsigned get_max_mshr_used(shader_core_ctx_t* shader);
+
+void gpu_print_stat()
+{  
+   unsigned i;
+   int j,k;
+    
+   printf("gpu_sim_cycle = %lld\n", gpu_sim_cycle);
+   printf("gpu_sim_insn = %lld\n", gpu_sim_insn);
+   printf("gpu_sim_no_ld_const_insn = %lld\n", gpu_sim_insn_no_ld_const);
+   printf("gpu_ipc = %12.4f\n", (float)gpu_sim_insn / gpu_sim_cycle);
+   printf("gpu_completed_thread = %lld\n", gpu_completed_thread);
+   printf("gpu_tot_sim_cycle = %lld\n", gpu_tot_sim_cycle);
+   printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn);
+   printf("gpu_tot_ipc = %12.4f\n", (float)gpu_tot_sim_insn / gpu_tot_sim_cycle);
+   printf("gpu_tot_completed_thread = %lld\n", gpu_tot_completed_thread);
+   printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta);
+   printf("gpgpu_n_sent_writes = %d\n", gpgpu_n_sent_writes);
+   printf("gpgpu_n_processed_writes = %d\n", gpgpu_n_processed_writes);
+
+   // performance counter for stalls due to congestion.
+   printf("gpu_stall_by_MSHRwb= %d\n", gpu_stall_by_MSHRwb);
+   printf("gpu_stall_shd_mem  = %d\n", gpu_stall_shd_mem );
+   printf("gpu_stall_wr_back  = %d\n", gpu_stall_wr_back );
+   printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull);
+   printf("gpu_stall_icnt2sh    = %d\n", gpu_stall_icnt2sh   );
+   printf("gpu_stall_sh2icnt    = %d\n", gpu_stall_sh2icnt   );
+   // performance counter that are not local to one shader
+   shader_print_accstats(stdout);
+
+   memlatstat_print();
+   printf("max return queue length = ");
+   for (unsigned i=0;i<gpu_n_shader;i++) {
+      printf("%d ", max_return_queue_length[i]);
+   }
+   printf("\n");
+   // merge misses
+   printf("merge misses = %d\n", mergemiss);
+   printf("L1 read misses = %d\n", L1_read_miss);
+   printf("L1 write misses = %d\n", L1_write_miss);
+   printf("L1 write hit on misses = %d\n", L1_write_hit_on_miss);
+   printf("L1 writebacks = %d\n", L1_writeback);
+   printf("L1 texture misses = %d\n", L1_texture_miss);
+   printf("L1 const misses = %d\n", L1_const_miss);
+   printf("L2_write_miss = %d\n", L2_write_miss);
+   printf("L2_write_hit = %d\n", L2_write_hit);
+   printf("L2_read_miss = %d\n", L2_read_miss);
+   printf("L2_read_hit = %d\n", L2_read_hit);
+   printf("made_read_mfs = %d\n", made_read_mfs);
+   printf("made_write_mfs = %d\n", made_write_mfs);
+   printf("freed_read_mfs = %d\n", freed_read_mfs);
+   printf("freed_L1write_mfs = %d\n", freed_L1write_mfs);
+   printf("freed_L2write_mfs = %d\n", freed_L2write_mfs);
+   printf("freed_dummy_read_mfs = %d\n", freed_dummy_read_mfs);
+
+   printf("gpgpu_n_mem_read_local = %d\n", gpgpu_n_mem_read_local);
+   printf("gpgpu_n_mem_write_local = %d\n", gpgpu_n_mem_write_local);
+   printf("gpgpu_n_mem_read_global = %d\n", gpgpu_n_mem_read_global);
+   printf("gpgpu_n_mem_write_global = %d\n", gpgpu_n_mem_write_global);
+   printf("gpgpu_n_mem_texture = %d\n", gpgpu_n_mem_texture);
+   printf("gpgpu_n_mem_const = %d\n", gpgpu_n_mem_const);
+
+   printf("max_n_mshr_used = ");
+   for (unsigned i=0; i< gpu_n_shader; i++) printf("%d ", get_max_mshr_used(sc[i]));
+   printf("\n");
+
+   if (gpgpu_cache_dl2_opt) {
+      L2c_print_stat( );
+   }
+   for (unsigned i=0;i<gpu_n_mem;i++) {
+      dram_print(dram[i],stdout);
+   }
+
+   for (i=0, j=0, k=0;i<gpu_n_shader;i++) {
+      shd_cache_print(sc[i]->L1cache,stdout);
+      j+=sc[i]->L1cache->miss;
+      k+=sc[i]->L1cache->access;
+   }
+   printf("L1 Data Cache Total Miss Rate = %0.3f\n", (float)j/k);
+
+   for (i=0,j=0,k=0;i<gpu_n_shader;i++) {
+      shd_cache_print(sc[i]->L1texcache,stdout);
+      j+=sc[i]->L1texcache->miss;
+      k+=sc[i]->L1texcache->access;
+   }
+   printf("L1 Texture Cache Total Miss Rate = %0.3f\n", (float)j/k);
+
+   for (i=0,j=0,k=0;i<gpu_n_shader;i++) {
+      shd_cache_print(sc[i]->L1constcache,stdout);
+      j+=sc[i]->L1constcache->miss;
+      k+=sc[i]->L1constcache->access;
+   }
+   printf("L1 Const Cache Total Miss Rate = %0.3f\n", (float)j/k);
+
+   if (gpgpu_cache_dl2_opt) {
+      L2c_print_cache_stat();
+   }
+   printf("n_regconflict_stall = %d\n", n_regconflict_stall);
+
+   if (gpgpu_simd_model == DWF) {
+      dwf_print_stat(stdout);
+   }
+
+   if (gpgpu_simd_model == POST_DOMINATOR) {
+      printf("num_warps_issuable:");
+      for (unsigned i=0;i<(gpu_n_warp_per_shader+1);i++) {
+         printf("%d ", num_warps_issuable[i]);
+      }
+      printf("\n");
+   }
+   if (gpgpu_strict_simd_wrbk) {
+      printf("warp_conflict_at_writeback = %d\n", warp_conflict_at_writeback);
+   }
+
+   printf("gpgpu_commit_pc_beyond_two = %d\n", gpgpu_commit_pc_beyond_two);
+
+   print_shader_cycle_distro( stdout );
+
+   print_thread_pc_histogram( stdout );
+
+   if (gpgpu_cflog_interval != 0) {
+      spill_log_to_file (stdout, 1, gpu_sim_cycle);
+      insn_warp_occ_print(stdout);
+   }
+   if ( gpgpu_ptx_instruction_classification ) {
+      StatDisp( g_inst_classification_stat[g_ptx_kernel_count]);
+      StatDisp( g_inst_op_classification_stat[g_ptx_kernel_count]);
+   }
+   time_vector_print();
+
+   fflush(stdout);
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// Wrapper function for shader cores' memory system: 
+////////////////////////////////////////////////////////////////////////////////////
+
+// a hack to make the size of a packet discrete multiples of the interconnect's flit_size.
+static inline
+unsigned int fill_to_next_flit(unsigned int size) 
+{
+   assert (g_network_mode == INTERSIM);
+   return size;
+}
+
+
+
+unsigned char check_icnt_has_buffer(unsigned long long int *addr, int *bsize, 
+                                    int n_addr, int sid )
+{
+   addrdec_t tlx;
+   static unsigned int *req_buffer = NULL;
+   //the req_buf size can be equal to gpu_n_mem ; gpu_n_shader is added to make it compatible
+   //with the case where a mem controller is sending to shd 
+   if (!req_buffer) req_buffer = (unsigned int*)malloc((gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int));
+   memset(req_buffer, 0, (gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int));
+
+   // aggregate all buffer requirement of all memory accesses by dram chips
+   for (int i=0; i< n_addr; i++) {
+      addrdec_tlx(addr[i],&tlx);
+      req_buffer[tlx.chip] += fill_to_next_flit(bsize[i]);
+   }
+
+   int tpc_id = sid / gpu_concentration;
+
+   return icnt_has_buffer(tpc_id, req_buffer);
+}
+
+unsigned char single_check_icnt_has_buffer(int chip, int sid, unsigned char is_write )
+{
+   static unsigned int *req_buffer = NULL;
+   //the req_buf size can be equal to gpu_n_mem ; gpu_n_shader is added to make it compatible
+   //with the case where a mem controller is sending to shd 
+   if (!req_buffer) req_buffer = (unsigned int*)malloc((gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int));
+   memset(req_buffer, 0, (gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int));
+
+   // aggregate all buffer requirement of all memory accesses by dram chips
+
+   int b_size;
+   if (is_write)
+      b_size = sc[sid]->L1cache->line_sz;
+   else
+      b_size = READ_PACKET_SIZE;
+   req_buffer[chip] += fill_to_next_flit(b_size);
+
+   int tpc_id = sid / gpu_concentration;
+
+   return icnt_has_buffer(tpc_id, req_buffer);
+}
+
+int max_n_addr = 0;
+
+// Check the memory system for buffer availability
+unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid )
+{
+   //requests should be single always now
+   int rsize = bsize;
+   //maintain similar functionality with fq_push, if its a read, bsize is the load size, not the request's size
+   if (!write) {
+       rsize = READ_PACKET_SIZE;
+   }
+   return check_icnt_has_buffer(&addr, &rsize, 1, sid);
+}
+
+// Takes in memory address and their parameters and pushes to the fetch queue 
+unsigned char fq_push(unsigned long long int addr, int bsize, unsigned char write, partial_write_mask_t partial_write_mask, 
+                      int sid, int wid, mshr_entry* mshr, int cache_hits_waiting,
+                      enum mem_access_type mem_acc, address_type pc) 
+{
+   mem_fetch_t *mf;
+
+   mf = (mem_fetch_t*) calloc(1,sizeof(mem_fetch_t));
+   mf->request_uid = g_next_request_uid++;
+   mf->addr = addr;
+   mf->nbytes_L1 = bsize;
+   mf->sid = sid;
+   mf->source_node = sid / gpu_concentration;
+   mf->wid = wid;
+   mf->cache_hits_waiting = cache_hits_waiting;
+   mf->txbytes_L1 = 0;
+   mf->rxbytes_L1 = 0;  
+   mf->mshr = mshr;
+   if (mshr) mshr->mf = (void*)mf; // for debugging
+   mf->write = write;
+
+   if (write)
+      made_write_mfs++;
+   else
+      made_read_mfs++;
+   memlatstat_start(mf);
+   addrdec_tlx(addr,&mf->tlx);
+   mf->bank = mf->tlx.bk;
+   mf->chip = mf->tlx.chip;
+   if (gpgpu_cache_dl2_opt)
+      mf->nbytes_L2 = L2c_get_linesize( dram[mf->tlx.chip] );
+   else
+      mf->nbytes_L2 = 0;
+   mf->txbytes_L2 = 0;
+   mf->rxbytes_L2 = 0;  
+
+   mf->write_mask = partial_write_mask;
+   if (!write) assert(partial_write_mask == NO_PARTIAL_WRITE);
+
+   // stat collection codes 
+   mf->mem_acc = mem_acc;
+   mf->pc = pc;
+
+   switch (mem_acc) {
+   case CONST_ACC_R: gpgpu_n_mem_const++; break;
+   case TEXTURE_ACC_R: gpgpu_n_mem_texture++; break;
+   case GLOBAL_ACC_R: gpgpu_n_mem_read_global++; break;
+   case GLOBAL_ACC_W: gpgpu_n_mem_write_global++; break;
+   case LOCAL_ACC_R: gpgpu_n_mem_read_local++; break;
+   case LOCAL_ACC_W: gpgpu_n_mem_write_local++; break;
+   default: assert(0);
+   }
+
+   return(issue_mf_from_fq(mf));
+
+}
+
+int issue_mf_from_fq(mem_fetch_t *mf){
+   int destination; // where is the next level of memory?
+   destination = mf->tlx.chip;
+   int tpc_id = mf->sid / gpu_concentration;
+
+   if (mf->mshr) mshr_update_status(mf->mshr,IN_ICNT2MEM);
+   if (!mf->write) {
+      mf->type = RD_REQ;
+      assert( mf->timestamp == (gpu_sim_cycle+gpu_tot_sim_cycle) );
+      time_vector_update(mf->mshr->insts[0].uid, MR_ICNT_PUSHED, gpu_sim_cycle+gpu_tot_sim_cycle, mf->type );
+      icnt_push(tpc_id, mem2device(destination), (void*)mf, READ_PACKET_SIZE);
+   } else {
+      mf->type = WT_REQ;
+      icnt_push(tpc_id, mem2device(destination), (void*)mf, mf->nbytes_L1);
+      gpgpu_n_sent_writes++;
+      assert( mf->timestamp == (gpu_sim_cycle+gpu_tot_sim_cycle) );
+      time_vector_update(mf->request_uid, MR_ICNT_PUSHED, gpu_sim_cycle+gpu_tot_sim_cycle, mf->type ) ;
+   }
+
+   return 0;
+}
+
+extern void mshr_return_from_mem(shader_core_ctx_t * shader, mshr_entry_t* mshr);
+
+inline void fill_shd_L1_with_new_line(shader_core_ctx_t * sc, mem_fetch_t * mf) {
+   unsigned long long int repl_addr = -1;
+   // When the data arrives, it flags all the appropriate MSHR
+   // entries accordingly (by checking the address in each entry ) 
+   memlatstat_read_done(mf);
+
+   mshr_return_from_mem(sc, mf->mshr);
+
+   if (mf->mshr->istexture) {
+       shd_cache_fill(sc->L1texcache,mf->addr,sc->gpu_cycle);
+       repl_addr = -1;
+   } else if (mf->mshr->isconst) {
+       shd_cache_fill(sc->L1constcache,mf->addr,sc->gpu_cycle);
+       repl_addr = -1;
+   } else {
+       if (!gpgpu_no_dl1) {
+           //if we are doing a writeback cache we may have marked off a mask in the mshr
+           //only write into the cache unmasked bytes.
+           //since this doesn't affect timing we don't actually do it.
+           repl_addr = shd_cache_fill(sc->L1cache,mf->addr,sc->gpu_cycle);
+      }
+   }
+
+   freed_read_mfs++;
+   free(mf);
+}
+
+unsigned char fq_pop(int tpc_id) 
+{
+   mem_fetch_t *mf;
+
+   mf = (mem_fetch_t*) icnt_pop(tpc_id);
+
+   // if there is a memory fetch request coming back, forward it to the proper shader core
+   if (mf) {
+      assert(mf->type == REPLY_DATA);
+      time_vector_update(mf->mshr->insts[0].uid ,MR_2SH_FQ_POP,gpu_sim_cycle+gpu_tot_sim_cycle, mf->type ) ;
+      fill_shd_L1_with_new_line(sc[mf->sid], mf);
+   }
+   return 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+
+int issue_block2core( shader_core_ctx_t *shdr, int grid_num  ) 
+{
+   int tid, nthreads_2beissued, more_threads;
+   int  nthreads_in_block= 0;
+   int start_thread = 0;
+   int end_thread = shdr->n_threads;
+   int cta_id=-1;
+   int cta_size=0;
+   int padded_cta_size;
+
+   cta_size = ptx_sim_cta_size();
+   padded_cta_size = cta_size;
+
+   assert(gpgpu_spread_blocks_across_cores); //should be if  muliple CTA per shader supported
+
+   for (unsigned i=0;i<max_cta_per_shader(shdr);i++ ) { //try to find next empty cta slot
+      if (shdr->cta_status[i]==0) { //
+         cta_id=i;
+         break;
+      }
+   }
+   assert( cta_id!=-1);//must have found a CTA to run
+   if (padded_cta_size%warp_size) {
+      padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+   }
+   start_thread = cta_id * padded_cta_size;
+   end_thread  = start_thread +  cta_size;
+   shader_reinit(shdr,start_thread, end_thread); 
+
+   // issue threads in blocks (if it is specified)
+   warp_set_t warps;
+   for (int i = start_thread; i<end_thread; i++) {  //setup the block
+      unsigned warp_id = i/warp_size;
+      shdr->thread[i].cta_id = cta_id;
+      nthreads_in_block +=  ptx_sim_init_thread(&shdr->thread[i].ptx_thd_info,shdr->sid,i,cta_size-(i-start_thread),shdr->n_threads/*cta_size*/,shdr,cta_id,warp_id);
+      warps.set( warp_id );
+   }
+   shdr->allocate_barrier( cta_id, warps );
+
+   shader_init_CTA(shdr, start_thread, end_thread);
+   nthreads_2beissued =  nthreads_in_block;
+   shdr->cta_status[cta_id]+=nthreads_2beissued;
+   assert( nthreads_2beissued ); //we should have not reached this point if there is no more thread to -
+
+   assert( (unsigned) nthreads_2beissued <= shdr->n_threads); //confirm threads to be issued is less than or equal to number of threads supported by microarchitecture
+
+   int n_cta_issued= nthreads_2beissued/cta_size ;//+ nthreads_2beissued%cta_size; 
+   shdr->n_active_cta +=  n_cta_issued;
+   shader_CTA_count_log(shdr->sid, n_cta_issued);
+   g_total_cta_left-= n_cta_issued;
+
+   more_threads = 1;
+   if (gpgpu_spread_blocks_across_cores) {
+      nthreads_2beissued += start_thread;
+   }
+   printf("Shader %d initializing CTA #%d with hw tids from %d to %d @(%lld,%lld)", 
+          shdr->sid, cta_id, start_thread, nthreads_2beissued, gpu_sim_cycle, gpu_tot_sim_cycle );
+   printf(" shdr->not_completed = %d\n", shdr->not_completed);
+
+   for (tid=start_thread;tid<nthreads_2beissued;tid++) {
+
+      // reset complete flag for stream
+      shdr->not_completed += 1;
+      assert( shdr->warp[tid/warp_size].n_completed > 0 );
+      assert( shdr->warp[tid/warp_size].n_completed <= warp_size);
+      shdr->warp[tid/warp_size].n_completed--;
+
+      // set avail4fetch flag to ready
+      shdr->thread[tid].avail4fetch = 1;
+      assert( shdr->warp[tid/warp_size].n_avail4fetch < warp_size );
+      shdr->warp[tid/warp_size].n_avail4fetch++;
+
+      g_nthreads_issued++;
+   }
+
+   if (!nthreads_in_block) more_threads = 0;
+   return more_threads; //if there are no more threads to be issued, return 0
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+// wrapper code to to create an illusion of a memory controller with L2 cache.
+// 
+int mem_ctrl_full( int mc_id ) 
+{
+   if (gpgpu_cache_dl2_opt) {
+      return L2c_full( dram[mc_id] );
+   } else {
+      return( gpgpu_dram_sched_queue_size && dram_full(dram[mc_id]) );
+   }
+}
+
+//#define DEBUG_PARTIAL_WRITES
+void mem_ctrl_push( int mc_id, mem_fetch_t* mf ) 
+{
+   if (gpgpu_cache_dl2_opt) {
+      L2c_push(dram[mc_id], mf);
+   } else {
+      addrdec_t tlx;
+      addrdec_tlx(mf->addr, &tlx);
+#if 0 //old chunking no longer valid.
+      if (gpgpu_partial_write_mask && mf->write) {
+         assert( gpgpu_no_dl1 ); // gpgpu_partial_write_mask is not supported with caches for now
+      }
+#endif //#if 0 //old chunking no longer valid 
+      dram_push(dram[mc_id], 
+                tlx.bk, tlx.row, tlx.col, 
+                mf->nbytes_L1, mf->write, 
+                mf->wid, mf->sid, mf->cache_hits_waiting, mf->addr, mf);
+      memlatstat_dram_access(mf, mc_id, tlx.bk);
+      if (mf->mshr) mshr_update_status(mf->mshr,IN_DRAM_REQ_QUEUE);
+   }
+}
+
+void* mem_ctrl_pop( int mc_id ) 
+{
+   mem_fetch_t* mf;
+   if (gpgpu_cache_dl2_opt) {
+      mf = L2c_pop(dram[mc_id]);
+      if (mf && mf->mshr && mf->mshr->insts[0].callback.function) {
+         dram_callback_t* cb = &(mf->mshr->insts[0].callback);
+         cb->function(cb->instruction, cb->thread);
+      }
+      return mf;
+   } else {
+      mf = static_cast<mem_fetch_t*> (dq_pop(dram[mc_id]->returnq)); //dram_pop(dram[mc_id]);
+      if (mf) mf->type = REPLY_DATA;
+      if (mf && mf->mshr && mf->mshr->insts[0].callback.function) {
+         dram_callback_t* cb = &(mf->mshr->insts[0].callback);
+         cb->function(cb->instruction, cb->thread);
+      }
+      return mf;
+   }
+}
+
+void* mem_ctrl_top( int mc_id ) 
+{
+   mem_fetch_t* mf;
+   if (gpgpu_cache_dl2_opt) {
+      return L2c_top(dram[mc_id]);
+   } else {
+      mf = static_cast<mem_fetch_t*> (dq_top(dram[mc_id]->returnq));//dram_top(dram[mc_id]);
+      if (mf) mf->type = REPLY_DATA;
+      return mf ;//dram_top(dram[mc_id]);
+   }
+}
+
+void get_dram_output ( dram_t* dram_p ) 
+{
+   mem_fetch_t* mf;
+   mem_fetch_t* mf_top;
+   mf_top = (mem_fetch_t*) dram_top(dram_p); //test
+   if (mf_top) {
+      if (mf_top->type == DUMMY_READ) {
+         dram_pop(dram_p);
+         free(mf_top);
+         freed_dummy_read_mfs++;
+         return;
+      }
+   }
+   if (gpgpu_cache_dl2_opt) {
+      L2c_get_dram_output( dram_p );
+   } else {
+      if ( dq_full(dram_p->returnq) ) return;
+      mf = (mem_fetch_t*) dram_pop(dram_p);
+      assert (mf_top==mf );
+      if (mf) {
+         dq_push(dram_p->returnq, mf);
+         if (mf->mshr) mshr_update_status(mf->mshr,IN_DRAMRETURN_Q);
+      }
+   }
+}
+
+void dram_log (int task ) {
+   static void ** mrqq_Dist; //memory request queue inside DRAM  
+   if (task == CREATELOG) {
+      mrqq_Dist = (void **)     calloc(gpu_n_mem,sizeof(void*));
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         if (dram[i]->queue_limit)
+            mrqq_Dist[i]      = StatCreate("mrqq_length",1,dram[i]->queue_limit);
+         else //queue length is unlimited; 
+            mrqq_Dist[i]      = StatCreate("mrqq_length",1,64); //track up to 64 entries
+      }
+   } else if (task == SAMPLELOG) {
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         StatAddSample(mrqq_Dist[i], dram_que_length(dram[i]));   
+      }
+   } else if (task == DUMPLOG) {
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         printf ("Queue Length DRAM[%d] ",i);StatDisp(mrqq_Dist[i]);
+      } 
+   }
+}
+
+void dramqueue_latency_log_dump()
+{
+   for (unsigned i=0;i<gpu_n_mem;i++) {
+      printf ("(LOGB2)Latency DRAM[%d] ",i);StatDisp(dram[i]->mrqq->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i);StatDisp(dram[i]->rwq->lat_stat);   
+   }
+}
+
+//Find next clock domain and increment its time
+inline int next_clock_domain(void) 
+{
+   double smallest = min3(core_time,icnt_time,dram_time);
+   int mask = 0x00;
+   if (gpgpu_cache_dl2_opt  //when no-L2 it will never be L2's turn
+       && ( l2_time <= smallest) ) {
+      smallest = l2_time;
+      mask |= L2 ;
+      l2_time += l2_period;
+   }
+   if ( icnt_time <= smallest ) {
+      mask |= ICNT;
+      icnt_time += icnt_period;
+   }
+   if ( dram_time <= smallest ) {
+      mask |= DRAM;
+      dram_time += dram_period;
+   }
+   if ( core_time <= smallest ) {
+      mask |= CORE;
+      core_time += core_period;
+   }
+   return mask;
+}
+
+extern time_t simulation_starttime;
+void gpu_sim_loop( int grid_num ) 
+{
+   int clock_mask = next_clock_domain();
+
+   // shader core loading (pop from ICNT into shader core) follows CORE clock
+   if (clock_mask & CORE ) {
+      for (int i=0;i<gpu_n_tpc;i++) {
+         fq_pop(i); 
+      }
+   }
+
+   if (clock_mask & ICNT) {
+      // pop from memory controller to interconnect
+      static unsigned int *rt_size = NULL;
+      if (!rt_size) rt_size = (unsigned int*) malloc ((gpu_n_tpc+gpu_n_mem)*sizeof(unsigned int));
+      memset(rt_size, 0, (gpu_n_tpc+gpu_n_mem)*sizeof(unsigned int));
+
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+
+         mem_fetch_t* mf;
+
+         mf = (mem_fetch_t*) mem_ctrl_top(i); //(returns L2_top or DRAM returnq top)
+
+         if (mf) {
+            mf->source_node = mem2device(i);
+            assert( mf->type != RD_REQ && mf->type != WT_REQ ); // never should a request come out from L2 or dram
+            if (!mf->write) {
+               int return_dev = -1;
+               return_dev = mf->sid / gpu_concentration;
+               assert(return_dev != -1);
+               // check icnt resource for READ data return
+               rt_size[return_dev] = mf->nbytes_L1;
+               if ( icnt_has_buffer( mem2device(i), rt_size) ) {
+                  if (mf->mshr) mshr_update_status(mf->mshr,IN_ICNT2SHADER);
+                  memlatstat_icnt2sh_push(mf);
+                  time_vector_update(mf->mshr->insts[0].uid ,MR_2SH_ICNT_PUSHED,gpu_sim_cycle+gpu_tot_sim_cycle,RD_REQ);
+                  icnt_push( mem2device(i), return_dev, mf, mf->nbytes_L1);
+                  mem_ctrl_pop(i);
+               } else {
+                  gpu_stall_icnt2sh++;
+               }
+               rt_size[return_dev] = 0; // clean up for the next dram_pop
+            } else {
+               time_vector_update(mf->request_uid ,MR_2SH_ICNT_PUSHED,gpu_sim_cycle+gpu_tot_sim_cycle,WT_REQ ) ;
+               mem_ctrl_pop(i);
+               free(mf);
+               freed_L1write_mfs++;
+               gpgpu_n_processed_writes++;
+            }
+         }
+      }
+   }
+
+   if (clock_mask & DRAM) {
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         get_dram_output ( dram[i] ); 
+      }          
+      // Issue the dram command (scheduler + delay model) 
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         dram_issueCMD(dram[i]);
+      }
+      dram_log(SAMPLELOG);   
+   }
+
+   // L2 operations follow L2 clock domain
+   if (clock_mask & L2) {
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         L2c_process_dram_output ( dram[i], i ); // pop from dram
+         L2c_push_miss_to_dram ( dram[i] );  //push to dram
+         L2c_service_mem_req ( dram[i], i );   // pop(push) from(to)  icnt2l2(l2toicnt) queues; service l2 requests 
+      }
+      if (gpgpu_cache_dl2_opt) { // L2 cache enabled
+         for (unsigned i=0;i<gpu_n_mem;i++) {
+            L2c_update_stat( dram[i] ); 
+         }
+      }
+      if (gpgpu_cache_dl2_opt) { //take a sample of l2c queue lengths
+         L2c_log(SAMPLELOG); 
+      }
+   }
+
+   if (clock_mask & ICNT) {
+      // pop memory request from ICNT and 
+      // push it to the proper memory controller (L2 or DRAM controller)
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+
+         if ( mem_ctrl_full(i) ) {
+            gpu_stall_dramfull++;
+            continue;
+         }
+
+         mem_fetch_t* mf;    
+         mf = (mem_fetch_t*) icnt_pop( mem2device(i) );
+
+         if (mf) {
+            if (mf->type==RD_REQ) {
+               time_vector_update(mf->mshr->insts[0].uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;             
+            } else {
+               time_vector_update(mf->request_uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;
+            }
+            memlatstat_icnt2mem_pop(mf);
+            mem_ctrl_push( i, mf );
+         }
+      }
+      icnt_transfer( );
+   }
+
+   if (clock_mask & CORE) {
+      // L1 cache + shader core pipeline stages 
+      for (unsigned i=0;i<gpu_n_shader;i++) {
+         if (sc[i]->not_completed || more_thread)
+            shader_cycle(sc[i], i, grid_num);
+         sc[i]->gpu_cycle++;
+      }
+      gpu_sim_cycle++;
+
+      for (unsigned i=0;i<gpu_n_shader && more_thread;i++) {
+         if (gpgpu_spread_blocks_across_cores) {
+            int cta_issue_count = 1;
+            if ( ( (unsigned) (sc[i]->n_active_cta + cta_issue_count) <= max_cta_per_shader(sc[i]) )
+                 && g_total_cta_left ) {
+               int j;
+               for (j=0;j<cta_issue_count;j++) {
+                  issue_block2core(sc[i], grid_num);
+               }
+               if (!g_total_cta_left) {
+                  more_thread = 0;
+               }
+               assert( g_total_cta_left > -1 );
+            }
+         } else {
+            if (!(sc[i]->not_completed))
+               more_thread = issue_block2core(sc[i], grid_num);
+         }
+      }
+
+
+      // Flush the caches once all of threads are completed.
+      if (gpgpu_flush_cache) {
+         int all_threads_complete = 1 ; 
+         for (unsigned i=0;i<gpu_n_shader;i++) {
+            if (sc[i]->not_completed == 0) {
+               shader_cache_flush(sc[i]);
+            } else {
+               all_threads_complete = 0 ; 
+            }
+         }
+         if (all_threads_complete) {
+            printf("Flushed L1 caches...\n");
+            if (gpgpu_cache_dl2_opt) {
+               int dlc = 0;
+               for (unsigned i=0;i<gpu_n_mem;i++) {
+                  dlc = L2c_cache_flush(dram[i]);
+                  printf("Dirty lines flushed from L2 %d is %d  \n", i, dlc  );
+               }
+            }
+         }
+      }
+
+      if (!(gpu_sim_cycle % gpu_stat_sample_freq)) {
+         time_t days, hrs, minutes, sec;
+         time_t curr_time;
+         time(&curr_time);
+         unsigned long long  elapsed_time = MAX(curr_time - simulation_starttime, 1);
+         days    = elapsed_time/(3600*24);
+         hrs     = elapsed_time/3600 - 24*days;
+         minutes = elapsed_time/60 - 60*(hrs + 24*days);
+         sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days));
+         printf("cycles: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
+                gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, 
+                (double)gpu_sim_insn/(double)gpu_sim_cycle,
+                (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time),
+                (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec,
+                ctime(&curr_time));
+         fflush(stdout);
+         memlatstat_lat_pw();
+         visualizer_printstat();
+         if (gpgpu_runtime_stat && (gpu_runtime_stat_flag != 0) ) {
+            if (gpu_runtime_stat_flag & GPU_RSTAT_BW_STAT) {
+               for (unsigned i=0;i<gpu_n_mem;i++) {
+                  dram_print_stat(dram[i],stdout);
+               }
+               printf("maxmrqlatency = %d \n", max_mrq_latency);
+               printf("maxmflatency = %d \n", max_mf_latency);
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_DWF_MAP) {
+               printf("DWF_MS: ");
+               for (unsigned i=0;i<gpu_n_shader;i++) {
+                  printf("%u ",acc_dyn_pcs[i]);
+               }
+               printf("\n");
+               print_thread_pc( stdout );
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_SHD_INFO) {
+               shader_print_runtime_stat( stdout );
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_WARP_DIS) {
+               print_shader_cycle_distro( stdout );
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_L1MISS) {
+               shader_print_l1_miss_stat( stdout );
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_PDOM ) {
+               if (pdom_sched_type) {
+                  printf ("pdom_original_warps_count %d \n",n_pdom_sc_orig_stat );
+                  printf ("pdom_single_warps_count %d \n",n_pdom_sc_single_stat );
+               }
+            }
+            if (gpu_runtime_stat_flag & GPU_RSTAT_SCHED ) {
+               printf("Average Num. Warps Issuable per Shader:\n");
+               for (unsigned i=0;i<gpu_n_shader;i++) {
+                  printf("%2.2f ", (float) num_warps_issuable_pershader[i]/ gpu_stat_sample_freq);
+                  num_warps_issuable_pershader[i] = 0;
+               }
+               printf("\n");
+            }
+         }
+      }
+
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         acc_mrq_length[i] += dram_que_length(dram[i]);
+      }
+      if (!(gpu_sim_cycle % 20000)) {
+         // deadlock detection 
+         if (gpu_deadlock_detect && gpu_sim_insn == last_gpu_sim_insn) {
+            gpu_deadlock = 1;
+         } else {
+            last_gpu_sim_insn = gpu_sim_insn;
+         }
+      }
+      try_snap_shot(gpu_sim_cycle);
+      spill_log_to_file (stdout, 0, gpu_sim_cycle);
+   }
+}
+
+void dump_regs(unsigned sid, unsigned tid)
+{
+   if ( sid >= gpu_n_shader ) {
+      printf("shader %u is out of range\n",sid);
+      return;
+   }
+   if ( tid >= gpu_n_thread_per_shader ) {
+      printf("thread %u is out of range\n",tid);
+      return;
+   }
+
+   shader_core_ctx_t *s = sc[sid];
+
+   ptx_dump_regs( s->thread[tid].ptx_thd_info );
+}
+
+int ptx_thread_done( void *thr );
+
+void shader_dump_istream_state(shader_core_ctx_t *shader, FILE *fout )
+{
+   fprintf( fout, "\n");
+   for (unsigned t=0; t < gpu_n_thread_per_shader/warp_size; t++ ) {
+      int tid = t*warp_size;
+      if ( shader->warp[t].n_completed < warp_size ) {
+         fprintf( fout, "  %u:%3u fetch state = c:%u a4f:%u bw:%u (completed: ", shader->sid, tid, 
+                shader->warp[t].n_completed,
+                shader->warp[t].n_avail4fetch,
+                shader->warp[t].n_waiting_at_barrier  );
+
+         for (unsigned i = tid; i < (t+1)*warp_size; i++ ) {
+            if ( ptx_thread_done(shader->thread[i].ptx_thd_info) ) {
+               fprintf(fout,"1");
+            } else {
+               fprintf(fout,"0");
+            }
+            if ( (((i+1)%4) == 0) && (i+1) < (t+1)*warp_size ) {
+               fprintf(fout,",");
+            }
+         }
+         fprintf(fout,")\n");
+      }
+   }
+}
+
+void dump_pipeline_impl( int mask, int s, int m ) 
+{
+/*
+   You may want to use this function while running GPGPU-Sim in gdb.
+   One way to do that is add the following to your .gdbinit file:
+ 
+      define dp
+         call dump_pipeline_impl((0x40|0x4|0x1),$arg0,0)
+      end
+ 
+   Then, typing "dp 3" will show the contents of the pipeline for shader core 3.
+*/
+
+   printf("Dumping pipeline state...\n");
+   if(!mask) mask = 0xFFFFFFFF;
+   for (unsigned i=0;i<gpu_n_shader;i++) {
+      if(s != -1) {
+         i = s;
+      }
+      if(mask&1) shader_display_pipeline(sc[i], stdout, 1, mask & 0x2E );
+      if(mask&0x40) shader_dump_istream_state(sc[i], stdout);
+      if(mask&0x100) mshr_print(stdout, sc[i]);
+      if(s != -1) {
+         break;
+      }
+   }
+   if(mask&0x10000) {
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+         if(m != -1) {
+            i=m;
+         }
+         printf("DRAM / memory controller %u:\n", i);
+         if(mask&0x100000) dram_print_stat(dram[i],stdout);
+         if(mask&0x1000000)   dram_visualize( dram[i] );
+         if(m != -1) {
+            break;
+         }
+      }
+   }
+   fflush(stdout);
+}
+
+void dump_pipeline()
+{
+   dump_pipeline_impl(0,-1,-1);
+}
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
new file mode 100644
index 0000000..e30797e
--- /dev/null
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -0,0 +1,116 @@
+/* 
+ * gpu-sim.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan, Ivan Sham and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files 
+ * src/gpgpusim_entrypoint.c and src/simplesim-3.0/ are derived from the 
+ * SimpleScalar Toolset available from http://www.simplescalar.com/ 
+ * (property of SimpleScalar LLC) and the files src/intersim/ are derived 
+ * from Booksim (Simulator provided with the textbook "Principles and 
+ * Practices of Interconnection Networks" available from 
+ * http://cva.stanford.edu/books/ppin/).  As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA
+ * which is distributed seperately by NVIDIA under separate terms and
+ * conditions.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include "zlib.h"
+
+#include "../option_parser.h"
+#include "shader.h"
+#include "dram.h"
+#include "mem_fetch.h"
+
+#ifndef GPU_SIM_H
+#define GPU_SIM_H
+
+#define NUM_SHADERS 8
+#define STREAMS_PER_FILE 128
+
+//unsigned int run_gpu_sim(int grid_num);
+
+unsigned int get_converge_point(unsigned int pc, void *thd);
+
+#define GPU_RSTAT_SHD_INFO 0x1
+#define GPU_RSTAT_BW_STAT  0x2
+#define GPU_RSTAT_WARP_DIS 0x4
+#define GPU_RSTAT_DWF_MAP  0x8
+
+//gpgpu_interwarp_mshr_merge
+#define TEX_MSHR_MERGE 0x4
+#define CONST_MSHR_MERGE 0x2
+#define GLOBAL_MSHR_MERGE 0x1
+
+//Prints out a verbose L1 miss rate per thread for shader 0 
+#define GPU_RSTAT_L1MISS 0x10
+#define GPU_RSTAT_PDOM 0x20
+#define GPU_RSTAT_SCHED 0x40
+
+//options for gpgpu_memlatency_stat
+#define GPU_MEMLATSTAT_MC 0x2
+#define GPU_MEMLATSTAT_QUEUELOGS 0x4
+
+//void gpu_reg_options(option_parser_t opp);
+//void init_gpu();
+void gpu_print_stat();
+
+int mem_ctrl_full( int mc_id );
+
+void dramqueue_latency_log_dump();
+
+#endif
diff --git a/src/gpgpu-sim/histogram.h b/src/gpgpu-sim/histogram.h
new file mode 100644
index 0000000..2d39410
--- /dev/null
+++ b/src/gpgpu-sim/histogram.h
@@ -0,0 +1,128 @@
+/* 
+ * histogram.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef HISTOGRAM_H
+#define HISTOGRAM_H
+
+#ifdef __cplusplus
+
+#include <string>
+
+class binned_histogram {
+protected:
+
+   std::string m_name;
+   int m_nbins;
+   int *m_bins;        // bin boundaries
+   int *m_bin_cnts;    // counters
+   int m_maximum;      // the maximum sample
+
+public:
+
+   binned_histogram (std::string name = "", int nbins = 32, int* bins = NULL);
+
+   binned_histogram (const binned_histogram& other);
+
+   void reset_bins ();
+
+   void add2bin (int sample);
+
+   void fprint (FILE *fout);
+
+   virtual ~binned_histogram ();
+
+};
+
+class pow2_histogram : public binned_histogram {
+
+public:
+
+   pow2_histogram ( std::string name = "", int nbins = 32, int* bins = NULL);
+
+   ~pow2_histogram() {}
+
+   void add2bin (int sample);
+
+};
+
+class linear_histogram : public binned_histogram {
+
+private:
+
+   int m_stride;
+
+public:
+
+   linear_histogram (int stride = 1, const char *name = NULL, int nbins = 32, int* bins = NULL);
+
+   ~linear_histogram() {}
+
+   void add2bin (int sample);
+
+};
+
+#endif
+
+#endif /* HISTOGRAM_H */
diff --git a/src/gpgpu-sim/icnt_wrapper.cc b/src/gpgpu-sim/icnt_wrapper.cc
new file mode 100644
index 0000000..3e38269
--- /dev/null
+++ b/src/gpgpu-sim/icnt_wrapper.cc
@@ -0,0 +1,97 @@
+/* 
+ * icnt_wrapper.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "icnt_wrapper.h"
+#include <assert.h>
+#include "../intersim/interconnect_interface.h"
+
+icnt_has_buffer_p icnt_has_buffer;
+icnt_push_p       icnt_push;
+icnt_pop_p        icnt_pop;
+icnt_transfer_p   icnt_transfer;
+icnt_busy_p       icnt_busy;
+
+extern int   g_network_mode;
+extern char* g_network_config_filename;
+
+void icnt_init( unsigned int n_shader, unsigned int n_mem )
+{
+   switch (g_network_mode) {
+   
+   case INTERSIM:
+      init_interconnect(g_network_config_filename  ,n_shader, n_mem);
+      icnt_has_buffer = interconnect_has_buffer;
+      icnt_push       = interconnect_push;
+      icnt_pop        = interconnect_pop;
+      icnt_transfer   = advance_interconnect;
+      icnt_busy       = interconnect_busy;
+     break;
+
+   default:
+      assert(0);
+      break;
+   }
+}
diff --git a/src/gpgpu-sim/icnt_wrapper.h b/src/gpgpu-sim/icnt_wrapper.h
new file mode 100644
index 0000000..998a4f3
--- /dev/null
+++ b/src/gpgpu-sim/icnt_wrapper.h
@@ -0,0 +1,100 @@
+/* 
+ * icnt_wrapper.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef ICNT_WRAPPER_H
+#define ICNT_WRAPPER_H
+
+// functional interface to the interconnect
+typedef int (*icnt_has_buffer_p)(unsigned int input, unsigned int *size);
+typedef void (*icnt_push_p)(unsigned int input, unsigned int output, void* data, unsigned int size);
+typedef void* (*icnt_pop_p)(unsigned int output);
+typedef void (*icnt_transfer_p)( );
+typedef unsigned (*icnt_busy_p)( );
+typedef void (*icnt_drain_p)( );
+
+
+extern icnt_has_buffer_p icnt_has_buffer;
+extern icnt_push_p       icnt_push;
+extern icnt_pop_p        icnt_pop;
+extern icnt_transfer_p   icnt_transfer;
+extern icnt_busy_p       icnt_busy;
+extern icnt_drain_p      icnt_drain;
+
+/*
+// definition of valid gpu network mode.
+extern enum {
+   INTERSIM = 1,
+   N_NETWORK_MODE
+} gpu_network_mode;
+*/
+enum network_mode {
+   INTERSIM = 1,
+   N_NETWORK_MODE
+};
+
+void icnt_init( unsigned int n_shader, unsigned int n_mem );
+
+#endif
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
new file mode 100644
index 0000000..aef837b
--- /dev/null
+++ b/src/gpgpu-sim/l2cache.cc
@@ -0,0 +1,999 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <list>
+#include <set>
+#include "../tr1_hash_map.h" // for unordered_map failback
+
+#include "../option_parser.h"
+#include "mem_fetch.h"
+#include "dram.h"
+#include "gpu-cache.h"
+#include "histogram.h"
+#include "l2cache.h"
+#include "../intersim/statwraper.h"
+
+class L2c_mshr;
+class L2c_miss_tracker;
+class L2c_access_locality;
+
+mem_fetch_t* g_debug_mf = NULL;
+
+// L2 cache block (include the cache model + flow controls)
+struct L2cacheblk 
+{
+   shd_cache_t *L2cache;
+
+   delay_queue *cbtoL2queue; //latency 10
+   delay_queue *cbtoL2writequeue;
+   delay_queue *dramtoL2queue; //latency 10
+   delay_queue *dramtoL2writequeue;
+   delay_queue *L2todramqueue; //latency 0
+   delay_queue *L2todram_wbqueue; 
+   delay_queue *L2tocbqueue; //latency 0
+
+   mem_fetch_t *L2request; //request currently being serviced by the L2 Cache
+
+   L2c_mshr *m_mshr; // mshr model 
+   L2c_miss_tracker *m_missTracker; // tracker observing for redundant misses
+   L2c_access_locality *m_accessLocality; // tracking true locality of L2 Cache access 
+
+   L2cacheblk(size_t linesize);
+   ~L2cacheblk(); 
+};
+
+// external dependencies
+extern unsigned long long int addrdec_mask[5];
+extern dram_t **dram;
+extern int gpgpu_dram_sched_queue_size; 
+extern unsigned int gpu_n_shader;
+extern unsigned int gpu_n_mem;
+extern unsigned long long gpu_sim_cycle;
+extern unsigned made_write_mfs;
+extern unsigned freed_L1write_mfs;
+extern unsigned freed_L2write_mfs;
+extern unsigned int gpgpu_n_sent_writes;
+extern unsigned int gpgpu_n_processed_writes;
+extern unsigned g_next_request_uid;
+
+void memlatstat_icnt2sh_push(mem_fetch_t *mf);
+void memlatstat_dram_access(mem_fetch_t *mf, unsigned dram_id, unsigned bank);
+void memlatstat_start(mem_fetch_t *mf);
+unsigned memlatstat_done(mem_fetch_t *mf);
+
+// option
+char *gpgpu_L2_queue_config;
+int gpgpu_l2_readoverwrite = 0;
+int l2_ideal = 0;
+
+void L2c_options(option_parser_t opp)
+{
+   option_parser_register(opp, "-gpgpu_L2_queue", OPT_CSTR, &gpgpu_L2_queue_config, 
+                  "L2 data cache queue length and latency config",
+                  "0:0:0:0:0:0:10:10");
+
+   option_parser_register(opp, "-gpgpu_l2_readoverwrite", OPT_BOOL, &gpgpu_l2_readoverwrite, 
+                "Prioritize read requests over write requests for L2",
+                "0");
+
+   option_parser_register(opp, "-l2_ideal", OPT_BOOL, &l2_ideal, 
+                "Use a ideal L2 cache that always hit",
+                "0");
+}
+
+// stats
+unsigned L2_write_miss = 0;
+unsigned L2_write_hit = 0;
+unsigned L2_read_hit = 0;
+unsigned L2_read_miss = 0;
+unsigned int *L2_cbtoL2length;
+unsigned int *L2_cbtoL2writelength;
+unsigned int *L2_L2tocblength;
+unsigned int *L2_dramtoL2length;
+unsigned int *L2_dramtoL2writelength;
+unsigned int *L2_L2todramlength;
+
+////////////////////////////////////////////////
+// L2 MSHR model
+
+class L2c_mshr 
+{
+private:
+   typedef std::list<const mem_fetch_t*> mem_fetch_list;
+   typedef tr1_hash_map<address_type, mem_fetch_list> L2missGroup;
+   L2missGroup m_L2missgroup; // structure tracking redundant dram access
+
+   struct active_chain {
+      address_type cacheTag;
+      mem_fetch_list *list;
+      active_chain() : cacheTag(0xDEADBEEF), list(NULL) { }
+   };
+   active_chain m_active_mshr_chain; 
+   size_t m_linesize; // L2 cache line size
+
+   const size_t m_n_entries; // total number of entries available
+   size_t m_entries_used; // number of entries in use
+
+   int m_n_miss; 
+   int m_n_miss_serviced_by_dram;
+   int m_n_mshr_hits;
+   size_t m_max_entries_used; 
+   
+   address_type cache_tag(const mem_fetch_t *mf) const 
+   {
+      // return mf->addr;
+      return (mf->addr & ~(m_linesize - 1));
+   }
+
+public:
+   L2c_mshr(size_t linesize, size_t n_entries = 64) 
+   : m_linesize(linesize), m_n_entries(n_entries), m_entries_used(0), 
+     m_n_miss(0), m_n_miss_serviced_by_dram(0), m_n_mshr_hits(0), m_max_entries_used(0) { }
+  
+   // add a cache miss to MSHR, return true if this access is hit another existing entry and merges with it
+   bool new_miss(const mem_fetch_t *mf);
+
+   // notify MSHR that a new cache line has been fetched, activate the associated MSHR chain
+   void miss_serviced(const mem_fetch_t *mf);
+
+   // probe if there are pending hits left in this MSHR chain
+   bool mshr_chain_empty();
+
+   // peek the first entry in the active MSHR chain
+   mem_fetch_t *mshr_chain_top();
+
+   // pop the first entry in the active MSHR chain
+   void mshr_chain_pop(); 
+
+   void print(FILE *fout = stdout); 
+   void print_stat(FILE *fout = stdout); 
+};
+
+bool L2c_mshr::new_miss(const mem_fetch_t *mf)
+{
+   address_type cacheTag = cache_tag(mf);
+   mem_fetch_list &missGroup = m_L2missgroup[cacheTag];
+
+   bool mshr_hit = not missGroup.empty();
+
+   missGroup.push_front(mf);
+   
+   m_n_miss += 1;
+   if (mshr_hit) 
+      m_n_mshr_hits += 1;
+   m_entries_used += 1;
+   m_max_entries_used = std::max(m_max_entries_used, m_entries_used);
+
+   return mshr_hit;
+}
+
+void L2c_mshr::miss_serviced(const mem_fetch_t *mf) 
+{
+   assert(m_active_mshr_chain.list == NULL);
+   address_type cacheTag = cache_tag(mf);
+   L2missGroup::iterator missGroup = m_L2missgroup.find(cacheTag);
+   if (missGroup == m_L2missgroup.end() || mf->type == L2_WTBK_DATA) {
+      assert(mf->type == L2_WTBK_DATA); // only this returning mem req can be missed by the MSHR
+      return; 
+   } 
+   assert(missGroup->first == cacheTag);
+
+   m_active_mshr_chain.cacheTag = cacheTag;
+   m_active_mshr_chain.list = &(missGroup->second);
+
+   m_n_miss_serviced_by_dram += 1;
+}
+
+bool L2c_mshr::mshr_chain_empty()
+{
+   return (m_active_mshr_chain.list == NULL);
+}
+
+mem_fetch_t *L2c_mshr::mshr_chain_top()
+{
+   const mem_fetch_t *mf = m_active_mshr_chain.list->back();
+   assert(cache_tag(mf) == m_active_mshr_chain.cacheTag);
+
+   return const_cast<mem_fetch_t*>(mf);
+}
+
+void L2c_mshr::mshr_chain_pop()
+{
+   m_entries_used -= 1;
+   m_active_mshr_chain.list->pop_back();
+   if (m_active_mshr_chain.list->empty()) {
+      address_type cacheTag = m_active_mshr_chain.cacheTag;
+      m_L2missgroup.erase(cacheTag);
+      m_active_mshr_chain.list = NULL;
+   }
+}
+
+void L2c_mshr::print(FILE *fout)
+{
+   fprintf(fout, "L2c MSHR: n_entries_used = %zu\n", m_entries_used);
+   L2missGroup::iterator missGroup;
+   for (missGroup = m_L2missgroup.begin(); missGroup != m_L2missgroup.end(); ++missGroup) {
+      fprintf(fout, "%#08x: ", missGroup->first); 
+      mem_fetch_list &mf_list = missGroup->second; 
+      for (mem_fetch_list::iterator imf = mf_list.begin(); imf != mf_list.end(); ++imf) {
+         fprintf(fout, "%p:%d ", *imf, (*imf)->request_uid);
+      }
+      fprintf(fout, "\n");
+   }
+}
+
+void L2c_mshr::print_stat(FILE *fout)
+{
+   fprintf(fout, "L2c MSHR: max_entry = %zu, n_miss = %d, n_mshr_hits = %d, n_serviced_by_dram %d\n", 
+           m_max_entries_used, m_n_miss, m_n_mshr_hits, m_n_miss_serviced_by_dram);
+}
+
+////////////////////////////////////////////////
+// track redundant dram access generated by L2 cache
+class L2c_miss_tracker
+{
+private:
+   typedef std::set<mem_fetch_t*> mem_fetch_set;
+   typedef tr1_hash_map<address_type, mem_fetch_set> L2missGroup;
+   L2missGroup m_L2missgroup; // structure tracking redundant dram access
+   size_t m_linesize; // L2 cache line size
+
+   typedef tr1_hash_map<address_type, int> L2redundantCnt; 
+   L2redundantCnt m_L2redundantCnt; 
+
+   int m_totalL2redundantAcc;
+
+   address_type cache_tag(const mem_fetch_t *mf) const 
+   {
+      // return mf->addr;
+      return (mf->addr & ~(m_linesize - 1));
+   }
+
+public:
+   L2c_miss_tracker(size_t linesize) : m_linesize(linesize), m_totalL2redundantAcc(0) { }
+   void new_miss(mem_fetch_t *mf);
+   void miss_serviced(mem_fetch_t *mf);
+
+   void print(FILE *fout, bool brief = true);
+   void print_stat(FILE *fout, bool brief = true);
+
+};
+
+void L2c_miss_tracker::new_miss(mem_fetch_t *mf)
+{
+   address_type cacheTag = cache_tag(mf);
+   mem_fetch_set &missGroup = m_L2missgroup[cacheTag];
+
+   if (missGroup.size() != 0) {
+      m_L2redundantCnt[cacheTag] += 1;
+      m_totalL2redundantAcc += 1;
+   }
+
+   missGroup.insert(mf);
+}
+
+void L2c_miss_tracker::miss_serviced(mem_fetch_t *mf)
+{
+   address_type cacheTag = cache_tag(mf);
+   L2missGroup::iterator iMissGroup = m_L2missgroup.find(cacheTag);
+   if (iMissGroup == m_L2missgroup.end()) return; // this is possible for write miss 
+   mem_fetch_set &missGroup = iMissGroup->second;
+
+   missGroup.erase(mf);
+
+   // remove the miss group if it goes empty
+   if (missGroup.empty()) {
+      m_L2missgroup.erase(iMissGroup);
+   }
+}
+
+void L2c_miss_tracker::print(FILE *fout, bool brief)
+{
+   L2missGroup::iterator iMissGroup;
+   for (iMissGroup = m_L2missgroup.begin(); iMissGroup != m_L2missgroup.end(); ++iMissGroup) {
+      fprintf(fout, "%#08x: ", iMissGroup->first); 
+      for (mem_fetch_set::iterator iMemSet = iMissGroup->second.begin(); iMemSet != iMissGroup->second.end(); ++iMemSet) { 
+         fprintf(fout, "%p ", *iMemSet);
+      }
+      fprintf(fout, "\n");
+   }
+}
+
+void L2c_miss_tracker::print_stat(FILE *fout, bool brief)
+{
+   fprintf(fout, "RedundantMiss = %d\n", m_totalL2redundantAcc);
+   if (brief == true) return;
+   fprintf(fout, "  Detail:");
+   for (L2redundantCnt::iterator iL2rc = m_L2redundantCnt.begin(); iL2rc != m_L2redundantCnt.end(); ++iL2rc) {
+      fprintf(fout, "%#08x:%d ", iL2rc->first, iL2rc->second);
+   }
+   fprintf(fout, "\n");
+}
+
+////////////////////////////////////////////////
+// track all locality of L2 cache access
+class L2c_access_locality
+{
+private:
+   size_t m_linesize; // L2 cache line size
+
+   typedef tr1_hash_map<address_type, int> L2accCnt; 
+   L2accCnt m_L2accCnt; 
+
+   int m_totalL2accAcc;
+
+   address_type cache_tag(const mem_fetch_t *mf) const 
+   {
+      // return mf->addr;
+      return (mf->addr & ~(m_linesize - 1));
+   }
+
+public:
+   L2c_access_locality(size_t linesize) : m_linesize(linesize), m_totalL2accAcc(0) { }
+   void access(mem_fetch_t *mf);
+
+   void print_stat(FILE *fout, bool brief = true);
+
+};
+
+void L2c_access_locality::access(mem_fetch_t *mf)
+{
+   address_type cacheTag = cache_tag(mf);
+   m_L2accCnt[cacheTag] += 1;
+   m_totalL2accAcc += 1;
+}
+
+void L2c_access_locality::print_stat(FILE *fout, bool brief)
+{
+   float access_locality = (float) m_totalL2accAcc / m_L2accCnt.size();
+   fprintf(fout, "Access Locality = %d / %zu (%f) \n", m_totalL2accAcc, m_L2accCnt.size(), access_locality);
+   if (brief == true) return;
+   fprintf(fout, "  Detail:");
+   pow2_histogram locality_histo(" Hits");
+   for (L2accCnt::iterator iL2rc = m_L2accCnt.begin(); iL2rc != m_L2accCnt.end(); ++iL2rc) {
+      locality_histo.add2bin(iL2rc->second);
+      // fprintf(fout, "%#08x:%d\n", iL2rc->first, iL2rc->second);
+   }
+   locality_histo.fprint(fout);
+   fprintf(fout, "\n");
+}
+
+L2cacheblk::L2cacheblk(size_t linesize)
+: m_mshr(new L2c_mshr(linesize)), 
+  m_missTracker(new L2c_miss_tracker(linesize)), 
+  m_accessLocality(new L2c_access_locality(linesize)) 
+{ }
+
+L2cacheblk::~L2cacheblk()
+{
+   delete m_mshr;
+   delete m_missTracker;
+   delete m_accessLocality; 
+}
+
+
+//////////////////////////////////////////////// 
+// L2 access functions
+
+// L2 Cache Creation 
+void L2c_create ( dram_t* dram_p, const char* cache_opt )
+{
+   unsigned int shd_n_set;
+   unsigned int shd_linesize;
+   unsigned int shd_n_assoc;
+   unsigned char shd_policy;
+
+   unsigned int L2c_cb_L2_length;
+   unsigned int L2c_cb_L2w_length;
+   unsigned int L2c_L2_dm_length;
+   unsigned int L2c_dm_L2_length;
+   unsigned int L2c_dm_L2w_length;
+   unsigned int L2c_L2_cb_length;
+   unsigned int L2c_L2_cb_minlength;
+   unsigned int L2c_L2_dm_minlength;
+
+   sscanf(cache_opt,"%d:%d:%d:%c", 
+          &shd_n_set, &shd_linesize, &shd_n_assoc, &shd_policy);
+
+   L2cacheblk *p_L2c = new L2cacheblk(shd_linesize);
+
+   char L2c_name[32];
+   snprintf(L2c_name, 32, "L2c_%03d", dram_p->id);
+   p_L2c->L2cache = shd_cache_create(L2c_name, 
+                                      shd_n_set, shd_n_assoc, shd_linesize, 
+				     shd_policy, 16, ~addrdec_mask[CHIP], 
+				     write_through); //write_through maintains old behavior for now
+
+   sscanf(gpgpu_L2_queue_config,"%d:%d:%d:%d:%d:%d:%d:%d", 
+          &L2c_cb_L2_length, &L2c_cb_L2w_length, &L2c_L2_dm_length, 
+          &L2c_dm_L2_length, &L2c_dm_L2w_length, &L2c_L2_cb_length,
+          &L2c_L2_cb_minlength, &L2c_L2_dm_minlength );
+   //(<name>,<latency>,<min_length>,<max_length>)
+   p_L2c->cbtoL2queue        = dq_create("cbtoL2queue",       0,0,L2c_cb_L2_length); 
+   p_L2c->cbtoL2writequeue   = dq_create("cbtoL2writequeue",  0,0,L2c_cb_L2w_length); 
+   p_L2c->L2todramqueue      = dq_create("L2todramqueue",     0,L2c_L2_dm_minlength,L2c_L2_dm_length);
+   p_L2c->dramtoL2queue      = dq_create("dramtoL2queue",     0,0,L2c_dm_L2_length);
+   p_L2c->dramtoL2writequeue = dq_create("dramtoL2writequeue",0,0,L2c_dm_L2w_length);
+   p_L2c->L2tocbqueue        = dq_create("L2tocbqueue",       0,L2c_L2_cb_minlength,L2c_L2_cb_length);
+
+   p_L2c->L2todram_wbqueue   = dq_create("L2todram_wbqueue",  0,L2c_L2_dm_minlength,
+                                          L2c_L2_dm_minlength + gpgpu_dram_sched_queue_size + L2c_dm_L2_length);
+
+   p_L2c->L2request = NULL; 
+
+   assert(dram_p->m_L2cache == NULL);
+   dram_p->m_L2cache = reinterpret_cast<void*>(p_L2c);
+}
+
+unsigned L2c_get_linesize( dram_t *dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+   return p_L2c->L2cache->line_sz; 
+}
+
+int L2c_full( dram_t *dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+   return(dq_full(p_L2c->cbtoL2queue) || dq_full(p_L2c->cbtoL2writequeue));
+}
+
+void L2c_push( dram_t *dram_p, mem_fetch_t *mf )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   if (gpgpu_l2_readoverwrite && mf->write)
+      dq_push(p_L2c->cbtoL2writequeue, mf);
+   else
+      dq_push(p_L2c->cbtoL2queue, mf);
+   p_L2c->m_accessLocality->access(mf); 
+   if (mf->mshr) mshr_update_status(mf->mshr, IN_CBTOL2QUEUE);
+}
+
+mem_fetch_t* L2c_pop( dram_t *dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   mem_fetch_t *mf;
+   mf = (mem_fetch_t*)dq_pop(p_L2c->L2tocbqueue);
+
+   return mf;
+}
+
+mem_fetch_t* L2c_top( dram_t *dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   return (mem_fetch_t*)dq_top(p_L2c->L2tocbqueue);
+}
+
+void L2c_qlen ( dram_t *dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   printf("\n");
+   printf("cb->L2{%d}\tcb->L2w{%d}\tL2->cb{%d}\n", 
+          p_L2c->cbtoL2queue->length, 
+          p_L2c->cbtoL2writequeue->length, 
+          p_L2c->L2tocbqueue->length);
+   printf("dm->L2{%d}\tdm->L2w{%d}\tL2->dm{%d}\tL2->wb_dm{%d}\n", 
+          p_L2c->dramtoL2queue->length, 
+          p_L2c->dramtoL2writequeue->length, 
+          p_L2c->L2todramqueue->length,
+          p_L2c->L2todram_wbqueue->length);
+}
+
+// service memory request in icnt-to-L2 queue, writing to L2 as necessary
+// (if L2 writeback miss, writeback to memory) 
+void L2c_service_mem_req ( dram_t* dram_p, int dm_id )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   mem_fetch_t* mf;
+
+   if (!p_L2c->L2request) {
+      //if not servicing L2 cache request..
+      p_L2c->L2request = (mem_fetch_t*) dq_pop(p_L2c->cbtoL2queue); //..then get one
+      if (!p_L2c->L2request) {
+         p_L2c->L2request = (mem_fetch_t*) dq_pop(p_L2c->cbtoL2writequeue);
+      }
+   }
+
+   mf = p_L2c->L2request;
+
+   if (!mf) return;
+
+   switch (mf->type) {
+   case RD_REQ:
+   case WT_REQ: {
+         shd_cache_line_t *hit_cacheline = shd_cache_access(p_L2c->L2cache,
+                                                            mf->addr,
+                                                            4, mf->write,
+                                                            gpu_sim_cycle);
+
+         if (hit_cacheline || l2_ideal) { //L2 Cache Hit; reads are sent as a single command and need to be stored
+            if (!mf->write) { //L2 Cache Read
+               if ( dq_full(p_L2c->L2tocbqueue) ) {
+                  p_L2c->L2cache->access--;
+               } else {
+                  mf->type = REPLY_DATA;
+                  dq_push(p_L2c->L2tocbqueue, mf);
+                  // at this point, should first check if earlier L2 miss is ready to be serviced
+                  // if so, service earlier L2 miss first
+                  p_L2c->L2request = NULL; //finished servicing
+                  L2_read_hit++;
+                  memlatstat_icnt2sh_push(mf);
+                  if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TOCBQUEUE_HIT);
+               }
+            } else { //L2 Cache Write aka servicing L1 Writeback
+               p_L2c->L2request = NULL;    
+               L2_write_hit++;
+               freed_L1write_mfs++;
+               free(mf); //writeback from L1 successful
+               gpgpu_n_processed_writes++;
+            }
+         } else {
+            // L2 Cache Miss; issue commands accordingly
+            if ( dq_full(p_L2c->L2todramqueue) ) {
+               p_L2c->L2cache->miss--;
+               p_L2c->L2cache->access--;
+            } else {
+               // if a miss hit the mshr, that means there is another inflight request for the same data
+               // this miss just need to access the cache later when this request is serviced
+               bool mshr_hit = p_L2c->m_mshr->new_miss(mf);
+               if (not mshr_hit) {
+                  if (!mf->write) {
+                     dq_push(p_L2c->L2todramqueue, mf);
+                  } else {
+                     // if request is writeback from L1 and misses, 
+                     // then redirect mf writes to dram (no write allocate)
+                     mf->nbytes_L2 = mf->nbytes_L1 - READ_PACKET_SIZE;
+                     dq_push(p_L2c->L2todramqueue, mf);
+                  }
+               }
+               if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TODRAMQUEUE);
+               p_L2c->L2request = NULL;
+            }
+         }
+      }
+      break;
+   default: assert(0);
+   }
+}
+
+// service memory request in L2todramqueue, pushing to dram 
+void L2c_push_miss_to_dram ( dram_t* dram_p )
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   mem_fetch_t* mf;
+
+   if ( gpgpu_dram_sched_queue_size && dram_full(dram_p) ) return;
+
+   mf = (mem_fetch_t*) dq_pop(p_L2c->L2todram_wbqueue); //prioritize writeback
+   if (!mf) mf = (mem_fetch_t*) dq_pop(p_L2c->L2todramqueue);
+   if (mf) {
+      if (mf->write) {
+         L2_write_miss++;
+      } else {
+         L2_read_miss++;
+      }
+      p_L2c->m_missTracker->new_miss(mf);
+      memlatstat_dram_access(mf, dram_p->id, mf->tlx.bk);
+      dram_push(dram_p,
+                mf->tlx.bk, mf->tlx.row, mf->tlx.col,
+                mf->nbytes_L2, mf->write,
+                mf->wid, mf->sid, mf->cache_hits_waiting, mf->addr, mf);
+      if (mf->mshr) mshr_update_status(mf->mshr, IN_DRAM_REQ_QUEUE);
+   }
+}
+
+//Service writes that are finished in Dram 
+//only updates the stats and frees the mf
+void dramtoL2_service_write(mem_fetch_t * mf) {
+   freed_L2write_mfs++;
+   free(mf);
+   gpgpu_n_processed_writes++;
+}
+
+// pop completed memory request from dram and push it to dram-to-L2 queue 
+void L2c_get_dram_output ( dram_t* dram_p ) 
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   mem_fetch_t* mf;
+   mem_fetch_t* mf_top;
+   if ( dq_full(p_L2c->dramtoL2queue) || dq_full(p_L2c->dramtoL2writequeue) ) return;
+   mf_top = (mem_fetch_t*) dram_top(dram_p); //test
+   mf = (mem_fetch_t*) dram_pop(dram_p);
+   assert (mf_top==mf );
+   if (mf) {
+      if (gpgpu_l2_readoverwrite && mf->write)
+         dq_push(p_L2c->dramtoL2writequeue, mf);
+      else
+         dq_push(p_L2c->dramtoL2queue, mf);
+      if (mf->mshr) mshr_update_status(mf->mshr, IN_DRAMTOL2QUEUE);
+   }
+}
+
+// service memory request in dramtoL2queue, writing to L2 as necessary
+// (may cause cache eviction and subsequent writeback) 
+void L2c_process_dram_output ( dram_t* dram_p, int dm_id ) 
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   static mem_fetch_t **L2dramout = NULL; 
+   static unsigned long long int *wb_addr = NULL;
+   if (!L2dramout) L2dramout = (mem_fetch_t**)calloc(gpu_n_mem, sizeof(mem_fetch_t*));
+   if (!wb_addr) {
+      wb_addr = (unsigned long long int*)calloc(gpu_n_mem, sizeof(unsigned long long int));
+      for (unsigned i = 0; i < gpu_n_mem; i++) wb_addr[i] = -1;
+   }
+
+   if (L2dramout[dm_id] == NULL) {
+      // pop from mshr chain if it is not empty, otherwise, pop a new cacheline from dram output queue
+      if (p_L2c->m_mshr->mshr_chain_empty() == false) {
+         L2dramout[dm_id] = p_L2c->m_mshr->mshr_chain_top();
+         p_L2c->m_mshr->mshr_chain_pop();
+      } else {
+         L2dramout[dm_id] = (mem_fetch_t*) dq_pop(p_L2c->dramtoL2queue);
+         if (!L2dramout[dm_id]) L2dramout[dm_id] = (mem_fetch_t*) dq_pop(p_L2c->dramtoL2writequeue);
+
+         if (L2dramout[dm_id] != NULL) {
+            p_L2c->m_mshr->miss_serviced(L2dramout[dm_id]);
+
+            if (p_L2c->m_mshr->mshr_chain_empty() == false) { // possible if this is a L2 writeback
+               L2dramout[dm_id] = p_L2c->m_mshr->mshr_chain_top();
+               p_L2c->m_mshr->mshr_chain_pop();
+            }
+         }
+      }
+   }
+
+   mem_fetch_t* mf = L2dramout[dm_id];
+   if (mf) {
+      if (!mf->write) { //service L2 read miss
+
+         // it is a pre-fill dramout mf
+         if (wb_addr[dm_id] == (unsigned long long int)-1) {
+            if ( dq_full(p_L2c->L2tocbqueue) ) goto RETURN;
+
+            if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TOCBQUEUE_MISS);
+
+            //only transfer across icnt once the whole line has been received by L2 cache
+            mf->type = REPLY_DATA;
+            dq_push(p_L2c->L2tocbqueue, mf);
+
+            assert(mf->sid <= (int)gpu_n_shader);           
+            shd_cache_line_t *fetch_line_exist = shd_cache_probe(p_L2c->L2cache, mf->addr);
+            if (fetch_line_exist == NULL) {
+               wb_addr[dm_id] = L2_shd_cache_fill(p_L2c->L2cache, mf->addr, gpu_sim_cycle );
+            }
+         }
+         // only perform a write on cache eviction (write-back policy)
+         // it is the 1st or nth time trial to writeback
+         if (wb_addr[dm_id] != (unsigned long long int)-1) {
+            // performing L2 writeback (no false sharing for memory-side cache)
+            int wb_succeed = L2c_write_back(wb_addr[dm_id], p_L2c->L2cache->line_sz, dm_id ); 
+            if (!wb_succeed) goto RETURN; //try again next cycle
+         }
+
+         p_L2c->m_missTracker->miss_serviced(mf);
+         L2dramout[dm_id] = NULL;
+         wb_addr[dm_id] = -1;
+      } else { //service L2 write miss
+         p_L2c->m_missTracker->miss_serviced(mf);
+         dramtoL2_service_write(mf);
+         L2dramout[dm_id] = NULL;
+         wb_addr[dm_id] = -1;
+      }
+   }
+   RETURN:   
+   assert (L2dramout[dm_id] || wb_addr[dm_id] == (unsigned long long int)-1);
+}
+
+// Writeback from L2 to DRAM: 
+// - Takes in memory address and their parameters and pushes to dram request queue
+// - This is used only for L2 writeback 
+unsigned char L2c_write_back(unsigned long long int addr, int bsize, int dram_id ) 
+{
+   addrdec_t tlx;
+   addrdec_tlx(addr,&tlx);
+
+   assert(dram[dram_id]->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[dram_id]->m_L2cache);
+
+   if ( dq_full(p_L2c->L2todram_wbqueue) ) return 0;
+
+   mem_fetch_t *mf;
+
+   mf = (mem_fetch_t*) malloc(sizeof(mem_fetch_t));
+   made_write_mfs++;
+   mf->request_uid = g_next_request_uid++;
+   mf->addr = addr;
+   mf->nbytes_L1 = bsize + READ_PACKET_SIZE;
+   mf->txbytes_L1 = 0;
+   mf->rxbytes_L1 = 0;  
+   mf->nbytes_L2 = bsize;
+   mf->sid = gpu_n_shader; // (gpu_n_shader+1);
+   mf->wid = 0;
+   mf->txbytes_L2 = 0;
+   mf->rxbytes_L2 = 0;
+   mf->mshr = NULL;
+   mf->pc = -1; // disable ptx_file_line_stats
+   mf->write = 1; // it is writeback
+   mf->mem_acc = L2_WRBK_ACC; 
+   memlatstat_start(mf);
+   mf->tlx = tlx;
+   mf->bank = mf->tlx.bk;
+   mf->chip = mf->tlx.chip;
+
+
+   //writeback
+   mf->type = L2_WTBK_DATA;
+   if (!dq_push(p_L2c->L2todram_wbqueue, mf)) assert(0);
+   gpgpu_n_sent_writes++;
+   return 1;
+}
+
+unsigned int L2c_cache_flush ( dram_t* dram_p) {
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   shd_cache_t *cp = p_L2c->L2cache; 
+   int dirty_lines_flushed = 0 ;
+   for (unsigned i = 0; i < cp->nset * cp->assoc ; i++) {
+      if ( (cp->lines[i].status & (DIRTY|VALID)) == (DIRTY|VALID) ) {
+         dirty_lines_flushed++;
+      }
+      cp->lines[i].status &= ~VALID;
+      cp->lines[i].status &= ~DIRTY;
+   }
+   return dirty_lines_flushed;
+}
+
+void L2c_init_stat()
+{
+   L2_cbtoL2length = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+   L2_cbtoL2writelength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+   L2_L2tocblength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+   L2_dramtoL2length = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+   L2_dramtoL2writelength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+   L2_L2todramlength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int));
+}
+
+void L2c_update_stat( dram_t* dram_p)
+{
+   assert(dram_p->m_L2cache != NULL);
+   L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram_p->m_L2cache);
+
+   int i = dram_p->id;
+
+   if (p_L2c->cbtoL2queue->length > L2_cbtoL2length[i])
+      L2_cbtoL2length[i] = p_L2c->cbtoL2queue->length;
+   if (p_L2c->cbtoL2writequeue->length > L2_cbtoL2writelength[i])
+      L2_cbtoL2writelength[i] = p_L2c->cbtoL2writequeue->length;
+   if (p_L2c->L2tocbqueue->length > L2_L2tocblength[i])
+      L2_L2tocblength[i] = p_L2c->L2tocbqueue->length;
+   if (p_L2c->dramtoL2queue->length > L2_dramtoL2length[i])
+      L2_dramtoL2length[i] = p_L2c->dramtoL2queue->length;
+   if (p_L2c->dramtoL2writequeue->length > L2_dramtoL2writelength[i])
+      L2_dramtoL2writelength[i] = p_L2c->dramtoL2writequeue->length;
+   if (p_L2c->L2todramqueue->length > L2_L2todramlength[i])
+      L2_L2todramlength[i] = p_L2c->L2todramqueue->length;
+}
+
+void L2c_print_stat( )
+{
+   unsigned i;
+
+   printf("                                     ");
+   for (i=0;i<gpu_n_mem;i++) {
+      printf(" dram[%d]", i);
+   }
+   printf("\n");
+
+   printf("cbtoL2 queue maximum length         ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_cbtoL2length[i]);
+   }
+   printf("\n");
+
+   printf("cbtoL2 write queue maximum length   ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_cbtoL2writelength[i]);
+   }
+   printf("\n");
+
+   printf("L2tocb queue maximum length         =");
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_L2tocblength[i]);
+   }
+   printf("\n");
+
+   printf("dramtoL2 queue maximum length       =");
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_dramtoL2length[i]);
+   }
+   printf("\n");
+
+   printf("dramtoL2 write queue maximum length ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_dramtoL2writelength[i]);
+   }
+   printf("\n");
+
+   printf("L2todram queue maximum length       =");
+   for (i=0;i<gpu_n_mem;i++) {
+      printf("%8d", L2_L2todramlength[i]);
+   }
+   printf("\n");
+}
+
+void L2c_print_cache_stat()
+{
+   unsigned i;
+   int j, k;
+   for (i=0,j=0,k=0;i<gpu_n_mem;i++) {
+      assert(dram[i]->m_L2cache != NULL);
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+
+      shd_cache_print(p_L2c->L2cache,stdout);
+      j += p_L2c->L2cache->miss;
+      k += p_L2c->L2cache->access;
+      p_L2c->m_mshr->print_stat(stdout); 
+      p_L2c->m_missTracker->print_stat(stdout);
+      p_L2c->m_accessLocality->print_stat(stdout, false);
+   }
+   printf("L2 Cache Total Miss Rate = %0.3f\n", (float)j/k);
+}
+
+void L2c_print_debug( )
+{
+   unsigned i;
+
+   printf("                                     ");
+   for (i=0;i<gpu_n_mem;i++) {
+      printf(" dram[%d]", i);
+   }
+   printf("\n");
+
+   printf("cbtoL2 queue length         ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->cbtoL2queue->length);
+   }
+   printf("\n");
+
+   printf("cbtoL2 write queue length   ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->cbtoL2writequeue->length);
+   }
+   printf("\n");
+
+   printf("L2tocb queue length         =");
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->L2tocbqueue->length);
+   }
+   printf("\n");
+
+   printf("dramtoL2 queue length       =");
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->dramtoL2queue->length);
+   }
+   printf("\n");
+
+   printf("dramtoL2 write queue length ="); 
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->dramtoL2writequeue->length);
+   }
+   printf("\n");
+
+   printf("L2todram queue length       =");
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->L2todramqueue->length);
+   }
+   printf("\n");
+
+   printf("L2todram writeback queue length       =");
+   for (i=0;i<gpu_n_mem;i++) {
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+      printf("%8d", p_L2c->L2todram_wbqueue->length);
+   }
+   printf("\n");
+}
+
+#define CREATELOG 111
+#define SAMPLELOG 222
+#define DUMPLOG 333
+
+void L2c_log(int task)
+{
+   unsigned i;
+   static void ** cbtol2_Dist   ;  
+   static void ** cbtoL2wr_Dist  ;  
+   static void ** L2tocb_Dist     ; 
+   static void ** dramtoL2_Dist   ;
+   static void ** dramtoL2wr_Dist  ;
+   static void ** L2todram_Dist    ;
+   static void ** L2todram_wb_Dist ;
+   if (task == CREATELOG) {
+      cbtol2_Dist = (void **)     calloc(gpu_n_mem,sizeof(void*));
+      cbtoL2wr_Dist = (void **)    calloc(gpu_n_mem,sizeof(void*));
+      L2tocb_Dist =   (void **)   calloc(gpu_n_mem,sizeof(void*));
+      dramtoL2_Dist =   (void **)calloc(gpu_n_mem,sizeof(void*));
+      dramtoL2wr_Dist  =  (void **)calloc(gpu_n_mem,sizeof(void*));
+      L2todram_Dist    = (void **)calloc(gpu_n_mem,sizeof(void*));
+      L2todram_wb_Dist = (void **)calloc(gpu_n_mem,sizeof(void*));
+
+      for (i=0;i<gpu_n_mem;i++) {
+         assert(dram[i]->m_L2cache != NULL);
+         L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+
+         cbtol2_Dist[i]      = StatCreate("cbtoL2",1,p_L2c->cbtoL2queue->max_len);
+         cbtoL2wr_Dist[i]    = StatCreate("cbtoL2write",1,p_L2c->cbtoL2writequeue->max_len);
+         L2tocb_Dist[i]      = StatCreate("L2tocb",1,p_L2c->L2tocbqueue->max_len);
+         dramtoL2_Dist[i]    = StatCreate("dramtoL2",1,p_L2c->dramtoL2queue->max_len);
+         dramtoL2wr_Dist[i]  = StatCreate("dramtoL2write",1,p_L2c->dramtoL2writequeue->max_len);
+         L2todram_Dist[i]    = StatCreate("L2todram",1,p_L2c->L2todramqueue->max_len);
+         L2todram_wb_Dist[i] = StatCreate("L2todram_wb",1,p_L2c->L2todram_wbqueue->max_len);
+      }
+   } else if (task == SAMPLELOG) {
+      for (i=0;i<gpu_n_mem;i++) {
+         assert(dram[i]->m_L2cache != NULL);
+         L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+
+         StatAddSample(cbtol2_Dist[i],       p_L2c->cbtoL2queue->length);
+         StatAddSample(cbtoL2wr_Dist[i],     p_L2c->cbtoL2writequeue->length);
+         StatAddSample(L2tocb_Dist[i],       p_L2c->L2tocbqueue->length);
+         StatAddSample(dramtoL2_Dist[i],     p_L2c->dramtoL2queue->length);
+         StatAddSample(dramtoL2wr_Dist[i],   p_L2c->dramtoL2writequeue->length);
+         StatAddSample(L2todram_Dist[i],     p_L2c->L2todramqueue->length);
+         StatAddSample(L2todram_wb_Dist[i],  p_L2c->L2todram_wbqueue->length);
+      }
+   } else if (task == DUMPLOG) {
+      for (i=0;i<gpu_n_mem;i++) {
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(cbtol2_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(cbtoL2wr_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(L2tocb_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(dramtoL2_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(dramtoL2wr_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(L2todram_Dist[i]);
+         printf ("Queue Length DRAM[%d] ",i); StatDisp(L2todram_wb_Dist[i]);
+      } 
+   }
+}
+
+void L2c_latency_log_dump()
+{
+   unsigned i;
+   for (i=0;i<gpu_n_mem;i++) {
+      assert(dram[i]->m_L2cache != NULL);
+      L2cacheblk *p_L2c = reinterpret_cast<L2cacheblk*>(dram[i]->m_L2cache);
+
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->cbtoL2queue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->cbtoL2writequeue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2tocbqueue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->dramtoL2queue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->dramtoL2writequeue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2todramqueue->lat_stat);
+      printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2todram_wbqueue->lat_stat);
+   }
+}
+
+
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
new file mode 100644
index 0000000..449d47a
--- /dev/null
+++ b/src/gpgpu-sim/l2cache.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include "dram.h"
+
+// L2 Cache Creation 
+void L2c_create ( dram_t* dram_p, const char* cache_opt );
+
+void L2c_qlen ( dram_t *dram_p );
+
+// service memory request in icnt-to-L2 queue, writing to L2 as necessary
+// (if L2 writeback miss, writeback to memory) 
+void L2c_service_mem_req ( dram_t* dram_p, int dm_id );
+
+// service memory request in L2todramqueue, pushing to dram 
+void L2c_push_miss_to_dram ( dram_t* dram_p );
+
+// pop completed memory request from dram and push it to dram-to-L2 queue 
+void L2c_get_dram_output ( dram_t* dram_p );
+
+// service memory request in dramtoL2queue, writing to L2 as necessary
+// (may cause cache eviction and subsequent writeback) 
+void L2c_process_dram_output ( dram_t* dram_p, int dm_id );
+
+// Writeback from L2 to DRAM: 
+// - Takes in memory address and their parameters and pushes to dram request queue
+// - This is used only for L2 writeback 
+unsigned char L2c_write_back(unsigned long long int addr, int bsize, int dram_id );
+
+unsigned int L2c_cache_flush ( dram_t* dram_p);
+
+unsigned L2c_get_linesize( dram_t *dram_p );
+
+// probe L2 cache for fullness 
+int L2c_full( dram_t *dram_p );
+void L2c_push( dram_t *dram_p, mem_fetch_t *mf );
+mem_fetch_t* L2c_pop( dram_t *dram_p );
+mem_fetch_t* L2c_top( dram_t *dram_p );
+
+void L2c_init_stat();
+void L2c_update_stat( dram_t* dram_p);
+void L2c_print_stat();
+void L2c_print_cache_stat();
+void L2c_print_debug();
+void L2c_log(int task);
+void L2c_latency_log_dump();
+
+void L2c_options(option_parser_t opp);
+
diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h
new file mode 100644
index 0000000..0e12a2d
--- /dev/null
+++ b/src/gpgpu-sim/mem_fetch.h
@@ -0,0 +1,110 @@
+/* 
+ * mem_fetch.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef MEM_FETCH_H
+#define MEM_FETCH_H
+
+#include "shader.h"
+#include "addrdec.h"
+
+enum mf_type {
+   RD_REQ = 0,
+   WT_REQ,
+   REPLY_DATA, // send to shader
+   L2_WTBK_DATA,
+   DUMMY_READ, //used in write mask    
+   N_MF_TYPE
+};
+
+typedef struct {
+   unsigned request_uid;
+   unsigned long long int addr;
+   int nbytes_L1;
+   int txbytes_L1;
+   int rxbytes_L1;
+   int nbytes_L2;
+   int txbytes_L2;
+   int rxbytes_L2;
+   int sid; //shader core id
+   int wid; //warp id
+   int cache_hits_waiting;
+   mshr_entry* mshr;
+   address_type pc;
+   unsigned char write;
+   enum mem_access_type mem_acc;
+   unsigned int timestamp; //set to gpu_sim_cycle at struct creation
+   unsigned int timestamp2; //set to gpu_sim_cycle when pushed onto icnt to shader; only used for reads
+   unsigned int icnt_receive_time; //set to gpu_sim_cycle + interconnect_latency when fixed icnt latency mode is enabled
+   unsigned char bank;
+   unsigned char chip;
+   addrdec_t tlx;
+   enum mf_type type;
+   partial_write_mask_t write_mask;
+   int source_node; //memory node id when sending from mem to shader
+                    //same as sid when sending from shader 2 mem 
+} mem_fetch_t;
+
+#endif
diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h
new file mode 100644
index 0000000..2fa6b1c
--- /dev/null
+++ b/src/gpgpu-sim/mem_latency_stat.h
@@ -0,0 +1,544 @@
+/* 
+ * mem_latency_stat.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+
+#ifndef MEM_LATENCY_STAT_H
+#define MEM_LATENCY_STAT_H
+
+extern unsigned long long  gpu_sim_cycle;
+extern unsigned int gpu_n_mem;
+extern unsigned int gpu_n_shader;
+extern int gpgpu_dram_sched_queue_size;
+extern int gpgpu_dram_scheduler;
+extern unsigned int gpu_mem_n_bk;
+#ifdef MEM_LATENCY_STAT_IMPL
+   #define EXTERN_DEF
+#else
+   #define EXTERN_DEF extern
+#endif
+
+EXTERN_DEF int gpgpu_memlatency_stat = FALSE;
+
+EXTERN_DEF unsigned max_mrq_latency;
+EXTERN_DEF unsigned max_dq_latency;
+EXTERN_DEF unsigned max_mf_latency;
+EXTERN_DEF unsigned max_icnt2mem_latency;
+EXTERN_DEF unsigned max_icnt2sh_latency;
+EXTERN_DEF unsigned mrq_lat_table[32];
+EXTERN_DEF unsigned dq_lat_table[32];
+EXTERN_DEF unsigned mf_lat_table[32];
+EXTERN_DEF unsigned icnt2mem_lat_table[24];
+EXTERN_DEF unsigned icnt2sh_lat_table[24];
+EXTERN_DEF unsigned mf_lat_pw_table[32]; //table storing values of mf latency Per Window
+EXTERN_DEF unsigned mf_num_lat_pw;
+EXTERN_DEF unsigned mf_tot_lat_pw; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window
+EXTERN_DEF unsigned long long int mf_total_lat;
+EXTERN_DEF unsigned long long int ** mf_total_lat_table; //mf latency sums[dram chip id][bank id]
+EXTERN_DEF unsigned ** mf_max_lat_table; //mf latency sums[dram chip id][bank id]
+EXTERN_DEF unsigned num_mfs;
+EXTERN_DEF unsigned int ***bankwrites; //bankwrites[shader id][dram chip id][bank id]
+EXTERN_DEF unsigned int ***bankreads; //bankreads[shader id][dram chip id][bank id]
+EXTERN_DEF unsigned int **totalbankwrites; //bankwrites[dram chip id][bank id]
+EXTERN_DEF unsigned int **totalbankreads; //bankreads[dram chip id][bank id]
+EXTERN_DEF unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id]
+EXTERN_DEF unsigned int *requests_by_warp;
+EXTERN_DEF unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp
+EXTERN_DEF unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache
+EXTERN_DEF unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen 
+EXTERN_DEF unsigned *mf_num_lat_pw_perwarp;
+EXTERN_DEF unsigned *mf_tot_lat_pw_perwarp; //total latency summed up per window per warp. divide by mf_num_lat_pw_perwarp to obtain average latency Per Window
+EXTERN_DEF unsigned long long int *mf_total_lat_perwarp;
+EXTERN_DEF unsigned *num_mfs_perwarp;
+EXTERN_DEF unsigned *acc_mrq_length;
+
+EXTERN_DEF unsigned ***mem_access_type_stats; // dram access type classification
+
+
+void memlatstat_init( )
+{
+   unsigned i,j;
+
+   max_mrq_latency = 0;
+   max_dq_latency = 0;
+   max_mf_latency = 0;
+   max_icnt2mem_latency = 0;
+   max_icnt2sh_latency = 0;
+   memset(mrq_lat_table, 0, sizeof(unsigned)*32);
+   memset(dq_lat_table, 0, sizeof(unsigned)*32);
+   memset(mf_lat_table, 0, sizeof(unsigned)*32);
+   memset(icnt2mem_lat_table, 0, sizeof(unsigned)*24);
+   memset(icnt2sh_lat_table, 0, sizeof(unsigned)*24);
+   memset(mf_lat_pw_table, 0, sizeof(unsigned)*32);
+   mf_num_lat_pw = 0;
+   mf_num_lat_pw_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int));
+   mf_tot_lat_pw_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int));
+   mf_total_lat_perwarp = (unsigned long long int *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned long long int));
+   num_mfs_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int));
+   acc_mrq_length = (unsigned *) calloc(gpu_n_mem, sizeof(unsigned int));
+   mf_tot_lat_pw = 0; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window
+   mf_total_lat = 0;
+   num_mfs = 0;
+   printf("*** Initializing Memory Statistics ***\n");
+   requests_by_warp = (unsigned int*) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int));
+   totalbankreads = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   totalbankwrites = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   totalbankaccesses = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+   mf_total_lat_table = (unsigned long long int **) calloc(gpu_n_mem, sizeof(unsigned long long *));
+   mf_max_lat_table = (unsigned **) calloc(gpu_n_mem, sizeof(unsigned *));
+   bankreads = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**));
+   bankwrites = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**));
+   MCB_accesses = (unsigned int*) calloc(gpu_n_mem*4, sizeof(unsigned int));
+   num_MCBs_accessed = (unsigned int*) calloc(gpu_n_mem*4+1, sizeof(unsigned int));
+   if (gpgpu_dram_sched_queue_size) {
+      position_of_mrq_chosen = (unsigned int*) calloc(gpgpu_dram_sched_queue_size, sizeof(unsigned int));
+   } else
+      position_of_mrq_chosen = (unsigned int*) calloc(1024, sizeof(unsigned int));
+   for (i=0;i<gpu_n_shader ;i++ ) {
+      bankreads[i] = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+      bankwrites[i] = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*));
+      for (j=0;j<gpu_n_mem ;j++ ) {
+         bankreads[i][j] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+         bankwrites[i][j] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      }
+   }
+
+   for (i=0;i<gpu_n_mem ;i++ ) {
+      totalbankreads[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      totalbankwrites[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      totalbankaccesses[i] = (unsigned int*) calloc(gpu_mem_n_bk, sizeof(unsigned int));
+      mf_total_lat_table[i] = (unsigned long long int*) calloc(gpu_mem_n_bk, sizeof(unsigned long long int));
+      mf_max_lat_table[i] = (unsigned *) calloc(gpu_mem_n_bk, sizeof(unsigned));
+   }
+
+   mem_access_type_stats = (unsigned ***) malloc(NUM_MEM_ACCESS_TYPE * sizeof(unsigned **));
+   for (i = 0; i < NUM_MEM_ACCESS_TYPE; i++) {
+      int j;
+      mem_access_type_stats[i] = (unsigned **) calloc(gpu_n_mem, sizeof(unsigned*));
+      for (j=0; (unsigned) j< gpu_n_mem; j++) {
+         mem_access_type_stats[i][j] = (unsigned *) calloc((gpu_mem_n_bk+1), sizeof(unsigned*));
+      }
+   }
+}
+
+void memlatstat_start(mem_fetch_t *mf)
+{
+   mf->timestamp = gpu_sim_cycle + gpu_tot_sim_cycle;
+   mf->timestamp2 = 0;
+}
+
+// recorder the total latency
+unsigned memlatstat_done(mem_fetch_t *mf)
+{
+   unsigned mf_latency;
+   unsigned wid = mf->sid*gpu_n_warp_per_shader + mf->wid;
+   mf_latency = (gpu_sim_cycle+gpu_tot_sim_cycle) - mf->timestamp;
+   mf_num_lat_pw++;
+   mf_num_lat_pw_perwarp[wid]++;
+   mf_tot_lat_pw_perwarp[wid] += mf_latency;
+   mf_tot_lat_pw += mf_latency;
+   check_time_vector_update(mf->mshr->insts[0].uid,MR_2SH_FQ_POP,mf_latency, mf->type ) ;
+   mf_lat_table[LOGB2(mf_latency)]++;
+   shader_mem_lat_log(mf->sid, mf_latency);
+   mf_total_lat_table[mf->chip][mf->bank] += mf_latency;
+   if (mf_latency > max_mf_latency)
+      max_mf_latency = mf_latency;
+   return mf_latency;
+}
+
+void memlatstat_icnt2sh_push(mem_fetch_t *mf)
+{
+   mf->timestamp2 = gpu_sim_cycle+gpu_tot_sim_cycle;
+}
+
+void memlatstat_read_done(mem_fetch_t *mf)
+{
+   if (gpgpu_memlatency_stat) {
+      unsigned mf_latency = memlatstat_done(mf);
+
+      if (mf_latency > mf_max_lat_table[mf->chip][mf->bank]) {
+         mf_max_lat_table[mf->chip][mf->bank] = mf_latency;
+      }
+
+      unsigned icnt2sh_latency;
+      icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->timestamp2;
+      icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++;
+      if (icnt2sh_latency > max_icnt2sh_latency)
+         max_icnt2sh_latency = icnt2sh_latency;
+   }
+}
+
+void memlatstat_dram_access(mem_fetch_t *mf, unsigned dram_id, unsigned bank)
+{
+   assert(dram_id < gpu_n_mem);
+   assert(bank < gpu_mem_n_bk);
+   if (gpgpu_memlatency_stat) { 
+      if (mf->write) {
+         if ( (unsigned) mf->sid  <  gpu_n_shader  ) {   //do not count L2_writebacks here 
+            bankwrites[mf->sid][dram_id][bank]++;
+            shader_mem_acc_log( mf->sid, dram_id, bank, 'w');
+         }
+         totalbankwrites[dram_id][bank]++;
+      } else {
+         bankreads[mf->sid][dram_id][bank]++;
+         shader_mem_acc_log( mf->sid, dram_id, bank, 'r');
+         totalbankreads[dram_id][bank]++;
+      }
+
+      if (mf->pc != (unsigned) -1) {
+         ptx_file_line_stats_add_dram_traffic(mf->pc, 1);
+      }
+      
+      mem_access_type_stats[mf->mem_acc][dram_id][bank]++;
+   }
+}
+
+void memlatstat_icnt2mem_pop(mem_fetch_t *mf)
+{
+   if (gpgpu_memlatency_stat) {
+      unsigned icnt2mem_latency;
+      icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->timestamp;
+      icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++;
+      if (icnt2mem_latency > max_icnt2mem_latency)
+         max_icnt2mem_latency = icnt2mem_latency;
+   }
+}
+
+void memlatstat_lat_pw( )
+{
+   unsigned i;
+   if (mf_num_lat_pw && gpgpu_memlatency_stat) {
+      assert(mf_tot_lat_pw);
+      mf_total_lat = mf_tot_lat_pw;
+      num_mfs = mf_num_lat_pw;
+      mf_lat_pw_table[LOGB2(mf_tot_lat_pw/mf_num_lat_pw)]++;
+      mf_tot_lat_pw = 0;
+      mf_num_lat_pw = 0;
+   }
+   for (i=0;i < ((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1); i++) {
+      if (mf_num_lat_pw_perwarp[i] && gpgpu_memlatency_stat) {
+         assert(mf_tot_lat_pw_perwarp[i]);
+         mf_total_lat_perwarp[i] += mf_tot_lat_pw_perwarp[i];
+         num_mfs_perwarp[i] += mf_num_lat_pw_perwarp[i];
+         //mf_lat_pw_table[LOGB2(mf_tot_lat_pw/mf_num_lat_pw)]++;
+         mf_tot_lat_pw_perwarp[i] = 0;
+         mf_num_lat_pw_perwarp[i] = 0;
+      }
+   }
+}
+
+
+void memlatstat_print( )
+{
+   unsigned i,j,k,l,m;
+   unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses;
+
+   if (gpgpu_memlatency_stat) {
+      printf("maxmrqlatency = %d \n", max_mrq_latency);
+      printf("maxdqlatency = %d \n", max_dq_latency);
+      printf("maxmflatency = %d \n", max_mf_latency);
+      if (num_mfs) {
+         printf("averagemflatency = %lld \n", mf_total_lat/num_mfs);
+      }
+      printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency);
+      printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency);
+      printf("mrq_lat_table:");
+      for (i=0; i< 32; i++) {
+         printf("%d \t", mrq_lat_table[i]);
+      }
+      printf("\n");
+      printf("dq_lat_table:");
+      for (i=0; i< 32; i++) {
+         printf("%d \t", dq_lat_table[i]);
+      }
+      printf("\n");
+      printf("mf_lat_table:");
+      for (i=0; i< 32; i++) {
+         printf("%d \t", mf_lat_table[i]);
+      }
+      printf("\n");
+      printf("icnt2mem_lat_table:");
+      for (i=0; i< 24; i++) {
+         printf("%d \t", icnt2mem_lat_table[i]);
+      }
+      printf("\n");
+      printf("icnt2sh_lat_table:");
+      for (i=0; i< 24; i++) {
+         printf("%d \t", icnt2sh_lat_table[i]);
+      }
+      printf("\n");
+      printf("mf_lat_pw_table:");
+      for (i=0; i< 32; i++) {
+         printf("%d \t", mf_lat_pw_table[i]);
+      }
+      printf("\n");
+
+      /*MAXIMUM CONCURRENT ACCESSES TO SAME ROW*/
+      printf("maximum concurrent accesses to same row:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            printf("%9d ",max_conc_access2samerow[i][j]);
+         }
+         printf("\n");
+      }
+
+      /*MAXIMUM SERVICE TIME TO SAME ROW*/
+      printf("maximum service time to same row:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            printf("%9d ",max_servicetime2samerow[i][j]);
+         }
+         printf("\n");
+      }
+
+      /*AVERAGE ROW ACCESSES PER ACTIVATE*/
+      int total_row_accesses = 0;
+      int total_num_activates = 0;
+      printf("average row accesses per activate:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            total_row_accesses += row_access[i][j];
+            total_num_activates += num_activates[i][j];
+            printf("%9f ",(float) row_access[i][j]/num_activates[i][j]);
+         }
+         printf("\n");
+      }
+      printf("average row locality = %d/%d = %f\n", total_row_accesses, total_num_activates, (float)total_row_accesses/total_num_activates);
+      /*MEMORY ACCESSES*/
+      k = 0;
+      l = 0;
+      m = 0;
+      max_bank_accesses = 0;
+      max_chip_accesses = 0;
+      min_bank_accesses = 0xFFFFFFFF;
+      min_chip_accesses = 0xFFFFFFFF;
+      printf("number of total memory accesses made:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            l = totalbankaccesses[i][j];
+            if (l < min_bank_accesses)
+               min_bank_accesses = l;
+            if (l > max_bank_accesses)
+               max_bank_accesses = l;
+            k += l;
+            m += l;
+            printf("%9d ",l);
+         }
+         if (m < min_chip_accesses)
+            min_chip_accesses = m;
+         if (m > max_chip_accesses)
+            max_chip_accesses = m;
+         m = 0;
+         printf("\n");
+      }
+      printf("total accesses: %d\n", k);
+      if (min_bank_accesses)
+         printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses);
+      else
+         printf("min_bank_accesses = 0!\n");
+      if (min_chip_accesses)
+         printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses);
+      else
+         printf("min_chip_accesses = 0!\n");
+
+      /*READ ACCESSES*/
+      k = 0;
+      l = 0;
+      m = 0;
+      max_bank_accesses = 0;
+      max_chip_accesses = 0;
+      min_bank_accesses = 0xFFFFFFFF;
+      min_chip_accesses = 0xFFFFFFFF;
+      printf("number of total read accesses:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            l = totalbankreads[i][j];
+            if (l < min_bank_accesses)
+               min_bank_accesses = l;
+            if (l > max_bank_accesses)
+               max_bank_accesses = l;
+            k += l;
+            m += l;
+            printf("%9d ",l);
+         }
+         if (m < min_chip_accesses)
+            min_chip_accesses = m;
+         if (m > max_chip_accesses)
+            max_chip_accesses = m;
+         m = 0;
+         printf("\n");
+      }
+      printf("total reads: %d\n", k);
+      if (min_bank_accesses)
+         printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses);
+      else
+         printf("min_bank_accesses = 0!\n");
+      if (min_chip_accesses)
+         printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses);
+      else
+         printf("min_chip_accesses = 0!\n");
+
+      /*WRITE ACCESSES*/
+      k = 0;
+      l = 0;
+      m = 0;
+      max_bank_accesses = 0;
+      max_chip_accesses = 0;
+      min_bank_accesses = 0xFFFFFFFF;
+      min_chip_accesses = 0xFFFFFFFF;
+      printf("number of total write accesses:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            l = totalbankwrites[i][j];
+            if (l < min_bank_accesses)
+               min_bank_accesses = l;
+            if (l > max_bank_accesses)
+               max_bank_accesses = l;
+            k += l;
+            m += l;
+            printf("%9d ",l);
+         }
+         if (m < min_chip_accesses)
+            min_chip_accesses = m;
+         if (m > max_chip_accesses)
+            max_chip_accesses = m;
+         m = 0;
+         printf("\n");
+      }
+      printf("total reads: %d\n", k);
+      if (min_bank_accesses)
+         printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses);
+      else
+         printf("min_bank_accesses = 0!\n");
+      if (min_chip_accesses)
+         printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses);
+      else
+         printf("min_chip_accesses = 0!\n");
+
+
+      /*AVERAGE MF LATENCY PER BANK*/
+      printf("average mf latency per bank:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            k = totalbankwrites[i][j] + totalbankreads[i][j];
+            if (k)
+               printf("%10lld", mf_total_lat_table[i][j] / k);
+            else
+               printf("    none  ");
+         }
+         printf("\n");
+      }
+
+      /*MAXIMUM MF LATENCY PER BANK*/
+      printf("maximum mf latency per bank:\n");
+      for (i=0;i<gpu_n_mem ;i++ ) {
+         printf("dram[%d]: ", i);
+         for (j=0;j<4 ;j++ ) {
+            printf("%10d", mf_max_lat_table[i][j]);
+         }
+         printf("\n");
+      }
+   }
+
+   if (gpgpu_memlatency_stat & GPU_MEMLATSTAT_MC) {
+      printf("\nNumber of Memory Banks Accessed per Memory Operation per Warp (from 0):\n");
+      unsigned long long accum_MCBs_accessed = 0;
+      unsigned long long tot_mem_ops_per_warp = 0;
+      for (i=0;i<= gpu_n_mem*4 ; i++ ) {
+         accum_MCBs_accessed += i*num_MCBs_accessed[i];
+         tot_mem_ops_per_warp += num_MCBs_accessed[i];
+         printf("%d\t", num_MCBs_accessed[i]);
+      }
+
+      printf("\nAverage # of Memory Banks Accessed per Memory Operation per Warp=%f\n", (float)accum_MCBs_accessed/tot_mem_ops_per_warp);
+
+      //printf("\nAverage Difference Between First and Last Response from Memory System per warp = ");
+
+
+      printf("\nposition of mrq chosen\n");
+
+      if (!gpgpu_dram_sched_queue_size)
+         j = 1024;
+      else
+         j = gpgpu_dram_sched_queue_size;
+      k=0;l=0;
+      for (i=0;i< j; i++ ) {
+         printf("%d\t", position_of_mrq_chosen[i]);
+         k += position_of_mrq_chosen[i];
+         l += i*position_of_mrq_chosen[i];
+      }
+      printf("\n");
+      printf("\naverage position of mrq chosen = %f\n", (float)l/k);
+   }
+}
+
+#endif /*MEM_LATENCY_STAT_H*/
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
new file mode 100644
index 0000000..8f3f700
--- /dev/null
+++ b/src/gpgpu-sim/shader.cc
@@ -0,0 +1,3811 @@
+/* 
+ * shader.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan, Ivan Sham, Henry Wong, Dan O'Connor, Henry Tran and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <float.h>
+#include "shader.h"
+#include "gpu-sim.h"
+#include "addrdec.h"
+#include "dram.h"
+#include "dwf.h"
+#include "warp_tracker.h"
+#include "cflogger.h"
+#include "gpu-misc.h"
+#include "../cuda-sim/ptx_sim.h"
+#include "../cuda-sim/ptx-stats.h"
+#include "../cuda-sim/dram_callback.h"
+#include "mem_fetch.h"
+#include <string.h>
+
+#define PRIORITIZE_MSHR_OVER_WB 1
+#define MAX(a,b) (((a)>(b))?(a):(b))
+
+extern unsigned int mergemiss;
+extern unsigned int L1_write_miss;
+extern unsigned int L1_read_miss;
+extern unsigned int L1_write_hit_on_miss;
+extern unsigned int L1_writeback;
+extern unsigned int L1_texture_miss;
+extern unsigned int L1_const_miss;
+
+extern unsigned int finished_trace;
+extern int gpgpu_perfect_mem;
+extern int gpgpu_no_dl1;
+extern char *gpgpu_cache_texl1_opt;
+extern char *gpgpu_cache_constl1_opt;
+extern char *gpgpu_cache_dl1_opt;
+extern unsigned int gpu_n_thread_per_shader;
+extern unsigned int gpu_n_mshr_per_shader;
+extern unsigned int gpu_n_shader;
+extern unsigned int gpu_n_mem;
+extern int gpgpu_reg_bankconflict;
+extern int gpgpu_dram_sched_queue_size;
+extern int gpgpu_memlatency_stat;
+extern dram_t **dram;
+extern int *num_warps_issuable;
+extern int *num_warps_issuable_pershader;
+
+extern unsigned long long  gpu_sim_insn;
+extern unsigned long long  gpu_sim_insn_no_ld_const;
+extern unsigned long long  gpu_sim_insn_last_update;
+extern unsigned long long  gpu_completed_thread;
+extern unsigned long long  gpu_sim_cycle;
+extern shader_core_ctx_t **sc;
+extern unsigned int gpgpu_pre_mem_stages;
+extern int gpgpu_no_divg_load;
+extern unsigned int gpgpu_thread_swizzling;
+extern unsigned int gpgpu_strict_simd_wrbk;
+extern unsigned int warp_conflict_at_writeback;
+extern unsigned int gpgpu_commit_pc_beyond_two;
+extern int gpgpu_spread_blocks_across_cores;
+extern int gpgpu_cflog_interval;
+
+extern unsigned int gpu_stall_by_MSHRwb;
+extern unsigned int gpu_stall_shd_mem;
+extern unsigned int gpu_stall_sh2icnt;
+
+enum mem_stage_access_type {
+   C_MEM,
+   T_MEM,
+   S_MEM,
+   G_MEM_LD,
+   L_MEM_LD,
+   G_MEM_ST,
+   L_MEM_ST,
+   N_MEM_STAGE_ACCESS_TYPE
+};
+
+enum mem_stage_stall_type {
+   NO_RC_FAIL = 0, 
+   BK_CONF,
+   MSHR_RC_FAIL,
+   ICNT_RC_FAIL,
+   COAL_STALL,
+   WB_ICNT_RC_FAIL,
+   WB_CACHE_RSRV_FAIL,
+   N_MEM_STAGE_STALL_TYPE
+};
+unsigned int gpu_stall_shd_mem_breakdown[N_MEM_STAGE_ACCESS_TYPE][N_MEM_STAGE_STALL_TYPE] = { {0} };
+
+unsigned warp_size = 4; 
+int pipe_simd_width;
+extern unsigned int **totalbankaccesses; //bankaccesses[shader id][dram chip id][bank id]
+extern unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp
+extern unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache
+extern unsigned int *max_return_queue_length;
+
+unsigned int *shader_cycle_distro;
+
+unsigned int g_waiting_at_barrier = 0;
+
+unsigned int gpgpu_shmem_size = 16384;
+unsigned int gpgpu_shader_registers = 8192;
+unsigned int gpgpu_shader_cta = 8;
+
+unsigned int gpgpu_n_load_insn = 0;
+unsigned int gpgpu_n_store_insn = 0;
+unsigned int gpgpu_n_shmem_insn = 0;
+unsigned int gpgpu_n_tex_insn = 0;
+unsigned int gpgpu_n_const_insn = 0;
+unsigned int gpgpu_n_param_insn = 0;
+unsigned int gpgpu_multi_unq_fetches = 0;
+
+extern int gpgpu_cache_wt_through;
+
+int gpgpu_shmem_bkconflict = 0;
+unsigned int gpgpu_n_shmem_bkconflict = 0;
+int gpgpu_n_shmem_bank = 16;
+
+int gpgpu_cache_bkconflict = 0;
+unsigned int gpgpu_n_cache_bkconflict = 0;
+unsigned int gpgpu_n_cmem_portconflict = 0;
+int gpgpu_n_cache_bank = 16;
+
+extern int gpu_runtime_stat_flag;
+int gpgpu_warpdistro_shader = -1;
+
+int gpgpu_interwarp_mshr_merge = 0;
+int gpgpu_n_intrawarp_mshr_merge = 0;
+
+extern int gpgpu_partial_write_mask;
+int gpgpu_n_partial_writes = 0;
+
+extern int gpgpu_n_mem_write_local;
+extern int gpgpu_n_mem_write_global;
+
+#ifndef MhZ
+   #define MhZ *1000000
+#endif
+extern double core_freq;
+extern double icnt_freq;
+extern double dram_freq;
+extern double l2_freq;
+
+int gpgpu_shmem_port_per_bank = 4;
+int gpgpu_cache_port_per_bank = 4;
+int gpgpu_const_port_per_bank = 4;
+int gpgpu_shmem_pipe_speedup = 2;
+
+unsigned int gpu_max_cta_per_shader = 8;
+unsigned int gpu_padded_cta_size = 32;
+int gpgpu_local_mem_map = 1;
+
+
+extern int pdom_sched_type;
+extern int n_pdom_sc_orig_stat;
+extern int n_pdom_sc_single_stat;
+extern int gpgpu_cuda_sim;
+extern unsigned long long  gpu_tot_sim_cycle;
+
+extern unsigned g_max_regs_per_thread;
+void ptx_decode_inst( void *thd, unsigned *op, int *i1, int *i2, int *i3, int *i4, int *o1, int *o2, int *o3, int *o4, int *vectorin, int *vectorout, int *arch_reg  );
+unsigned ptx_get_inst_op( void *thd);
+void ptx_exec_inst( void *thd, address_type *addr, unsigned *space, unsigned *data_size, dram_callback_t* callback, unsigned warp_active_mask);
+int  ptx_branch_taken( void *thd );
+void ptx_sim_free_sm( void** thread_info );
+unsigned ptx_sim_init_thread( void** thread_info, int sid, unsigned tid,unsigned threads_left,unsigned num_threads, core_t *core, unsigned hw_cta_id, unsigned hw_warp_id);
+unsigned ptx_sim_cta_size();
+const struct gpgpu_ptx_sim_kernel_info* ptx_sim_kernel_info();
+void set_option_gpgpu_spread_blocks_across_cores(int option);
+int ptx_thread_done( void *thr );
+unsigned ptx_thread_donecycle( void *thr );
+int ptx_thread_get_next_pc( void *thd );
+void* ptx_thread_get_next_finfo( void *thd );
+int ptx_thread_at_barrier( void *thd );
+int ptx_thread_all_at_barrier( void *thd );
+unsigned long long ptx_thread_get_cta_uid( void *thd );
+void ptx_thread_reset_barrier( void *thd );
+void ptx_thread_release_barrier( void *thd );
+void ptx_print_insn( address_type pc, FILE *fp );
+int ptx_set_tex_cache_linesize( unsigned linesize);
+void time_vector_update(unsigned int uid,int slot ,long int cycle,int type);
+
+
+
+/////////////////////////////////////////////////////////////////////////////
+/*-------------------------------------------------------------------------*/
+/*-------------------------------------------------------------------------*/
+
+static const char* MSHR_Status_str[] = {
+   "INITIALIZED",
+   "IN_ICNT2MEM",
+   "IN_ICNTOL2QUEUE",
+   "IN_L2TODRAMQUEUE",
+   "IN_DRAM_REQ_QUEUE",
+   "IN_DRAMTOL2QUEUE",
+   "IN_L2TOICNTQUEUE_HIT",
+   "IN_L2TOICNTQUEUE_MISS",
+   "IN_ICNT2SHADER",
+   "FETCHED",
+};
+
+// a helper function that deduce if a mshr contains an atomic operation 
+bool isatomic(mshr_entry_t *mshr)
+{
+   return (mshr->insts[0].callback.function != NULL);
+}
+
+#include <map>
+#include <utility>
+#include <algorithm>
+// a class that speeds up mshr lookup with a C++ multimap 
+class mshr_lookup {
+private:
+   typedef std::multimap<unsigned long long int, mshr_entry*> mshr_lut_t;
+   mshr_lut_t m_lut; // multiple mshr entries can have the same tag
+private:
+   void insert(mshr_entry* mshr) 
+   {
+      using namespace std;
+      unsigned long long int tag_addr = mshr->addr;
+      m_lut.insert(make_pair(tag_addr, mshr));
+   }
+
+   mshr_entry* lookup(unsigned long long int addr) const 
+   {
+      using namespace std;
+      // mshr_lut_t::const_iterator i_lut = m_lut.find(tag_addr);
+      pair<mshr_lut_t::const_iterator, mshr_lut_t::const_iterator> i_range = m_lut.equal_range(addr);
+      if (i_range.first == i_range.second) {
+         return NULL;
+      } else {
+         mshr_lut_t::const_iterator i_lut = i_range.first; 
+         mshr_entry* mshr_hit = i_lut->second;
+         //follow match to end of merge chain:
+         //this won't really work for different sized requests, ie can't merge a 64b request to a 32b
+         while (mshr_hit->merged_requests) {
+            mshr_hit = mshr_hit->merged_requests;
+         }
+         return mshr_hit;
+      }
+   }     
+
+   void remove(mshr_entry* mshr)
+   {
+      using namespace std;
+      std::pair<mshr_lut_t::iterator, mshr_lut_t::iterator> i_range = m_lut.equal_range(mshr->addr);
+
+      assert(i_range.first != i_range.second);
+
+      for (mshr_lut_t::iterator i_lut = i_range.first; i_lut != i_range.second; ++i_lut) {
+         if (i_lut->second == mshr) {
+            m_lut.erase(i_lut);
+            break;
+         }
+      }
+   }
+public:
+   //checks if we should do mshr merging for this mshr
+   bool can_merge(mshr_entry_t * mshr)
+   {
+      if (mshr->iswrite) return false; // can't merge a write
+      if (isatomic(mshr)) return false; // can't merge a atomic operation
+      bool interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & GLOBAL_MSHR_MERGE;
+      if (mshr->istexture) {
+         interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & TEX_MSHR_MERGE;
+      } else if (mshr->isconst) {
+         interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & CONST_MSHR_MERGE;
+      }
+      return interwarp_mshr_merge;
+   }
+
+   void mshr_fast_lookup_insert(mshr_entry* mshr) 
+   {        
+      if (!can_merge(mshr)) return;  
+      insert(mshr);
+   }
+   
+   void mshr_fast_lookup_remove(mshr_entry* mshr) 
+   {
+      if (!can_merge(mshr)) return;  
+      remove(mshr);
+   }
+   
+   mshr_entry* shader_get_mergeable_mshr(mshr_entry_t* mshr)
+   {
+      if (!can_merge(mshr)) return NULL;
+      return lookup(mshr->addr);
+   }
+};
+
+class mem_access_t;
+int is_tex ( int space );
+int is_const ( int space );
+int is_local ( int space );
+#include <iostream>
+class mshr_shader_unit{
+public:
+   mshr_shader_unit(unsigned max_mshr): m_max_mshr(max_mshr), m_max_mshr_used(0){
+      m_mshrs.resize(max_mshr);
+      for (std::vector<mshr_entry_t>::iterator i = m_mshrs.begin(); i != m_mshrs.end(); i++) m_free_list.push_back(i);
+   }
+   bool has_mshr(unsigned num){return (num <= m_free_list.size());}
+   unsigned mshr_used(){ return m_max_mshr - m_free_list.size();}
+   mshr_entry_t* add_mshr(mem_access_t &access, inst_t* warp);
+
+   //return queue access; (includes texture pipeline return)
+   mshr_entry_t* return_head(){
+      if (has_return()) 
+         return &(*(choose_return_queue().front())); 
+      else 
+         return NULL;
+      }
+   //return queue pop; (includes texture pipeline return)
+   void pop_return_head() {
+      free_mshr(return_head()->this_mshr);
+      choose_return_queue().pop_front(); 
+   }
+
+   static void mshr_update_status(mshr_entry *mshr, enum mshr_status new_status );
+   void mshr_return_from_mem(mshr_entry *mshr);
+   void check_mshr(mshr_entry *mshr){
+      assert(find(m_free_list.begin(),m_free_list.end(),mshr->this_mshr)==m_free_list.end());
+      assert(mshr->insts.size());
+   }
+   unsigned get_max_mshr_used(){return m_max_mshr_used;}  
+   void print(FILE* fp, shader_core_ctx_t* shader);
+private:
+   typedef std::vector<mshr_entry_t> mshr_storage_type;//list might be less complicated, but slower?
+   mshr_storage_type m_mshrs; 
+   std::deque< mshr_storage_type::iterator > m_free_list;
+   std::deque< mshr_storage_type::iterator > m_mshr_return_queue;
+   std::deque< mshr_storage_type::iterator > m_texture_mshr_pipeline;
+   unsigned m_max_mshr;
+   unsigned m_max_mshr_used;
+   mshr_lookup m_mshr_lookup;
+
+   mshr_entry_t *alloc_free_mshr(bool istexture){
+      assert(!m_free_list.empty());
+      std::vector<mshr_entry_t>::iterator i = m_free_list.back();
+      m_free_list.pop_back();
+      i->this_mshr = i;
+      if (istexture) {
+         //put in texture pipeline
+         m_texture_mshr_pipeline.push_back(i);
+      }
+      if (mshr_used() > m_max_mshr_used) m_max_mshr_used = mshr_used();
+      return &(*i);
+   }
+   void free_mshr(std::vector<mshr_entry_t>::iterator &i){
+      //clean up up for next time, since not reallocating memory.
+      m_mshr_lookup.mshr_fast_lookup_remove(&(*i)); //need to remove before clearing insts, as they are accessed
+      i->insts.clear(); //add expects this to be clear
+      m_free_list.push_back(i);
+   }
+   bool has_return() { return (not m_mshr_return_queue.empty()) or ((not m_texture_mshr_pipeline.empty()) and m_texture_mshr_pipeline.front()->fetched());}
+   std::deque< std::vector<mshr_entry_t>::iterator > & choose_return_queue() {
+      //prioritize a ready texture over a global/const...
+      if ((not m_texture_mshr_pipeline.empty()) and m_texture_mshr_pipeline.front()->fetched()) return m_texture_mshr_pipeline;
+      assert(!m_mshr_return_queue.empty());
+      return m_mshr_return_queue;
+   }
+};
+
+
+
+void mshr_shader_unit::mshr_update_status(mshr_entry *mshr, enum mshr_status new_status ) {
+   mshr->status = new_status;
+#if DEBUGL1MISS 
+   printf("cycle %d Addr %x  %d \n",gpu_sim_cycle,CACHE_TAG_OF_64(mshr->addr),new_status);
+#endif
+   mshr_entry * merged_req = mshr->merged_requests;
+   while (merged_req) {
+      merged_req->status = new_status;
+      merged_req = merged_req->merged_requests;
+   }
+}
+
+inline void mshr_shader_unit::mshr_return_from_mem(mshr_entry *mshr){
+   mshr_update_status(mshr, FETCHED);
+   if (not mshr->istexture) {
+       //place in return queue
+       m_mshr_return_queue.push_back(mshr->this_mshr);
+       //place all merged requests in return queue
+       mshr_entry * merged_req = mshr->merged_requests;
+       while (merged_req) {
+          m_mshr_return_queue.push_back(merged_req->this_mshr);
+          merged_req = merged_req->merged_requests;
+       } 
+   }
+}
+
+void mshr_return_from_mem(shader_core_ctx_t * shader, mshr_entry_t* mshr){
+    shader->mshr_unit->mshr_return_from_mem(mshr);
+}
+
+unsigned get_max_mshr_used(shader_core_ctx_t * shader){
+   return shader->mshr_unit->get_max_mshr_used();
+}
+
+
+void mshr_print(FILE* fp, shader_core_ctx_t *shader) {
+   shader->mshr_unit->print(fp, shader);
+}
+
+void mshr_shader_unit::print(FILE* fp, shader_core_ctx_t* shader){
+   for (mshr_storage_type::iterator it = m_mshrs.begin(); it != m_mshrs.end(); it++) {
+      //valid if not in free list;
+      if (find(m_free_list.begin(),m_free_list.end(), it) == m_free_list.end()) {
+         mshr_entry *mshr = &(*it);
+         fprintf(fp, "MSHR(%d): %s Addr:0x%llx Fetched:%d Merged:%d Status:%s\n", 
+		 shader->sid, 
+                 (mshr->iswrite)? "=>" : "<=",
+                 mshr->addr, mshr->fetched(), 
+                 (mshr->merged_requests != NULL or mshr->merged_on_other_reqest), MSHR_Status_str[mshr->status]);
+         for (unsigned i = 0; i < mshr->insts.size(); i++) {
+            fprintf(fp,"\tthread: UID:%d HW:%d ReqAddr:0x%llx\n", mshr->insts[i].uid, mshr->insts[i].hw_thread_id, mshr->insts[i].memreqaddr);
+         }
+      }
+   }
+}
+
+void mshr_update_status(mshr_entry* mshr, enum mshr_status new_status) {
+   mshr_entry *merged_req;
+   mshr->status = new_status;
+#if DEBUGL1MISS 
+   printf("cycle %d Addr %x  %d \n",gpu_sim_cycle,CACHE_TAG_OF_64(mshr->addr),new_status);
+#endif
+   merged_req = mshr->merged_requests;
+   while (merged_req) {
+      merged_req->status = new_status;
+      merged_req = merged_req->merged_requests;
+   }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////
+/*-------------------------------------------------------------------------*/
+
+inst_t create_nop_inst() // just because C++ does not have designated initializer list....
+{
+   inst_t nop_inst;
+   nop_inst.pc = 0;
+   nop_inst.op=NO_OP; 
+   memset(nop_inst.out, 0, sizeof(nop_inst.out)); 
+   memset(nop_inst.in, 0, sizeof(nop_inst.in)); 
+   nop_inst.is_vectorin=0; 
+   nop_inst.is_vectorout=0;
+   nop_inst.memreqaddr=0; 
+   nop_inst.reg_bank_access_pending=0;
+   nop_inst.reg_bank_conflict_stall_checked=0,
+   nop_inst.hw_thread_id=-1; 
+   nop_inst.wlane=-1;
+   nop_inst.uid = (unsigned)-1;
+   nop_inst.priority = (unsigned)-1;
+   nop_inst.ptx_thd_info = NULL;
+   nop_inst.warp_active_mask = 0;
+   nop_inst.ts_cycle = 0;
+   nop_inst.id_cycle = 0;
+   nop_inst.ex_cycle = 0;
+   nop_inst.mm_cycle = 0;
+   return nop_inst;
+}
+
+static inst_t nop_inst = create_nop_inst();
+
+int log2i(int n) {
+   int lg;
+   lg = -1;
+   while (n) {
+      n>>=1;lg++;
+   }
+   return lg;
+}
+
+extern unsigned int gpu_n_warp_per_shader;
+
+shader_core_ctx_t* shader_create( const char *name, int sid,
+                                  unsigned int n_threads,
+                                  unsigned int n_mshr,
+                                  fq_push_t fq_push,
+                                  fq_has_buffer_t fq_has_buffer,
+                                  unsigned int model )
+{
+   shader_core_ctx_t *sc;
+   int i;
+   unsigned int shd_n_set;
+   unsigned int shd_linesize;
+   unsigned int shd_n_assoc;
+   unsigned char shd_policy;
+
+   unsigned int l1tex_cache_n_set; //L1 texture cache parameters
+   unsigned int l1tex_cache_linesize;
+   unsigned int l1tex_cache_n_assoc;
+   unsigned char l1tex_cache_policy;
+
+   unsigned int l1const_cache_n_set; //L1 constant cache parameters
+   unsigned int l1const_cache_linesize;
+   unsigned int l1const_cache_n_assoc;
+   unsigned char l1const_cache_policy;
+
+   if ( gpgpu_cuda_sim ) {
+      unsigned cta_size = ptx_sim_cta_size();
+      if ( cta_size > n_threads ) {
+         printf("Execution error: Shader kernel CTA (block) size is too large for microarch config.\n");
+         printf("                 This can cause problems with applications that use __syncthreads.\n");
+         printf("                 CTA size (x*y*z) = %u, n_threads = %u\n", cta_size, n_threads );
+         printf("                 => either change -gpgpu_shader argument in gpgpusim.config file or\n");
+         printf("                 modify the CUDA source to decrease the kernel block size.\n");
+         abort();
+      }
+   }
+
+   sc = new shader_core_ctx_t( gpu_n_warp_per_shader, gpgpu_shader_cta );
+
+   sc->name = name;
+   sc->sid = sid;
+
+   sc->RR_k = 0;
+
+   sc->model = model;
+
+   sc->pipeline_reg = (inst_t**) calloc(N_PIPELINE_STAGES, sizeof(inst_t*));
+   for (int j = 0; j<N_PIPELINE_STAGES; j++) {
+      sc->pipeline_reg[j] = (inst_t*) calloc(warp_size, sizeof(inst_t));
+      for (unsigned i=0; i<warp_size; i++) {
+         sc->pipeline_reg[j][i] = nop_inst;
+      }
+   }
+
+   if (gpgpu_pre_mem_stages) {
+      sc->pre_mem_pipeline = (inst_t**) calloc(gpgpu_pre_mem_stages+1, sizeof(inst_t*));
+      for (unsigned j = 0; j<=gpgpu_pre_mem_stages; j++) {
+         sc->pre_mem_pipeline[j] = (inst_t*) calloc(pipe_simd_width, sizeof(inst_t));
+         for (int i=0; i<pipe_simd_width; i++) {
+            sc->pre_mem_pipeline[j][i] = nop_inst;
+         }
+      }
+   }
+   sc->n_threads = n_threads;
+   sc->thread = (thread_ctx_t*) calloc(sizeof(thread_ctx_t), n_threads);
+   sc->not_completed = 0;
+
+   unsigned n_warp = (n_threads/warp_size) + ((n_threads%warp_size)?1:0);
+   sc->warp.resize(n_warp, shd_warp_t(warp_size));
+   for (unsigned j = 0; j < n_warp; j++) {
+      sc->warp[j].wid = j;
+   }
+
+   sc->n_active_cta = 0;
+   for (i = 0; i<MAX_CTA_PER_SHADER; i++  ) {
+      sc->cta_status[i]=0;
+   }
+   //Warp variable initializations
+   sc->next_warp = 0;
+   sc->branch_priority = 0;
+   sc->max_branch_priority = (int*) malloc(sizeof(int)*n_threads);
+
+
+   for (unsigned i = 0; i<n_threads; i++) {
+      sc->max_branch_priority[i] = INT_MAX;
+      sc->thread[i].id = i;
+
+      sc->thread[i].warp_priority = sc->max_branch_priority[i];
+      sc->thread[i].avail4fetch = 0;
+      sc->thread[i].m_waiting_at_barrier = 0;
+
+      sc->thread[i].ptx_thd_info = NULL;
+      sc->thread[i].cta_id = -1;
+   }
+
+   sscanf(gpgpu_cache_dl1_opt,"%d:%d:%d:%c", 
+          &shd_n_set, &shd_linesize, &shd_n_assoc, &shd_policy);
+   sscanf(gpgpu_cache_texl1_opt,"%d:%d:%d:%c", 
+          &l1tex_cache_n_set, &l1tex_cache_linesize, &l1tex_cache_n_assoc, &l1tex_cache_policy);
+   sscanf(gpgpu_cache_constl1_opt,"%d:%d:%d:%c", 
+          &l1const_cache_n_set, &l1const_cache_linesize, &l1const_cache_n_assoc, &l1const_cache_policy);
+#define STRSIZE 32
+   char L1c_name[STRSIZE];
+   char L1texc_name[STRSIZE];
+   char L1constc_name[STRSIZE];
+   snprintf(L1c_name, STRSIZE, "L1c_%03d", sc->sid);
+   sc->L1cache = shd_cache_create(L1c_name,shd_n_set,shd_n_assoc,shd_linesize,shd_policy,1,0, 
+				  gpgpu_cache_wt_through?write_through:write_back);
+   shd_cache_bind_logger(sc->L1cache, sc->sid, get_shader_normal_cache_id());
+   snprintf(L1texc_name, STRSIZE, "L1texc_%03d", sc->sid);
+   sc->L1texcache = shd_cache_create(L1texc_name,l1tex_cache_n_set,l1tex_cache_n_assoc,l1tex_cache_linesize,l1tex_cache_policy,1,0, no_writes );
+   shd_cache_bind_logger(sc->L1texcache, sc->sid, get_shader_texture_cache_id());
+   snprintf(L1constc_name, STRSIZE, "L1constc_%03d", sc->sid);
+   sc->L1constcache = shd_cache_create(L1constc_name,l1const_cache_n_set,l1const_cache_n_assoc,l1const_cache_linesize,l1const_cache_policy,1,0, no_writes );
+   shd_cache_bind_logger(sc->L1constcache, sc->sid, get_shader_constant_cache_id());
+   //at this point, should set the parameters used by addressing schemes of all textures
+   ptx_set_tex_cache_linesize(l1tex_cache_linesize);
+
+   sc->mshr_unit = new mshr_shader_unit(gpu_n_mshr_per_shader);
+
+   sc->fq_push = fq_push;
+   sc->fq_has_buffer = fq_has_buffer;
+
+   sc->pdom_warp = (pdom_warp_ctx_t*)calloc(n_threads / warp_size, sizeof(pdom_warp_ctx_t));
+   for (unsigned i = 0; i < n_threads / warp_size; ++i) {
+      sc->pdom_warp[i].m_stack_top = 0;
+      sc->pdom_warp[i].m_pc = (address_type*)calloc(warp_size * 2, sizeof(address_type));
+      sc->pdom_warp[i].m_calldepth = (unsigned int*)calloc(warp_size * 2, sizeof(unsigned int));
+      sc->pdom_warp[i].m_active_mask = (unsigned int*)calloc(warp_size * 2, sizeof(unsigned int));
+      sc->pdom_warp[i].m_recvg_pc = (address_type*)calloc(warp_size * 2, sizeof(address_type));
+      sc->pdom_warp[i].m_branch_div_cycle = (unsigned long long *)calloc(warp_size * 2, sizeof(unsigned long long ));
+
+      memset(sc->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type));
+      memset(sc->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int));
+      memset(sc->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int));
+      memset(sc->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type));
+   }
+
+   sc->waiting_at_barrier = 0;
+
+   sc->last_issued_thread = sc->n_threads - 1;
+
+   sc->using_dwf = (sc->model == DWF);
+
+   sc->using_rrstage = (sc->model == DWF);
+
+   sc->using_commit_queue = (sc->model == DWF
+                             || sc->model == POST_DOMINATOR || sc->model == NO_RECONVERGE);
+
+   if (sc->using_commit_queue) {
+      sc->thd_commit_queue = dq_create("thd_commit_queue", 0, 0, 0);
+   }
+
+   sc->shmem_size = gpgpu_shmem_size;
+   sc->n_registers = gpgpu_shader_registers;
+   sc->n_cta = gpgpu_shader_cta;
+
+   sc->shader_memory_new_instruction_processed = false;
+
+   return sc;
+}
+
+
+unsigned shader_reinit(shader_core_ctx_t *sc, int start_thread, int end_thread ) 
+{
+   int i;
+   unsigned result=0;
+
+   if ( gpgpu_cuda_sim ) {
+      unsigned cta_size = ptx_sim_cta_size();
+      if ( cta_size > sc->n_threads ) {
+         printf("Execution error: Shader kernel CTA (block) size is too large for microarch config.\n");
+         printf("                 This can cause problems with applications that use __syncthreads.\n");
+         printf("                 CTA size (x*y*z) = %u, n_threads = %u\n", cta_size, sc->n_threads );
+         printf("                 => either change -gpgpu_shader argument in gpgpusim.config file or\n");
+         printf("                 modify the CUDA source to decrease the kernel block size.\n");
+         abort();
+      }
+   }
+
+   sc->next_warp = 0;
+   sc->branch_priority = 0;
+
+   for (i = start_thread; i<end_thread; i++)
+      ptx_sim_free_sm(&sc->thread[i].ptx_thd_info);
+
+   for (i = start_thread; i<end_thread; i++) {
+      sc->max_branch_priority[i] = INT_MAX;
+      sc->thread[i].warp_priority = sc->max_branch_priority[i];
+      sc->thread[i].n_insn = 0;
+      sc->thread[i].cta_id = -1;
+   }
+
+   for (unsigned i = start_thread / warp_size; i < end_thread / warp_size; ++i) {
+      sc->warp[i].reset(warp_size);
+      sc->pdom_warp[i].m_stack_top = 0;
+      memset(sc->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type));
+      memset(sc->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int));
+      memset(sc->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int));
+      memset(sc->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type));
+      memset(sc->pdom_warp[i].m_branch_div_cycle, 0, warp_size * 2 * sizeof(unsigned long long ));
+   }
+
+   sc->waiting_at_barrier = 0;
+   sc->last_issued_thread = end_thread - 1; 
+
+   if (sc->using_commit_queue) {
+      if (!gpgpu_spread_blocks_across_cores) //assertion no longer holds with multiple blocks per core  
+         assert(dq_empty(sc->thd_commit_queue));
+   }
+   sc->pending_shmem_bkacc = 0;
+   sc->pending_cache_bkacc = 0;
+   sc->pending_cmem_acc = 0;
+
+   //do not reset this here, shader memory may be in the middle of processing another cta's instruction.
+   //sc->shader_memory_new_instruction_processed = false;
+
+   return result;
+}
+
+// initialize a CTA in the shader core, currently only useful for PDOM and DWF
+
+void shader_init_CTA(shader_core_ctx_t *shader, int start_thread, int end_thread)
+{
+   int i;
+   int n_thread = end_thread - start_thread;
+   address_type start_pc = ptx_thread_get_next_pc(shader->thread[start_thread].ptx_thd_info);
+   if (shader->model == POST_DOMINATOR) {
+      int start_warp = start_thread / warp_size;
+      int end_warp = end_thread / warp_size + ((end_thread % warp_size)? 1 : 0);
+      for (i = start_warp; i < end_warp; ++i) {
+         shader->pdom_warp[i].m_stack_top = 0;
+         memset(shader->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type));
+         memset(shader->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int));
+         memset(shader->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int));
+         memset(shader->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type));
+         memset(shader->pdom_warp[i].m_branch_div_cycle, 0, warp_size * 2 * sizeof(unsigned long long ));
+         shader->pdom_warp[i].m_pc[0] = start_pc;
+         shader->pdom_warp[i].m_calldepth[0] = 1;
+         int t = 0;
+         for (t = 0; t < (int)warp_size; t++) {
+            if ( i * (int)warp_size + t < end_thread ) {
+               shader->pdom_warp[i].m_active_mask[0] |= (1 << t);
+            }
+         }
+      }
+   } else if (shader->model == DWF) {
+      dwf_init_CTA(shader->sid, start_thread, n_thread, start_pc);
+   }
+
+   for (i = start_thread; i<end_thread; i++) {
+      shader->thread[i].in_scheduler = 1;
+   } 
+}
+
+
+
+
+// register id for unused register slot in instruction
+#define DNA       (0)
+
+unsigned g_next_shader_inst_uid=1;
+
+// check to see if the fetch stage need to be stalled
+int shader_fetch_stalled(shader_core_ctx_t *shader)
+{
+   int n_warp_parts = warp_size/pipe_simd_width;
+
+   if (shader->warp_part2issue < n_warp_parts) {
+      return 1;
+   }
+
+   for (unsigned i=0; i<warp_size; i++) {
+      if (shader->pipeline_reg[TS_IF][i].hw_thread_id != -1 ) {
+         return 1;  // stalled 
+      }
+   }
+   for (int i=0; i<pipe_simd_width; i++) {
+      if (shader->pipeline_reg[IF_ID][i].hw_thread_id != -1 ) {
+         return 1;  // stalled 
+      }
+   }
+
+   shader->warp_part2issue = 0; // reset pointer to first warp part
+   shader->new_warp_TS = 1;
+
+   return 0; // not stalled
+}
+
+// initalize the pipeline stage register to nops
+void shader_clear_stage_reg(shader_core_ctx_t *shader, int stage)
+{
+   for (unsigned i=0; i<warp_size; i++) {
+      shader->pipeline_reg[stage][i] = nop_inst;
+   }
+}
+
+// return the next pc of a thread 
+address_type shader_thread_nextpc(shader_core_ctx_t *shader, int tid)
+{
+   assert( gpgpu_cuda_sim );
+   address_type pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
+   return pc;
+}
+
+// issue thread to the warp 
+// tid - thread id, warp_id - used by PDOM, wlane - position in warp
+void shader_issue_thread(shader_core_ctx_t *shader, int tid, int wlane, unsigned active_mask )
+{
+   if ( gpgpu_cuda_sim ) {
+      shader->pipeline_reg[TS_IF][wlane].hw_thread_id = tid;
+      shader->pipeline_reg[TS_IF][wlane].wlane = wlane;
+      shader->pipeline_reg[TS_IF][wlane].pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
+      shader->pipeline_reg[TS_IF][wlane].ptx_thd_info = shader->thread[tid].ptx_thd_info;
+      shader->pipeline_reg[TS_IF][wlane].memreqaddr = 0;
+      shader->pipeline_reg[TS_IF][wlane].reg_bank_conflict_stall_checked = 0;
+      shader->pipeline_reg[TS_IF][wlane].reg_bank_access_pending = 0;
+      shader->pipeline_reg[TS_IF][wlane].uid = g_next_shader_inst_uid++;
+      shader->pipeline_reg[TS_IF][wlane].warp_active_mask = active_mask;
+      shader->pipeline_reg[TS_IF][wlane].ts_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+   }
+   assert( shader->thread[tid].avail4fetch > 0 );
+   shader->thread[tid].avail4fetch--;
+   assert( shader->warp[wid_from_hw_tid(tid,warp_size)].n_avail4fetch > 0 );
+   shader->warp[wid_from_hw_tid(tid,warp_size)].n_avail4fetch--;
+}
+
+void update_max_branch_priority(shader_core_ctx_t *shader, unsigned warp_hw_id, unsigned grid_num )
+{
+   int temp_max = 0;
+   // This means that a group of threads has completed,
+   // hence need to update max_priority
+   for (unsigned i = 0; i<warp_size; i++) {
+      if ( !ptx_thread_done( shader->thread[hw_tid_from_wid(warp_hw_id,warp_size,i)].ptx_thd_info ) ) {
+         if (shader->thread[hw_tid_from_wid(warp_hw_id,warp_size,i)].warp_priority>=temp_max) {
+            temp_max = shader->thread[hw_tid_from_wid(warp_hw_id,warp_size,i)].warp_priority;
+         }
+      }
+   }
+   for (unsigned i = 0; i<warp_size; i++) {
+      shader->max_branch_priority[hw_tid_from_wid(warp_hw_id,warp_size,i)] = temp_max;
+   }
+}
+
+void shader_fetch_simd_no_reconverge(shader_core_ctx_t *shader, unsigned int shader_number, int grid_num )
+{
+   int i;
+   int tid;
+   int new_tid = 0;
+   address_type pc = 0;
+   int warp_ok = 0;
+   int n_warp = shader->n_threads/warp_size;
+   int complete = 0;
+
+   assert(gpgpu_cuda_sim);
+
+   // First, check to see if entire program is completed, 
+   // if it is, then break out of loop
+   for (unsigned i=0; i<shader->n_threads; i++) {
+      if (!ptx_thread_done( shader->thread[i].ptx_thd_info )) {
+         complete = 0;
+         break;
+      } else {
+         complete = 1;
+      }
+   }
+   if (complete) {
+      // printf("Shader has completed program.\n");
+      return;
+   }
+
+   if (shader_fetch_stalled(shader)) {
+      return; 
+   }
+   shader_clear_stage_reg(shader, TS_IF);
+
+   // Finds a warp where all threads in it are available for fetching 
+   // simultaneously(all threads are not yet in pipeline, or, the ones 
+   // that are not available, are completed already
+   for (i=0; i<n_warp; i++) {
+      int n_completed = shader->warp[shader->next_warp].n_completed;
+      int n_avail4fetch = shader->warp[shader->next_warp].n_avail4fetch;
+      if (((n_completed) == (int)warp_size) ||
+          ((n_completed + n_avail4fetch) < (int)warp_size) ) {
+         //All threads in this warp have completed, hence go to next warp
+         //Or, some of the threads are still in pipeline
+         warp_ok = 0; // hey look, it's a silent register update / store instruction! (this operation is redundant)
+         shader->next_warp = (shader->next_warp+1)%n_warp;
+      } else {
+         int n_waiting_at_barrier = shader->warp[shader->next_warp].n_waiting_at_barrier;
+         if ( n_waiting_at_barrier >= (int)warp_size ) {
+            warp_ok = 0; // hey look, it's a silent register update / store instruction! (this operation is redundant)
+            continue;
+         }
+         warp_ok = 1;
+         break;
+      }
+   }
+   // None of the instructions from inside the warp can be scheduled -> should   
+   // probably just stall, ie nops into pipeline
+   if (!warp_ok) {
+      shader_clear_stage_reg(shader, TS_IF);  // NOTE: is this needed?
+      shader->next_warp = (shader->next_warp+1)%n_warp;  // NOTE: this is not round-robin.
+      return;
+   }
+
+   tid = warp_size*shader->next_warp;
+
+   for (i = 0; i<(int)warp_size; i++) {
+      if (shader->thread[tid+i].warp_priority == shader->max_branch_priority[tid+i]) {
+         pc = shader_thread_nextpc(shader, tid+i);
+         new_tid = tid+i;
+         break;
+      }
+   }
+   //Determine which instructions inside this 'warp' will be scheduled together at this run
+   //If they are cannot be scheduled together then 'save' their branch priority
+   for (i = 0; i<(int)warp_size; i++) {
+      if (!ptx_thread_done( shader->thread[tid+i].ptx_thd_info )) {
+         address_type next_pc;
+         next_pc = shader_thread_nextpc(shader, tid+i);
+         if (next_pc != pc ||
+             shader->thread[tid+i].warp_priority != shader->max_branch_priority[tid+i] ||
+             shader->thread[tid+i].m_waiting_at_barrier) {
+            if (!ptx_thread_done( shader->thread[tid+i].ptx_thd_info )) {
+               if ( !shader->thread[tid + i].m_waiting_at_barrier ) {
+                  shader->thread[tid + i].warp_priority = shader->branch_priority;
+               }
+            }
+         } else {
+            shader_issue_thread(shader, tid+i, i,(unsigned)-1); 
+         }
+      }
+   }
+   shader->branch_priority++;
+
+   shader->next_warp = (shader->next_warp+1)%n_warp;
+}
+
+int pdom_sched_find_next_warp (shader_core_ctx_t *shader,int pdom_sched_policy, int* ready_warps
+                               , int ready_warp_count, int* last_warp, int w_comp_c, int w_pipe_c, int w_barr_c)
+{
+   int n_warp = shader->n_threads/warp_size;
+   int i=0;
+   int selected_warp = ready_warps[0];
+   int found =0; 
+
+   switch (pdom_sched_policy) {
+   case 0: 
+      selected_warp = ready_warps[0]; //first ok warp found
+      found=1;
+      break;
+   case 1  ://random
+      selected_warp = ready_warps[rand()%ready_warp_count];
+      found=1;  
+      break;
+   case 8  :// execute the first available warp which is after the warp execued last time
+      found=0;
+      selected_warp = (last_warp[shader->sid] + 1 ) % n_warp;
+      while (!found) {
+         for (i=0;i<ready_warp_count;i++) {
+            if (selected_warp==ready_warps[i]) {
+               found=1;
+            }
+         }
+         if (!found)
+            selected_warp = (selected_warp + 1 ) % n_warp;
+      }
+      break;         
+   default:
+      assert(0);
+   }
+   if (found) {
+      if (ready_warp_count==1) {
+         n_pdom_sc_single_stat++;
+      } else {
+         n_pdom_sc_orig_stat++;
+      }
+      return selected_warp;
+   } else {
+      return -1;
+   }
+}
+
+void shader_fetch_simd_postdominator(shader_core_ctx_t *shader, unsigned int shader_number, int grid_num) {
+   int i;
+   int warp_ok = 0;
+   int n_warp = shader->n_threads/warp_size;
+   int complete = 0;
+   int tmp_warp;
+   int warp_id;
+
+   address_type check_pc = -1;
+
+   assert(gpgpu_cuda_sim);
+
+   // First, check to see if entire program is completed, 
+   // if it is, then break out of loop
+   for (unsigned i=0; i<shader->n_threads; i++) {
+      if (!ptx_thread_done( shader->thread[i].ptx_thd_info )) {
+         complete = 0;
+         break;
+      } else {
+         complete = 1;
+      }
+   }
+   if (complete) {
+      return;
+   }
+
+   if (shader_fetch_stalled(shader)) {
+      return; 
+   }
+   shader_clear_stage_reg(shader, TS_IF);
+
+   int ready_warp_count = 0;
+   int w_comp_c = 0 ;
+   int w_pipe_c = 0 ;
+   int w_barr_c = 0 ;
+   static int * ready_warps = NULL;
+   static int * tmp_ready_warps = NULL;
+   if (!ready_warps) {
+      ready_warps = (int*)calloc(n_warp,sizeof(int));
+   }
+   if (!tmp_ready_warps) {
+      tmp_ready_warps = (int*)calloc(n_warp,sizeof(int));
+   }
+   for (i=0; i<n_warp; i++) {
+      ready_warps[i]=-1;
+      tmp_ready_warps[i]=-1;
+   }
+
+   static int* last_warp; //keeps track of last warp issued per shader
+   if (!last_warp) {
+      last_warp = (int*)calloc(gpu_n_shader,sizeof(int));
+   }
+
+
+   // Finds a warp where all threads in it are available for fetching 
+   // simultaneously(all threads are not yet in pipeline, or, the ones 
+   // that are not available, are completed already
+   for (i=0; i<n_warp; i++) {
+      int n_completed = shader->warp[shader->next_warp].n_completed;
+      int n_avail4fetch = shader->warp[shader->next_warp].n_avail4fetch;
+
+      if ((n_completed) == (int)warp_size) {
+         //All threads in this warp have completed 
+         w_comp_c++;
+      } else if ((n_completed+n_avail4fetch) < (int)warp_size) {
+         //some of the threads are still in pipeline
+         w_pipe_c++;
+      } else if ( shader->warp_waiting_at_barrier(shader->next_warp) ) {
+         w_barr_c++;
+      } else {
+         // A valid warp is found at this point
+         tmp_ready_warps[ready_warp_count] = shader->next_warp;
+         ready_warp_count++;
+      }
+      shader->next_warp = (shader->next_warp + 1) % n_warp;
+   }
+   for (i=0;i<ready_warp_count;i++) {
+      ready_warps[i]=tmp_ready_warps[i];
+   }
+
+   num_warps_issuable[ready_warp_count]++;
+   num_warps_issuable_pershader[shader->sid]+= ready_warp_count;
+
+   if (ready_warp_count) {
+      tmp_warp = pdom_sched_find_next_warp (shader, pdom_sched_type ,ready_warps
+                                            , ready_warp_count, last_warp, w_comp_c, w_pipe_c ,w_barr_c);
+      if (tmp_warp != -1) {
+         shader->next_warp = tmp_warp;
+         warp_ok=1;  
+      }
+   }
+
+   static int no_warp_issued; 
+   // None of the instructions from inside the warp can be scheduled -> should  
+   // probably just stall, ie nops into pipeline
+   if (!warp_ok) {
+      shader_clear_stage_reg(shader, TS_IF);  
+      shader->next_warp = (shader->next_warp+1) % n_warp;  
+      no_warp_issued = 1 ; 
+      return;
+   }
+
+   /************************************************************/
+   //at this point we have a warp to execute which is pointed to by
+   //shader->next_warp
+
+   warp_id = shader->next_warp;
+   last_warp[shader->sid] = warp_id;
+   int wtid = warp_size*shader->next_warp;
+
+   pdom_warp_ctx_t *scheduled_warp = &(shader->pdom_warp[warp_id]);
+
+   int stack_top = scheduled_warp->m_stack_top;
+
+   address_type top_pc = scheduled_warp->m_pc[stack_top];
+   unsigned int top_active_mask = scheduled_warp->m_active_mask[stack_top];
+   address_type top_recvg_pc = scheduled_warp->m_recvg_pc[stack_top];
+
+   assert(top_active_mask != 0);
+
+   const address_type null_pc = 0;
+   int warp_diverged = 0;
+   address_type new_recvg_pc = null_pc;
+   while (top_active_mask != 0) {
+
+      // extract a group of threads with the same next PC among the active threads in the warp
+      address_type tmp_next_pc = null_pc;
+      unsigned int tmp_active_mask = 0;
+      void *first_active_thread=NULL;
+      for (i = warp_size - 1; i >= 0; i--) {
+         unsigned int mask = (1 << i);
+         if ((top_active_mask & mask) == mask) { // is this thread active?
+            if (ptx_thread_done( shader->thread[wtid+i].ptx_thd_info )) {
+               top_active_mask &= ~mask; // remove completed thread from active mask
+            } else if (tmp_next_pc == null_pc) {
+               first_active_thread = shader->thread[wtid+i].ptx_thd_info;
+               tmp_next_pc = shader_thread_nextpc(shader, wtid+i);
+               tmp_active_mask |= mask;
+               top_active_mask &= ~mask;
+            } else if (tmp_next_pc == shader_thread_nextpc(shader, wtid+i)) {
+               tmp_active_mask |= mask;
+               top_active_mask &= ~mask;
+            }
+         }
+      }
+
+      // discard the new entry if its PC matches with reconvergence PC
+      // that automatically reconverges the entry 
+      if (tmp_next_pc == top_recvg_pc) continue;
+
+      // this new entry is not converging
+      // if this entry does not include thread from the warp, divergence occurs
+      if (top_active_mask != 0 && warp_diverged == 0) {
+         warp_diverged = 1;
+         // modify the existing top entry into a reconvergence entry in the pdom stack
+         new_recvg_pc = get_converge_point(top_pc,first_active_thread);
+         if (new_recvg_pc != top_recvg_pc) {
+            scheduled_warp->m_pc[stack_top] = new_recvg_pc;
+            scheduled_warp->m_branch_div_cycle[stack_top] = gpu_sim_cycle;
+            stack_top += 1;
+            scheduled_warp->m_branch_div_cycle[stack_top] = 0;
+         }
+      }
+
+      // discard the new entry if its PC matches with reconvergence PC
+      if (warp_diverged && tmp_next_pc == new_recvg_pc) continue;
+
+      // update the current top of pdom stack
+      scheduled_warp->m_pc[stack_top] = tmp_next_pc;
+      scheduled_warp->m_active_mask[stack_top] = tmp_active_mask;
+      if (warp_diverged) {
+         scheduled_warp->m_calldepth[stack_top] = 0;
+         scheduled_warp->m_recvg_pc[stack_top] = new_recvg_pc;
+      } else {
+         scheduled_warp->m_recvg_pc[stack_top] = top_recvg_pc;
+      }
+      stack_top += 1; // set top to next entry in the pdom stack
+   }
+   scheduled_warp->m_stack_top = stack_top - 1;
+
+   assert(scheduled_warp->m_stack_top >= 0);
+   assert(scheduled_warp->m_stack_top < (int)warp_size * 2);
+
+   // schedule threads according to active mask on the top of pdom stack
+   for (i = 0; i < (int)warp_size; i++) {
+      unsigned int mask = (1 << i);
+      if ((scheduled_warp->m_active_mask[scheduled_warp->m_stack_top] & mask) == mask) {
+         assert (!ptx_thread_done( shader->thread[wtid+i].ptx_thd_info ));
+         shader_issue_thread(shader, wtid+i, i, scheduled_warp->m_active_mask[scheduled_warp->m_stack_top]);
+      }
+   }
+   shader->next_warp = (shader->next_warp+1)%n_warp;
+
+   // check if all issued threads have the same pc
+   for (i = 0; i < (int) warp_size; i++) {
+      if ( shader->pipeline_reg[TS_IF][i].hw_thread_id != -1 ) {
+         if ( check_pc == (unsigned)-1 ) {
+            check_pc = shader->pipeline_reg[TS_IF][i].pc;
+         } else {
+            assert( check_pc == shader->pipeline_reg[TS_IF][i].pc );
+         }
+      }
+   }
+}
+
+
+void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc )
+{
+   unsigned warp_id = tid/warp_size;
+   pdom_warp_ctx_t *warp_info = &(sc[sid]->pdom_warp[warp_id]);
+   unsigned idx = warp_info->m_stack_top;
+   *pc = warp_info->m_pc[idx];
+   *rpc = warp_info->m_recvg_pc[idx];
+}
+
+void shader_fetch_mimd( shader_core_ctx_t *shader, unsigned int shader_number ) 
+{
+   unsigned int last_issued_thread = 0;
+
+   if (shader_fetch_stalled(shader)) {
+      return; 
+   }
+   shader_clear_stage_reg(shader, TS_IF);
+
+   // some form of barrel processing: 
+   // - checking availability from the thread after the last issued thread
+   for (int i=0, j=0;i<(int)shader->n_threads && j< (int) warp_size;i++) {
+      int thd_id = (i + shader->last_issued_thread + 1) % shader->n_threads;
+      if (shader->thread[thd_id].avail4fetch && !shader->thread[thd_id].m_waiting_at_barrier ) {
+         shader_issue_thread(shader, thd_id, j,(unsigned)-1);
+         last_issued_thread = thd_id;
+         j++;
+      }
+   }
+   shader->last_issued_thread = last_issued_thread;
+}
+
+// seperate the incoming warp into multiple warps with seperate pcs
+int split_warp_by_pc(int *tid_in, shader_core_ctx_t *shader, int **tid_split, address_type *pc) {
+   unsigned n_pc = 0;
+   static int *pc_cnt = NULL; // count the number of threads with the same pc
+
+   assert(tid_in);
+   assert(tid_split);
+   assert(pc);
+   memset(pc,0,sizeof(address_type)*warp_size);
+
+   if (!pc_cnt) pc_cnt = (int*) malloc(sizeof(int)*warp_size);
+   memset(pc_cnt,0,sizeof(int)*warp_size);
+
+   // go through each thread in the given warp
+   for (unsigned i=0; i< warp_size; i++) {
+      if (tid_in[i] < 0) continue;
+      int matched = 0;
+      address_type thd_pc;
+      thd_pc = shader_thread_nextpc(shader, tid_in[i]);
+
+      // check to see if the pc has occured before
+      for (unsigned j=0; j<n_pc; j++) {
+         if (thd_pc == pc[j]) {
+            tid_split[j][pc_cnt[j]] = tid_in[i];
+            pc_cnt[j]++;
+            matched = 1;
+            break;
+         }
+      }
+      // if not, put the tid in a seperate warp
+      if (!matched) {
+         assert(n_pc < warp_size);
+         tid_split[n_pc][0] = tid_in[i];
+         pc[n_pc] = thd_pc;
+         pc_cnt[n_pc] = 1;
+         n_pc++;
+      }
+   }
+   return n_pc;
+}
+
+// see if this warp just executed the barrier instruction 
+int warp_reached_barrier(int *tid_in, shader_core_ctx_t *shader)
+{
+   int reached_barrier = 0;
+   for (unsigned i=0; i<warp_size; i++) {
+      if (tid_in[i] < 0) continue;
+      if (shader->thread[tid_in[i]].m_reached_barrier) {
+         reached_barrier = 1;
+         break;
+      }
+   }
+   return reached_barrier;
+}
+
+// seperate the incoming warp into multiple warps with seperate pcs and cta
+int split_warp_by_cta(int *tid_in, shader_core_ctx_t *shader, int **tid_split, address_type *pc, int *cta) {
+   unsigned n_pc = 0;
+   static int *pc_cnt = NULL; // count the number of threads with the same pc
+
+   assert(tid_in);
+   assert(tid_split);
+   assert(pc);
+   memset(pc,0,sizeof(address_type)*warp_size);
+
+   if (!pc_cnt) pc_cnt = (int*) malloc(sizeof(int)*warp_size);
+   memset(pc_cnt,0,sizeof(int)*warp_size);
+
+   // go through each thread in the given warp
+   for (unsigned i=0; i<warp_size; i++) {
+      if (tid_in[i] < 0) continue;
+      int matched = 0;
+      address_type thd_pc;
+      thd_pc = shader_thread_nextpc(shader, tid_in[i]);
+
+      int thd_cta = ptx_thread_get_cta_uid( shader->thread[tid_in[i]].ptx_thd_info );
+
+      // check to see if the pc has occured before
+      for (unsigned j=0; j<n_pc; j++) {
+         if (thd_pc == pc[j] && thd_cta == cta[j]) {
+            tid_split[j][pc_cnt[j]] = tid_in[i];
+            pc_cnt[j]++;
+            matched = 1;
+            break;
+         }
+      }
+      // if not, put the tid in a seperate warp
+      if (!matched) {
+         assert(n_pc < warp_size);
+         tid_split[n_pc][0] = tid_in[i];
+         pc[n_pc] = thd_pc;
+         cta[n_pc] = thd_cta;
+         pc_cnt[n_pc] = 1;
+         n_pc++;
+      }
+   }
+   return n_pc;
+}
+
+void shader_fetch_simd_dwf( shader_core_ctx_t *shader, unsigned int shader_number ) {
+
+   static int *tid_in = NULL;
+   static int *tid_out = NULL;
+
+   if (!tid_in) {
+      tid_in = (int*) malloc(sizeof(int)*warp_size);
+      memset(tid_in, -1, sizeof(int)*warp_size);
+   }
+   if (!tid_out) {
+      tid_out = (int*) malloc(sizeof(int)*warp_size);
+      memset(tid_out, -1, sizeof(int)*warp_size);
+   }
+
+
+   static int **tid_split = NULL;
+   if (!tid_split) {
+      tid_split = (int**)malloc(sizeof(int*)*warp_size);
+      tid_split[0] = (int*)malloc(sizeof(int)*warp_size*warp_size);
+      for (unsigned i=1; i<warp_size; i++) {
+         tid_split[i] = tid_split[0] + warp_size * i;
+      }
+   }
+
+   static address_type *thd_pc = NULL;
+   if (!thd_pc) thd_pc = (address_type*)malloc(sizeof(address_type)*warp_size);
+   static int *thd_cta = NULL;
+   if (!thd_cta) thd_cta = (int*)malloc(sizeof(int)*warp_size);
+
+   int warpupdate_bw = 1;
+   while (!dq_empty(shader->thd_commit_queue) && warpupdate_bw > 0) {
+      // grab a committed warp, split it into multiple BRUs (tid_split) by PC
+      int *tid_commit = (int*)dq_pop(shader->thd_commit_queue);
+      memset(tid_split[0], -1, sizeof(int)*warp_size*warp_size);
+      memset(thd_pc, 0, sizeof(address_type)*warp_size);
+      memset(thd_cta, -1, sizeof(int)*warp_size);
+
+      int reached_barrier = warp_reached_barrier(tid_commit, shader);
+
+      unsigned n_warp_update;
+      if (reached_barrier) {
+         n_warp_update = split_warp_by_cta(tid_commit, shader, tid_split, thd_pc, thd_cta);
+      } else {
+         n_warp_update = split_warp_by_pc(tid_commit, shader, tid_split, thd_pc);
+      }
+
+      if (n_warp_update > 2) gpgpu_commit_pc_beyond_two++;
+      warpupdate_bw -= n_warp_update;
+      // put the splitted warp updates into the DWF scheduler
+      for (unsigned i=0;i<n_warp_update;i++) {
+         for (unsigned j=0;j<warp_size;j++) {
+            if (tid_split[i][j] < 0) continue;
+            assert(shader->thread[tid_split[i][j]].avail4fetch);
+            assert(!shader->thread[tid_split[i][j]].in_scheduler);
+            shader->thread[tid_split[i][j]].in_scheduler = 1;
+         }
+         dwf_clear_accessed(shader->sid);
+         if (reached_barrier) {
+            dwf_update_warp_at_barrier(shader->sid, tid_split[i], thd_pc[i], thd_cta[i]);
+         } else {
+            dwf_update_warp(shader->sid, tid_split[i], thd_pc[i]);
+         }
+      }
+
+      free_commit_warp(tid_commit);
+   }
+
+   // Track the #PC right after the warps are input to the scheduler
+   dwf_update_statistics(shader->sid);
+   dwf_clear_policy_access(shader->sid);
+
+   if (shader_fetch_stalled(shader)) {
+      return; 
+   }
+   shader_clear_stage_reg(shader, TS_IF);
+
+   address_type scheduled_pc;
+   dwf_issue_warp(shader->sid, tid_out, &scheduled_pc);
+
+   for (unsigned i=0; i<warp_size; i++) {
+      int issue_tid = tid_out[i];
+      if (issue_tid >= 0) {
+         shader_issue_thread(shader, issue_tid, i, (unsigned)-1);
+         shader->thread[issue_tid].in_scheduler = 0;
+         shader->thread[issue_tid].m_reached_barrier = 0;
+         shader->last_issued_thread = issue_tid;
+         assert(shader->pipeline_reg[TS_IF][i].pc == scheduled_pc);
+      }
+   }   
+}
+
+void print_shader_cycle_distro( FILE *fout ) 
+{
+   fprintf(fout, "Warp Occupancy Distribution:\n");
+   fprintf(fout, "Stall:%d\t", shader_cycle_distro[0]);
+   fprintf(fout, "W0_Idle:%d\t", shader_cycle_distro[1]);
+   fprintf(fout, "W0_Mem:%d", shader_cycle_distro[2]);
+   for (unsigned i = 3; i < warp_size + 3; i++) {
+      fprintf(fout, "\tW%d:%d", i-2, shader_cycle_distro[i]);
+   }
+   fprintf(fout, "\n");
+}
+
+void inflight_memory_insn_add( shader_core_ctx_t *shader, inst_t *mem_insn)
+{
+   if (enable_ptx_file_line_stats) {
+      ptx_file_line_stats_add_inflight_memory_insn(shader->sid, mem_insn->pc);
+   }
+}
+
+void inflight_memory_insn_sub( shader_core_ctx_t *shader, inst_t *mem_insn)
+{
+   if (enable_ptx_file_line_stats) {
+      ptx_file_line_stats_sub_inflight_memory_insn(shader->sid, mem_insn->pc);
+   }
+}
+
+void report_exposed_memory_latency( shader_core_ctx_t *shader )
+{
+   if (enable_ptx_file_line_stats) {
+      ptx_file_line_stats_commit_exposed_latency(shader->sid, 1);
+   }
+}
+
+static int gpgpu_warp_occ_detailed = 0;
+static int **warp_occ_detailed = NULL;
+
+void check_stage_pcs( shader_core_ctx_t *shader, unsigned stage );
+void check_pm_stage_pcs( shader_core_ctx_t *shader, unsigned stage );
+
+void shader_fetch( shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) 
+{
+   assert(shader->model < NUM_SIMD_MODEL);
+   int n_warp_parts = warp_size/pipe_simd_width;
+
+   // check if decode stage is stalled
+   int decode_stalled = 0;
+   for (int i = 0; i < pipe_simd_width; i++) {
+      if (shader->pipeline_reg[IF_ID][i].hw_thread_id != -1 )
+         decode_stalled = 1;
+   }
+
+   if (shader->gpu_cycle % n_warp_parts == 0) {
+
+      if (shader->model == POST_DOMINATOR || shader->model == NO_RECONVERGE) {
+         int warpupdate_bw = 1; // number of warps to be unlocked per scheduler cycle
+         while (!dq_empty(shader->thd_commit_queue) && warpupdate_bw > 0) {
+            // grab a committed warp and unlock it here
+            int *tid_commit = (int*)dq_pop(shader->thd_commit_queue);
+            for (unsigned i = 0; i < warp_size; i++) {
+               if (tid_commit[i] >= 0) {
+                  shader->thread[tid_commit[i]].avail4fetch++;
+                  assert(shader->thread[tid_commit[i]].avail4fetch <= 1);
+                  assert( shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch < (unsigned)warp_size );
+                  shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch++;
+               }
+            }
+            warpupdate_bw -= 1;
+            free_commit_warp(tid_commit);
+         }
+      }
+
+      switch (shader->model) {
+      case NO_RECONVERGE:
+         shader_fetch_simd_no_reconverge(shader, shader_number, grid_num );
+         break;
+      case POST_DOMINATOR:
+         shader_fetch_simd_postdominator(shader, shader_number, grid_num);
+         break;
+      case MIMD:
+         shader_fetch_mimd(shader, shader_number);
+         break;
+      case DWF:
+         shader_fetch_simd_dwf(shader, shader_number);
+         break;
+      default:
+         fprintf(stderr, "Unknown scheduler: %d\n", shader->model);
+         assert(0);
+         break;
+      }
+
+      static int *tid_out = NULL;
+      if (!tid_out) {
+         tid_out = (int*) malloc(sizeof(int) * warp_size);
+      }
+      memset(tid_out, -1, sizeof(int)*warp_size);
+
+      if (!shader_cycle_distro) {
+         shader_cycle_distro = (unsigned int*) calloc(warp_size + 3, sizeof(unsigned int));
+      }
+
+      if (gpgpu_no_divg_load && shader->new_warp_TS && !decode_stalled) {
+         int n_thd_in_warp = 0;
+         address_type pc_out = 0xDEADBEEF;
+         for (unsigned i=0; i<warp_size; i++) {
+            tid_out[i] = shader->pipeline_reg[TS_IF][i].hw_thread_id;
+            if (tid_out[i] >= 0) {
+               n_thd_in_warp += 1;
+               pc_out = shader->pipeline_reg[TS_IF][i].pc;
+            }
+         }
+         wpt_register_warp(tid_out, shader);
+         if (gpu_runtime_stat_flag & GPU_RSTAT_DWF_MAP) {
+            track_thread_pc( shader->sid, tid_out, pc_out );
+         }
+         if (gpgpu_cflog_interval != 0) {
+            insn_warp_occ_log( shader->sid, pc_out, n_thd_in_warp);
+            shader_warp_occ_log( shader->sid, n_thd_in_warp);
+         }
+         if ( gpgpu_warpdistro_shader < 0 || shader->sid == gpgpu_warpdistro_shader ) {
+            shader_cycle_distro[n_thd_in_warp + 2] += 1;
+            if (n_thd_in_warp == 0) {
+               if (shader->pending_mem_access == 0) shader_cycle_distro[1]++;
+            }
+         }
+         shader->new_warp_TS = 0;
+
+         if (enable_ptx_file_line_stats && n_thd_in_warp > 0) {
+            //ptx_file_line_stats_add_warp_issued(pc_out);
+            //ptx_file_line_stats_add_warp_occ_total(pc_out, n_thd_in_warp);
+         }
+
+         if ( gpgpu_warp_occ_detailed && 
+              n_thd_in_warp && (shader->model == POST_DOMINATOR) ) {
+            int n_warp = gpu_n_thread_per_shader / warp_size;
+            if (!warp_occ_detailed) {
+               warp_occ_detailed = (int**) malloc(sizeof(int*) * gpu_n_shader * n_warp);
+               warp_occ_detailed[0] = (int*) calloc(sizeof(int), gpu_n_shader * n_warp * warp_size);
+               for (unsigned i = 0; i < n_warp * gpu_n_shader; i++) {
+                  warp_occ_detailed[i] = warp_occ_detailed[0] + i * warp_size;
+               }
+            }
+
+            int wid = -1;
+            for (unsigned i=0; i<warp_size; i++) {
+               if (tid_out[i] >= 0) wid = tid_out[i] / warp_size;
+            }
+            assert(wid != -1);
+            warp_occ_detailed[shader->sid * n_warp + wid][n_thd_in_warp - 1] += 1;
+
+            if (shader->sid == 0 && wid == 16 && 0) {
+               printf("wtrace[%08x] ", pc_out);
+               for (unsigned i=0; i<warp_size; i++) {
+                  printf("%03d ", tid_out[i]);
+               }
+               printf("\n");
+            }
+         }
+      } else {
+         if ( gpgpu_warpdistro_shader < 0 || shader->sid == gpgpu_warpdistro_shader ) {
+            shader_cycle_distro[0] += 1;
+         }
+      }
+
+      if (!decode_stalled) {
+         for (unsigned i = 0; i < warp_size; i++) {
+            int tid_tsif = shader->pipeline_reg[TS_IF][i].hw_thread_id;
+            address_type pc_out = shader->pipeline_reg[TS_IF][i].pc;
+            cflog_update_thread_pc(shader->sid, tid_tsif, pc_out);
+         }
+      }
+
+      if (enable_ptx_file_line_stats && !decode_stalled) {
+         int TS_stage_empty = 1;
+         for (unsigned i = 0; i < warp_size; i++) {
+            if (shader->pipeline_reg[TS_IF][i].hw_thread_id >= 0) {
+               TS_stage_empty = 0;
+               break;
+            }
+         }
+         if (TS_stage_empty) {
+            report_exposed_memory_latency(shader);
+         }
+      }
+   }
+
+   // if not, send the warp part to decode stage
+   if (!decode_stalled && shader->warp_part2issue < n_warp_parts) {
+      check_stage_pcs(shader,TS_IF);
+      for (int i = 0; i < pipe_simd_width; i++) {
+         int wlane_idx = shader->warp_part2issue * pipe_simd_width + i;
+         shader->pipeline_reg[IF_ID][i] = shader->pipeline_reg[TS_IF][wlane_idx];
+         shader->pipeline_reg[IF_ID][i].if_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+         shader->pipeline_reg[TS_IF][wlane_idx] = nop_inst;
+      }
+      shader->warp_part2issue += 1;
+   }
+}
+
+inline int is_load ( op_type op ) {
+   return op == LOAD_OP;
+}
+
+inline int is_store ( op_type op ) {
+   return op == STORE_OP;
+}
+
+inline int is_tex ( int space ) {
+   return((space) == TEX_DIRECTIVE);
+}
+
+inline int is_const ( int space ) {
+   return((space) == CONST_DIRECTIVE || (space) == PARAM_DIRECTIVE);
+}
+
+inline int is_local ( int space ) {
+   return((space) == LOCAL_DIRECTIVE);
+}
+
+inline int is_param ( int space ) {
+   return((space) == PARAM_DIRECTIVE);
+}
+
+inline int is_shared ( int space ) {
+   return((space) == SHARED_DIRECTIVE);
+}
+
+inline int shmem_bank ( address_type addr ) {
+   return((int)(addr/((address_type)WORD_SIZE)) % gpgpu_n_shmem_bank);
+}
+
+inline int cache_bank ( address_type addr, shader_core_ctx_t *shader ) {
+   return(int)( addr >> (address_type)shader->L1cache->line_sz_log2 ) & ( gpgpu_n_cache_bank - 1 );
+}
+
+inline address_type coalesced_segment(address_type addr, unsigned segment_size_lg2bytes)
+{
+   return  (addr >> segment_size_lg2bytes);
+}
+
+
+inline address_type translate_local_memaddr(address_type localaddr, shader_core_ctx_t *shader, int tid)
+{
+   // During functional execution, each thread sees its own memory space for local memory, but these
+   // need to be mapped to a shared address space for timing simulation.  We do that mapping here.
+   localaddr -= 0x100;
+   localaddr /=4;
+   if (gpgpu_local_mem_map) {
+      // Dnew = D*nTpC*nCpS*nS + nTpC*C + T%nTpC
+      // C = S + nS*(T/nTpC)
+      // D = data index; T = thread; C = CTA; S = shader core; p = per
+      // keep threads in a warp contiguous
+      // then distribute across memory space by CTAs from successive shader cores first, 
+      // then by successive CTA in same shader core
+      localaddr *= gpu_padded_cta_size * gpu_max_cta_per_shader * gpu_n_shader;
+      localaddr += gpu_padded_cta_size * (shader->sid + gpu_n_shader * (tid / gpu_padded_cta_size));
+      localaddr += tid % gpu_padded_cta_size; 
+   } else {
+      // legacy mapping that maps the same address in the local memory space of all threads 
+      // to a single contiguous address region 
+      localaddr *= gpu_n_shader * gpu_n_thread_per_shader;
+      localaddr += (gpu_n_thread_per_shader*shader->sid) + tid;
+   }
+   localaddr *= 4;
+   localaddr += 0x100;
+
+   return localaddr;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// Register Bank Conflict Structures
+
+int gpgpu_reg_bank_conflict_model = 0;
+
+#define MAX_REG_BANKS 32
+unsigned int gpgpu_num_reg_banks=8; // this needs to be less than MAX_REG_BANKS
+
+#define MAX_BANK_CONFLICT 8 /* tex can have four source and four destination regs */
+
+class reg_bank_access {
+public:
+  reg_bank_access():tot(0),rd(0),wr(0){
+    for (unsigned i = 0; i < 4; i++) rd_regs[i] = -1; 
+  }
+	unsigned tot;
+	unsigned rd;
+	unsigned wr;
+	int rd_regs[4];
+}; 
+
+reg_bank_access g_reg_bank_access[MAX_REG_BANKS];
+
+// just to use as "shorthand" for clearing accesses each cycle
+static const struct reg_bank_access empty_reg_bank_access;
+
+unsigned int gpu_reg_bank_conflict_stalls = 0;
+/////////////////////////////////////////////////////////////////////////////////////////
+
+void shader_decode( shader_core_ctx_t *shader, 
+                    unsigned int shader_number,
+                    unsigned int grid_num ) {
+
+   address_type addr;
+   dram_callback_t callback;
+   op_type op = NO_OP;
+   register int is_write;
+   int tid;
+   int i1, i2, i3, i4, o1, o2, o3, o4; //4 outputs needed for texture fetches in cuda-sim
+   int i;
+   int touched_priority=0;
+   int warp_tid=0;
+   unsigned space, data_size;
+   int vectorin, vectorout;
+   int arch_reg[MAX_REG_OPERANDS] = { -1 };
+   address_type regs_regs_PC = 0xDEADBEEF;
+   address_type warp_current_pc = 0x600DBEEF;
+   address_type warp_next_pc = 0x600DBEEF;
+   int       warp_diverging = 0;
+   unsigned warp_id = -1;
+   unsigned cta_id = -1;
+
+   // stalling for register bank conflict 
+   if ( gpgpu_reg_bank_conflict_model ) {
+      for (i=0; i<pipe_simd_width;i++) {
+         if ( shader->pipeline_reg[IF_ID][i].reg_bank_conflict_stall_checked ) {
+            if ( shader->pipeline_reg[IF_ID][i].reg_bank_access_pending > 0 ) {
+               assert( shader->pipeline_reg[IF_ID][i].reg_bank_access_pending <= 8 );
+               shader->pipeline_reg[IF_ID][i].reg_bank_access_pending--;
+               gpu_reg_bank_conflict_stalls++;
+               return;
+            }
+         }
+      }
+   }
+
+   for (i=0; i<pipe_simd_width;i++) {
+      int next_stage = (shader->using_rrstage)? ID_RR:ID_EX;
+      if (shader->pipeline_reg[next_stage][i].hw_thread_id != -1 ) {
+         return;  /* stalled */
+      }
+   }
+
+   check_stage_pcs(shader,IF_ID);
+
+   // decode the instruction 
+   int first_valid_thread = -1;
+   for (i=0; i<pipe_simd_width;i++) {
+
+      if (shader->pipeline_reg[IF_ID][i].hw_thread_id == -1 )
+         continue; /* bubble */
+
+      /* get the next instruction to execute from fetch stage */
+      tid = shader->pipeline_reg[IF_ID][i].hw_thread_id;
+      if (first_valid_thread == -1) {
+         first_valid_thread = i;
+         warp_id = tid/warp_size;
+         assert( !shader->warp_waiting_at_barrier(warp_id) );
+         cta_id = shader->thread[tid].cta_id;
+      }
+
+      if ( gpgpu_cuda_sim ) {
+         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg);
+         shader->pipeline_reg[IF_ID][i].op = op;
+         shader->pipeline_reg[IF_ID][i].pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
+         shader->pipeline_reg[IF_ID][i].ptx_thd_info = shader->thread[tid].ptx_thd_info;
+
+      } else {
+         abort();
+      }
+      // put the info into the shader instruction structure 
+      // - useful in tracking instruction dependency (not needed for now)
+      shader->pipeline_reg[IF_ID][i].in[0] = i1;
+      shader->pipeline_reg[IF_ID][i].in[1] = i2;
+      shader->pipeline_reg[IF_ID][i].in[2] = i3;
+      shader->pipeline_reg[IF_ID][i].in[3] = i4;
+      shader->pipeline_reg[IF_ID][i].out[0] = o1;
+      shader->pipeline_reg[IF_ID][i].out[1] = o2;
+      shader->pipeline_reg[IF_ID][i].out[2] = o3;
+      shader->pipeline_reg[IF_ID][i].out[3] = o4;
+
+   }
+
+   // checking for register bank conflict and stall accordingly
+   if ( gpgpu_reg_bank_conflict_model && 
+        first_valid_thread != -1 && 
+        !shader->pipeline_reg[first_valid_thread][IF_ID].reg_bank_conflict_stall_checked ) 
+   {
+      for (i = 4; i < 8; i++) {
+         if( arch_reg[i] != -1 ) {
+            assert( arch_reg[i] >=0 );
+            assert( gpgpu_num_reg_banks <= MAX_REG_BANKS );
+            int skip = 0;
+            int bank = arch_reg[i] % gpgpu_num_reg_banks;
+            int opndreg = shader->pipeline_reg[first_valid_thread][IF_ID].in[i-4];
+            assert(opndreg >= 0); 
+            int j;
+            for (j = 0; j < 4; j++) {
+	            if (g_reg_bank_access[bank].rd_regs[j] == -1)
+		            break;
+	            else if (g_reg_bank_access[bank].rd_regs[j] == opndreg) {
+		            // two operands reading from same register in same bank, can be merged into a single read
+		            skip = 1;
+		            break;
+	            }
+            }
+            if (!skip) {
+	            g_reg_bank_access[bank].tot++;
+	            g_reg_bank_access[bank].rd++;
+	            g_reg_bank_access[bank].rd_regs[j] = opndreg;
+            }
+         }
+      }
+      
+      unsigned max_access=0;   
+      inst_t* conflict_inst = &shader->pipeline_reg[first_valid_thread][IF_ID];
+      for(unsigned r = 0; r < gpgpu_num_reg_banks; r++ ) {
+         if( g_reg_bank_access[r].tot > max_access )
+            max_access = g_reg_bank_access[r].tot;
+         g_reg_bank_access[r] = empty_reg_bank_access;
+      }
+      if( max_access >= 1 ) {
+         assert( max_access <= MAX_REG_OPERANDS );
+         conflict_inst->reg_bank_access_pending = max_access - 1;
+         if( max_access > 1 ) {
+            conflict_inst->reg_bank_conflict_stall_checked = 1;
+            return; // stall pipeline
+         }
+      }
+      shader->pipeline_reg[first_valid_thread][IF_ID].reg_bank_conflict_stall_checked = 1;
+   }
+
+   // execute the instruction functionally
+   for (i=0; i<pipe_simd_width;i++) {
+      if (shader->pipeline_reg[IF_ID][i].hw_thread_id == -1 )
+         continue; /* bubble */
+      /* get the next instruction to execute from fetch stage */
+      tid = shader->pipeline_reg[IF_ID][i].hw_thread_id;
+      if ( gpgpu_cuda_sim ) {
+         int arch_reg[MAX_REG_OPERANDS];
+         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg );
+         ptx_exec_inst( shader->thread[tid].ptx_thd_info, &addr, &space, &data_size, &callback, shader->pipeline_reg[IF_ID][i].warp_active_mask );
+         shader->pipeline_reg[IF_ID][i].callback = callback;
+         shader->pipeline_reg[IF_ID][i].space = space;
+         if (is_local(space) && (is_load(op) || is_store(op))) {
+            addr = translate_local_memaddr(addr, shader, tid);
+         }
+         shader->pipeline_reg[IF_ID][i].is_vectorin = vectorin;
+         shader->pipeline_reg[IF_ID][i].is_vectorout = vectorout;
+         shader->pipeline_reg[IF_ID][i].data_size = data_size;
+         warp_current_pc = shader->pipeline_reg[IF_ID][i].pc;
+         memcpy( shader->pipeline_reg[IF_ID][i].arch_reg, arch_reg, sizeof(arch_reg) );
+         regs_regs_PC = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
+      }
+
+      shader->pipeline_reg[IF_ID][i].memreqaddr = addr;
+      if ( op == LOAD_OP ) {
+         shader->pipeline_reg[IF_ID][i].inst_type = LOAD_OP;
+      } else if ( op == STORE_OP ) {
+         shader->pipeline_reg[IF_ID][i].inst_type = STORE_OP;
+      }
+
+      if ( gpgpu_cuda_sim && ptx_thread_at_barrier( shader->thread[tid].ptx_thd_info ) ) {
+         if (shader->model == DWF) {
+            shader->thread[tid].m_waiting_at_barrier=1;
+            shader->thread[tid].m_reached_barrier=1; // not reset at barrier release, but at the issue after that
+            shader->warp[wid_from_hw_tid(tid,warp_size)].n_waiting_at_barrier++;
+            shader->waiting_at_barrier++;
+            int cta_uid = ptx_thread_get_cta_uid( shader->thread[tid].ptx_thd_info );
+            dwf_hit_barrier( shader->sid, cta_uid );
+         
+            int release = ptx_thread_all_at_barrier( shader->thread[tid].ptx_thd_info ); //test if all threads arrived at the barrier
+            if ( release ) { //All threads arrived at barrier...releasing
+               int cta_uid = ptx_thread_get_cta_uid( shader->thread[tid].ptx_thd_info );
+               for ( unsigned t=0; t < gpu_n_thread_per_shader; ++t ) {
+                  if ( !ptx_thread_at_barrier( shader->thread[t].ptx_thd_info ) )
+                     continue;
+                  int other_cta_uid = ptx_thread_get_cta_uid( shader->thread[t].ptx_thd_info );
+                  if ( other_cta_uid == cta_uid ) { //reseting @barrier tracking info
+                     shader->warp[wid_from_hw_tid(t,warp_size)].n_waiting_at_barrier=0;
+                     shader->thread[t].m_waiting_at_barrier=0;
+                     ptx_thread_reset_barrier( shader->thread[t].ptx_thd_info );
+                     shader->waiting_at_barrier--;
+                  }
+               }
+               if (shader->model == DWF) {
+                  dwf_release_barrier( shader->sid, cta_uid );
+               }
+               ptx_thread_release_barrier( shader->thread[tid].ptx_thd_info );
+            }
+         }
+      } else {
+         assert( !shader->thread[tid].m_waiting_at_barrier );
+      }
+
+      // put the info into the shader instruction structure 
+      // - useful in tracking instruction dependency (not needed for now)
+      shader->pipeline_reg[IF_ID][i].in[0] = i1;
+      shader->pipeline_reg[IF_ID][i].in[1] = i2;
+      shader->pipeline_reg[IF_ID][i].in[2] = i3;
+      shader->pipeline_reg[IF_ID][i].in[3] = i4;
+      shader->pipeline_reg[IF_ID][i].out[0] = o1;
+      shader->pipeline_reg[IF_ID][i].out[1] = o2;
+      shader->pipeline_reg[IF_ID][i].out[2] = o3;
+      shader->pipeline_reg[IF_ID][i].out[3] = o4;
+
+      if ( op == STORE_OP ) {
+         is_write = TRUE;
+      }
+
+      if ( op == BRANCH_OP ) {
+         int taken=0;
+         assert( gpgpu_cuda_sim );
+         taken = ptx_branch_taken(shader->thread[tid].ptx_thd_info);
+      }
+
+      // go to the next instruction 
+      // - done implicitly in ptx_exec_inst()
+      
+      // branch divergence detection
+      if (warp_next_pc != regs_regs_PC) {
+         if (warp_next_pc == 0x600DBEEF) {
+            warp_next_pc = regs_regs_PC;
+         } else {
+            warp_diverging = 1;
+         }
+      }
+
+      // direct the instruction to the appropriate next stage (config dependent)
+      int next_stage = (shader->using_rrstage)? ID_RR:ID_EX;
+      shader->pipeline_reg[next_stage][i] = shader->pipeline_reg[IF_ID][i];
+      shader->pipeline_reg[next_stage][i].id_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+      shader->pipeline_reg[IF_ID][i] = nop_inst;
+   }
+
+   if( op == BARRIER_OP ) {
+      shader->set_at_barrier(cta_id,warp_id);
+   }
+
+   if ( shader->model == NO_RECONVERGE && touched_priority ) {
+      update_max_branch_priority(shader,warp_tid,grid_num);
+   }
+   shader->n_diverge += warp_diverging;
+   if (warp_diverging == 1) {
+       assert(warp_current_pc != 0x600DBEEF); // guard against empty warp causing warp divergence
+       ptx_file_line_stats_add_warp_divergence(warp_current_pc, 1);
+   }
+}
+
+unsigned int n_regconflict_stall = 0;
+
+
+int regfile_hash(signed thread_number, unsigned simd_size, unsigned n_banks) {
+   if (gpgpu_thread_swizzling) {
+      signed warp_ID = thread_number / simd_size;
+      return((thread_number + warp_ID) % n_banks);
+   } else {
+      return(thread_number % n_banks);
+   }
+}
+
+int gpgpu_n_reg_banks = 8;
+void shader_preexecute( shader_core_ctx_t *shader, 
+                        unsigned int shader_number ) {
+   int i;
+   static int *thread_warp = NULL;
+   int n_access_per_cycle = pipe_simd_width / gpgpu_n_reg_banks;
+
+   if (!thread_warp) {
+      thread_warp = (int*) malloc(sizeof(int) * pipe_simd_width);
+   }
+
+   for (i=0; i<pipe_simd_width; i++) {
+      if (shader->pipeline_reg[RR_EX][i].hw_thread_id != -1 ) {
+         //stalled, but can still service a register read
+         if (shader->RR_k) {
+            shader->RR_k--;  
+         }
+         return;  // stalled 
+      }
+   }
+
+   // if there is still register read to service, stall
+   if (shader->RR_k > 1) {
+      shader->RR_k--;
+      return; 
+   }
+
+   // if RR_k == 1, it was stalled previously and the register read is now done
+   if (!shader->RR_k && gpgpu_reg_bankconflict) {
+      int max_reg_bank_acc = 0;
+      for (i=0; i<pipe_simd_width; i++) {
+         thread_warp[i] = 0;
+      }
+      for (i=0; i<pipe_simd_width; i++) {
+         if (shader->pipeline_reg[ID_RR][i].hw_thread_id != -1 )
+            thread_warp[regfile_hash(shader->pipeline_reg[ID_RR][i].hw_thread_id, 
+                                     warp_size, gpgpu_n_reg_banks)]++;
+      }
+      for (i=0; i<pipe_simd_width; i++) {
+         if (thread_warp[i] > max_reg_bank_acc ) {
+            max_reg_bank_acc = thread_warp[i];
+         }
+      }
+      // calculate the number of cycles needed for each register bank to fulfill all accesses
+      shader->RR_k = (max_reg_bank_acc / n_access_per_cycle) + ((max_reg_bank_acc % n_access_per_cycle)? 1 : 0);
+   }
+
+   // if there are more than one access cycle needed at a bank, stall
+   if (shader->RR_k > 1) {
+      n_regconflict_stall++;
+      shader->RR_k--;
+      return; 
+   }
+
+   check_stage_pcs(shader,ID_RR);
+
+   shader->RR_k = 0; //setting RR_k to 0 to indicate RF conflict check next cycle
+   for (i=0; i<pipe_simd_width;i++) {
+      if (shader->pipeline_reg[ID_RR][i].hw_thread_id == -1 )
+         continue; //bubble 
+      shader->pipeline_reg[ID_EX][i] = shader->pipeline_reg[ID_RR][i];
+      shader->pipeline_reg[ID_RR][i] = nop_inst;
+   }
+
+}
+
+
+void shader_execute( shader_core_ctx_t *shader, 
+                     unsigned int shader_number ) {
+
+   int i;
+
+   for (i=0; i<pipe_simd_width; i++) {
+      if (gpgpu_pre_mem_stages) {
+         if (shader->pre_mem_pipeline[0][i].hw_thread_id != -1 ) {
+            //printf("stalled in shader_execute\n");
+            return;  // stalled 
+         }
+      } else {
+         if (shader->pipeline_reg[EX_MM][i].hw_thread_id != -1 )
+            return;  // stalled 
+      }
+   }
+
+   check_stage_pcs(shader,ID_EX);
+
+   for (i=0; i<pipe_simd_width; i++) {
+      if (shader->pipeline_reg[ID_EX][i].hw_thread_id == -1 )
+         continue; // bubble 
+      if (gpgpu_pre_mem_stages) {
+         shader->pre_mem_pipeline[0][i] = shader->pipeline_reg[ID_EX][i];
+         shader->pre_mem_pipeline[0][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+      } else {
+         shader->pipeline_reg[EX_MM][i] = shader->pipeline_reg[ID_EX][i];
+         shader->pipeline_reg[EX_MM][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+      }
+      shader->pipeline_reg[ID_EX][i] = nop_inst;
+   }  
+
+   if (!gpgpu_pre_mem_stages) {
+      // inform memory stage that a new instruction has arrived 
+      shader->shader_memory_new_instruction_processed = 0;
+   }
+}
+
+void shader_pre_memory( shader_core_ctx_t *shader, 
+                        unsigned int shader_number ) {
+   int i,j;
+
+
+   for (j = gpgpu_pre_mem_stages; j > 0; j--) {
+      for (i=0; i<pipe_simd_width; i++) {
+         if (shader->pre_mem_pipeline[j][i].hw_thread_id != -1 ) {
+            return; 
+         }
+      }
+      check_pm_stage_pcs(shader,j-1);
+      for (i=0; i<pipe_simd_width; i++) {
+         shader->pre_mem_pipeline[j][i] = shader->pre_mem_pipeline[j - 1][i];
+         shader->pre_mem_pipeline[j - 1][i] = nop_inst;
+      }
+   }
+   check_pm_stage_pcs(shader,gpgpu_pre_mem_stages);
+   for (i=0;i<pipe_simd_width ;i++ )
+      shader->pipeline_reg[EX_MM][i] = shader->pre_mem_pipeline[gpgpu_pre_mem_stages][i];
+
+   // inform memory stage that a new instruction has arrived 
+   shader->shader_memory_new_instruction_processed = 0;
+
+   if (gpgpu_pre_mem_stages) {
+      for (i=0; i<pipe_simd_width; i++)
+         shader->pre_mem_pipeline[0][i] = nop_inst;
+   }
+}
+
+int gpgpu_coalesce_arch = 13;
+
+enum memory_path {
+   NO_MEM_PATH = 0,
+   SHARED_MEM_PATH,
+   GLOBAL_MEM_PATH,
+   TEXTURE_MEM_PATH,
+   CONSTANT_MEM_PATH,
+   NUM_MEM_PATHS //not a mem path
+};
+
+static unsigned next_access_uid = 0;   
+
+class mem_access_t{
+public:
+   mem_access_t(): uid(next_access_uid++),addr(0),req_size(0),order(0),_quarter_count_all(0),warp_indices(),space(0),path(NO_MEM_PATH),isatomic(false),cache_hit(false),cache_checked(false),recheck_cache(false),iswrite(false),need_wb(false),wb_addr(0),reserved_mshr(NULL){};
+   bool operator<(const mem_access_t &other) const {return (order > other.order);}//this is reverse
+   unsigned uid;
+   address_type addr; //address of the segment to load.
+   unsigned req_size; //bytes
+   unsigned order; // order of accesses, based on banks.
+   union{
+   unsigned _quarter_count_all;
+   char quarter_count[4]; //access counts to each quarter of segment, for compaction;
+   };
+   std::vector<unsigned> warp_indices; //warp indicies for this request.
+   unsigned space;
+   memory_path path;
+   bool isatomic;
+   bool cache_hit;
+   bool cache_checked;
+   bool recheck_cache;
+   bool iswrite;
+   bool need_wb;
+   address_type wb_addr; //address to wb too if necessary.
+   mshr_entry_t* reserved_mshr;
+};
+
+mshr_entry_t* mshr_shader_unit::add_mshr(mem_access_t &access, inst_t* warp)
+{
+   static unsigned next_request_uid = 1;
+   mshr_entry_t* mshr = alloc_free_mshr(is_tex(access.space));
+   //note no constructor was called, all entries must be reinitialized!
+   mshr->request_uid = next_request_uid++;
+   mshr->status = INITIALIZED;
+   mshr->addr = access.addr;
+   mshr->mf = NULL;
+   mshr->merged_on_other_reqest = false;
+   mshr->merged_requests =NULL;
+   mshr->iswrite = access.iswrite;
+   assert(access.warp_indices.size()); //code assumes at least one instruction attached to mshr.
+   for (unsigned i = 0; i < access.warp_indices.size(); i++) {
+      mshr->insts.push_back(warp[access.warp_indices[i]]);
+   }
+   mshr->islocal = is_local(access.space);
+   mshr->isconst = is_const(access.space);
+   mshr->istexture = is_tex(access.space);
+   if (gpgpu_interwarp_mshr_merge) {
+      mshr_entry_t* mergehit = m_mshr_lookup.shader_get_mergeable_mshr(mshr);
+      if (mergehit) {
+         //merge this request;
+         mergehit->merged_requests = mshr;
+         mshr->merged_on_other_reqest = true;
+         if (mergehit->fetched()) mshr_return_from_mem(mshr);
+      }
+      m_mshr_lookup.mshr_fast_lookup_insert(mshr);
+   }
+   return mshr;
+}
+
+
+inline address_type line_size_based_tag_func(address_type address, unsigned line_size)
+{
+   return ((address) & (~((address_type)line_size - 1)));
+}
+
+inline address_type null_tag_func(address_type address, unsigned line_size){
+   return address; //no modification: each address is its own tag. Equivalent to line_size_based_tag_func(address,1), but line_size ignored.
+}
+
+// only 1 bank
+inline int null_bank_func(address_type add, unsigned line_size)
+{
+   return 1;
+}
+
+inline int shmem_bank_func(address_type add, unsigned line_size)
+{
+   return shmem_bank(add);
+}
+
+inline int dcache_bank_func(address_type add, unsigned line_size)
+{
+   if (gpgpu_no_dl1) return 1; //no banks
+   else return (add / line_size) & (gpgpu_n_cache_bank - 1);
+}
+
+#include <bitset>
+void check_accessq(  shader_core_ctx_t *shader, std::vector<mem_access_t> &accessq ){
+   std::bitset<32> check = 0;
+   for (unsigned i = 0; i < accessq.size(); i++) {
+      if (shader) {
+         std::cout << shader->sid << ":" << i << " space " << accessq[i].space << " " << gpu_sim_cycle <<  std::endl;
+         assert(accessq[i].space == (unsigned) shader->pipeline_reg[EX_MM][accessq[i].warp_indices[0]].space);
+      }
+      for (unsigned j = 0; j < accessq[i].warp_indices.size(); j++) {
+         if (check[accessq[i].warp_indices[j]]) {
+            std::cout << "OOOPS" << std::endl; //good line for breakpoint
+         }else{check[accessq[i].warp_indices[j]] = true;}
+      }
+   }
+}
+
+// This speciallized function calculates the list of independant memory accesses, sorted by access order
+// Acesses to same tag line are coalesced.
+// will neither coalesce nor overlap bank accesses accross warp parts.
+template < int (*bank_func)(address_type add, unsigned line_size), address_type (*tag_func)(address_type add, unsigned line_size) >
+inline void get_memory_access_list(inst_t* insns, unsigned char* paths, memory_path path, unsigned warp_parts, unsigned line_size, bool limit_broadcast,std::vector<mem_access_t> &accessq)
+{
+   // calculates the memory accesses for a generic cache with banks and tags.
+   // can be used for coalesescing 
+
+   //tracks bank accesses for sorting into generations;
+   static std::map<unsigned,unsigned> bank_accs;
+   bank_accs.clear();
+   //keep track of broadcasts with unique orders if limit_broadcast
+   //the normally calculated orders will never be greater than pipe_simd_width;
+   unsigned broadcast_order =  pipe_simd_width;
+
+   unsigned qbegin = accessq.size();
+   unsigned qpartbegin = qbegin;
+   unsigned mem_pipe_size = pipe_simd_width / warp_parts;
+   for (unsigned part = 0; part < (unsigned)pipe_simd_width; part += mem_pipe_size) {
+      for (unsigned i = part; i < part + mem_pipe_size; i++) {
+         if (paths[i] != path) continue; //skip instructions from other memory paths
+         address_type segment = (*tag_func)(insns[i].memreqaddr, line_size);
+         unsigned quarter=0;
+         if ( line_size>=4 ) {
+           quarter = (insns[i].memreqaddr / (line_size/4)) & 3;
+         } 
+         //check if we are already loading this segment. 
+         bool isatomic = (insns[i].callback.function != NULL);
+         unsigned match = 0;
+         if (not isatomic) { //atomics must have own request
+            for (unsigned j = qpartbegin; j < accessq.size(); j++) {
+               if (segment == accessq[j].addr and not accessq[j].isatomic) {
+                  //match
+                  accessq[j].quarter_count[quarter]++;
+                  accessq[j].warp_indices.push_back(i);
+		  if (limit_broadcast) accessq[j].order = ++broadcast_order; //do proadcast in its own cycle.
+                  match = 1;
+                  break;
+               }
+            }
+         }
+         if (!match) {
+            //needs its own request
+            accessq.push_back(mem_access_t());
+            accessq.back().addr = segment; 
+            accessq.back().space = insns[i].space;
+            accessq.back().path = path;
+            accessq.back().isatomic = isatomic;
+            accessq.back().iswrite = is_store(insns[i].op);
+            accessq.back().req_size = line_size;
+            accessq.back().quarter_count[quarter]++;
+            accessq.back().warp_indices.push_back(i);
+
+            //Determine Bank Conflicts.
+            unsigned bank = (*bank_func)(insns[i].memreqaddr, line_size);
+            //ensure no concurrent bank access accross warp parts. 
+            // ie. order will be less than part for all previous loads in previous parts, so:
+            if (bank_accs[bank] < part) bank_accs[bank]=part; 
+            accessq.back().order = bank_accs[bank];
+            bank_accs[bank]++;
+         }
+      }
+      qpartbegin = accessq.size(); //don't coalesce accross warp parts
+   }
+   //sort requests into order accorting to order (orders will not necessarily be consequtive if multiple parts)
+   std::stable_sort(accessq.begin()+qbegin,accessq.end()); //this is a reverse sort, least order last, but doesn't really matter where consumed.
+}  
+
+
+void shader_memory_shared_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector<mem_access_t> &accessq)
+{
+   get_memory_access_list<&shmem_bank_func, &null_tag_func>(shader->pipeline_reg[EX_MM], paths, SHARED_MEM_PATH, 
+                          gpgpu_shmem_pipe_speedup, 
+                          1, //shared memory doesn't care about line_size, needs to be at least 1;
+			  true, //limit broadcasts to single cycle. 
+                          accessq);
+   //thats it :)
+}
+
+void shader_memory_const_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector<mem_access_t> &accessq)
+{
+   unsigned qbegin = accessq.size();
+   get_memory_access_list<&null_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, CONSTANT_MEM_PATH, 
+                          1, //warp parts
+                          shader->L1constcache->line_sz,
+			  false, //no broadcast limit.
+                          accessq);
+   //do cache checks here for each request, could be done later for more accurate timing of cache accesses, but probably uneccesary; 
+   for (unsigned i = qbegin; i < accessq.size(); i++) {
+      if (is_param(accessq[i].space)) {
+         accessq[i].cache_hit = true;
+      } else {
+         cache_request_status status = shd_cache_access_wb(shader->L1constcache,
+                                                            accessq[i].addr,
+                                                            WORD_SIZE, //this field is ingored.
+                                                            0, //should always be a read
+                                                            shader->gpu_cycle,
+							    NULL/*should never writeback*/);
+         accessq[i].cache_hit = (status == HIT);
+         if (gpgpu_perfect_mem) accessq[i].cache_hit = true;
+	 if (accessq[i].cache_hit) L1_const_miss++;
+      } 
+      accessq[i].cache_checked = true;
+   }
+}
+
+void shader_memory_texture_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector<mem_access_t> &accessq)
+{
+   unsigned qbegin = accessq.size();
+   get_memory_access_list<&null_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, TEXTURE_MEM_PATH, 
+                          1, //warp parts
+                          shader->L1texcache->line_sz,
+			  false, //no broadcast limit.
+                          accessq);
+   //do cache checks here for each request, could be done later for more accurate timing of cache accesses, but probably uneccesary; 
+   for (unsigned i = qbegin; i < accessq.size(); i++) {
+      cache_request_status status = shd_cache_access_wb(shader->L1texcache,
+                                                         accessq[i].addr,
+                                                         WORD_SIZE, //this field is ignored.
+                                                         0, //should always be a read
+                                                         shader->gpu_cycle,
+							 NULL /*should never writeback*/);
+      accessq[i].cache_hit = (status == HIT);
+      if (gpgpu_perfect_mem) accessq[i].cache_hit = true;
+      if (accessq[i].cache_hit) L1_texture_miss++;
+      accessq[i].cache_checked = true;
+   }
+}
+
+void shader_memory_global_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector<mem_access_t> &accessq)
+{
+   unsigned qbegin = accessq.size();
+   unsigned warp_parts = 1;
+   unsigned line_size = shader->L1cache->line_sz;
+   if (gpgpu_coalesce_arch == 13) {
+      warp_parts = 2;
+      if(gpgpu_no_dl1) {
+         int valindex = -1;
+         for (int i = 0; i < pipe_simd_width; i++) {
+            if (paths[i] == GLOBAL_MEM_PATH) {
+               valindex = i;
+               break;
+            }
+         }
+         assert(valindex != -1);
+         // line size is dependant on instruction;
+         //assume first valid thread instruction is the same as the rest.
+         switch (shader->pipeline_reg[EX_MM][valindex].data_size) {
+         case 1:
+            line_size = 32;
+            break;
+         case 2:
+            line_size = 64;
+            break;
+         case 4:
+         case 8:
+            line_size = 128;
+            break;
+         default:
+            assert(0);
+         }
+      }
+   }          
+   get_memory_access_list<&dcache_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, GLOBAL_MEM_PATH, 
+                          warp_parts, //warp parts
+                          line_size,
+                          false, //no broadcast limit.
+                          accessq);
+
+   for (unsigned i = qbegin; i < accessq.size(); i++) {
+      if (gpgpu_coalesce_arch == 13 and gpgpu_no_dl1) {
+         //if there is no l1 cache it makes sense to do coalescing here.
+         //reduce memory request sizes.
+         char* quarter_counts = accessq[i].quarter_count;
+         bool low = quarter_counts[0] or quarter_counts[1];
+         bool high = quarter_counts[2] or quarter_counts[3];
+         if (accessq[i].req_size == 128) {
+            if (low xor high) { //can reduce size
+               accessq[i].req_size = 64;
+               if (high) accessq[i].addr += 64;
+               low = quarter_counts[0] or quarter_counts[2]; //set low and high for next pass
+               high = quarter_counts[1] or quarter_counts[3];
+            }
+         }
+         if (accessq[i].req_size == 64) {
+            if (low xor high) { //can reduce size
+               accessq[i].req_size = 32;
+               if (high) accessq[i].addr += 32;
+            }
+         }
+      }
+   }
+}
+      
+
+
+mem_stage_stall_type send_mem_request(shader_core_ctx_t *shader, mem_access_t &access){
+   inst_t* warp = shader->pipeline_reg[EX_MM];
+   inst_t* req_head = warp + access.warp_indices[0];
+
+   if (access.need_wb) {
+      //fill out and send a writeback
+      unsigned req_size = shader->L1cache->line_sz + WRITE_PACKET_SIZE;
+      if (!(shader->fq_has_buffer(access.wb_addr, req_size, true, shader->sid))) {
+         gpu_stall_sh2icnt++; 
+         return WB_ICNT_RC_FAIL;
+      }
+
+      shader->fq_push( access.wb_addr,
+                       req_size,
+                       true, NO_PARTIAL_WRITE, shader->sid, -1, NULL, 
+                       0, 
+		       is_local(access.space)?LOCAL_ACC_W:GLOBAL_ACC_W, //space of cache line is same as new request
+		        -1);
+      L1_writeback++;
+      access.need_wb = false; 
+   }
+
+   bool requires_mshr = (shader->model != MIMD) and (not access.iswrite);
+
+   //this decoding here might belong elsewhere
+   unsigned code;
+   mem_access_type  access_type;
+   switch(access.space) {
+   case CONST_DIRECTIVE:
+   case PARAM_DIRECTIVE:
+      code = CONSTC;
+      access_type = CONST_ACC_R;   
+      break;
+   case TEX_DIRECTIVE:
+      code = TEXTC;
+      access_type = TEXTURE_ACC_R;   
+      break;
+   case GLOBAL_DIRECTIVE:
+      code = DCACHE;
+      access_type = (access.iswrite)? GLOBAL_ACC_W: GLOBAL_ACC_R;   
+      break;
+   case LOCAL_DIRECTIVE:
+      code = DCACHE;
+      access_type = (access.iswrite)? LOCAL_ACC_W: LOCAL_ACC_R;   
+      break;
+   default:
+      assert(0); // NOT A MEM SPACE;
+      break; 
+   }
+
+   //reserve mshr
+   if (requires_mshr and not access.reserved_mshr) {
+
+      // can allocate mshr?
+      if (not shader->mshr_unit->has_mshr(1)) {
+         //no mshr available;
+         return MSHR_RC_FAIL;
+      }
+
+      access.reserved_mshr = shader->mshr_unit->add_mshr(access, warp);
+      access.recheck_cache = false; //we have an mshr now, so have checked cache in same cycle as checking mshrs, so have merged if necessary.
+   }
+
+   //require inct if access is this far without reserved mshr, or has and mshr but not merged with another request
+   bool requires_icnt = (not access.reserved_mshr) or (not access.reserved_mshr->merged_on_other_reqest);
+
+   if (requires_icnt) {
+
+      //calculate request size for icnt check (and later send);
+      unsigned request_size = access.req_size;
+      if (access.iswrite) {
+         if (requires_mshr) {
+            //needs information for a load back into cache.
+            request_size += READ_PACKET_SIZE + WRITE_MASK_SIZE;
+         } else {
+            //plain write
+            request_size += WRITE_PACKET_SIZE + WRITE_MASK_SIZE;
+         }
+      }
+
+
+      // can allocate icnt?
+      //unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid);
+      if (!(shader->fq_has_buffer(access.addr, request_size, access.iswrite, shader->sid))) {
+         gpu_stall_sh2icnt++;
+	 //std::cout<< "failed to push " << request_size << " bytes" << std::endl;
+         return ICNT_RC_FAIL;
+      }
+
+      //send over interconnect
+
+      unsigned cache_hits_waiting = 0; //fixme do we really want to be passing this in?
+
+      partial_write_mask_t  write_mask = NO_PARTIAL_WRITE;
+      if (access.iswrite) {
+         for (unsigned i=0;i < access.warp_indices.size();i++) {
+            unsigned w = access.warp_indices[i];
+            int data_offset = warp[w].memreqaddr & ((unsigned long long int)access.req_size - 1);
+            for (unsigned b = data_offset; b < data_offset + warp[w].data_size; b++) write_mask.set(b);
+         }
+         if (write_mask.count() != access.req_size) {
+            gpgpu_n_partial_writes++;
+         }
+      }
+
+      //typedef unsigned char (*fq_push_t)(unsigned long long int addr, int bsize, unsigned char readwrite,
+      //                             unsigned long long int partial_write_mask, 
+      //                             int sid, int wid, mshr_entry* mshr, int cache_hits_waiting,  
+      //                             enum mem_access_type mem_acc, address_type pc);
+      shader->fq_push( access.addr,
+                       request_size,
+                       access.iswrite, write_mask, shader->sid, req_head->hw_thread_id/warp_size, access.reserved_mshr, 
+                       cache_hits_waiting, access_type, req_head->pc);
+
+   }
+
+
+   //book keeping for mshr since this request is done (sent/accounted for) at this point;
+   if (requires_mshr) {
+
+      for (unsigned i = 0; i < access.warp_indices.size(); i++) {
+         unsigned o = access.warp_indices[i];
+         shader->pending_mem_access++;
+         inflight_memory_insn_add(shader, &warp[o]);
+         
+#if 0    //old stats
+         if (i > 0) { //maintain old stats (yes/no?)
+            shader->thread[warp[o].hw_thread_id].n_l1_mrghit_ac++; 
+            shd_cache_mergehit(shader->L1texcache, warp[o].memreqaddr); //fixme;
+         }
+#endif   
+      }
+
+      if (not access.iswrite) {
+         // set the pipeline instructions in this request to noops, they all wait for memory;
+         for (unsigned i = 0; i < access.warp_indices.size(); i++) {
+            unsigned o = access.warp_indices[i];
+            shader->pipeline_reg[EX_MM][o] = nop_inst;
+            
+         }
+      }
+   }
+
+   return NO_RC_FAIL;
+}     
+
+
+bool shader_memory_shared_cycle( shader_core_ctx_t *shader, std::vector<mem_access_t> &accessq, 
+                            mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){
+   //consume port number orders from the top of the queue;
+   for (unsigned i = 0; i < (unsigned) gpgpu_shmem_port_per_bank; i++) {
+      if (accessq.empty()) break;
+      unsigned current_order = accessq.back().order;
+      //consume all requests of the same order (concurrent bank requests)
+      while ((not accessq.empty()) and accessq.back().order == current_order) accessq.pop_back();
+   }
+   if (not accessq.empty()) {
+      rc_fail = BK_CONF;
+      fail_type = S_MEM;
+      gpgpu_n_shmem_bkconflict++;
+   }
+   return accessq.empty(); //done if empty.
+}
+
+//generic memory access queue processing, accessq must be sorted by order
+//--that is, requests of similar order are expected to be contiguous in the queueu.
+//if you want to use this for shared memory, make sure they are marked as cashe hits (not default)
+// cycle_exec may be called multiple times if memory fails. typically used for cache checks
+template < mem_stage_stall_type (*cycle_exec)(shader_core_ctx_t*, mem_access_t&) >
+inline mem_stage_stall_type shader_memory_generic_process_queue( shader_core_ctx_t *shader, 
+                                                         unsigned ports_per_bank, unsigned memory_send_max, 
+                                                         std::vector<mem_access_t> &accessq ){
+   mem_stage_stall_type rc_fail = NO_RC_FAIL; 
+   // number of requests to sent to memory this cycle
+   unsigned mem_req_count = 0;
+   //consume port number orders from the top of the queue;
+   for (unsigned i = 0; i < ports_per_bank; i++) {
+      if (accessq.empty()) break;
+      unsigned current_order = accessq.back().order;
+      //consume all requests of the same order (concurrent bank requests)
+      //stop when things that go to memory exceed a per cycle limit.
+      while ((not accessq.empty()) and accessq.back().order == current_order and rc_fail == NO_RC_FAIL) {
+         rc_fail = (*cycle_exec)(shader, accessq.back());
+         if (rc_fail != NO_RC_FAIL) break; //can't complete this request this cycle.
+         if (not accessq.back().cache_hit){
+            if (mem_req_count < memory_send_max) {
+               mem_req_count++;
+               rc_fail = send_mem_request(shader, accessq.back()); //try to get mshr, icnt, send;
+            }
+            else {
+                rc_fail = COAL_STALL; //not really a coal stall, its a too many memory request stall;
+            }
+            if (rc_fail != NO_RC_FAIL) break; //can't complete this request this cycle.
+         }
+         accessq.pop_back();
+      }
+   }
+   if (not accessq.empty() and rc_fail == NO_RC_FAIL) {
+      //no resource failed so must be a bank comflict.
+      rc_fail = BK_CONF;
+   }
+   return rc_fail;
+}  
+
+mem_stage_stall_type ccache_check(shader_core_ctx_t *shader, mem_access_t& access){ /*done in process queue*/ return NO_RC_FAIL;}
+
+bool shader_memory_constant_cycle( shader_core_ctx_t *shader, std::vector<mem_access_t> &accessq, 
+                            mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){
+
+   mem_stage_stall_type fail = shader_memory_generic_process_queue<ccache_check>( shader, gpgpu_const_port_per_bank, 
+                                                                   1, //memory send max per cycle
+                                                                   accessq );
+   if (fail != NO_RC_FAIL){ 
+      rc_fail = fail; //keep other fails if this didn't fail.
+      fail_type = C_MEM;
+      if (rc_fail == BK_CONF or rc_fail == COAL_STALL) {
+         gpgpu_n_cmem_portconflict++; //coal stalls aren't really a bank conflict, but this maintains previous behavior.
+      }
+   }
+   return accessq.empty(); //done if empty.
+}
+
+mem_stage_stall_type tcache_check(shader_core_ctx_t *shader, mem_access_t& access){ /*done in process queue*/ return NO_RC_FAIL;}
+
+bool shader_memory_texture_cycle( shader_core_ctx_t *shader, std::vector<mem_access_t> &accessq, 
+                            mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){
+
+   mem_stage_stall_type fail = shader_memory_generic_process_queue<tcache_check>(shader, 1, //how is tex memory banked? 
+                                                                   1, //memory send max per cycle
+                                                                   accessq );
+   if (fail != NO_RC_FAIL){ 
+      rc_fail = fail; //keep other fails if this didn't fail.
+      fail_type = T_MEM;
+   }
+   return accessq.empty(); //done if empty.
+}
+
+
+mem_stage_stall_type dcache_check(shader_core_ctx_t *shader, mem_access_t& access){
+   if (access.cache_checked and not access.recheck_cache) return NO_RC_FAIL;
+   if (!gpgpu_no_dl1 && !gpgpu_perfect_mem) { 
+      //check cache
+      cache_request_status status = shd_cache_access_wb(shader->L1cache,
+                                                         access.addr,
+                                                         WORD_SIZE, //this field is ignored.
+                                                         access.iswrite,
+                                                         shader->gpu_cycle,
+                                                         &access.wb_addr);
+      if (status == RESERVATION_FAIL) {
+         access.cache_checked = false;
+         return WB_CACHE_RSRV_FAIL;
+      }
+      access.cache_hit = (status == HIT); //if HIT_W_WT then still send to memory so "MISS" 
+      if (status == MISS_W_WB) access.need_wb = true;
+      if (status == WB_HIT_ON_MISS and access.iswrite) 
+      {
+         //write has hit a reserved cache line
+         //it has writen its data into the cache line, so no need to go to memory
+         access.cache_hit = true;
+         L1_write_hit_on_miss++;
+         // here we would search the MSHRs for the originating read, 
+         // and mask off the writen bytes, so they are not overwritten in the cache when it comes back
+         // --- don't actually do this since we are pretending.
+         // MSHR will still forward the unmasked value to its dependant reads. 
+         // if doing stall on use, must stall this thread after this write (otherwise, inproper values may be forwarded to future reads).
+      }
+      if (status == WB_HIT_ON_MISS and not access.iswrite) {
+         //read has hit on a reserved cache line, 
+         //we need to make sure cache check happens on same cycle as a mshr merge happens, otherwise we might miss it coming back
+         access.recheck_cache = true;
+      }
+      access.cache_checked = true;
+   } else {
+      access.cache_hit = false;
+   }
+
+   if (gpgpu_perfect_mem) access.cache_hit = true;
+   
+   //atomics always go to memory 
+   if (access.isatomic) {
+      if (!gpgpu_perfect_mem) {
+         access.cache_hit = false;
+      } else {
+         //unless perfect mem, in which case, the callback can only be done here 
+         dram_callback_t &atom_exec = shader->pipeline_reg[EX_MM][access.warp_indices[0]].callback;
+         atom_exec.function(atom_exec.instruction, atom_exec.thread);
+      }
+   }
+
+   if (!access.cache_hit) { 
+      if (access.iswrite) L1_write_miss++;
+      else L1_read_miss++;
+   }
+   return NO_RC_FAIL;
+}
+
+bool shader_memory_global_cycle( shader_core_ctx_t *shader, std::vector<mem_access_t> &accessq, 
+                            mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){
+   mem_stage_stall_type fail = shader_memory_generic_process_queue<&dcache_check>(shader, gpgpu_cache_port_per_bank, 
+                                                                   1, //memory send max per cycle
+                                                                   accessq );
+   if (fail != NO_RC_FAIL) {
+      rc_fail = fail; //keep other fails if this didn't fail.
+      //need to determine load/store, local/global:
+      bool iswrite = accessq.back().iswrite;
+      if (is_local(accessq.back().space)) {
+         fail_type = (iswrite)?L_MEM_ST:L_MEM_LD;
+      } else {
+         fail_type = (iswrite)?G_MEM_ST:G_MEM_LD;
+      }
+
+      if (rc_fail == BK_CONF or rc_fail == COAL_STALL) {
+         gpgpu_n_cache_bkconflict++;
+      }
+   }
+   return accessq.empty(); //done if empty.
+}
+
+inline void mem_instruction_stats(inst_t* warp){
+   //there must be a better way to count these
+   for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) {
+      if (warp[i].hw_thread_id == -1) continue; //bubble 
+         //this breaks some encapsulation: the is_[space] functions, if you change those, change this.
+      bool store = is_store(warp[i].op);
+      switch (warp[i].space) {
+      case SHARED_DIRECTIVE:
+         gpgpu_n_shmem_insn++;
+         break;
+      case CONST_DIRECTIVE:
+         gpgpu_n_const_insn++;
+         break;
+      case PARAM_DIRECTIVE:
+         gpgpu_n_param_insn++;
+         break;
+      case TEX_DIRECTIVE:
+         gpgpu_n_tex_insn++;
+         break;
+      case GLOBAL_DIRECTIVE:
+      case LOCAL_DIRECTIVE:
+         if (store){ 
+            gpgpu_n_store_insn++;
+         } else {
+            gpgpu_n_load_insn++;
+         }
+         break;
+      default:
+         //assert(0); //unknown mem space.
+         break; //not a mem instruction
+      }
+   }
+}
+
+struct shader_queues_t{
+   std::vector<mem_access_t> shared;
+   std::vector<mem_access_t> constant;
+   std::vector<mem_access_t> texture;
+   std::vector<mem_access_t> global;
+};
+
+void shader_memory_queue(shader_core_ctx_t *shader, shader_queues_t *accessqs)
+{
+      //classify memory according to type;
+      static unsigned char *path = NULL;
+      if (!path) path = (unsigned char*)malloc(pipe_simd_width * sizeof(unsigned char));
+      memset(path, 0, pipe_simd_width * sizeof(unsigned char));
+      //static std::vector<char> path;
+      //path.clear(); path.resize(p, NO_MEM_PATH);
+
+      static unsigned type_counts[NUM_MEM_PATHS];
+      memset(type_counts, 0, sizeof(type_counts));
+      //static std::vector<unsigned> type_counts;
+      //type_counts.clear(); type_counts.resize(NUM_MEM_PATHS);
+
+      for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) {
+         if (shader->pipeline_reg[EX_MM][i].hw_thread_id == -1) continue; //bubble 
+         //this breaks some encapsulation: the is_[space] functions; if you change those, change this.
+         switch (shader->pipeline_reg[EX_MM][i].space) {
+         case SHARED_DIRECTIVE:
+            path[i] = SHARED_MEM_PATH;
+            type_counts[SHARED_MEM_PATH]++;
+            break;
+         case CONST_DIRECTIVE:
+         case PARAM_DIRECTIVE:
+            path[i] = CONSTANT_MEM_PATH;
+            type_counts[CONSTANT_MEM_PATH]++;   
+            break;
+         case TEX_DIRECTIVE:
+            path[i] = TEXTURE_MEM_PATH;
+            type_counts[TEXTURE_MEM_PATH]++;
+            break;
+         case GLOBAL_DIRECTIVE:
+         case LOCAL_DIRECTIVE:
+            path[i] = GLOBAL_MEM_PATH;
+            type_counts[GLOBAL_MEM_PATH]++;
+            break;
+         default:
+            //path[i] = NO_MEM_PATH;
+            break; //not a mem instruction
+         }
+      }
+
+      //instruction counting:
+      mem_instruction_stats(shader->pipeline_reg[EX_MM]);
+      
+
+      if (type_counts[SHARED_MEM_PATH]) shader_memory_shared_process_inst(shader, path, accessqs->shared);
+      if (type_counts[CONSTANT_MEM_PATH]) shader_memory_const_process_inst(shader, path, accessqs->constant);
+      if (type_counts[TEXTURE_MEM_PATH]) shader_memory_texture_process_inst(shader, path, accessqs->texture);
+      if (type_counts[GLOBAL_MEM_PATH]) shader_memory_global_process_inst(shader, path, accessqs->global);
+
+}
+
+
+void shader_memory( shader_core_ctx_t *shader, unsigned int shader_number )
+{
+   enum mem_stage_stall_type rc_fail = NO_RC_FAIL; // resource allocation
+
+   //these should be local to the shader structure but can't because it is included in non c++ files.
+   //so provide static storage for it here
+   static std::vector<shader_queues_t> shader_memory_queues;
+   if (shader_memory_queues.size() == 0) {
+      shader_memory_queues.resize(gpu_n_shader);
+      for (unsigned i = 0; i < gpu_n_shader; i++) {
+         shader_memory_queues[i].shared.reserve(pipe_simd_width);
+         shader_memory_queues[i].constant.reserve(pipe_simd_width);
+         shader_memory_queues[i].texture.reserve(pipe_simd_width);
+         shader_memory_queues[i].global.reserve(pipe_simd_width);
+      }
+   }
+   shader_queues_t *accessqs = &(shader_memory_queues[shader->sid]);
+
+   if (shader->shader_memory_new_instruction_processed == 0) {
+      shader->shader_memory_new_instruction_processed = 1; //only do this once per pipeline occupant
+      shader_memory_queue(shader, accessqs);
+   }
+
+   bool done = true;
+   mem_stage_access_type type; 
+
+   done &= shader_memory_shared_cycle(shader, accessqs->shared, rc_fail, type);
+   done &= shader_memory_constant_cycle(shader, accessqs->constant, rc_fail, type);
+   done &= shader_memory_texture_cycle(shader, accessqs->texture, rc_fail, type);
+   done &= shader_memory_global_cycle(shader, accessqs->global, rc_fail, type);
+
+   //wb stalled?
+   int wb_stalled = 0; // check if next stage is stalled
+   for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) {
+      if (shader->pipeline_reg[MM_WB][i].hw_thread_id != -1 ) {
+         wb_stalled = 1;
+         break;
+      }
+   }
+
+   if (!done) {
+      assert(rc_fail != NO_RC_FAIL);
+      //log stall types
+      gpu_stall_shd_mem++;
+      gpu_stall_shd_mem_breakdown[type][rc_fail]++;
+   }
+
+   if (!done or wb_stalled) return;
+
+   // this memory stage is done and not stalled by wb
+   // pipeline forward
+
+   check_stage_pcs(shader,EX_MM);
+   // and pass instruction from EX_MM to MM_WB for cache hit
+   for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) {
+      if (shader->pipeline_reg[EX_MM][i].hw_thread_id == -1 )
+         continue; // bubble 
+      shader->pipeline_reg[MM_WB][i] = shader->pipeline_reg[EX_MM][i];
+      shader->pipeline_reg[MM_WB][i].mm_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+      shader->pipeline_reg[EX_MM][i] = nop_inst;
+   }    
+
+   // reflect the change to EX|MM pipeline register to the pre_mem stage
+   if (gpgpu_pre_mem_stages) {
+      check_stage_pcs(shader,EX_MM);
+      for (unsigned i=0;i< (unsigned)pipe_simd_width ;i++ )
+         shader->pre_mem_pipeline[gpgpu_pre_mem_stages][i] = shader->pipeline_reg[EX_MM][i];
+   }
+}
+
+int writeback_l1_miss =0 ;
+
+
+void register_cta_thread_exit(shader_core_ctx_t *shader, int tid )
+{
+   if (gpgpu_cuda_sim && gpgpu_spread_blocks_across_cores) {
+      unsigned padded_cta_size = ptx_sim_cta_size();
+      if (padded_cta_size%warp_size) {
+         padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+      }
+      int cta_num = tid/padded_cta_size;
+      assert( shader->cta_status[cta_num] > 0 );
+      shader->cta_status[cta_num]--;
+      if (!shader->cta_status[cta_num]) {
+         shader->n_active_cta--;
+         shader->deallocate_barrier(cta_num);
+         shader_CTA_count_unlog(shader->sid, 1);
+         printf("Shader %d finished CTA #%d (%lld,%lld)\n", shader->sid, cta_num, gpu_sim_cycle, gpu_tot_sim_cycle );
+      }
+   }
+}
+
+#if 0
+//this function is unecessary, cache is properly dirtied by fill with cache line function in gpu-sim.cc
+void dirty_cache_lines(shader_core_ctx_t *shader, mshr_entry_t* mshr){
+   shd_cache_line_t *hit_cacheline;
+   if (mshr->istexture) {
+      hit_cacheline = shd_cache_access(shader->L1texcache, 
+                                       mshr->addr, WORD_SIZE, 
+                                       mshr->iswrite, //should always be 0
+                                       shader->gpu_cycle); 
+      shd_cache_undo_stats( shader->L1texcache, !hit_cacheline );
+   } else if (mshr->isconst) {
+      hit_cacheline = shd_cache_access(shader->L1constcache, 
+                                       mshr->addr, WORD_SIZE, 
+                                       mshr->iswrite, //should always be 0
+                                       shader->gpu_cycle); 
+      shd_cache_undo_stats( shader->L1constcache, !hit_cacheline );
+   } else if (!gpgpu_no_dl1) {
+      hit_cacheline = shd_cache_access(shader->L1cache, 
+                                       mshr->addr, WORD_SIZE, 
+                                       mshr->iswrite, 
+                                       shader->gpu_cycle); 
+      shd_cache_undo_stats( shader->L1constcache, !hit_cacheline );
+      if (!hit_cacheline) {
+         writeback_l1_miss++;
+      }
+   }
+}
+#endif
+
+typedef struct {
+   unsigned pc;
+   unsigned long latency;
+   void *ptx_thd_info;
+} insn_latency_info;
+
+void obtain_insn_latency_info(insn_latency_info *latinfo, inst_t *insn)
+{
+   latinfo->pc = insn->pc;
+   latinfo->latency = gpu_tot_sim_cycle + gpu_sim_cycle - insn->ts_cycle;
+   latinfo->ptx_thd_info = insn->ptx_thd_info;
+}
+
+unsigned   gpu_n_max_mshr_writeback=1;
+void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) 
+{
+   static int *unlock_tid = NULL;
+   static int *freed_warp = NULL;
+   static int *mshr_tid = NULL;
+   static int *pl_tid = NULL;
+   static int *done_tid = NULL;
+   insn_latency_info *unlock_lat_info = NULL;
+   static insn_latency_info *mshr_lat_info = NULL;
+   static insn_latency_info *pl_lat_info = NULL;
+
+   mshr_entry *mshr_head = NULL;
+
+   int tid;
+   op_type op;
+   int o1, o2, o3, o4;
+   bool stalled_by_MSHR = false;
+   bool writeback_by_MSHR = false;
+   bool w2rf = false;
+
+   if ( unlock_tid == NULL ) {
+      unlock_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
+      mshr_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
+      pl_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
+      done_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
+      freed_warp = (int *) malloc(sizeof(int)*pipe_simd_width);
+      mshr_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width);
+      pl_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width);
+   }
+   memset(unlock_tid, -1, sizeof(int)*pipe_simd_width);
+   memset(mshr_tid,   -1, sizeof(int)*pipe_simd_width);
+   memset(pl_tid,     -1, sizeof(int)*pipe_simd_width);
+   memset(done_tid,   -1, sizeof(int)*pipe_simd_width);
+   memset(freed_warp,  0, sizeof(int)*pipe_simd_width);
+   unlock_lat_info = NULL;
+
+   check_stage_pcs(shader,MM_WB);
+
+   /* Generate Condition for instruction writeback to register file.
+      A load miss *instruction* does not reach writeback until the data is fetched */
+   for (int i=0; i<pipe_simd_width; i++) {
+      tid = shader->pipeline_reg[MM_WB][i].hw_thread_id;
+      w2rf |= (tid >= 0); 
+      pl_tid[i] = tid;
+   }
+
+   //check mshrs for commit;
+   //in future do req bank checking here;
+   unsigned mshr_threads_unlocked = 0;
+   for (unsigned i = 0; i < gpu_n_max_mshr_writeback; i++) {
+      mshr_head = shader->mshr_unit->return_head();
+      if (mshr_head) {
+         //bail if we can't unlock anymore threads, needs to be implemented better.
+         if (mshr_threads_unlocked + mshr_head->insts.size() > (unsigned) pipe_simd_width) break;//todo, do this right
+         assert(!gpgpu_strict_simd_wrbk);//implementation commented out below 
+/*
+         //for stalling in the middle of an mshr writeback due to mshr having threads from multiple warps, does this happen anymore?
+         static unsigned next_mshr_index = 0;
+         int  mshr_warpid = -1; 
+         bool mshr_not_blocked_by_samewarp = true;
+         //use below: for (j = next_mshr_index; ...
+*/
+         assert (mshr_head->insts.size());
+         for (unsigned j = 0; j < mshr_head->insts.size(); j++) {
+            inst_t &insn = mshr_head->insts[j];
+            time_vector_update(insn.uid,MR_WRITEBACK,gpu_sim_cycle+gpu_tot_sim_cycle,RD_REQ);
+            obtain_insn_latency_info(&mshr_lat_info[mshr_threads_unlocked], &(mshr_head->insts[j]));
+            inflight_memory_insn_sub(shader, &mshr_head->insts[j]);
+            assert (insn.hw_thread_id >= 0);
+            unlock_tid[mshr_threads_unlocked] = insn.hw_thread_id;
+            shader->pending_mem_access--;
+            // for ensuring that we don't unlock more than the code allows, needs to be fixed.
+            mshr_threads_unlocked++;
+/*
+            next_mshr_index++;
+            if ((shader->model == POST_DOMINATOR || shader->model == NO_RECONVERGE) && gpgpu_strict_simd_wrbk) {
+               // restricting the threads from mshr to be in the same warp
+               if (mshr_warpid == -1) {
+                  mshr_warpid = mshr_head->insts[j].hw_thread_id / warp_size;
+               } else if ((unsigned)mshr_warpid != (mshr_head->insts[j].hw_thread_id / warp_size)) {
+                  warp_conflict_at_writeback++;
+                  mshr_not_blocked_by_samewarp = false;
+                  break;
+               }
+            }
+*/
+         }
+         //done with it since garanteed to wb
+/*
+         if (mshr_not_blocked_by_samewarp) {
+*/
+         //this mshr is done this cycle, can pop it
+         shader->mshr_unit->pop_return_head();
+/*
+            //reset for next mshr
+            next_mshr_index = 0;
+         }
+*/
+         writeback_by_MSHR = true;
+         unlock_lat_info = mshr_lat_info; 
+         if (w2rf) {
+            stalled_by_MSHR = true;
+         }
+         assert(mshr_threads_unlocked);
+      }
+   }
+   if (stalled_by_MSHR) {
+      gpu_stall_by_MSHRwb++;
+   }
+
+   if (!writeback_by_MSHR) { //!writeback_by_MSHR
+      for (int i=0; i<pipe_simd_width; i++) {
+         op  = shader->pipeline_reg[MM_WB][i].op;
+         tid = shader->pipeline_reg[MM_WB][i].hw_thread_id;
+         o1  = shader->pipeline_reg[MM_WB][i].out[0];
+         o2  = shader->pipeline_reg[MM_WB][i].out[1];
+         o3  = shader->pipeline_reg[MM_WB][i].out[2];
+         o4  = shader->pipeline_reg[MM_WB][i].out[3];
+
+         unlock_tid[i] = pl_tid[i];
+         obtain_insn_latency_info(&pl_lat_info[i], &shader->pipeline_reg[MM_WB][i]);
+      }
+      unlock_lat_info = pl_lat_info;
+   }
+   int thd_unlocked = 0;
+   for (int i=0; i<pipe_simd_width; i++) {
+      // NOTE: no need to check for next-stage stall at the last stage
+      if (unlock_tid[i] >= 0 ) { // not unlocking an invalid thread (ie. due to a bubble)
+         // thread completed if it is going to fetching beyond code boundry 
+         if ( gpgpu_cuda_sim && ptx_thread_done(shader->thread[unlock_tid[i]].ptx_thd_info) ) {
+
+            finished_trace += 1;
+            shader->not_completed -= 1;
+            gpu_completed_thread += 1;
+
+            int warp_id = wid_from_hw_tid(unlock_tid[i],warp_size);
+            if (!(shader->warp[warp_id].n_completed < (unsigned)warp_size)) {
+               printf("shader[%d]->warp[%d].n_completed = %d; warp_size = %d\n", 
+                      shader->sid,warp_id, shader->warp[warp_id].n_completed, warp_size);
+            }
+            assert( shader->warp[warp_id].n_completed < (unsigned)warp_size );
+            shader->warp[warp_id].n_completed++;
+            if ( shader->model == NO_RECONVERGE ) {
+               update_max_branch_priority(shader,warp_id,grid_num);
+            }
+
+            if (gpgpu_no_divg_load) {
+               int amask = wpt_signal_complete(unlock_tid[i], shader);
+               freed_warp[i] = (amask != 0)? 1 : 0;
+            } else {
+               register_cta_thread_exit(shader, unlock_tid[i] );
+            }
+         } else { //thread is not finished yet
+            // program is not finished yet, allow more fetch 
+            if (gpgpu_no_divg_load) {
+               freed_warp[i] = wpt_signal_avail(unlock_tid[i], shader);
+            } else {
+               shader->thread[unlock_tid[i]].avail4fetch++;
+               assert(shader->thread[unlock_tid[i]].avail4fetch <= 1);
+               assert( shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch < (unsigned)warp_size );
+               shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch++;
+               thd_unlocked = 1;
+            }
+         }
+
+         // At any rate, a real instruction is committed
+         // - don't count cache miss
+         if ( shader->pipeline_reg[MM_WB][i].inst_type != NO_OP_FLAG ) {
+            gpu_sim_insn++;
+            if ( !is_const(shader->pipeline_reg[MM_WB][i].space) ) 
+               gpu_sim_insn_no_ld_const++;
+            gpu_sim_insn_last_update = gpu_sim_cycle;
+            shader->num_sim_insn++;
+            shader->thread[unlock_tid[i]].n_insn++;
+            shader->thread[unlock_tid[i]].n_insn_ac++;
+         }
+
+         if (enable_ptx_file_line_stats) {
+            unsigned pc = unlock_lat_info[i].pc;
+            unsigned long latency = unlock_lat_info[i].latency;
+            ptx_file_line_stats_add_latency(unlock_lat_info[i].ptx_thd_info, pc, latency);
+         }
+      }
+   }
+   if (shader->using_commit_queue && thd_unlocked) {
+      int *tid_unlocked = alloc_commit_warp();
+      memcpy(tid_unlocked, unlock_tid, sizeof(int)*pipe_simd_width); //NOTE: this maybe warp_size
+      dq_push(shader->thd_commit_queue,(void*)tid_unlocked);
+   }
+
+
+   /* The pipeline can be stalled by MSHR */
+   if (!stalled_by_MSHR) {
+      for (int i=0; i<pipe_simd_width; i++) {
+         shader->pipeline_reg[WB_RT][i] = shader->pipeline_reg[MM_WB][i];
+         shader->pipeline_reg[MM_WB][i] = nop_inst;
+      }
+   }
+}
+
+
+void shader_print_runtime_stat( FILE *fout ) {
+   unsigned i;
+
+   fprintf(fout, "SHD_INSN: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      fprintf(fout, "%u ",sc[i]->num_sim_insn);
+   }
+   fprintf(fout, "\n");
+   fprintf(fout, "SHD_THDS: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      fprintf(fout, "%u ",sc[i]->not_completed);
+   }
+   fprintf(fout, "\n");
+   fprintf(fout, "SHD_DIVG: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      fprintf(fout, "%u ",sc[i]->n_diverge);
+   }
+   fprintf(fout, "\n");
+
+   fprintf(fout, "THD_INSN: ");
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      fprintf(fout, "%d ", sc[0]->thread[i].n_insn);
+   }
+   fprintf(fout, "\n");
+}
+
+
+void shader_print_l1_miss_stat( FILE *fout ) {
+   unsigned i;
+
+   fprintf(fout, "THD_INSN_AC: ");
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      fprintf(fout, "%d ", sc[0]->thread[i].n_insn_ac);
+   }
+   fprintf(fout, "\n");
+
+   fprintf(fout, "T_L1_Mss: "); //l1 miss rate per thread
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      fprintf(fout, "%d ", sc[0]->thread[i].n_l1_mis_ac);
+   }
+   fprintf(fout, "\n");
+
+   fprintf(fout, "T_L1_Mgs: "); //l1 merged miss rate per thread
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      fprintf(fout, "%d ", sc[0]->thread[i].n_l1_mis_ac - sc[0]->thread[i].n_l1_mrghit_ac);
+   }
+   fprintf(fout, "\n");
+
+   fprintf(fout, "T_L1_Acc: "); //l1 access per thread
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      fprintf(fout, "%d ", sc[0]->thread[i].n_l1_access_ac);
+   }
+   fprintf(fout, "\n");
+
+   //per warp
+   int temp =0; 
+   fprintf(fout, "W_L1_Mss: "); //l1 miss rate per warp
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      temp += sc[0]->thread[i].n_l1_mis_ac;
+      if (i%warp_size == (unsigned)(warp_size-1)) {
+         fprintf(fout, "%d ", temp);
+         temp = 0;
+      }
+   }
+   fprintf(fout, "\n");
+   temp=0;
+   fprintf(fout, "W_L1_Mgs: "); //l1 merged miss rate per warp
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      temp += (sc[0]->thread[i].n_l1_mis_ac - sc[0]->thread[i].n_l1_mrghit_ac);
+      if (i%warp_size == (unsigned)(warp_size-1)) {
+         fprintf(fout, "%d ", temp);
+         temp = 0;
+      }
+   }
+   fprintf(fout, "\n");
+   temp =0;
+   fprintf(fout, "W_L1_Acc: "); //l1 access per warp
+   for (i=0; i<gpu_n_thread_per_shader; i++) {
+      temp += sc[0]->thread[i].n_l1_access_ac;
+      if (i%warp_size == (unsigned)(warp_size-1)) {
+         fprintf(fout, "%d ", temp);
+         temp = 0;
+      }
+   }
+   fprintf(fout, "\n");
+
+}
+
+void shader_print_stage(shader_core_ctx_t *shader, unsigned int stage, 
+                        FILE *fout, int stage_width, int print_mem, int mask ) 
+{
+   int i, j, warp_id = -1;
+
+   for (i=0; i<stage_width; i++) {
+      if (shader->pipeline_reg[stage][i].hw_thread_id > -1) {
+         warp_id = shader->pipeline_reg[stage][i].hw_thread_id / warp_size;
+         break;
+      }
+   }
+   i = (i>=stage_width)? 0 : i;
+
+   fprintf(fout,"0x%04x ", shader->pipeline_reg[stage][i].pc );
+
+   if( mask & 2 ) {
+      fprintf(fout, "(" );
+      for (j=0; j<stage_width; j++)
+         fprintf(fout, "%03d ", shader->pipeline_reg[stage][j].hw_thread_id);
+      fprintf(fout, "): ");
+   } else {
+      fprintf(fout, "w%02d[", warp_id);
+      for (j=0; j<stage_width; j++) 
+         fprintf(fout, "%c", ((shader->pipeline_reg[stage][j].hw_thread_id != -1)?'1':'0') );
+      fprintf(fout, "]: ");
+   }
+
+   if( warp_id != -1 && shader->model == POST_DOMINATOR ) {
+      pdom_warp_ctx_t *warp=&(shader->pdom_warp[warp_id]);
+      if( warp->m_recvg_pc[warp->m_stack_top] == (unsigned)-1 ) {
+         fprintf(fout," rp:--- ");
+      } else {
+         fprintf(fout," rp:0x%03x ", warp->m_recvg_pc[warp->m_stack_top] );
+      }
+   }
+
+   ptx_print_insn( shader->pipeline_reg[stage][i].pc, fout );
+
+   if( mask & 0x10 ) {
+      if ( (shader->pipeline_reg[stage][i].op == STORE_OP ||
+            shader->pipeline_reg[stage][i].op == LOAD_OP) && print_mem )
+         fprintf(fout, "  mem: 0x%016llx", shader->pipeline_reg[stage][i].memreqaddr);
+   }
+   fprintf(fout, "\n");
+}
+
+void shader_print_pre_mem_stages(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask ) 
+{
+   int i, j;
+   int warp_id;
+
+   if (!gpgpu_pre_mem_stages) return;
+
+   for (unsigned pms = 0; pms <= gpgpu_pre_mem_stages - 1; pms++) {
+      fprintf(fout, "PM[%01d] = ", pms);
+
+      warp_id = -1;
+
+      for (i=0; i<pipe_simd_width; i++) {
+         if (shader->pre_mem_pipeline[pms][i].hw_thread_id > -1) {
+            warp_id = shader->pre_mem_pipeline[pms][i].hw_thread_id / warp_size;
+            break;
+         }
+      }
+      i = (i>=pipe_simd_width)? 0 : i;
+
+      fprintf(fout,"0x%04x ", shader->pre_mem_pipeline[pms][i].pc );
+
+      if( mask & 2 ) {
+         fprintf(fout, "(" );
+         for (j=0; j<pipe_simd_width; j++)
+            fprintf(fout, "%03d ", shader->pre_mem_pipeline[pms][j].hw_thread_id);
+         fprintf(fout, "): ");
+      } else {
+         fprintf(fout, "w%02d[", warp_id);
+         for (j=0; j<pipe_simd_width; j++)
+            fprintf(fout, "%c", ((shader->pre_mem_pipeline[pms][j].hw_thread_id != -1)?'1':'0') );
+         fprintf(fout, "]: ");
+      }
+
+      if( warp_id != -1 && shader->model == POST_DOMINATOR ) {
+         pdom_warp_ctx_t *warp=&(shader->pdom_warp[warp_id]);
+         if( warp->m_recvg_pc[warp->m_stack_top] == (unsigned)-1 ) {
+            printf(" rp:--- ");
+         } else {
+            printf(" rp:0x%03x ", warp->m_recvg_pc[warp->m_stack_top] );
+         }
+      }
+
+      ptx_print_insn( shader->pre_mem_pipeline[pms][i].pc, fout );
+
+      if( mask & 0x10 ) {
+         if ( ( shader->pre_mem_pipeline[pms][i].op == LOAD_OP ||
+                shader->pre_mem_pipeline[pms][i].op == STORE_OP ) && print_mem )
+            fprintf(fout, "  mem: 0x%016llx", shader->pre_mem_pipeline[pms][i].memreqaddr);
+      }
+      fprintf(fout, "\n");
+   }
+}
+
+const char * ptx_get_fname( unsigned PC );
+
+void shader_display_pipeline(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask ) 
+{
+   // call this function from within gdb to print out status of pipeline
+   // if you encounter a bug, or to visualize pipeline operation
+   // (this is a good way to "verify" your pipeline model makes sense!)
+
+   fprintf(fout, "=================================================\n");
+   fprintf(fout, "shader %u at cycle %Lu+%Lu (%u threads running)\n", shader->sid, 
+           gpu_tot_sim_cycle, gpu_sim_cycle, shader->not_completed);
+   fprintf(fout, "=================================================\n");
+
+   if ( (mask & 4) && shader->model == POST_DOMINATOR ) {
+      fprintf(fout,"warp status:\n");
+      unsigned n = shader->n_threads / warp_size;
+      for (unsigned i=0; i < n; i++) {
+         unsigned nactive = 0;
+         for (unsigned j=0; j<warp_size; j++ ) {
+            unsigned tid = i*warp_size + j;
+            int done = ptx_thread_done( shader->thread[tid].ptx_thd_info );
+            nactive += (ptx_thread_done( shader->thread[tid].ptx_thd_info )?0:1);
+            if ( done && (mask & 8) ) {
+               unsigned done_cycle = ptx_thread_donecycle( shader->thread[tid].ptx_thd_info );
+               if ( done_cycle ) {
+                  printf("\n w%02u:t%03u: done @ cycle %u", i, tid, done_cycle );
+               }
+            }
+         }
+         if ( nactive == 0 ) {
+            continue;
+         }
+         pdom_warp_ctx_t *warp=&(shader->pdom_warp[i]);
+         for ( int k=0; k <= warp->m_stack_top; k++ ) {
+            if ( k==0 ) {
+               fprintf(fout, "w%02d (%2u thds active): %2u ", i, nactive, k );
+            } else {
+               fprintf(fout, "                      %2u ", k );
+            }
+            for (unsigned m=1,j=0; j<warp_size; j++, m<<=1)
+               fprintf(fout, "%c", ((warp->m_active_mask[k] & m)?'1':'0') );
+            fprintf(fout, " pc: %4u", warp->m_pc[k] );
+            if ( warp->m_recvg_pc[k] == (unsigned)-1 ) {
+               fprintf(fout," rp: ---- cd: %2u ", warp->m_calldepth[k] );
+            } else {
+               fprintf(fout," rp: %4u cd: %2u ", warp->m_recvg_pc[k], warp->m_calldepth[k] );
+            }
+            if ( warp->m_branch_div_cycle[k] != 0 ) {
+               fprintf(fout," bd@%6u ", (unsigned) warp->m_branch_div_cycle[k] );
+            } else {
+               fprintf(fout,"           " );
+            }
+            //fprintf(fout," func=\'%s\' ", ptx_get_fname( warp->m_pc[k] ) );
+            ptx_print_insn( warp->m_pc[k], fout );
+            fprintf(fout,"\n");
+         }
+      }
+      fprintf(fout,"\n");
+   }
+
+   if ( mask & 0x20 ) {
+      fprintf(fout, "TS/IF = ");
+      shader_print_stage(shader, TS_IF, fout, warp_size, print_mem, mask);
+   }
+
+   fprintf(fout, "IF/ID = ");
+   shader_print_stage(shader, IF_ID, fout, pipe_simd_width, print_mem, mask );
+
+   if (shader->using_rrstage) {
+      fprintf(fout, "ID/RR = ");
+      shader_print_stage(shader, ID_RR, fout, pipe_simd_width, print_mem, mask);
+   }
+
+   fprintf(fout, "ID/EX = ");
+   shader_print_stage(shader, ID_EX, fout, pipe_simd_width, print_mem, mask);
+
+   shader_print_pre_mem_stages(shader, fout, print_mem, mask);
+
+   if (!gpgpu_pre_mem_stages)
+      fprintf(fout, "EX/MEM= ");
+   else
+      fprintf(fout, "PM/MEM= ");
+   shader_print_stage(shader, EX_MM, fout, pipe_simd_width, print_mem, mask);
+
+   fprintf(fout, "MEM/WB= ");
+   shader_print_stage(shader, MM_WB, fout, pipe_simd_width, print_mem, mask);
+
+   fprintf(fout, "\n");
+}
+
+void shader_dump_thread_state(shader_core_ctx_t *shader, FILE *fout )
+{
+   fprintf( fout, "\n");
+   for ( unsigned w = 0; w < gpu_n_thread_per_shader/warp_size; w++ ) {
+      int tid = w*warp_size;
+      if ( shader->warp[w].n_completed < (unsigned)warp_size ) {
+         fprintf( fout, "  %u:%3u fetch state = c:%u a4f:%u bw:%u (completed: ", shader->sid, tid, 
+                  shader->warp[w].n_completed,
+                  shader->warp[w].n_avail4fetch,
+                  shader->warp[w].n_waiting_at_barrier );
+
+         for ( unsigned i = tid; i < (w+1)*warp_size; i++ ) {
+            if ( gpgpu_cuda_sim && ptx_thread_done(shader->thread[i].ptx_thd_info) ) {
+               fprintf(fout,"1");
+            } else {
+               fprintf(fout,"0");
+            }
+            if ( (((i+1)%4) == 0) && (i+1) < (w+1)*warp_size ) {
+               fprintf(fout,",");
+            }
+         }
+         fprintf(fout,")\n");
+      }
+   }
+}
+
+void shader_dp(shader_core_ctx_t *shader, int print_mem) {
+   shader_display_pipeline(shader, stdout, print_mem, 7 );
+}
+
+
+unsigned int max_cta_per_shader( shader_core_ctx_t *shader)
+{
+   unsigned int result;
+   unsigned int padded_cta_size;
+
+   padded_cta_size = ptx_sim_cta_size();
+   if (padded_cta_size%warp_size) {
+      padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+      //printf("padded_cta_size=%u\n", padded_cta_size);
+   }
+
+   //Limit by n_threads/shader
+   unsigned int result_thread = shader->n_threads / padded_cta_size;
+
+   const struct gpgpu_ptx_sim_kernel_info *kernel_info = ptx_sim_kernel_info();
+
+   //Limit by shmem/shader
+   unsigned int result_shmem = (unsigned)-1;
+   if (kernel_info->smem > 0)
+      result_shmem = shader->shmem_size / kernel_info->smem;
+
+   //Limit by register count, rounded up to multiple of 4.
+   unsigned int result_regs = (unsigned)-1;
+   if (kernel_info->regs > 0)
+      result_regs = shader->n_registers / (padded_cta_size * ((kernel_info->regs+3)&~3));
+
+   //Limit by CTA
+   unsigned int result_cta = shader->n_cta;
+
+   result = result_thread;
+   result = gs_min2(result, result_shmem);
+   result = gs_min2(result, result_regs);
+   result = gs_min2(result, result_cta);
+
+   static const struct gpgpu_ptx_sim_kernel_info* last_kinfo = NULL;
+   if (last_kinfo != kernel_info) {   //Only print out stats if kernel_info struct changes
+      last_kinfo = kernel_info;
+      printf ("CTA/core = %u, limited by:", result);
+      if (result == result_thread) printf (" threads");
+      if (result == result_shmem) printf (" shmem");
+      if (result == result_regs) printf (" regs");
+      if (result == result_cta) printf (" cta_limit");
+      printf ("\n");
+   }
+
+   if (result < 1) {
+      printf ("Error: max_cta_per_shader(\"%s\") returning %d. Kernel requires more resources than shader has?\n", shader->name, result);
+      abort();
+   }
+   return result;
+}
+
+void shader_cycle( shader_core_ctx_t *shader, 
+                   unsigned int shader_number,
+                   int grid_num ) 
+{
+
+   // last pipeline stage
+   shader_writeback(shader, shader_number, grid_num);
+
+   // three parallel stages (only one does something on a given cycle)
+   //shader_const_memory   (shader, shader_number);
+   shader_memory   (shader, shader_number);
+   //shader_texture_memory   (shader, shader_number);
+
+   // empty stage
+   if (gpgpu_pre_mem_stages)
+      shader_pre_memory(shader, shader_number);
+
+   shader_execute  (shader, shader_number);
+   if (shader->using_rrstage) {
+      // model register bank conflicts 
+      // (see Fung et al. MICRO'07 paper or ACM TACO paper)
+      shader_preexecute (shader, shader_number);
+   }
+
+   shader_decode   (shader, shader_number, grid_num);
+
+   shader_fetch    (shader, shader_number, grid_num);
+}
+
+// performance counter that are not local to one shader
+void shader_print_accstats( FILE* fout ) 
+{
+   fprintf(fout, "gpgpu_n_load_insn  = %d\n", gpgpu_n_load_insn);
+   fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn);
+   fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn);
+   fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn);
+   fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn);
+   fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn);
+
+   fprintf(fout, "gpgpu_n_shmem_bkconflict = %d\n", gpgpu_n_shmem_bkconflict);
+   fprintf(fout, "gpgpu_n_cache_bkconflict = %d\n", gpgpu_n_cache_bkconflict);   
+
+   fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge);
+   fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict);
+
+   fprintf(fout, "gpgpu_n_writeback_l1_miss = %d\n", writeback_l1_miss);
+
+   fprintf(fout, "gpgpu_n_partial_writes = %d\n", gpgpu_n_partial_writes);
+
+   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]);
+   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem][coal_stall] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][COAL_STALL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]);
+   fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]);
+
+   fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls);
+
+   if (warp_occ_detailed) {
+      int n_warp = gpu_n_thread_per_shader / warp_size;
+
+      for (unsigned s = 0; s<gpu_n_shader; s++)
+         for (int w = 0; w<n_warp; w++) {
+            fprintf(fout, "wod[%d][%d]=", s, w);
+            for (unsigned t = 0; t<warp_size; t++) {
+               fprintf(fout, "%d ", warp_occ_detailed[s * n_warp + w][t]);
+            }
+            fprintf(fout, "\n");
+         }
+   }
+}
+
+// Flushes all content of the cache to memory
+
+void shader_cache_flush(shader_core_ctx_t* sc) 
+{
+   unsigned int i;
+   unsigned int set;
+   unsigned long long int flush_addr;
+
+   shd_cache_t *cp = sc->L1cache;
+   shd_cache_line_t *pline;
+
+   for (i=0; i<cp->nset*cp->assoc; i++) {
+      pline = &(cp->lines[i]);
+      set = i / cp->assoc;
+      if ((pline->status & (DIRTY|VALID)) == (DIRTY|VALID)) {
+         flush_addr = pline->addr;
+
+         sc->fq_push(flush_addr, sc->L1cache->line_sz, 1, NO_PARTIAL_WRITE, sc->sid, 0, NULL, 0, GLOBAL_ACC_W, -1);
+
+         pline->status &= ~VALID;
+         pline->status &= ~DIRTY;
+      } else if (pline->status & VALID) {
+         pline->status &= ~VALID;
+      }
+   }
+}
+
+barrier_set_t::barrier_set_t( unsigned max_warps_per_core, unsigned max_cta_per_core )
+{
+   m_max_warps_per_core = max_warps_per_core;
+   m_max_cta_per_core = max_cta_per_core;
+   if( max_warps_per_core > WARP_PER_CTA_MAX ) {
+      printf("ERROR ** increase WARP_PER_CTA_MAX in shader.h from %u to >= %u or warps per cta in gpgpusim.config\n",
+             WARP_PER_CTA_MAX, max_warps_per_core );
+      exit(1);
+   }
+   m_warp_active.reset();
+   m_warp_at_barrier.reset();
+}
+
+// during cta allocation
+void barrier_set_t::allocate_barrier( unsigned cta_id, warp_set_t warps )
+{
+   assert( cta_id < m_max_cta_per_core );
+   cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id);
+   assert( w == m_cta_to_warps.end() ); // cta should not already be active or allocated barrier resources
+   m_cta_to_warps[cta_id] = warps;
+   assert( m_cta_to_warps.size() <= m_max_cta_per_core ); // catch cta's that were not properly deallocated
+  
+   m_warp_active |= warps;
+   m_warp_at_barrier &= ~warps;
+}
+
+// during cta deallocation
+void barrier_set_t::deallocate_barrier( unsigned cta_id )
+{
+   cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id);
+   if( w == m_cta_to_warps.end() )
+      return;
+   warp_set_t warps = w->second;
+   warp_set_t at_barrier = warps & m_warp_at_barrier;
+   assert( at_barrier.any() == false ); // no warps stuck at barrier
+   warp_set_t active = warps & m_warp_active;
+   assert( active.any() == false ); // no warps in CTA still running
+   m_warp_active &= ~warps;
+   m_warp_at_barrier &= ~warps;
+   m_cta_to_warps.erase(w);
+}
+
+// individual warp hits barrier
+void barrier_set_t::warp_reaches_barrier( unsigned cta_id, unsigned warp_id )
+{
+   cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id);
+
+   if( w == m_cta_to_warps.end() ) { // cta is active
+      printf("ERROR ** cta_id %u not found in barrier set on cycle %llu+%llu...\n", cta_id, gpu_tot_sim_cycle, gpu_sim_cycle );
+      dump();
+      abort();
+   }
+   assert( w->second.test(warp_id) == true ); // warp is in cta
+
+   m_warp_at_barrier.set(warp_id);
+
+   warp_set_t warps_in_cta = w->second;
+   warp_set_t at_barrier = warps_in_cta & m_warp_at_barrier;
+   warp_set_t active = warps_in_cta & m_warp_active;
+
+   if( at_barrier == active ) {
+      // all warps have reached barrier, so release waiting warps...
+      m_warp_at_barrier &= ~at_barrier;
+   }
+}
+
+// fetching a warp
+bool barrier_set_t::available_for_fetch( unsigned warp_id ) const
+{
+   return m_warp_active.test(warp_id) && m_warp_at_barrier.test(warp_id);
+}
+
+// warp reaches exit 
+void barrier_set_t::warp_exit( unsigned warp_id )
+{
+   // caller needs to verify all threads in warp are done, e.g., by checking PDOM stack to 
+   // see it has only one entry during exit_impl()
+   m_warp_active.reset(warp_id);
+}
+
+// assertions
+bool barrier_set_t::warp_waiting_at_barrier( unsigned warp_id )
+{ 
+   return m_warp_at_barrier.test(warp_id);
+}
+
+void barrier_set_t::dump() const
+{
+   printf( "barrier set information\n");
+   printf( "  m_max_cta_per_core = %u\n",  m_max_cta_per_core );
+   printf( "  m_max_warps_per_core = %u\n", m_max_warps_per_core );
+   printf( "  cta_to_warps:\n");
+   
+   cta_to_warp_t::const_iterator i;
+   for( i=m_cta_to_warps.begin(); i!=m_cta_to_warps.end(); i++ ) {
+      unsigned cta_id = i->first;
+      warp_set_t warps = i->second;
+      printf("    cta_id %u : %s\n", cta_id, warps.to_string().c_str() );
+   }
+   printf("  warp_active: %s\n", m_warp_active.to_string().c_str() );
+   printf("  warp_at_barrier: %s\n", m_warp_at_barrier.to_string().c_str() );
+   fflush(stdout); 
+}
+
+shader_core_ctx::shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core )
+   : m_barriers( max_warps_per_cta, max_cta_per_core )
+{
+}
+
+void shader_core_ctx::set_at_barrier( unsigned cta_id, unsigned warp_id )
+{
+   m_barriers.warp_reaches_barrier(cta_id,warp_id);
+}
+
+void shader_core_ctx::warp_exit( unsigned warp_id )
+{
+   m_barriers.warp_exit( warp_id );
+}
+
+bool shader_core_ctx::warp_waiting_at_barrier( unsigned warp_id )
+{
+   return m_barriers.warp_waiting_at_barrier(warp_id);
+}
+
+void shader_core_ctx::allocate_barrier( unsigned cta_id, warp_set_t warps )
+{
+   m_barriers.allocate_barrier(cta_id,warps);
+}
+
+void shader_core_ctx::deallocate_barrier( unsigned cta_id )
+{
+   m_barriers.deallocate_barrier(cta_id);
+}
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
new file mode 100644
index 0000000..8333202
--- /dev/null
+++ b/src/gpgpu-sim/shader.h
@@ -0,0 +1,524 @@
+/* 
+ * shader.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan, Ivan Sham, Henry Wong, Dan O'Connor, Henry Tran and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <limits.h>
+#include <assert.h>
+#include <map>
+
+#include "../cuda-sim/ptx.tab.h"
+#include "../cuda-sim/dram_callback.h"
+
+#include "gpu-cache.h"
+#include "delayqueue.h"
+#include "stack.h"
+#include "dram.h"
+#include "../abstract_hardware_model.h"
+
+#ifndef SHADER_H
+#define SHADER_H
+
+#define NO_OP_FLAG            0xFF
+
+//READ_PACKET_SIZE: bytes: 6 address (flit can specify chanel so this gives up to ~2GB/channel, so good for now), 2 bytes [shaderid + mshrid](14 bits) + req_size(0-2 bits (if req_size variable) - so up to 2^14 = 16384 mshr total
+#define READ_PACKET_SIZE 8
+//WRITE_PACKET_SIZE: bytes: 6 address, 2 miscelaneous. 
+#define WRITE_PACKET_SIZE 8
+
+#include <bitset>
+const unsigned partial_write_mask_bits = 128; //must be at least size of largest memory access.
+typedef std::bitset<partial_write_mask_bits> partial_write_mask_t;
+
+#define WRITE_MASK_SIZE 8
+#define NO_PARTIAL_WRITE (partial_write_mask_t())
+
+//this is used a lot of places where it maybe should be more variable?
+#define WORD_SIZE 4
+
+//Set a hard limit of 32 CTAs per shader [cuda only has 8]
+#define MAX_CTA_PER_SHADER 32
+
+typedef unsigned op_type;
+
+enum {
+   NO_RECONVERGE = 0,
+   POST_DOMINATOR = 1,
+   MIMD = 2,
+   DWF = 3,
+   NUM_SIMD_MODEL
+};
+
+//Defines number of threads grouped together to be executed together
+
+
+typedef struct {
+
+   address_type pc;
+
+   op_type op;
+   int space;
+
+   unsigned long long int memreqaddr;
+   //Each instruction keeps track of which hardware thread it came from
+   short hw_thread_id;
+   short wlane;
+
+   /* reg label of the instruction */
+   unsigned out[4];
+   unsigned in[4];
+   unsigned char is_vectorin;
+   unsigned char is_vectorout;
+   int arch_reg[8]; // register number for bank conflict evaluation
+   unsigned data_size; // what is the size of the word being operated on?
+
+   int reg_bank_access_pending;
+   int reg_bank_conflict_stall_checked; // flag to turn off register bank conflict checker to avoid double stalling
+
+   unsigned char inst_type;
+
+   unsigned priority;
+
+   unsigned uid;
+
+   void *ptx_thd_info; 
+   dram_callback_t callback;
+   unsigned warp_active_mask;
+   unsigned long long  ts_cycle;
+   unsigned long long  if_cycle;
+   unsigned long long  id_cycle;
+   unsigned long long  ex_cycle;
+   unsigned long long  mm_cycle;
+
+} inst_t;
+
+typedef struct {
+
+   void *ptx_thd_info; // pointer to the functional state of the thread in cuda-sim
+
+   int avail4fetch; // 1 if its instrucion can be fetch into the pipeline, 0 otherwise 
+   int warp_priority;
+
+   int id;
+
+   //unsigned n_completed;         // number of threads in warp completed -- set for first thread in each warp
+   //unsigned n_avail4fetch;       // number of threads in warp available to fetch -- set for first thread in each warp
+   //int n_waiting_at_barrier;  // number of threads in warp that have reached the barrier
+   unsigned in_scheduler;     // used by dynamic warp formation for error check
+
+   int         m_waiting_at_barrier;
+   int         m_reached_barrier;
+
+   unsigned n_insn;
+   unsigned n_insn_ac;
+   unsigned n_l1_mis_ac,
+   n_l1_mrghit_ac,
+   n_l1_access_ac; //used to collect "per thread" l1 miss statistics 
+                   // ac stands for accumulative.
+   unsigned cta_id; // which hardware CTA does this thread belong to?
+} thread_ctx_t;
+
+struct shd_warp_t
+{
+    shd_warp_t(unsigned warp_size){reset(warp_size); assert(warp_size <= bitset_size);}
+    void reset(unsigned warp_size){n_completed = warp_size; n_avail4fetch = n_waiting_at_barrier = 0; threads_completed.reset(); threads_functionally_executed.reset();}
+
+    unsigned wid;
+    unsigned n_completed; // number of threads in warp completed
+    unsigned n_avail4fetch; // number of threads in warp available to fetch 
+    int n_waiting_at_barrier; // number of threads in warp that have reached the barrier
+
+    const static unsigned bitset_size = 32;
+    std::bitset<bitset_size> threads_completed;
+    std::bitset<bitset_size> threads_functionally_executed;
+};
+
+inline unsigned hw_tid_from_wid(unsigned wid, unsigned warp_size, unsigned i){return wid * warp_size + i;};
+inline unsigned wid_from_hw_tid(unsigned tid, unsigned warp_size){return tid/warp_size;};
+
+typedef struct {
+
+   int m_stack_top;
+
+   address_type *m_pc;
+   unsigned int *m_active_mask;
+   address_type *m_recvg_pc;
+   unsigned int *m_calldepth;
+
+   unsigned long long  *m_branch_div_cycle;
+
+} pdom_warp_ctx_t; // bounded stack that implements pdom reconvergence (see MICRO'07 paper)
+
+
+enum mshr_status {
+   INITIALIZED = 0,
+   IN_ICNT2MEM,
+   IN_CBTOL2QUEUE,
+   IN_L2TODRAMQUEUE,
+   IN_DRAM_REQ_QUEUE,
+   IN_DRAMRETURN_Q,
+   IN_DRAMTOL2QUEUE,
+   IN_L2TOCBQUEUE_HIT,
+   IN_L2TOCBQUEUE_MISS,
+   IN_ICNT2SHADER,
+   FETCHED,
+   NUM_MSHR_STATUS
+};
+
+//used to stages that time_vector will keep track of their timing 
+enum mem_req_stat {
+   MR_UNUSED,
+   MR_FQPUSHED,
+   MR_ICNT_PUSHED,
+   MR_ICNT_INJECTED,
+   MR_ICNT_AT_DEST,
+   MR_DRAMQ, //icnt_pop at dram side and mem_ctrl_push
+   MR_DRAM_PROCESSING_START,
+   MR_DRAM_PROCESSING_END,
+   MR_DRAM_OUTQ,
+   MR_2SH_ICNT_PUSHED, // icnt_push and mem_ctl_pop //STORES END HERE!
+   MR_2SH_ICNT_INJECTED,
+   MR_2SH_ICNT_AT_DEST,
+   MR_2SH_FQ_POP, //icnt_pop called inside fq_pop
+   MR_RETURN_Q,
+   MR_WRITEBACK, //done
+   NUM_MEM_REQ_STAT
+};
+#include <vector>
+typedef struct mshr_entry_t {
+#ifdef _GLIBCXX_DEBUG
+    //satisfy cxx debug conditions on iterators, needs to be nonsingular to copy, which messes completely with structures containing them.
+    mshr_entry_t(){
+        static std::vector<mshr_entry_t> dummy_vector;
+        this_mshr = dummy_vector.begin(); //initialize it to something nonsingular so it can be copied. 
+    }
+#endif
+private:
+   friend class mshr_shader_unit;
+   std::vector<mshr_entry_t>::iterator this_mshr; //to ease tracking and update.
+public:
+   unsigned request_uid;
+    
+   /* memory address of the data */
+   unsigned long long int addr;
+
+   // instructions are stored here.
+   std::vector<inst_t> insts;
+
+   /* Current stage of the load: fetched or not? */
+   bool fetched(){return status == FETCHED;};
+
+   bool iswrite;
+
+   bool merged_on_other_reqest; //true if waiting for another mshr - this mshr doesn't send a memory request
+   struct mshr_entry_t *merged_requests; //mshrs waiting on this mshr
+
+   enum mshr_status status; 
+
+   void *mf; // link to corresponding memory fetch structure
+
+   //unsigned space; //does below.
+   bool istexture; //if it's a request from the texture cache
+   bool isconst; //if it's a request from the constant cache
+   bool islocal; //if it's a request to the local memory of a thread
+
+   bool wt_no_w2cache; //in write_through, sometimes need to prevent writing back returning data into cache, because its been written in the meantime. 
+} mshr_entry;
+
+enum mem_access_type { 
+   GLOBAL_ACC_R = 0, 
+   LOCAL_ACC_R = 1, 
+   CONST_ACC_R = 2, 
+   TEXTURE_ACC_R = 3, 
+   GLOBAL_ACC_W = 4, 
+   LOCAL_ACC_W = 5,
+   L2_WRBK_ACC = 6, 
+   NUM_MEM_ACCESS_TYPE = 7
+};
+
+
+/* A pointer to the function that glues the shader with the memory hiearchy */
+typedef unsigned char (*fq_push_t)(unsigned long long int addr, int bsize, unsigned char readwrite,
+                                   partial_write_mask_t, 
+                                   int sid, int wid, mshr_entry* mshr, int cache_hits_waiting,  
+                                   enum mem_access_type mem_acc, address_type pc);
+
+typedef unsigned char (*fq_has_buffer_t)(unsigned long long int addr, int bsize, bool write, int sid);
+
+const unsigned WARP_PER_CTA_MAX = 32;
+typedef std::bitset<WARP_PER_CTA_MAX> warp_set_t;
+
+class barrier_set_t {
+public:
+   barrier_set_t( unsigned max_warps_per_core, unsigned max_cta_per_core );
+
+   // during cta allocation
+   void allocate_barrier( unsigned cta_id, warp_set_t warps );
+
+   // during cta deallocation
+   void deallocate_barrier( unsigned cta_id );
+
+   typedef std::map<unsigned, warp_set_t >  cta_to_warp_t;
+
+   // individual warp hits barrier
+   void warp_reaches_barrier( unsigned cta_id, unsigned warp_id );
+
+   // fetching a warp
+   bool available_for_fetch( unsigned warp_id ) const;
+
+   // warp reaches exit 
+   void warp_exit( unsigned warp_id );
+
+   // assertions
+   bool warp_waiting_at_barrier( unsigned warp_id );
+
+   // debug
+   void dump() const;
+
+private:
+   unsigned m_max_cta_per_core;
+   unsigned m_max_warps_per_core;
+
+   cta_to_warp_t m_cta_to_warps; 
+   warp_set_t m_warp_active;
+   warp_set_t m_warp_at_barrier;
+};
+
+class mshr_shader_unit;
+  
+extern unsigned int warp_size; 
+
+typedef struct shader_core_ctx : public core_t 
+{
+   shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core );
+
+	virtual void set_at_barrier( unsigned cta_id, unsigned warp_id );
+   virtual void warp_exit( unsigned warp_id );
+   virtual bool warp_waiting_at_barrier( unsigned warp_id );
+   void allocate_barrier( unsigned cta_id, warp_set_t warps );
+   void deallocate_barrier( unsigned cta_id );
+
+////
+   
+   const char *name;
+   int sid;
+
+   // array of the threads running on this shader core 
+   thread_ctx_t *thread;
+   unsigned int n_threads;
+   unsigned int last_issued_thread;
+
+   //per warp information array
+   std::vector<shd_warp_t> warp;
+
+   barrier_set_t m_barriers;
+
+   //Keeps track of which warp of instructions to fetch/execute
+   int next_warp; 
+
+   // number of threads to be completed ( ==0 when all thread on this core completed) 
+   int not_completed; 
+   // number of Cuda Thread Arrays (blocks) currently running on this shader.
+   int n_active_cta;
+   //Keep track of multiple CTAs in shader 
+   int cta_status[MAX_CTA_PER_SHADER]; 
+   // registers holding the instruction between pipeline stages. 
+   // see below for definition of pipeline stages
+   inst_t** pipeline_reg;
+   inst_t** pre_mem_pipeline;
+   int warp_part2issue; // which part of warp to issue to pipeline 
+   int new_warp_TS; // new warp at TS pipeline register
+
+   shd_cache_t *L1cache;
+   shd_cache_t *L1texcache;
+   shd_cache_t *L1constcache;
+
+   // pointer to memory access wrapping function 
+   fq_push_t fq_push;
+   fq_has_buffer_t fq_has_buffer;
+
+   // simulation cycles happened to the shader, kept for cacheline replacement
+   unsigned int gpu_cycle;
+   // number of instructions committed by this shader core
+   unsigned int num_sim_insn;
+
+   // reconvergence
+   unsigned int model;
+
+   // Structure is used to keep track of the branching within the warp of instructions.
+   // As a group of instructions is grouped together from different threads to be executed, when
+   // a branch does occur, then the sub-set that does not get run will be given the value of warp_priority,
+   // and warp_priority will increase. Each time a sub-set branches further, a similar scheme is used.
+   // When a sub-set completes fully, then this table will determine which next sub-set to finish, which 
+   // will be the next largest value in the table.
+   int branch_priority;  
+   int* max_branch_priority; //Keeps track of the maximum priority of the threads running within a warp. need n_threads number of these
+
+   // pdom reconvergence context for each warp
+   pdom_warp_ctx_t *pdom_warp;
+
+   int waiting_at_barrier; // number of threads current waiting at a barrier in this shader.
+   int RR_k; //counter for register read pipeline
+
+   int using_dwf; //is the scheduler using dynamic warp formation
+   int using_rrstage; //is the pipeline using an extra stage for register read
+   int using_commit_queue; //is the scheduler using commit_queue?
+
+   delay_queue *thd_commit_queue;
+
+   int pending_shmem_bkacc; // 0 = check conflict for new insn
+   int pending_cache_bkacc; // 0 = check conflict for new insn
+
+   bool shader_memory_new_instruction_processed;
+
+   int pending_mem_access; // number of memory access to be serviced (use for W0 classification)
+
+   int pending_cmem_acc; //number of accesses to differrnt addresses in the constant memory cache
+
+   unsigned int n_diverge; // number of divergence occurred in this shader
+
+   //Shader core resources
+   unsigned int shmem_size;
+   unsigned int n_registers;   //registers available in the shader core 
+   unsigned int n_cta;      //Limit on number of concurrent CTAs in shader core
+
+   //void *req_hist; //not used anywhere
+
+   mshr_shader_unit *mshr_unit;
+} shader_core_ctx_t;
+
+
+shader_core_ctx_t* shader_create( const char *name, int sid, unsigned int n_threads, 
+                                  unsigned int n_mshr, fq_push_t fq_push, fq_has_buffer_t fq_has_buffer, unsigned int model);
+unsigned shader_reinit(shader_core_ctx_t *sc, int start_thread, int end_thread);
+void shader_init_CTA(shader_core_ctx_t *shader, int start_thread, int end_thread);
+
+void shader_fetch( shader_core_ctx_t *shader, 
+                   unsigned int shader_number,
+                   int grid_num );
+void shader_decode( shader_core_ctx_t *shader, 
+                    unsigned int shader_number,
+                    unsigned int grid_num );
+void shader_preexecute( shader_core_ctx_t *shader, 
+                        unsigned int shader_number );
+void shader_execute( shader_core_ctx_t *shader, 
+                     unsigned int shader_number );
+void shader_pre_memory( shader_core_ctx_t *shader, 
+                        unsigned int shader_number );
+void shader_const_memory( shader_core_ctx_t *shader, 
+                          unsigned int shader_number );
+void shader_texture_memory( shader_core_ctx_t *shader, 
+                            unsigned int shader_number );
+void shader_memory( shader_core_ctx_t *shader, 
+                    unsigned int shader_number );
+void shader_writeback( shader_core_ctx_t *shader, 
+                       unsigned int shader_number,
+                       int grid_num );
+
+void shader_display_pipeline(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask3bit );
+void shader_dump_thread_state(shader_core_ctx_t *shader, FILE *fout );
+void shader_cycle( shader_core_ctx_t *shader, 
+                   unsigned int shader_number,
+                   int grid_num );
+
+void mshr_print(FILE *fp, shader_core_ctx_t *shader);
+
+void mshr_update_status(mshr_entry* mshr, enum mshr_status new_status);
+
+mshr_entry* fetchMSHR(delay_queue** mshr, shader_core_ctx_t* sc);
+mshr_entry* shader_check_mshr4tag(shader_core_ctx_t* sc, unsigned long long int addr,int mem_type);
+void shader_update_mshr(shader_core_ctx_t* sc, unsigned long long int fetched_addr, unsigned int mshr_idx, int mem_type );
+void shader_visualizer_dump(FILE *fp, shader_core_ctx_t* sc);
+
+void init_mshr_pool();
+mshr_entry* alloc_mshr_entry();
+void free_mshr_entry( mshr_entry * );
+
+void shader_clean(shader_core_ctx_t *sc, unsigned int n_threads);
+void shader_cache_flush(shader_core_ctx_t* sc);
+
+// print out the accumulative statistics for shaders (those that are not local to one shader)
+void shader_print_accstats( FILE* fout );
+void shader_print_runtime_stat( FILE *fout );
+void shader_print_l1_miss_stat( FILE *fout );
+
+//return the maximum CTAs that can be running at the same on shader 
+//based on on the current kernel's CTA size and is 1 if mutiple CTA per block is not supported
+unsigned int max_cta_per_shader( shader_core_ctx_t *shader);
+
+#define N_PIPELINE_STAGES 7
+#define TS_IF 0
+#define IF_ID 1
+#define ID_RR 2
+#define ID_EX 3
+#define RR_EX 3
+#define EX_MM 4
+#define MM_WB 5
+#define WB_RT 6
+
+
+#endif /* SHADER_H */
diff --git a/src/gpgpu-sim/stack.cc b/src/gpgpu-sim/stack.cc
new file mode 100644
index 0000000..7f7cc70
--- /dev/null
+++ b/src/gpgpu-sim/stack.cc
@@ -0,0 +1,127 @@
+/* 
+ * stack.c
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * Ivan Sham and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+* 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "stack.h"
+
+#include <stdlib.h>
+#include <assert.h>
+
+void push_stack(Stack *S, address_type val) {
+   assert(S->top < S->max_size);
+   S->v[S->top] = val;
+   (S->top)++;
+
+}
+
+address_type pop_stack(Stack *S) {
+   (S->top)--;
+   return(S->v[S->top]);
+}
+
+address_type top_stack(Stack *S) {
+   assert(S->top >= 1);
+   return(S->v[S->top - 1]);
+}
+
+Stack* new_stack(int size) {
+   Stack* S;
+   S = (Stack*)malloc(sizeof(Stack));
+   S->max_size = size;
+   S->top = 0;
+   S->v = (address_type*)calloc(size, sizeof(address_type));
+   return S;
+}
+
+void free_stack(Stack *S) {
+   free(S->v);
+   free(S);
+}
+
+int size_stack(Stack *S) {
+   return S->top;
+}
+
+int full_stack(Stack *S) {
+   return S->top >= S->max_size;
+}
+
+int empty_stack(Stack *S) {
+   return S->top == 0;
+}
+
+int element_exist_stack(Stack *S, address_type value) {
+   int i;
+   for (i = 0; i < S->top; ++i) {
+      if (value == S->v[i]) {
+         return 1;
+      }
+   }
+   return 0;
+}
+
+void reset_stack(Stack *S) {
+   S->top = 0;
+}
diff --git a/src/gpgpu-sim/stack.h b/src/gpgpu-sim/stack.h
new file mode 100644
index 0000000..b00e25b
--- /dev/null
+++ b/src/gpgpu-sim/stack.h
@@ -0,0 +1,90 @@
+/* 
+ * stack.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * Ivan Sham and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files 
+ * src/gpgpusim_entrypoint.c and src/simplesim-3.0/ are derived from the 
+ * SimpleScalar Toolset available from http://www.simplescalar.com/ 
+ * (property of SimpleScalar LLC) and the files src/intersim/ are derived 
+ * from Booksim (Simulator provided with the textbook "Principles and 
+ * Practices of Interconnection Networks" available from 
+ * http://cva.stanford.edu/books/ppin/).  As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA
+ * which is distributed seperately by NVIDIA under separate terms and
+ * conditions.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef _MY_STACK_
+#define _MY_STACK_
+
+#include "../util.h"
+
+typedef struct {
+   address_type *v;
+   int max_size;
+   int top;
+} Stack;
+
+void push_stack(Stack *S, address_type val);
+address_type pop_stack(Stack *S);
+address_type top_stack(Stack *S);
+Stack* new_stack(int size);
+void free_stack(Stack *S);
+int size_stack(Stack *S);
+int full_stack(Stack *S);
+int empty_stack(Stack *S);
+int element_exist_stack(Stack *S, address_type value);
+void reset_stack(Stack *S);
+#endif // _MY_STACK_
diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc
new file mode 100644
index 0000000..2dc9c0b
--- /dev/null
+++ b/src/gpgpu-sim/stat-tool.cc
@@ -0,0 +1,1081 @@
+/* 
+ * stat-tool.cc
+ *
+ * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the University of British Columbia, Vancouver, 
+ * BC V6T 1Z4, All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <zlib.h>
+#include <string>
+
+// detect gcc 4.3 and use unordered map (part of c++0x)
+// unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its enabled.
+#if  defined( __GNUC__ ) and not defined( _GLIBCXX_DEBUG )
+#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3
+   #include <unordered_map>
+   #define my_hash_map std::unordered_map
+#else
+   #include <ext/hash_map>
+   namespace std {
+      using namespace __gnu_cxx;
+   }
+   #define my_hash_map std::hash_map
+#endif
+#else
+   #include <map>
+   #define my_hash_map std::map
+   #define USE_MAP
+#endif
+
+#include "histogram.h"
+
+binned_histogram::binned_histogram (std::string name, int nbins, int* bins) 
+   : m_name(name), m_nbins(nbins), m_bins(NULL), m_bin_cnts(new int[m_nbins]), m_maximum(0)
+{
+   if (bins) {
+      m_bins = new int[m_nbins];
+      for (int i = 0; i < nbins; i++) {
+         m_bins[i] = bins[i];
+      }
+   }
+
+   reset_bins();
+}
+
+binned_histogram::binned_histogram (const binned_histogram& other)
+   : m_name(other.m_name), m_nbins(other.m_nbins), m_bins(NULL), 
+     m_bin_cnts(new int[m_nbins]), m_maximum(0)
+{
+   for (int i = 0; i < m_nbins; i++) {
+      m_bin_cnts[i] = other.m_bin_cnts[i];
+   }
+}
+
+void binned_histogram::reset_bins () {
+   for (int i = 0; i < m_nbins; i++) {
+      m_bin_cnts[i] = 0;
+   }
+}
+
+void binned_histogram::add2bin (int sample) {
+   assert(0);
+   m_maximum = (sample > m_maximum)? sample : m_maximum;
+}
+
+void binned_histogram::fprint (FILE *fout) {
+   if (m_name.c_str() != NULL) fprintf(fout, "%s = ", m_name.c_str());
+   for (int i = 0; i < m_nbins; i++) {
+      fprintf(fout, "%d ", m_bin_cnts[i]);
+   }
+   fprintf(fout, "max=%d ", m_maximum);
+}
+
+binned_histogram::~binned_histogram () {
+   if (m_bins) delete[] m_bins;
+   delete[] m_bin_cnts;
+}
+
+pow2_histogram::pow2_histogram (std::string name, int nbins, int* bins) 
+   : binned_histogram (name, nbins, bins) {}
+
+void pow2_histogram::add2bin (int sample) {
+   assert(sample >= 0);
+   
+   int bin;
+   int v = sample;
+   register unsigned int shift;
+
+   bin =   (v > 0xFFFF) << 4; v >>= bin;
+   shift = (v > 0xFF  ) << 3; v >>= shift; bin |= shift;
+   shift = (v > 0xF   ) << 2; v >>= shift; bin |= shift;
+   shift = (v > 0x3   ) << 1; v >>= shift; bin |= shift;
+                                           bin |= (v >> 1);
+   bin += (sample > 0)? 1:0;
+   
+   m_bin_cnts[bin] += 1;
+   
+   m_maximum = (sample > m_maximum)? sample : m_maximum;
+}
+
+linear_histogram::linear_histogram (int stride, const char *name, int nbins, int* bins) 
+   : binned_histogram (name, nbins, bins), m_stride(stride)
+{
+}
+
+void linear_histogram::add2bin (int sample) {
+   assert(sample >= 0);
+
+   int bin = sample / m_stride;      
+   
+   m_bin_cnts[bin] += 1;
+   
+   m_maximum = (sample > m_maximum)? sample : m_maximum;
+}
+
+
+#include <list>
+#include <vector>
+#include <map>
+#include <algorithm>
+#include <string>
+#include "../util.h"
+
+#include "cflogger.h"
+
+/////////////////////////////////////////////////////////////////////////////////////
+// logger snapshot trigger: 
+// - automate the snap_shot part of loggers to avoid modifying simulation loop everytime 
+//   a new time-dependent stat is added
+/////////////////////////////////////////////////////////////////////////////////////
+
+class snap_shot_trigger {
+protected:
+   unsigned long long  m_snap_shot_interval;
+
+public:
+   snap_shot_trigger(unsigned long long  interval) : m_snap_shot_interval(interval) {}
+   virtual ~snap_shot_trigger() {}
+   
+   const unsigned long long & get_interval() const { return m_snap_shot_interval;}
+   
+   void try_snap_shot(unsigned long long  current_cycle) {
+      if ((current_cycle % m_snap_shot_interval == 0) && current_cycle != 0) {
+         snap_shot(current_cycle);
+      }
+   }
+   
+   virtual void snap_shot(unsigned long long  current_cycle) = 0;
+};
+
+static unsigned long long  min_snap_shot_interval = 0;
+static unsigned long long  next_snap_shot_cycle = 0;
+static std::list<snap_shot_trigger*> list_ss_trigger;
+
+void add_snap_shot_trigger (snap_shot_trigger* ss_trigger)
+{
+   // quick optimization assuming that all snap shot intervals are perfect multiples of each other
+   if (min_snap_shot_interval == 0 || min_snap_shot_interval > ss_trigger->get_interval()) {
+      min_snap_shot_interval = ss_trigger->get_interval();
+      next_snap_shot_cycle = min_snap_shot_interval; // assume that snap shots haven't started yet
+   }
+   list_ss_trigger.push_back(ss_trigger);
+}
+
+void remove_snap_shot_trigger (snap_shot_trigger* ss_trigger)
+{
+   list_ss_trigger.remove(ss_trigger);
+}
+
+void try_snap_shot (unsigned long long  current_cycle)
+{
+   if (min_snap_shot_interval == 0) return;
+   if (current_cycle != next_snap_shot_cycle) return;
+   
+   std::list<snap_shot_trigger*>::iterator ss_trigger_iter = list_ss_trigger.begin();
+   for(; ss_trigger_iter != list_ss_trigger.end(); ++ss_trigger_iter) {
+      (*ss_trigger_iter)->snap_shot(current_cycle); // WF: should be try_snap_shot
+   }
+   next_snap_shot_cycle = current_cycle + min_snap_shot_interval; // WF: stateful testing, maybe bad
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// spill log interface: 
+// - unified interface to spill log to file to avoid infinite memory usage for logging
+/////////////////////////////////////////////////////////////////////////////////////
+
+class spill_log_interface {
+   public:
+   spill_log_interface() {}
+   virtual ~spill_log_interface() {}
+   
+   virtual void spill(FILE *fout, bool final) = 0;
+};
+
+static unsigned long long  spill_interval = 0;
+static unsigned long long  next_spill_cycle = 0;
+static std::list<spill_log_interface*> list_spill_log;
+
+void add_spill_log (spill_log_interface* spill_log)
+{
+   list_spill_log.push_back(spill_log);
+}
+
+void remove_spill_log (spill_log_interface* spill_log)
+{
+   list_spill_log.remove(spill_log);
+}
+
+void set_spill_interval (unsigned long long  interval)
+{
+   spill_interval = interval;
+   next_spill_cycle = spill_interval;
+}
+
+void spill_log_to_file (FILE *fout, int final, unsigned long long  current_cycle)
+{
+   if (!final && spill_interval == 0) return;
+   if (!final && current_cycle <= next_spill_cycle) return;
+
+   fprintf(fout, "\n"); // ensure that the spill occurs at a new line
+   std::list<spill_log_interface*>::iterator i_spill_log = list_spill_log.begin();
+   for(; i_spill_log != list_spill_log.end(); ++i_spill_log) {
+      (*i_spill_log)->spill(fout, final); 
+   }
+   fflush(fout);
+
+   next_spill_cycle = current_cycle + spill_interval; // WF: stateful testing, maybe bad
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// thread control-flow locality logger
+/////////////////////////////////////////////////////////////////////////////////////
+unsigned translate_pc_to_ptxlineno(unsigned pc);
+class thread_insn_span {
+private: 
+   
+   typedef my_hash_map<address_type, int> span_count_map;
+   unsigned long long  m_cycle;
+   int m_n_insn;
+   span_count_map m_insn_span_count;
+   
+public:
+   
+   thread_insn_span(unsigned long long  cycle, int n_insn)
+      : m_cycle(cycle), m_n_insn(n_insn), 
+#ifdef USE_MAP
+        m_insn_span_count() 
+#else 
+        m_insn_span_count(n_insn * 2) 
+#endif
+   { }
+
+   ~thread_insn_span() { }
+   
+   thread_insn_span(const thread_insn_span& other)
+      : m_cycle(other.m_cycle), m_n_insn(other.m_n_insn), 
+        m_insn_span_count(other.m_insn_span_count) 
+   { }
+      
+   thread_insn_span& operator=(const thread_insn_span& other)
+   {
+      printf("thread_insn_span& operator=\n");
+      if (this != &other && m_n_insn != other.m_n_insn) {
+         m_n_insn = other.m_n_insn;
+         m_insn_span_count = other.m_insn_span_count;
+         m_cycle = other.m_cycle;
+      }
+      return *this;
+   }
+   
+   thread_insn_span& operator+=(const thread_insn_span& other)
+   {
+      assert(m_n_insn == other.m_n_insn); // no way to aggregate if they are different programs
+      span_count_map::const_iterator i_sc = other.m_insn_span_count.begin();
+      for (; i_sc != other.m_insn_span_count.end(); ++i_sc) {
+         m_insn_span_count[i_sc->first] += i_sc->second;
+      }
+      return *this;
+   }
+   
+   void set_span( address_type pc ) {
+      if( ((int)pc) >= 0 )
+         m_insn_span_count[pc] += 1;
+   }
+   
+   void reset(unsigned long long  cycle) {
+      m_cycle = cycle;
+      m_insn_span_count.clear(); 
+   }
+   
+   void print_span(FILE *fout) {
+      fprintf(fout, "%d: ", (int)m_cycle);
+      span_count_map::const_iterator i_sc = m_insn_span_count.begin();
+      for (; i_sc != m_insn_span_count.end(); ++i_sc) {
+         fprintf(fout, "%d ", i_sc->first);
+      }
+      fprintf(fout, "\n");
+   }
+
+   void print_histo(FILE *fout) {
+      fprintf(fout, "%d:", (int)m_cycle);
+      span_count_map::const_iterator i_sc = m_insn_span_count.begin();
+      for (; i_sc != m_insn_span_count.end(); ++i_sc) {
+         fprintf(fout, "%d ", i_sc->second);
+      }
+      fprintf(fout, "\n");
+   }
+
+   void print_sparse_histo(FILE *fout) {
+      int n_printed_entries = 0;
+      span_count_map::const_iterator i_sc = m_insn_span_count.begin();
+      for (; i_sc != m_insn_span_count.end(); ++i_sc) {
+         unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first);
+         fprintf(fout, "%u %d ", ptx_lineno, i_sc->second);
+         n_printed_entries++;
+      }
+      if (n_printed_entries == 0) {
+         fprintf(fout, "0 0 ");
+      }
+      fprintf(fout, "\n");
+   }
+
+   void print_sparse_histo(gzFile fout) {
+      int n_printed_entries = 0;
+      span_count_map::const_iterator i_sc = m_insn_span_count.begin();
+      for (; i_sc != m_insn_span_count.end(); ++i_sc) {
+         unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first);
+         gzprintf(fout, "%u %d ", ptx_lineno, i_sc->second);
+         n_printed_entries++;
+      }
+      if (n_printed_entries == 0) {
+         gzprintf(fout, "0 0 ");
+      }
+      gzprintf(fout, "\n");
+   }
+};
+
+class thread_CFlocality : public snap_shot_trigger, public spill_log_interface {
+private:
+   
+   std::string m_name;
+
+   int m_nthreads;
+   std::vector<address_type> m_thread_pc;
+   
+   unsigned long long  m_cycle;
+   thread_insn_span m_thd_span;
+   std::list<thread_insn_span> m_thd_span_archive;
+   
+public:
+   
+   thread_CFlocality(std::string name, unsigned long long  snap_shot_interval, 
+                     int nthreads, int n_insn, address_type start_pc, unsigned long long  start_cycle = 0)
+      : snap_shot_trigger(snap_shot_interval), m_name(name),
+        m_nthreads(nthreads), m_thread_pc(nthreads, start_pc), m_cycle(start_cycle),
+        m_thd_span(start_cycle, n_insn)
+   {
+      std::fill(m_thread_pc.begin(), m_thread_pc.end(), -1); // so that hw thread with no work assigned will not clobber results
+   }
+   
+   ~thread_CFlocality() {} 
+   
+   void update_thread_pc( int thread_id, address_type pc ) {
+      m_thread_pc[thread_id] = pc;
+      m_thd_span.set_span(pc);
+   }
+   
+   void snap_shot(unsigned long long  current_cycle) {
+      m_thd_span_archive.push_back(m_thd_span);
+      m_thd_span.reset(current_cycle);
+      for (int i = 0; i < (int)m_thread_pc.size(); i++) {
+         m_thd_span.set_span(m_thread_pc[i]);
+      }
+   }
+   
+   void spill(FILE *fout, bool final) {
+      std::list<thread_insn_span>::iterator lit = m_thd_span_archive.begin();
+      for (; lit != m_thd_span_archive.end(); lit = m_thd_span_archive.erase(lit) ) {
+         fprintf(fout, "%s-", m_name.c_str());
+         lit->print_histo(fout);
+      }
+      assert( m_thd_span_archive.empty() );
+      if (final) {
+         fprintf(fout, "%s-", m_name.c_str());
+         m_thd_span.print_histo(fout);
+      }
+   }
+   
+   void print_visualizer(FILE *fout) {
+      fprintf(fout, "%s: ", m_name.c_str());
+      if (m_thd_span_archive.empty()) {
+      
+         // visualizer do no require snap_shots
+         m_thd_span.print_sparse_histo(fout);
+         
+         // clean the thread span
+         m_thd_span.reset(0);
+         for (int i = 0; i < (int)m_thread_pc.size(); i++) {
+            m_thd_span.set_span(m_thread_pc[i]);
+         }
+      } else { 
+         assert(0); // TODO: implement fall back so that visualizer can work with snap shots
+      }
+   }
+   
+   void print_visualizer(gzFile fout) {
+      gzprintf(fout, "%s: ", m_name.c_str());
+      if (m_thd_span_archive.empty()) {
+      
+         // visualizer do no require snap_shots
+         m_thd_span.print_sparse_histo(fout);
+         
+         // clean the thread span
+         m_thd_span.reset(0);
+         for (int i = 0; i < (int)m_thread_pc.size(); i++) {
+            m_thd_span.set_span(m_thread_pc[i]);
+         }
+      } else { 
+         assert(0); // TODO: implement fall back so that visualizer can work with snap shots
+      }
+   }
+   
+   void print_span(FILE *fout) {
+      std::list<thread_insn_span>::iterator lit = m_thd_span_archive.begin();
+      for (; lit != m_thd_span_archive.end(); ++lit) {
+         fprintf(fout, "%s-", m_name.c_str());
+         lit->print_span(fout);
+      }
+      fprintf(fout, "%s-", m_name.c_str());
+      m_thd_span.print_span(fout);
+   }
+
+   void print_histo(FILE *fout) {
+      std::list<thread_insn_span>::iterator lit = m_thd_span_archive.begin();
+      for (; lit != m_thd_span_archive.end(); ++lit) {
+         fprintf(fout, "%s-", m_name.c_str());
+         lit->print_histo(fout);
+      }
+      fprintf(fout, "%s-", m_name.c_str());
+      m_thd_span.print_histo(fout);
+   }
+};
+
+static int n_thread_CFloggers = 0;
+static thread_CFlocality** thread_CFlogger = NULL;
+
+void create_thread_CFlogger( int n_loggers, int n_threads, int n_insn, address_type start_pc, unsigned long long  logging_interval) 
+{
+   destroy_thread_CFlogger();
+   
+   n_thread_CFloggers = n_loggers;
+   thread_CFlogger = new thread_CFlocality*[n_loggers];
+
+   std::string name_tpl("CFLog");
+   char buffer[32];
+   for (int i = 0; i < n_thread_CFloggers; i++) {
+      snprintf(buffer, 32, "%02d", i);
+      thread_CFlogger[i] = new thread_CFlocality( name_tpl + buffer, logging_interval, n_threads, n_insn, start_pc);
+      if (logging_interval != 0) {
+         add_snap_shot_trigger(thread_CFlogger[i]);
+         add_spill_log(thread_CFlogger[i]);
+      }
+   }
+}
+
+void destroy_thread_CFlogger( ) 
+{
+   if (thread_CFlogger != NULL) {
+      for (int i = 0; i < n_thread_CFloggers; i++) {
+         remove_snap_shot_trigger(thread_CFlogger[i]);
+         remove_spill_log(thread_CFlogger[i]);
+         delete thread_CFlogger[i];
+      }
+      delete thread_CFlogger;
+      thread_CFlogger = NULL;
+   }
+}
+
+void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ) 
+{
+   if (thread_id < 0) return;
+   thread_CFlogger[logger_id]->update_thread_pc(thread_id, pc);
+}
+
+void cflog_snapshot( int logger_id, unsigned long long  cycle ) 
+{
+   thread_CFlogger[logger_id]->snap_shot(cycle);
+}
+
+void cflog_print(FILE *fout) 
+{
+   for (int i = 0; i < n_thread_CFloggers; i++) {
+      thread_CFlogger[i]->print_histo(fout);
+   }
+}
+
+void cflog_visualizer_print(FILE *fout) 
+{
+   for (int i = 0; i < n_thread_CFloggers; i++) {
+      thread_CFlogger[i]->print_visualizer(fout);
+   }
+}
+
+void cflog_visualizer_gzprint(gzFile fout) 
+{
+   for (int i = 0; i < n_thread_CFloggers; i++) {
+      thread_CFlogger[i]->print_visualizer(fout);
+   }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-insn active thread distribution (warp occ) logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+class insn_warp_occ_logger{
+private:
+   int m_simd_width;
+   std::vector<linear_histogram> m_insn_warp_occ;
+   int m_id;
+   static int s_ids;
+
+public:
+   insn_warp_occ_logger(int simd_width, int n_insn)
+      : m_simd_width(simd_width), 
+        m_insn_warp_occ(n_insn, linear_histogram(1, "", m_simd_width)),
+        m_id(s_ids++) {}
+   
+   insn_warp_occ_logger(const insn_warp_occ_logger& other)
+      : m_simd_width(other.m_simd_width), 
+        m_insn_warp_occ(other.m_insn_warp_occ.size(), linear_histogram(1, "", m_simd_width)),
+        m_id(s_ids++) {}
+   
+   insn_warp_occ_logger& operator=(const insn_warp_occ_logger& p) {
+      printf("insn_warp_occ_logger Operator= called: %02d \n", m_id);
+      assert(0);
+      return *this;
+   }   
+
+   ~insn_warp_occ_logger() {}
+   
+   void set_id(int id) {
+      m_id = id;
+   }
+   
+   void log(address_type pc, int warp_occ) {
+      m_insn_warp_occ[pc].add2bin(warp_occ - 1);
+   }
+   
+   void print(FILE *fout) {
+      for (unsigned i = 0; i < m_insn_warp_occ.size(); i++) {
+         fprintf(fout, "InsnWarpOcc%02d-%d", m_id, i);
+         m_insn_warp_occ[i].fprint(fout);
+         fprintf(fout, "\n");
+      }
+   }
+};
+int insn_warp_occ_logger::s_ids = 0;
+
+static std::vector<insn_warp_occ_logger> iwo_logger;
+
+void insn_warp_occ_create( int n_loggers, int simd_width, int n_insn)
+{
+   iwo_logger.clear();
+   iwo_logger.assign(n_loggers, insn_warp_occ_logger(simd_width, n_insn));
+   for (unsigned i = 0; i < iwo_logger.size(); i++) {
+      iwo_logger[i].set_id(i);
+   }
+}
+
+void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ)
+{
+   if (warp_occ <= 0) return;
+   iwo_logger[logger_id].log(pc, warp_occ);
+}
+
+void insn_warp_occ_print( FILE *fout )
+{
+   for (unsigned i = 0; i < iwo_logger.size(); i++) {
+      iwo_logger[i].print(fout);
+   }
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// generic linear histogram logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+class linear_histogram_snapshot {
+private:
+   unsigned long long  m_cycle;
+   std::vector<int> m_linear_histogram;
+public:
+   linear_histogram_snapshot(int n_bins, unsigned long long  cycle) 
+      : m_cycle(cycle), 
+        m_linear_histogram(n_bins,0) 
+   { }
+   
+   linear_histogram_snapshot(const linear_histogram_snapshot& other) 
+      : m_cycle(other.m_cycle), 
+        m_linear_histogram(other.m_linear_histogram)
+   { }
+   
+   ~linear_histogram_snapshot() { }
+   
+   void addsample(int pos) {
+      assert((size_t)pos < m_linear_histogram.size());
+      m_linear_histogram[pos] += 1;
+   }
+   
+   void subsample(int pos) {
+      assert((size_t)pos < m_linear_histogram.size());
+      m_linear_histogram[pos] -= 1;
+   }
+   
+   void reset(unsigned long long  cycle) {
+      m_cycle = cycle;
+      m_linear_histogram.assign(m_linear_histogram.size(), 0);
+   }
+   
+   void set_cycle(unsigned long long  cycle) {
+      m_cycle = cycle;
+   }
+   
+   void print(FILE *fout) {
+      fprintf(fout, "%d = ", (int)m_cycle);
+      for (unsigned int i = 0; i < m_linear_histogram.size(); i++) {
+         fprintf(fout, "%d ", m_linear_histogram[i]);
+      }
+   }
+
+   void print_visualizer(FILE *fout) {
+      for (unsigned int i = 0; i < m_linear_histogram.size(); i++) {
+         fprintf(fout, "%d ", m_linear_histogram[i]);
+      }
+   }
+
+   void print_visualizer(gzFile fout) {
+      for (unsigned int i = 0; i < m_linear_histogram.size(); i++) {
+         gzprintf(fout, "%d ", m_linear_histogram[i]);
+      }
+   }
+};
+
+class linear_histogram_logger : public snap_shot_trigger, public spill_log_interface {
+private:
+   int m_n_bins;
+   linear_histogram_snapshot m_curr_lin_hist;
+   std::list<linear_histogram_snapshot> m_lin_hist_archive;
+   unsigned long long  m_cycle;
+   bool m_reset_at_snap_shot;
+   std::string m_name;
+   int m_id;
+   static int s_ids;
+
+public:
+   linear_histogram_logger(int n_bins, 
+                           unsigned long long  snap_shot_interval, 
+                           const char *name, 
+                           bool reset_at_snap_shot = true, 
+                           unsigned long long  start_cycle = 0)
+      : snap_shot_trigger(snap_shot_interval), 
+        m_n_bins(n_bins), 
+        m_curr_lin_hist(m_n_bins, start_cycle),
+        m_lin_hist_archive(),
+        m_cycle(start_cycle),
+        m_reset_at_snap_shot(reset_at_snap_shot), 
+        m_name(name),
+        m_id(s_ids++) {}
+   
+   linear_histogram_logger(const linear_histogram_logger& other) // WF: Buggy - Not really copying data over
+      : snap_shot_trigger(other.get_interval()), 
+        m_n_bins(other.m_n_bins), 
+        m_curr_lin_hist(m_n_bins, other.m_cycle),
+        m_lin_hist_archive(),
+        m_cycle(other.m_cycle),
+        m_reset_at_snap_shot(other.m_reset_at_snap_shot), 
+        m_name(other.m_name),
+        m_id(s_ids++) {}
+   
+   // using default assignment operator!
+   
+   ~linear_histogram_logger() {
+      // printf("Destroyer called: %s%02d \n", m_name.c_str(), m_id);
+      remove_snap_shot_trigger(this);
+      remove_spill_log(this);
+   }
+   
+   void set_id(int id) {
+      m_id = id;
+   }
+   
+   void log(int pos) {
+      m_curr_lin_hist.addsample(pos);
+   }
+   
+   void unlog(int pos) {
+      m_curr_lin_hist.subsample(pos);
+   }
+   
+   void snap_shot(unsigned long long  current_cycle) {
+      m_lin_hist_archive.push_back(m_curr_lin_hist);
+      if (m_reset_at_snap_shot) {
+         m_curr_lin_hist.reset(current_cycle);
+      } else {
+         m_curr_lin_hist.set_cycle(current_cycle);
+      }
+   }
+   
+   void spill(FILE *fout, bool final) {
+      std::list<linear_histogram_snapshot>::iterator iter = m_lin_hist_archive.begin();
+      for (; iter != m_lin_hist_archive.end(); iter = m_lin_hist_archive.erase(iter) ) {
+         fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0);
+         iter->print(fout);
+         fprintf(fout, "\n");
+      }
+      assert( m_lin_hist_archive.empty() );
+      if (final) {
+         fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0);
+         m_curr_lin_hist.print(fout);
+         fprintf(fout, "\n");
+      }
+   }
+   
+   void print(FILE *fout) {
+      std::list<linear_histogram_snapshot>::iterator iter = m_lin_hist_archive.begin();
+      for (; iter != m_lin_hist_archive.end(); ++iter) {
+         fprintf(fout, "%s%02d-", m_name.c_str(), m_id);
+         iter->print(fout);
+         fprintf(fout, "\n");
+      }
+      fprintf(fout, "%s%02d-", m_name.c_str(), m_id);
+      m_curr_lin_hist.print(fout);
+      fprintf(fout, "\n");
+   }
+
+   void print_visualizer(FILE *fout) {
+      assert(m_lin_hist_archive.empty()); // don't support snapshot for now
+      fprintf(fout, "%s", m_name.c_str());
+      if (m_id >= 0) {
+         fprintf(fout, "%02d: ", m_id);
+      } else {
+         fprintf(fout, ": ");
+      }
+      m_curr_lin_hist.print_visualizer(fout);
+      fprintf(fout, "\n");
+      if (m_reset_at_snap_shot) {
+         m_curr_lin_hist.reset(0);
+      } 
+   }
+
+   void print_visualizer(gzFile fout) {
+      assert(m_lin_hist_archive.empty()); // don't support snapshot for now
+      gzprintf(fout, "%s", m_name.c_str());
+      if (m_id >= 0) {
+         gzprintf(fout, "%02d: ", m_id);
+      } else {
+         gzprintf(fout, ": ");
+      }
+      m_curr_lin_hist.print_visualizer(fout);
+      gzprintf(fout, "\n");
+      if (m_reset_at_snap_shot) {
+         m_curr_lin_hist.reset(0);
+      } 
+   }
+};
+int linear_histogram_logger::s_ids = 0;
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-shadercore active thread distribution (warp occ) logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+static std::vector<linear_histogram_logger> s_warp_occ_logger;
+
+void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long  logging_interval)
+{
+   // simd_width + 1 to include the case with full warp
+   s_warp_occ_logger.assign(n_loggers, 
+                            linear_histogram_logger(simd_width + 1, logging_interval, "ShdrWarpOcc"));
+   for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) {
+      s_warp_occ_logger[i].set_id(i);
+      add_snap_shot_trigger(&(s_warp_occ_logger[i]));
+      add_spill_log(&(s_warp_occ_logger[i]));
+   }
+}
+
+void shader_warp_occ_log( int logger_id, int warp_occ)
+{
+   s_warp_occ_logger[logger_id].log(warp_occ);
+}
+
+void shader_warp_occ_snapshot( int logger_id, unsigned long long  current_cycle)
+{
+   s_warp_occ_logger[logger_id].snap_shot(current_cycle);
+}
+
+void shader_warp_occ_print( FILE *fout )
+{
+   for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) {
+      s_warp_occ_logger[i].print(fout);
+   }
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-shadercore memory-access logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+static int s_mem_acc_logger_n_dram = 0;
+static int s_mem_acc_logger_n_bank = 0;
+static std::vector<linear_histogram_logger> s_mem_acc_logger;
+
+void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long  logging_interval)
+{
+   // (n_bank + 1) to space data out; 2x to separate read and write
+   s_mem_acc_logger.assign(n_loggers, 
+                           linear_histogram_logger(2 * n_dram * (n_bank + 1), logging_interval, "ShdrMemAcc"));
+
+   s_mem_acc_logger_n_dram = n_dram;
+   s_mem_acc_logger_n_bank = n_bank;
+   for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) {
+      s_mem_acc_logger[i].set_id(i);
+      add_snap_shot_trigger(&(s_mem_acc_logger[i]));
+      add_spill_log(&(s_mem_acc_logger[i]));
+   }
+}
+
+void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw)
+{
+   if (s_mem_acc_logger_n_dram == 0) return;
+   int write_offset = 0;
+   switch(rw) {
+   case 'r': write_offset = 0; break;
+   case 'w': write_offset = (s_mem_acc_logger_n_bank + 1) * s_mem_acc_logger_n_dram; break;
+   default: assert(0); break;
+   }
+   s_mem_acc_logger[logger_id].log(dram_id * s_mem_acc_logger_n_bank + bank + write_offset);
+}
+
+void shader_mem_acc_snapshot( int logger_id, unsigned long long  current_cycle)
+{
+   s_mem_acc_logger[logger_id].snap_shot(current_cycle);
+}
+
+void shader_mem_acc_print( FILE *fout )
+{
+   for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) {
+      s_mem_acc_logger[i].print(fout);
+   }
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-shadercore memory-latency logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+static bool s_mem_lat_logger_used = false;
+static int s_mem_lat_logger_nbins = 48;     // up to 2^24 = 16M
+static std::vector<linear_histogram_logger> s_mem_lat_logger;
+
+void shader_mem_lat_create( int n_loggers, unsigned long long  logging_interval)
+{
+   s_mem_lat_logger.assign(n_loggers, 
+                           linear_histogram_logger(s_mem_lat_logger_nbins, logging_interval, "ShdrMemLat"));
+
+   for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) {
+      s_mem_lat_logger[i].set_id(i);
+      add_snap_shot_trigger(&(s_mem_lat_logger[i]));
+      add_spill_log(&(s_mem_lat_logger[i]));
+   }
+   
+   s_mem_lat_logger_used = true;
+}
+
+void shader_mem_lat_log( int logger_id, int latency)
+{
+   if (s_mem_lat_logger_used == false) return;
+   if (latency > (1<<(s_mem_lat_logger_nbins/2))) assert(0); // guard for out of bound bin
+   assert(latency > 0);
+   
+   int latency_bin;
+   
+   int bin; // LOG_2(latency)
+   int v = latency;
+   register unsigned int shift;
+
+   bin =   (v > 0xFFFF) << 4; v >>= bin;
+   shift = (v > 0xFF  ) << 3; v >>= shift; bin |= shift;
+   shift = (v > 0xF   ) << 2; v >>= shift; bin |= shift;
+   shift = (v > 0x3   ) << 1; v >>= shift; bin |= shift;
+                                           bin |= (v >> 1);
+   latency_bin = 2 * bin;
+   if (bin > 0) {
+      latency_bin += ((latency & (1 << (bin - 1))) != 0)? 1 : 0; // approx. for LOG_sqrt2(latency)
+   }
+
+   s_mem_lat_logger[logger_id].log(latency_bin);
+}
+
+void shader_mem_lat_snapshot( int logger_id, unsigned long long  current_cycle)
+{
+   s_mem_lat_logger[logger_id].snap_shot(current_cycle);
+}
+
+void shader_mem_lat_print( FILE *fout )
+{
+   for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) {
+      s_mem_lat_logger[i].print(fout);
+   }
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-shadercore cache-miss logger
+/////////////////////////////////////////////////////////////////////////////////////
+
+static int s_cache_access_logger_n_types = 0;
+static std::vector<linear_histogram_logger> s_cache_access_logger;
+
+enum cache_access_logger_types {
+   NORMAL, TEXTURE, CONSTANT
+};
+
+int get_shader_normal_cache_id() { return NORMAL; }
+int get_shader_texture_cache_id() { return TEXTURE; }
+int get_shader_constant_cache_id() { return CONSTANT; }
+
+void shader_cache_access_create( int n_loggers, int n_types, unsigned long long  logging_interval)
+{
+   // There are different type of cache (x2 for recording accesses and misses)
+   s_cache_access_logger.assign(n_loggers, 
+                                linear_histogram_logger(n_types * 2, logging_interval, "ShdrCacheMiss"));
+
+   s_cache_access_logger_n_types = n_types;
+   for (unsigned i = 0; i < s_cache_access_logger.size(); i++) {
+      s_cache_access_logger[i].set_id(i);
+      add_snap_shot_trigger(&(s_cache_access_logger[i]));
+      add_spill_log(&(s_cache_access_logger[i]));
+   }
+}
+
+void shader_cache_access_log( int logger_id, int type, int miss)
+{
+   if (s_cache_access_logger_n_types == 0) return;
+   if (logger_id < 0) return;
+   assert(type == NORMAL || type == TEXTURE || type == CONSTANT);
+   assert(miss == 0 || miss == 1);
+   
+   s_cache_access_logger[logger_id].log(2 * type + miss);
+}
+
+void shader_cache_access_unlog( int logger_id, int type, int miss)
+{
+   if (s_cache_access_logger_n_types == 0) return;
+   if (logger_id < 0) return;
+   assert(type == NORMAL || type == TEXTURE || type == CONSTANT);
+   assert(miss == 0 || miss == 1);
+   
+   s_cache_access_logger[logger_id].unlog(2 * type + miss);
+}
+
+void shader_cache_access_print( FILE *fout )
+{
+   for (unsigned i = 0; i < s_cache_access_logger.size(); i++) {
+      s_cache_access_logger[i].print(fout);
+   }
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////
+// per-shadercore CTA count logger (only make sense with gpgpu_spread_blocks_across_cores)
+/////////////////////////////////////////////////////////////////////////////////////
+
+static linear_histogram_logger *s_CTA_count_logger = NULL;
+
+void shader_CTA_count_create( int n_shaders, unsigned long long  logging_interval)
+{
+   // only need one logger to track all the shaders
+   if (s_CTA_count_logger != NULL) delete s_CTA_count_logger;
+   s_CTA_count_logger = new linear_histogram_logger(n_shaders, logging_interval, "ShdrCTACount", false);
+
+   s_CTA_count_logger->set_id(-1);
+   if (logging_interval != 0) {
+   add_snap_shot_trigger(s_CTA_count_logger);
+   add_spill_log(s_CTA_count_logger);
+}
+}
+
+void shader_CTA_count_log( int shader_id, int nCTAadded )
+{
+   if (s_CTA_count_logger == NULL) return;
+   
+   for (int i = 0; i < nCTAadded; i++) {
+      s_CTA_count_logger->log(shader_id);
+   }
+}
+
+void shader_CTA_count_unlog( int shader_id, int nCTAdone )
+{
+   if (s_CTA_count_logger == NULL) return;
+   
+   for (int i = 0; i < nCTAdone; i++) {
+      s_CTA_count_logger->unlog(shader_id);
+   }
+}
+
+void shader_CTA_count_print( FILE *fout )
+{
+   if (s_CTA_count_logger == NULL) return;
+   s_CTA_count_logger->print(fout);
+}
+
+void shader_CTA_count_visualizer_print( FILE *fout )
+{
+   if (s_CTA_count_logger == NULL) return;
+   s_CTA_count_logger->print_visualizer(fout);
+}
+
+void shader_CTA_count_visualizer_gzprint( gzFile fout )
+{
+   if (s_CTA_count_logger == NULL) return;
+   s_CTA_count_logger->print_visualizer(fout);
+}
+
diff --git a/src/gpgpu-sim/visualizer.cc b/src/gpgpu-sim/visualizer.cc
new file mode 100644
index 0000000..3772fc2
--- /dev/null
+++ b/src/gpgpu-sim/visualizer.cc
@@ -0,0 +1,602 @@
+/* 
+ * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung and the University of 
+ * British Columbia, Vancouver, BC V6T 1Z4, All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "gpu-sim.h"
+#include "../option_parser.h"
+#include <time.h>
+#include <string.h>
+#include <zlib.h>
+
+extern unsigned int gpu_n_shader;
+extern unsigned int gpu_n_mem;
+extern unsigned int gpu_mem_n_bk;
+extern shader_core_ctx_t **sc;
+extern dram_t **dram;
+extern unsigned int L1_read_miss;
+extern unsigned int L1_write_miss;
+extern unsigned int L1_texture_miss;
+extern unsigned int L1_const_miss;
+extern unsigned L2_write_miss;
+extern unsigned L2_write_hit;
+extern unsigned L2_read_hit;
+extern unsigned L2_read_miss;
+extern unsigned long long int mf_total_lat;
+extern unsigned num_mfs;
+extern unsigned long long  gpu_sim_cycle;
+extern unsigned long long  gpu_sim_insn;
+extern unsigned long long  gpu_tot_sim_insn;
+extern unsigned long long  gpu_completed_thread;
+extern unsigned int gpgpu_n_sent_writes;
+extern unsigned int gpgpu_n_processed_writes;
+extern unsigned int gpgpu_n_cache_bkconflict;
+extern unsigned int gpgpu_n_shmem_bkconflict;
+extern unsigned int gpu_stall_by_MSHRwb;
+extern unsigned int *max_return_queue_length;
+extern unsigned max_mrq_latency;
+extern unsigned max_dq_latency;
+extern unsigned max_mf_latency;
+extern unsigned max_icnt2mem_latency;
+extern unsigned max_icnt2sh_latency;
+extern int gpgpu_warpdistro_shader;
+extern unsigned ***mem_access_type_stats;
+
+extern unsigned int warp_size; 
+extern unsigned int *shader_cycle_distro;
+void time_vector_print_interval2file(FILE *outfile);
+void time_vector_print_interval2gzfile(gzFile outfile);
+void cflog_visualizer_gzprint(gzFile fout);
+void shader_CTA_count_visualizer_gzprint(gzFile fout);
+float shd_cache_windowed_cache_miss_rate(shd_cache_t*, int);
+void shd_cache_new_window(shd_cache_t*);
+
+int g_visualizer_enabled = 1;
+char *g_visualizer_filename = NULL;
+int g_visualizer_zlevel = 6;
+
+void visualizer_options(option_parser_t opp)
+{
+   option_parser_register(opp, "-visualizer_enabled", OPT_BOOL,
+                          &g_visualizer_enabled, "Turn on visualizer output (1=On, 0=Off)",
+                          "1");
+
+   option_parser_register(opp, "-visualizer_outputfile", OPT_CSTR, 
+                          &g_visualizer_filename, "Specifies the output log file for visualizer",
+                          NULL);
+
+   option_parser_register(opp, "-visualizer_zlevel", OPT_INT32,
+                          &g_visualizer_zlevel, "Compression level of the visualizer output log (0=no comp, 9=highest)",
+                          "6");
+
+}
+
+void visualizer_printstat()
+{
+   static unsigned int *last_shader_cycle_distro = NULL;
+   gzFile visualizer_file = NULL; // gzFile is basically a pointer to a struct, so it is fine to initialize it as NULL
+   unsigned i;
+   if ( !g_visualizer_enabled )
+      return;
+   if (!last_shader_cycle_distro)
+      last_shader_cycle_distro = (unsigned int*) calloc(warp_size + 3, sizeof(unsigned int));
+
+   if ( g_visualizer_filename == NULL ) {
+      time_t curr_time;
+      time(&curr_time);
+      char *date = ctime(&curr_time);
+      char *s = date;
+      while (*s) {
+         if (*s == ' ' || *s == '\t' || *s == ':') *s = '-';
+         if (*s == '\n' || *s == '\r' ) *s = 0;
+         s++;
+      }
+      char buf[1024];
+      snprintf(buf,1024,"gpgpusim_visualizer__%s.log.gz",date);
+      visualizer_file = gzopen(buf, "w");
+      if (visualizer_file == NULL) {
+         printf("error - could not open visualizer trace file.\n");
+         exit(1);
+      }
+      gzsetparams(visualizer_file, g_visualizer_zlevel, Z_DEFAULT_STRATEGY);
+      g_visualizer_filename = strdup(buf);
+   } else {
+      visualizer_file = gzopen(g_visualizer_filename,"a");
+      if (visualizer_file == NULL) {
+         printf("error - could not open visualizer trace file.\n");
+         exit(1);
+      }
+      gzsetparams(visualizer_file, g_visualizer_zlevel, Z_DEFAULT_STRATEGY);
+   }
+
+   // instruction count per shader core
+   gzprintf(visualizer_file, "shaderinsncount:  ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%u ",sc[i]->num_sim_insn);
+   }
+   gzprintf(visualizer_file, "\n");
+
+   // warp divergence per shader core
+   gzprintf(visualizer_file, "shaderwarpdiv: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%u ", sc[i]->n_diverge);
+   }
+   gzprintf(visualizer_file, "\n");
+
+   cflog_visualizer_gzprint(visualizer_file);
+   shader_CTA_count_visualizer_gzprint(visualizer_file);
+
+   // per shader core cache miss rate 
+   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1cache, 0));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1texcache, 0));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1constcache, 0));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1cache, 1));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1texcache, 1));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: ");
+   for (i=0;i<gpu_n_shader;i++) {
+      gzprintf(visualizer_file, "%0.4f ", shd_cache_windowed_cache_miss_rate(sc[i]->L1constcache, 1));
+   }
+   gzprintf(visualizer_file, "\n");
+
+   // reset for next interval
+   for (i=0;i<gpu_n_shader;i++) {
+      shd_cache_new_window(sc[i]->L1cache);
+      shd_cache_new_window(sc[i]->L1texcache);
+      shd_cache_new_window(sc[i]->L1constcache);
+   }
+
+   // dram specific statistics
+   for (i=0;i<gpu_n_mem;i++) {
+      gzprintf(visualizer_file, "dramncmd: %u %u\n",dram[i]->id, dram[i]->n_cmd_partial);  
+      gzprintf(visualizer_file, "dramnop: %u %u\n",dram[i]->id,dram[i]->n_nop_partial);
+      gzprintf(visualizer_file,"dramnact: %u %u\n",dram[i]->id,dram[i]->n_act_partial);
+      gzprintf(visualizer_file,"dramnpre: %u %u\n",dram[i]->id,dram[i]->n_pre_partial);
+      gzprintf(visualizer_file,"dramnreq: %u %u\n",dram[i]->id,dram[i]->n_req_partial);
+      gzprintf(visualizer_file,"dramavemrqs: %u %u\n",dram[i]->id,
+               dram[i]->n_cmd_partial?(dram[i]->ave_mrqs_partial/dram[i]->n_cmd_partial ):0);
+
+      // utilization and efficiency
+      gzprintf(visualizer_file,"dramutil: %u %u\n",  
+               dram[i]->id,dram[i]->n_cmd_partial?100*dram[i]->bwutil_partial/dram[i]->n_cmd_partial:0);
+      gzprintf(visualizer_file,"drameff: %u %u\n", 
+               dram[i]->id,dram[i]->n_activity_partial?100*dram[i]->bwutil_partial/dram[i]->n_activity_partial:0);
+
+      // reset for next interval
+      dram[i]->bwutil_partial = 0;
+      dram[i]->n_activity_partial = 0;
+      dram[i]->ave_mrqs_partial = 0; 
+      dram[i]->n_cmd_partial = 0;
+      dram[i]->n_nop_partial = 0;
+      dram[i]->n_act_partial = 0;
+      dram[i]->n_pre_partial = 0;
+      dram[i]->n_req_partial = 0;
+   }
+
+   // dram access type classification
+   for (i=0;i<gpu_n_mem;i++) {
+      unsigned int j;
+      for (j = 0; j < gpu_mem_n_bk; j++) {
+         gzprintf(visualizer_file,"dramglobal_acc_r: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[GLOBAL_ACC_R][dram[i]->id][j]);
+         gzprintf(visualizer_file,"dramglobal_acc_w: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[GLOBAL_ACC_W][dram[i]->id][j]);
+         gzprintf(visualizer_file,"dramlocal_acc_r: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[LOCAL_ACC_R][dram[i]->id][j]);
+         gzprintf(visualizer_file,"dramlocal_acc_w: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[LOCAL_ACC_W][dram[i]->id][j]);
+         gzprintf(visualizer_file,"dramconst_acc_r: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[CONST_ACC_R][dram[i]->id][j]);
+         gzprintf(visualizer_file,"dramtexture_acc_r: %u %u %u\n", dram[i]->id, j, 
+                  mem_access_type_stats[TEXTURE_ACC_R][dram[i]->id][j]);
+      }
+   }
+
+   // overall cache miss rates
+   gzprintf(visualizer_file, "Lonetexturemiss: %d\n", L1_texture_miss);
+   gzprintf(visualizer_file, "Loneconstmiss: %d\n", L1_const_miss);
+   gzprintf(visualizer_file, "Lonereadmiss: %d\n", L1_read_miss);
+   gzprintf(visualizer_file, "Lonewritemiss: %d\n", L1_write_miss);
+   gzprintf(visualizer_file, "Ltwowritemiss: %d\n", L2_write_miss);
+   gzprintf(visualizer_file, "Ltwowritehit: %d\n",  L2_write_hit);
+   gzprintf(visualizer_file, "Ltworeadmiss: %d\n", L2_read_miss);
+   gzprintf(visualizer_file, "Ltworeadhit: %d\n", L2_read_hit);
+
+   // latency stats
+   if (num_mfs) {
+      gzprintf(visualizer_file, "averagemflatency: %lld\n", mf_total_lat/num_mfs);
+   }
+
+   // other parameters for graphing
+   gzprintf(visualizer_file, "globalcyclecount: %lld\n", gpu_sim_cycle);
+   gzprintf(visualizer_file, "globalinsncount: %lld\n", gpu_sim_insn);
+   gzprintf(visualizer_file, "globaltotinsncount: %lld\n", gpu_tot_sim_insn);
+   gzprintf(visualizer_file, "gpucompletedthreads: %lld\n", gpu_completed_thread);
+   gzprintf(visualizer_file, "gpgpunsentwrites: %d\n", gpgpu_n_sent_writes);
+   gzprintf(visualizer_file, "gpgpunprocessedwrites: %d\n", gpgpu_n_processed_writes);
+   gzprintf(visualizer_file, "gpgpu_n_cache_bkconflict: %d\n", gpgpu_n_cache_bkconflict);
+   gzprintf(visualizer_file, "gpgpu_n_shmem_bkconflict: %d\n", gpgpu_n_shmem_bkconflict);     
+   gzprintf(visualizer_file, "gpu_stall_by_MSHRwb: %d\n", gpu_stall_by_MSHRwb);   
+
+   // warp divergence breakdown
+   time_vector_print_interval2gzfile(visualizer_file);
+   gzprintf(visualizer_file, "WarpDivergenceBreakdown:");
+   unsigned int total=0;
+   unsigned int cf = (gpgpu_warpdistro_shader==-1)?gpu_n_shader:1;
+   gzprintf(visualizer_file, " %d", (shader_cycle_distro[0] - last_shader_cycle_distro[0]) / cf );
+   gzprintf(visualizer_file, " %d", (shader_cycle_distro[2] - last_shader_cycle_distro[2]) / cf );
+   for (i=0; i<warp_size+3; i++) {
+      if ( i>=3 ) {
+         total += (shader_cycle_distro[i] - last_shader_cycle_distro[i]);
+         if ( ((i-3) % (warp_size/8)) == ((warp_size/8)-1) ) {
+            gzprintf(visualizer_file, " %d", total / cf );
+            total=0;
+         }
+      }
+      last_shader_cycle_distro[i] = shader_cycle_distro[i];
+   }
+   gzprintf(visualizer_file,"\n");
+
+   gzclose(visualizer_file);
+}
+
+#include <list>
+#include <vector>
+#include <iostream>
+#include <map>
+#include"../gpgpu-sim/shader.h"
+class my_time_vector {
+private:
+   std::map< unsigned int, std::vector<long int> > ld_time_map;
+   std::map< unsigned int, std::vector<long int> > st_time_map;
+   unsigned ld_vector_size;
+   unsigned st_vector_size;
+   std::vector<double>  ld_time_dist;
+   std::vector<double>  st_time_dist;
+
+   std::vector<double>  overal_ld_time_dist;
+   std::vector<double>  overal_st_time_dist;
+   int overal_ld_count;
+   int overal_st_count;
+
+public:
+   my_time_vector(int ld_size,int st_size){
+      ld_vector_size = ld_size;
+      st_vector_size = st_size;
+      ld_time_dist.resize(ld_size);
+      st_time_dist.resize(st_size);
+      overal_ld_time_dist.resize(ld_size);
+      overal_st_time_dist.resize(st_size);
+      overal_ld_count = 0;
+      overal_st_count= 0;
+   }
+   void update_ld(unsigned int uid,unsigned int slot, long int time) { 
+      if ( ld_time_map.find( uid )!=ld_time_map.end() ) {
+         ld_time_map[uid][slot]=time;
+      } else if (slot <= MR_2SH_FQ_POP ) {
+         std::vector<long int> time_vec;
+         time_vec.resize(ld_vector_size);
+         time_vec[slot] = time;
+         ld_time_map[uid] = time_vec;  
+      } else {
+         //It's a merged mshr! forget it
+      }
+   }
+   void update_st(unsigned int uid,unsigned int slot, long int time) { 
+      if ( st_time_map.find( uid )!=st_time_map.end() ) {
+         st_time_map[uid][slot]=time;
+      } else {
+         std::vector<long int> time_vec;
+         time_vec.resize(st_vector_size);
+         time_vec[slot] = time;
+         st_time_map[uid] = time_vec;  
+      } 
+   }
+   void check_ld_update(unsigned int uid,unsigned int slot, long int latency) { 
+      if ( ld_time_map.find( uid )!=ld_time_map.end() ) {
+         int our_latency = ld_time_map[uid][slot] - ld_time_map[uid][MR_ICNT_PUSHED];
+         assert( our_latency == latency);
+      } else if (slot <= MR_2SH_FQ_POP ) {
+         abort();
+      }
+   }
+   void check_st_update(unsigned int uid,unsigned int slot, long int latency) { 
+      if ( st_time_map.find( uid )!=st_time_map.end() ) {
+         int our_latency = st_time_map[uid][slot] - st_time_map[uid][MR_ICNT_PUSHED];
+         assert( our_latency == latency);
+      } else {
+         abort();
+      } 
+   }
+private:
+   void calculate_ld_dist(void) {
+      unsigned i,first;
+      long int last_update,diff;
+      int finished_count=0;
+      ld_time_dist.clear();
+      ld_time_dist.resize(ld_vector_size);
+      std::map< unsigned int, std::vector<long int> >::iterator iter, iter_temp;
+      iter =ld_time_map.begin() ;
+      while (iter != ld_time_map.end()) {
+         last_update=0;
+         first=-1;
+         if (!iter->second[MR_WRITEBACK]) {
+            //this request is not done yet skip it!
+            ++iter; 
+            continue;
+         }
+         while ( !last_update ) {
+            first++;
+            assert( first < iter->second.size() );
+            last_update = iter->second[first];
+         }
+
+         for ( i=first;i<ld_vector_size;i++ ) {
+            diff = iter->second[i] - last_update;
+            if ( diff>0 ) {
+               ld_time_dist[i]+=diff;
+               last_update = iter->second[i];               
+            }
+         }
+         iter_temp = iter;
+         iter++;
+         ld_time_map.erase(iter_temp);
+         finished_count++;
+      }
+      if ( finished_count ) {
+         for ( i=0;i<ld_vector_size;i++ ) {
+            overal_ld_time_dist[i] = (overal_ld_time_dist[i]*overal_ld_count + ld_time_dist[i]) /  (overal_ld_count + finished_count); 
+         }
+         overal_ld_count += finished_count;        
+         for ( i=0;i<ld_vector_size;i++ ) {
+            ld_time_dist[i]/=finished_count;
+         }
+      }
+   }
+
+   void calculate_st_dist(void) {
+      unsigned i,first;
+      long int last_update,diff;
+      int finished_count=0;
+      st_time_dist.clear();
+      st_time_dist.resize(st_vector_size);
+      std::map< unsigned int, std::vector<long int> >::iterator iter,iter_temp;
+      iter =st_time_map.begin() ;
+      while (  iter != st_time_map.end() ) {
+         last_update=0;
+         first=-1;
+         if (!iter->second[MR_2SH_ICNT_PUSHED]) {
+            //this request is not done yet skip it!
+            ++iter;
+            continue;
+         }
+         while ( !last_update ) {
+            first++;
+            assert( first < iter->second.size() );
+            last_update = iter->second[first];
+         }
+
+         for ( i=first;i<st_vector_size;i++ ) {
+            diff = iter->second[i] - last_update;
+            if ( diff>0 ) {
+               st_time_dist[i]+=diff;
+               last_update = iter->second[i];               
+            }
+         }
+         iter_temp = iter;
+         iter++;
+         st_time_map.erase(iter_temp);
+         finished_count++;       
+      }
+      if ( finished_count ) {
+         for ( i=0;i<st_vector_size;i++ ) {
+            overal_st_time_dist[i] = (overal_st_time_dist[i]*overal_st_count + st_time_dist[i]) /  (overal_st_count + finished_count); 
+         }
+         overal_st_count += finished_count;        
+         for ( i=0;i<st_vector_size;i++ ) {
+            st_time_dist[i]/=finished_count;
+         }
+      }
+   }
+
+public:
+   void clear_time_map_vectors(void) {   
+      ld_time_map.clear();
+      st_time_map.clear();
+   }
+   void print_all_ld(void) {
+      unsigned i;
+      std::map< unsigned int, std::vector<long int> >::iterator iter;
+      for ( iter =ld_time_map.begin() ; iter != ld_time_map.end(); ++iter ) {
+         std::cout<<"ld_uid"<<iter->first;
+         for ( i=0;i<ld_vector_size;i++ ) {
+            std::cout<<" "<<iter->second[i];
+         }
+         std::cout<< std::endl;
+      }
+   }
+
+   void print_all_st(void) {
+      unsigned i;
+      std::map< unsigned int, std::vector<long int> >::iterator iter;
+
+      for ( iter =st_time_map.begin() ; iter != st_time_map.end(); ++iter ) {
+         std::cout<<"st_uid"<<iter->first;
+         for ( i=0;i<st_vector_size;i++ ) {
+            std::cout<<" "<<iter->second[i];
+         }
+         std::cout<<std::endl;
+      }
+   }
+
+   void calculate_dist() {
+      calculate_ld_dist();
+      calculate_st_dist();
+   }
+   void print_dist(void) {
+      unsigned i; 
+      calculate_dist();
+      std::cout << "LD_mem_lat_dist " ;
+      for ( i=0;i<ld_vector_size;i++ ) {
+         std::cout <<" "<<(int)overal_ld_time_dist[i]; 
+      }
+      std::cout << std::endl;
+      std::cout << "ST_mem_lat_dist " ;
+      for ( i=0;i<st_vector_size;i++ ) {
+         std::cout <<" "<<(int)overal_st_time_dist[i];
+      }
+      std::cout << std::endl;
+   }
+   void print_to_file(FILE *outfile) {
+      unsigned i; 
+      calculate_dist();
+      fprintf (outfile,"LDmemlatdist:") ;
+      for ( i=0;i<ld_vector_size;i++ ) {
+         fprintf (outfile," %d", (int)ld_time_dist[i]); 
+      }
+      fprintf (outfile,"\n") ;
+      fprintf (outfile,"STmemlatdist:") ;
+      for ( i=0;i<st_vector_size;i++ ) {
+         fprintf (outfile," %d", (int)st_time_dist[i]); 
+      }
+      fprintf (outfile,"\n") ;
+   }   
+   void print_to_gzfile(gzFile outfile) {
+      unsigned i; 
+      calculate_dist();
+      gzprintf (outfile,"LDmemlatdist:") ;
+      for ( i=0;i<ld_vector_size;i++ ) {
+         gzprintf (outfile," %d", (int)ld_time_dist[i]); 
+      }
+      gzprintf (outfile,"\n") ;
+      gzprintf (outfile,"STmemlatdist:") ;
+      for ( i=0;i<st_vector_size;i++ ) {
+         gzprintf (outfile," %d", (int)st_time_dist[i]); 
+      }
+      gzprintf (outfile,"\n") ;
+   }   
+};
+
+my_time_vector* g_my_time_vector; 
+
+void time_vector_create(int ld_size,int st_size) {
+   g_my_time_vector = new my_time_vector(ld_size,st_size); 
+}                               
+
+
+void time_vector_print(void) {
+   g_my_time_vector->print_dist();
+}
+
+
+void time_vector_print_interval2file(FILE *outfile) {
+   g_my_time_vector->print_to_file(outfile);
+}
+
+
+void time_vector_print_interval2gzfile(gzFile outfile) {
+   g_my_time_vector->print_to_gzfile(outfile);
+}
+
+#include "../gpgpu-sim/mem_fetch.h"
+
+void time_vector_update(unsigned int uid,int slot ,long int cycle,int type) {
+   if ( (type == RD_REQ) || (type == REPLY_DATA) ) {
+      g_my_time_vector->update_ld( uid, slot,cycle);
+   } else if ( type == WT_REQ ) {
+      g_my_time_vector->update_st( uid, slot,cycle);
+   } else {
+      abort(); 
+   }
+}
+
+void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type) 
+{
+   if ( (type == RD_REQ) || (type == REPLY_DATA) ) {
+      g_my_time_vector->check_ld_update( uid, slot, latency );
+   } else if ( type == WT_REQ ) {
+      g_my_time_vector->check_st_update( uid, slot, latency );
+   } else {
+      abort(); 
+   }
+}
diff --git a/src/gpgpu-sim/warp_tracker.cc b/src/gpgpu-sim/warp_tracker.cc
new file mode 100644
index 0000000..19f0e12
--- /dev/null
+++ b/src/gpgpu-sim/warp_tracker.cc
@@ -0,0 +1,471 @@
+/* 
+ * waro_tracker.cc
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "warp_tracker.h"
+
+using namespace std;
+
+extern unsigned int warp_size;
+extern unsigned int gpu_n_shader;
+extern unsigned int gpu_n_thread_per_shader;
+
+#include <set>
+
+class warp_tracker {
+public:
+
+   int *tid; // the threads in this warp
+   int n_thd; // total number of threads in this warp
+   int n_notavail; // number of threads still not available
+   shader_core_ctx_t *shd; // reference to shader core
+
+   warp_tracker () {
+      tid = new int[warp_size];
+      memset(tid, -1, sizeof(int)*warp_size);
+      n_thd = 0;
+      n_notavail = 0;
+      shd = NULL;
+   }
+   warp_tracker( int *tid, shader_core_ctx_t * shd ) {
+      this->tid = new int[warp_size];
+      memcpy(this->tid, tid, sizeof(int)*warp_size);
+      this->n_thd = 0;
+      this->n_notavail = 0;
+      for (unsigned i=0; i<warp_size; i++) {
+         if (this->tid[i] >= 0) {
+            this->n_thd++;
+         }
+      }
+      this->n_notavail = this->n_thd;
+      this->shd = shd;
+   }
+   ~warp_tracker () {
+      delete[] tid;
+   }
+
+   // set the warp to be consist of the given threads
+   void set_warp ( int *tid, shader_core_ctx_t *shd) {
+      memcpy(this->tid, tid, sizeof(int)*warp_size);
+      this->n_thd = 0;
+      this->n_notavail = 0;
+      for (unsigned i=0; i<warp_size; i++) {
+         if (this->tid[i] >= 0) {
+            this->n_thd++;
+         }
+      }
+      this->n_notavail = this->n_thd;
+      this->shd = shd;
+   }
+
+   // signal that this thread is available for fetch
+   // if all threads in the warp are available, change all their status
+   // and return true
+   bool avail_thd ( int tid_in ) {
+      n_notavail--;
+      if (n_notavail) {
+         return false;
+      } else {
+
+         int thd_unlocked = 0;
+         if (shd->model == POST_DOMINATOR || shd->model == NO_RECONVERGE) {
+            thd_unlocked = 1;
+         } else {
+            // unlock the threads here if scheduler is not PDOM or NO-RECONV
+            for (unsigned i=0; i<warp_size; i++) {
+               if (this->tid[i] >= 0) {
+                  shd->thread[tid[i]].avail4fetch++;
+                  assert(shd->thread[tid[i]].avail4fetch <= 1);
+                  assert( shd->warp[tid[i]/warp_size].n_avail4fetch < warp_size );
+                  shd->warp[tid[i]/warp_size].n_avail4fetch++;
+                  thd_unlocked = 1;
+               }
+            }
+         }
+         if (shd->using_commit_queue && thd_unlocked) {
+            int *tid_unlocked = alloc_commit_warp();
+            memcpy(tid_unlocked, this->tid, sizeof(int)*warp_size);
+            dq_push(shd->thd_commit_queue,(void*)tid_unlocked);
+         }
+
+         return true;
+      }
+   }
+
+   // a bookkeeping method to allow a warp to be deallocated 
+   // when its threads have finished executing.
+   bool complete_thd ( int tid_in ) {
+      n_notavail--;
+      if (n_notavail) {
+         return false;
+      } else {
+         return true;
+      }
+   }
+};
+
+static warp_tracker ***warp_tracker_map;
+static unsigned **g_warp_tracker_map_setl_cycle;
+static warp_tracker *warp_tracker_pool = NULL;
+static list<warp_tracker*> free_wpt;
+
+warp_tracker* alloc_warp_tracker( int *tid_in, shader_core_ctx_t *shd ) 
+{
+   assert(!free_wpt.empty());
+   warp_tracker* wpt = free_wpt.front();
+   free_wpt.pop_front();
+
+   wpt->set_warp(tid_in, shd);
+
+   return wpt;
+}
+
+void free_warp_tracker(warp_tracker* wpt)
+{
+   free_wpt.push_back(wpt);
+}
+
+void init_warp_tracker( ) 
+{
+   unsigned int i;
+
+   warp_tracker_map = (warp_tracker ***)calloc(gpu_n_shader, sizeof(warp_tracker **));
+   g_warp_tracker_map_setl_cycle = (unsigned**)calloc(gpu_n_shader, sizeof(unsigned*));
+   for (i=0; i<gpu_n_shader; i++) {
+      warp_tracker_map[i] = (warp_tracker **)calloc(gpu_n_thread_per_shader, sizeof(warp_tracker *));
+      g_warp_tracker_map_setl_cycle[i] = (unsigned*)calloc(gpu_n_thread_per_shader, sizeof(unsigned));
+   }
+
+   // max possible number of warps is just when each thread has its own warp
+   warp_tracker_pool = new warp_tracker[gpu_n_shader * gpu_n_thread_per_shader]; 
+   printf("%d %d %d %d\n", warp_size, gpu_n_shader, gpu_n_thread_per_shader, 
+          warp_size * gpu_n_shader * gpu_n_thread_per_shader);
+   for (i=0; i<gpu_n_shader*gpu_n_thread_per_shader; i++) {
+      free_wpt.push_back(&(warp_tracker_pool[i]));
+   }
+   printf("%zd\n", free_wpt.size());
+}
+
+extern signed long long gpu_tot_sim_cycle;
+extern signed long long gpu_sim_cycle;
+
+void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd ) 
+{
+   int sid = shd->sid;
+   unsigned i;
+   int n_thd = 0;
+   for (i=0; i<warp_size; i++) {
+      if (tid_in[i] >= 0) n_thd++;
+   }
+
+   if (!n_thd) return;
+
+   warp_tracker *wpt = alloc_warp_tracker(tid_in, shd);
+
+   // assign the new warp_tracker to warp_tracker_map
+   for (i=0; i<warp_size; i++) {
+      if (tid_in[i] >= 0) {
+         assert( warp_tracker_map[sid][tid_in[i]] == NULL );
+         warp_tracker_map[sid][tid_in[i]] = wpt;
+         g_warp_tracker_map_setl_cycle[sid][tid_in[i]] = gpu_tot_sim_cycle + gpu_sim_cycle;
+      }
+   }
+}
+
+int wpt_signal_avail( int tid, shader_core_ctx_t *shd ) 
+{
+   int sid = shd->sid;
+   warp_tracker *wpt = warp_tracker_map[sid][tid];
+   assert(wpt != NULL);
+
+
+   // signal the warp tracker
+   if (wpt->avail_thd(tid)) {
+      // if the warp is ready to be fetched again, remove this warp_tracker
+      for (unsigned i=0; i<warp_size; i++) {
+         if (wpt->tid[i] >= 0) {
+            warp_tracker_map[sid][wpt->tid[i]] = NULL;
+            g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle;
+         }
+      }
+
+      free_warp_tracker( wpt );
+
+      return 1;
+   } else {
+      return 0;
+   }
+}
+
+void register_cta_thread_exit(shader_core_ctx_t *shader, int cta_num );
+
+int wpt_signal_complete( int tid, shader_core_ctx_t *shd ) 
+{
+   int sid = shd->sid;
+   warp_tracker *wpt = warp_tracker_map[sid][tid];
+   assert(wpt != NULL);
+
+   // signal the warp tracker
+   if (wpt->complete_thd(tid)) {
+      // if the warp has completed execution, remove this warp_tracker
+      int warp_mask = 0;
+      for (unsigned i=0; i<warp_size; i++) {
+         if (wpt->tid[i] >= 0) {
+            register_cta_thread_exit(shd, wpt->tid[i] );
+            warp_tracker_map[sid][wpt->tid[i]] = NULL;
+            g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle;
+            warp_mask |= (1 << i);
+         }
+      }
+
+      free_warp_tracker( wpt );
+
+      return warp_mask;
+   } else {
+      return 0;
+   }
+}
+
+//------------------------------------------------------------------------------------
+
+class thread_pc_tracker_class {
+public:
+   address_type *thd_pc; // tracks the pc of each thread
+   map<address_type, unsigned> pc_count;
+   unsigned acc_pc_count;
+   int simd_width;
+   static map<unsigned, unsigned> histogram;
+
+   thread_pc_tracker_class( ) {
+      this->acc_pc_count = 0;
+      this->simd_width = 0;
+      this->thd_pc = NULL;
+   }
+
+   thread_pc_tracker_class(int simd_width, int thread_count) {
+      this->acc_pc_count = 0;
+      this->simd_width = simd_width;
+      this->thd_pc = new address_type[thread_count];
+      memset( this->thd_pc, 0, sizeof(address_type)*thread_count);
+   }
+
+   void add_threads( int *tid, address_type pc ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] != -1) {
+            pc_count[pc] += 1; // automatically create a new entry if not exist
+            thd_pc[tid[i]] = pc;
+         }
+      }
+   }
+
+   void sub_threads( int *tid ) {
+      for (int i=0; i<simd_width; i++) {
+         if (tid[i] != -1) {
+            address_type pc = thd_pc[tid[i]];
+            if (pc == 0) break;
+            pc_count[pc] -= 1;
+            assert((int)pc_count[pc] >= 0);
+            if (pc_count[pc] == 0) pc_count.erase(pc); // manually erasing entries with 0 count
+         }
+      }
+   }
+
+   void update_acc_count( ) { 
+      acc_pc_count += pc_count.size(); 
+      histogram[pc_count.size()] += 1;
+   }
+
+   void set_threads_pc ( int *tid, address_type pc ) {
+      sub_threads(tid);
+      add_threads(tid, pc);
+      update_acc_count( );
+   }
+
+   unsigned get_acc_pc_count( ) { return acc_pc_count;}
+
+   unsigned count( ) { return pc_count.size();}
+
+   static void histo_print( FILE* fout ) {
+      if (histogram.empty()) return; // do not output anything if the histogram is empty
+      map<unsigned, unsigned>::iterator i;
+      fprintf(fout, "Thread PC Histogram: ");
+      for (i = histogram.begin(); i != histogram.end(); i++) {
+         fprintf(fout, "%d:%d ", i->first, i->second);
+      }
+      fprintf(fout, "\n");
+   }
+};
+
+map<unsigned, unsigned> thread_pc_tracker_class::histogram;
+
+thread_pc_tracker_class *thread_pc_tracker = NULL;
+
+void print_thread_pc_histogram( FILE *fout )
+{
+   thread_pc_tracker_class::histo_print(fout);
+}
+
+void print_thread_pc( FILE *fout )
+{
+   fprintf(fout, "SHD_PC_C: ");
+   for (unsigned i=0; i<gpu_n_shader; i++) {
+      fprintf(fout, "%d ", thread_pc_tracker[i].get_acc_pc_count() );
+   }
+   fprintf(fout, "\n");
+}
+
+void track_thread_pc( int shader_id, int *tid, address_type pc ) 
+{
+   if (!thread_pc_tracker) {
+      thread_pc_tracker = new thread_pc_tracker_class[gpu_n_shader];
+      for (unsigned i=0; i<gpu_n_shader; i++) {
+         thread_pc_tracker[i] = thread_pc_tracker_class(warp_size, gpu_n_thread_per_shader);
+      }
+   }
+   thread_pc_tracker[shader_id].set_threads_pc( tid, pc );
+}
+
+//------------------------------------------------------------------------------------
+
+static int *commit_warp_pool = NULL;
+static queue<int*> free_commit_warp_q;
+
+void init_commit_warp( )
+{
+   unsigned int num_warp = warp_size * gpu_n_shader * gpu_n_thread_per_shader;
+   commit_warp_pool = new int[num_warp];
+   for (unsigned int i=0; i<num_warp; i+=warp_size) {
+      free_commit_warp_q.push(&(commit_warp_pool[i]));
+   }
+}
+
+int* alloc_commit_warp( )
+{
+   if (!commit_warp_pool) {
+      init_commit_warp( );
+   }
+
+   assert(!free_commit_warp_q.empty());
+   int *new_commit_warp = free_commit_warp_q.front();
+   free_commit_warp_q.pop();
+
+   return new_commit_warp;
+}
+
+void free_commit_warp( int *commit_warp )
+{
+   free_commit_warp_q.push(commit_warp);
+}
+
+
+extern int pipe_simd_width;
+
+// uncomment to enable checking for warp consistency
+// #define CHECK_WARP_CONSISTENCY
+
+void check_stage_pcs( shader_core_ctx_t *shader, unsigned stage )
+{
+#ifdef CHECK_WARP_CONSISTENCY
+   address_type inst_pc = (address_type)-1;
+   unsigned tid;
+   if( shader->model == MIMD ) 
+      return;
+
+   std::set<unsigned> tids;
+
+   for ( int i = 0; i < pipe_simd_width; i++) {
+      if (shader->pipeline_reg[i][stage].hw_thread_id == -1 ) 
+         continue;
+      if ( inst_pc == (address_type)-1 ) 
+         inst_pc = shader->pipeline_reg[i][stage].pc;
+      tid = shader->pipeline_reg[i][stage].hw_thread_id;
+      assert( tids.find(tid) == tids.end() );
+      tids.insert(tid);
+      assert( inst_pc == shader->pipeline_reg[i][stage].pc );
+   }
+#endif
+}
+
+void check_pm_stage_pcs( shader_core_ctx_t *shader, unsigned stage )
+{
+#ifdef CHECK_WARP_CONSISTENCY
+   address_type inst_pc = (address_type)-1;
+   unsigned tid;
+   if( shader->model == MIMD ) 
+      return;
+
+   std::set<unsigned> tids;
+
+   for (int i = 0; i < pipe_simd_width; i++) {
+      if (shader->pre_mem_pipeline[i][stage].hw_thread_id == -1 ) 
+         continue;
+      if ( inst_pc == (address_type)-1 ) 
+         inst_pc = shader->pre_mem_pipeline[i][stage].pc;
+      tid = shader->pre_mem_pipeline[i][stage].hw_thread_id;
+      assert( tids.find(tid) == tids.end() );
+      tids.insert(tid);
+      assert( inst_pc == shader->pre_mem_pipeline[i][stage].pc );
+   }
+#endif
+}
diff --git a/src/gpgpu-sim/warp_tracker.h b/src/gpgpu-sim/warp_tracker.h
new file mode 100644
index 0000000..a4faa09
--- /dev/null
+++ b/src/gpgpu-sim/warp_tracker.h
@@ -0,0 +1,100 @@
+/* 
+ * waro_tracker.h
+ *
+ * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda and the 
+ * University of British Columbia
+ * Vancouver, BC  V6T 1Z4
+ * All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#ifndef warp_tracker_h_INCLUDED
+#define warp_tracker_h_INCLUDED
+
+#ifdef __cplusplus
+
+   #include <cstdio>
+   #include <cstdlib>
+   #include <cstring>
+   #include <cassert>
+   #include <map>
+   #include <list>
+   #include <deque>
+   #include <queue>
+
+#endif
+
+#include "../util.h"
+#include "shader.h"
+
+void init_warp_tracker( ); 
+
+void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd ); 
+
+int wpt_signal_avail( int tid, shader_core_ctx_t *shd ); 
+
+int wpt_signal_complete( int tid, shader_core_ctx_t *shd ); 
+
+void print_thread_pc_histogram( FILE *fout );
+void print_thread_pc( FILE *fout );
+void track_thread_pc( int shader_id, int *tid, address_type pc );
+
+int* alloc_commit_warp( );
+void free_commit_warp( int *commit_warp );
+
+#endif