- integrate changes from fermi-test (CL's under that path in range 7261-7418).

(add scoreboard logic from ptxplus branch and modified operand collector
with parallel ALU/SFU pipelines)

passing regressions

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 7419]

20 files changed, 1610 insertions, 458 deletions

diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 612c46d..9762674 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -26,11 +26,12 @@ typedef unsigned addr_t;
 
 // these are operations the timing model can see
 #define NO_OP -1
-#define ALU_OP 1000
-#define LOAD_OP 2000
-#define STORE_OP 3000
-#define BRANCH_OP 4000
-#define BARRIER_OP 5000
+#define ALU_OP     0x01000
+#define SFU_OP     0x02000
+#define LOAD_OP    0x04000
+#define STORE_OP   0x08000
+#define BRANCH_OP  0x10000
+#define BARRIER_OP 0x20000
 
 enum _memory_space_t {
    undefined_space=0,
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 9115774..2c317b2 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -524,13 +524,57 @@ void ptx_print_insn( address_type pc, FILE *fp )
    finfo->print_insn(pc,fp);
 }
 
+static void get_opcode_info( const ptx_instruction *pI, unsigned opcode, unsigned *cycles, unsigned *op_type )
+{
+   *op_type = ALU_OP;
+   *cycles = 1;
+   if ( opcode == LD_OP ) {
+      *op_type = LOAD_OP;
+   } else if ( opcode == ST_OP ) {
+      *op_type = STORE_OP;
+   } else if ( opcode == BRA_OP ) {
+      *op_type = BRANCH_OP;
+   } else if ( opcode == TEX_OP ) {
+      *op_type = LOAD_OP;
+   } else if ( opcode == ATOM_OP ) {
+      *op_type = LOAD_OP; // make atomics behave more like a load.
+   } else if ( opcode == BAR_OP ) {
+      *op_type = BARRIER_OP;
+   }
+
+   // Floating point instructions
+   if( opcode == RCP_OP || opcode == LG2_OP || opcode == RSQRT_OP ) {
+      *cycles = 4;
+      *op_type = SFU_OP;
+   } else if( opcode == SQRT_OP || opcode == SIN_OP || opcode == COS_OP || opcode == EX2_OP ) {
+      *cycles = 4;
+      *op_type = SFU_OP;
+   } else if( opcode == DIV_OP ) {
+      // Floating point only
+      if( pI->get_type() == F32_TYPE || pI->get_type() == F64_TYPE ) {
+         *cycles = 8;         
+         *op_type = SFU_OP;
+      }
+   }
+   // Integer instructions
+   if( opcode == MUL_OP ) {
+      if( pI->get_type() == B32_TYPE || pI->get_type() == U32_TYPE || pI->get_type() == S32_TYPE ) {
+         // 32-bit integer instruction
+         *cycles = 4;
+         *op_type = SFU_OP;
+      }
+   }
+}
+
 void function_info::ptx_decode_inst( ptx_thread_info *thread, 
                                      unsigned *op_type, 
                                      int *i1, int *i2, int *i3, int *i4, 
                                      int *o1, int *o2, int *o3, int *o4, 
                                      int *vectorin, 
                                      int *vectorout,
-                                     int *arch_reg )
+                                     int *arch_reg,
+                                     int *pred,
+                                     int *ar1, int *ar2 )
 {
    addr_t pc = thread->get_pc();
    unsigned index = pc - m_start_PC;
@@ -549,21 +593,10 @@ void function_info::ptx_decode_inst( ptx_thread_info *thread,
       break;
    }
 
-   *op_type = ALU_OP;
-   if ( opcode == LD_OP ) {
-      *op_type = LOAD_OP;
-   } else if ( opcode == ST_OP ) {
-      *op_type = STORE_OP;
-   } else if ( opcode == BRA_OP ) {
-      *op_type = BRANCH_OP;
-   } else if ( opcode == TEX_OP ) {
-      *op_type = LOAD_OP;
-   } else if ( opcode == ATOM_OP ) {
-      *op_type = LOAD_OP; // make atomics behave more like a load.
-   } else if ( opcode == BAR_OP ) {
-      *op_type = BARRIER_OP;
-   }
+   unsigned cycles;
+   get_opcode_info(pI,opcode,&cycles,op_type);
 
+   // Get register operands
    int n=0,m=0;
    ptx_instruction::const_iterator op=pI->op_iter_begin();
    for ( ; op != pI->op_iter_end(); op++, n++ ) { //process operands
@@ -607,6 +640,37 @@ void function_info::ptx_decode_inst( ptx_thread_info *thread,
          }
       }
    }
+
+   // Get predicate
+   if(pI->has_pred()) {
+	   const operand_info &p = pI->get_pred();
+	   *pred = p.reg_num();
+   }
+
+   // Get address registers inside memory operands.
+   // Assuming only one memory operand per instruction,
+   //  and maximum of two address registers for one memory operand.
+   if( pI->has_memory_read() || pI->has_memory_write() ) {
+	  ptx_instruction::const_iterator op=pI->op_iter_begin();
+	  for ( ; op != pI->op_iter_end(); op++, n++ ) { //process operands
+	     const operand_info &o = *op;
+
+	     // memory operand with addressing (ex. s[0x4] or g[$r1])
+	     if(o.is_memory_operand2()) {
+	    	 // memory operand with one address register (ex. g[$r1] or s[$r2+0x4])
+	    	 if(o.get_double_operand_type() == 0 && o.is_memory_operand()){
+             const symbol *base_addr = o.get_symbol();
+             if( base_addr->is_reg() ) 
+                *ar1 = base_addr->reg_num();
+	    	 }
+	    	 // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2])
+	    	 else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) {
+	    		 *ar1 = o.reg1_num();
+	    		 *ar2 = o.reg2_num();
+	    	 }
+	     }
+	  }
+   }
 }
 
 void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size )
@@ -773,11 +837,13 @@ unsigned datatype2size( unsigned data_type )
 }
 
 unsigned g_warp_active_mask;
+FILE *ptx_inst_debug_file;
 
 void function_info::ptx_exec_inst( ptx_thread_info *thread, 
                                    addr_t *addr, 
                                    memory_space_t *space, 
-                                   unsigned *data_size, 
+                                   unsigned *data_size,
+                                   unsigned *cycles,
                                    dram_callback_t* callback, 
                                    unsigned warp_active_mask  )
 {
@@ -832,10 +898,9 @@ void function_info::ptx_exec_inst( ptx_thread_info *thread,
    addr_t insn_memaddr = 0xFEEBDAED;
    memory_space_t insn_space = undefined_space;
    unsigned insn_data_size = 0;
-   if ( pI->get_opcode() == LD_OP || pI->get_opcode() == ST_OP || pI->get_opcode() == TEX_OP ) {
+   if ( (pI->has_memory_read()  || pI->has_memory_write()) ) {
       insn_memaddr = thread->last_eaddr();
       insn_space = thread->last_space();
-
       unsigned to_type = pI->get_type();
       insn_data_size = datatype2size(to_type);
    }
@@ -855,6 +920,47 @@ void function_info::ptx_exec_inst( ptx_thread_info *thread,
       callback->instruction = NULL;
    }
 
+   // Set number of cycles for this instruction
+   int opcode = pI->get_opcode(); //determine the opcode
+   unsigned op_type;
+   get_opcode_info(pI,opcode,cycles,&op_type);
+
+   if (pI->get_opcode() == TEX_OP) {
+      *addr = thread->last_eaddr();
+      *space = thread->last_space();
+
+      unsigned to_type = pI->get_type();
+      switch ( to_type ) {
+      case B8_TYPE:
+      case S8_TYPE:
+      case U8_TYPE: 
+         *data_size = 1; break;
+      case B16_TYPE:
+      case S16_TYPE:
+      case U16_TYPE:
+      case F16_TYPE: 
+         *data_size = 2; break;
+      case B32_TYPE:
+      case S32_TYPE:
+      case U32_TYPE:
+      case F32_TYPE: 
+         *data_size = 4; break;
+      case B64_TYPE:
+      case S64_TYPE:
+      case U64_TYPE:
+      case F64_TYPE: 
+         *data_size = 8; break;
+      default: assert(0); break;
+      }
+   }
+
+   // Output register information to file and stdout
+   if( g_ptx_inst_debug_to_file != 0 && 
+        (g_ptx_inst_debug_thread_uid == 0 || g_ptx_inst_debug_thread_uid == thread->get_uid()) ) {
+      thread->dump_modifiedregs(ptx_inst_debug_file);
+      thread->dump_regs(ptx_inst_debug_file);
+   }
+
    if ( g_debug_execution >= 6 ) {
       if ( ptx_debug_exec_dump_cond<6>(thread->get_uid(), pc) )
          thread->dump_modifiedregs();
@@ -1414,6 +1520,8 @@ void gpgpu_cuda_ptx_sim_main_func( const char *kernel_key, dim3 gridDim, dim3 bl
    time_t end_time, elapsed_time, days, hrs, minutes, sec;
    int i1, i2, i3, i4, o1, o2, o3, o4;
    int vectorin, vectorout;
+   int pred;
+   int ar1, ar2;
 
    gpgpu_cuda_ptx_sim_init_grid(kernel_key, args,gridDim,blockDim);
 
@@ -1498,6 +1606,7 @@ void gpgpu_cuda_ptx_sim_main_func( const char *kernel_key, dim3 gridDim, dim3 bl
 
                   unsigned op_type;
                   addr_t addr;
+                  unsigned cycles;
                   memory_space_t space;
                   int arch_reg[MAX_REG_OPERANDS] = { -1 };
                   unsigned data_size;
@@ -1505,8 +1614,8 @@ void gpgpu_cuda_ptx_sim_main_func( const char *kernel_key, dim3 gridDim, dim3 bl
                   unsigned warp_active_mask = (unsigned)-1; // vote instruction with diverged warps won't execute correctly
                                                             // in functional simulation mode
 
-                  g_func_info->ptx_decode_inst( thread, &op_type, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg );
-                  g_func_info->ptx_exec_inst( thread, &addr, &space, &data_size, &callback, warp_active_mask );
+                  g_func_info->ptx_decode_inst( thread, &op_type, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg, &pred, &ar1, &ar2 );
+                  g_func_info->ptx_exec_inst( thread, &addr, &space, &data_size, &cycles, &callback, warp_active_mask );
                }
             }
          }
@@ -1532,7 +1641,7 @@ void gpgpu_cuda_ptx_sim_main_func( const char *kernel_key, dim3 gridDim, dim3 bl
    fflush(stdout); 
 }
 
-void ptx_decode_inst( void *thd, unsigned *op, int *i1, int *i2, int *i3, int *i4, int *o1, int *o2, int *o3, int *o4, int *vectorin, int *vectorout, int *arch_reg  )
+void ptx_decode_inst( void *thd, unsigned *op, int *i1, int *i2, int *i3, int *i4, int *o1, int *o2, int *o3, int *o4, int *vectorin, int *vectorout, int *arch_reg, int *pred, int *ar1, int *ar2 )
 {
    *op = NO_OP;
    *o1 = 0;
@@ -1546,22 +1655,35 @@ void ptx_decode_inst( void *thd, unsigned *op, int *i1, int *i2, int *i3, int *i
    *vectorin = 0;
    *vectorout = 0;
    std::fill_n(arch_reg, MAX_REG_OPERANDS, -1);
+   *pred = 0;
+   *ar1 = 0;
+   *ar2 = 0;
 
    if ( thd == NULL )
       return;
 
    ptx_thread_info *thread = (ptx_thread_info *) thd;
    g_func_info = thread->func_info();
-   g_func_info->ptx_decode_inst(thread,op,i1,i2,i3,i4,o1,o2,o3,o4,vectorin,vectorout,arch_reg);
+   g_func_info->ptx_decode_inst(thread,op,i1,i2,i3,i4,o1,o2,o3,o4,vectorin,vectorout,arch_reg,pred,ar1,ar2);
+}
+
+extern "C" unsigned ptx_get_inst_op( void *thd)
+{
+   if ( thd == NULL )
+      return NO_OP;
+
+   ptx_thread_info *thread = (ptx_thread_info *) thd;
+   return(thread->func_info())->ptx_get_inst_op(thread);
 }
 
-void ptx_exec_inst( void *thd, address_type *addr, memory_space_t *space, unsigned *data_size, dram_callback_t* callback, unsigned warp_active_mask )
+void ptx_exec_inst( void *thd, address_type *addr, memory_space_t *space, unsigned *data_size, unsigned *cycles, dram_callback_t* callback, unsigned warp_active_mask )
 {
    if ( thd == NULL )
       return;
+   *cycles = 1;
    ptx_thread_info *thread = (ptx_thread_info *) thd;
    g_func_info = thread->func_info();
-   g_func_info->ptx_exec_inst( thread, addr, space, data_size, callback, warp_active_mask );
+   g_func_info->ptx_exec_inst( thread, addr, space, data_size, cycles, callback, warp_active_mask );
 }
 
 void ptx_dump_regs( void *thd )
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index baf3b63..6f931fc 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -89,11 +89,15 @@ void ptx_decode_inst( void *thd,
                       int *o4, 
                       int *vectorin, 
                       int *vectorout, 
-                      int *arch_reg );
+                      int *arch_reg, 
+                      int *pred,
+                      int *ar1, 
+                      int *ar2 );
 void ptx_exec_inst( void *thd, 
                     address_type *addr, 
                     memory_space_t *space, 
-                    unsigned *data_size, 
+                    unsigned *data_size,
+                    unsigned *cycles, 
                     dram_callback_t* callback, 
                     unsigned warp_active_mask );
 
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 47b3ebb..cbf4c44 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -2639,7 +2639,7 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread )
          thread->get_vector_operand_values(src1, ptx_regs, 2); 
          mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI);
          mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI);
-         free(ptx_regs);
+         delete [] ptx_regs;
       }
       if (vector_spec == V3_TYPE) {
          ptx_reg_t* ptx_regs = new ptx_reg_t[3]; 
@@ -2647,7 +2647,7 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread )
          mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI);
          mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI);
          mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI);
-         free(ptx_regs);
+         delete [] ptx_regs;
       }
       if (vector_spec == V4_TYPE) {
          ptx_reg_t* ptx_regs = new ptx_reg_t[4]; 
@@ -2656,7 +2656,7 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread )
          mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI);
          mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI);
          mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI);
-         free(ptx_regs);
+         delete [] ptx_regs;
       }
    }
    thread->m_last_effective_address = addr;
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 3a2a4e3..0887df0 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -565,6 +565,13 @@ public:
    bool is_label() const { return m_type == label_t;}
    bool is_builtin() const { return m_type == builtin_t;}
    bool is_memory_operand() const { return m_type == memory_t;}
+   // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004])
+   bool is_memory_operand2() const { 
+      /* TODO: modify after integrate with ptxplus*/ 
+      return m_type == memory_t; 
+      /*return (m_addr_space==1 || m_addr_space==2 || m_addr_space==3 || m_addr_space==4);*/ 
+   }
+
    bool is_literal() const { return m_type == int_t ||
       m_type == float_op_t ||
       m_type == double_op_t ||
@@ -621,6 +628,8 @@ public:
    bool is_neg_pred() const { return m_neg_pred; }
    bool is_valid() const { return m_valid; }
 
+   int get_double_operand_type() const { return 0; /* TODO: modify after integrate with ptxplus*/ }
+
 private:
    unsigned m_uid;
    bool m_valid;
@@ -828,6 +837,17 @@ public:
 
    unsigned warp_size() const { return m_warp_size; }
    int membar_level() const { return m_membar_level; }
+
+   bool has_memory_read() const {
+      if( m_opcode == LD_OP || m_opcode == TEX_OP ) 
+         return true;
+      return false;
+   }
+   bool has_memory_write() const {
+      if( m_opcode == ST_OP ) return true;
+      return false;
+   }
+
 private:
    basic_block_t        *m_basic_block;
    unsigned          m_uid;
@@ -958,8 +978,11 @@ public:
                          int *o4,
                          int *vectorin,
                          int *vectorout,
-                         int *arch_reg );
-   void ptx_exec_inst( ptx_thread_info *thd, addr_t *addr, memory_space_t *space, unsigned *data_size, dram_callback_t* callback, unsigned warp_active_mask  );
+                         int *arch_reg,
+                         int *pred,
+                         int *ar1, int *ar2 );
+   void ptx_exec_inst( ptx_thread_info *thd, addr_t *addr, memory_space_t *space, unsigned *data_size, unsigned *cycles, dram_callback_t* callback, unsigned warp_active_mask  );
+   unsigned ptx_get_inst_op( ptx_thread_info *thread );
    void add_param( const char *name, struct param_t value )
    {
       m_kernel_params[ name ] = value;
diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc
index b433fb3..685aedb 100644
--- a/src/cuda-sim/ptx_sim.cc
+++ b/src/cuda-sim/ptx_sim.cc
@@ -310,44 +310,49 @@ void ptx_thread_info::print_insn( unsigned pc, FILE * fp ) const
    m_func_info->print_insn(pc,fp);
 }
 
-static void print_reg( std::string name, ptx_reg_t value, symbol_table *symtab )
+static void print_reg( FILE *fp, std::string name, ptx_reg_t value, symbol_table *symtab )
 {
    const symbol *sym = symtab->lookup(name.c_str());
-   printf("  %8s   ", name.c_str() );
+   fprintf(fp,"  %8s   ", name.c_str() );
    if( sym == NULL ) {
-      printf("<unknown type> 0x%llx\n", (unsigned long long ) value.u64 );
+      fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 );
       return;
    }
    const type_info *t = sym->type();
    if( t == NULL ) {
-      printf("<unknown type> 0x%llx\n", (unsigned long long ) value.u64 );
+      fprintf(fp,"<unknown type> 0x%llx\n", (unsigned long long ) value.u64 );
       return;
    }
    type_info_key ti = t->get_key();
 
    switch ( ti.scalar_type() ) {
-   case S8_TYPE:  printf(".s8  %d\n", value.s8 );  break;
-   case S16_TYPE: printf(".s16 %d\n", value.s16 ); break;
-   case S32_TYPE: printf(".s32 %d\n", value.s32 ); break;
-   case S64_TYPE: printf(".s64 %Ld\n", value.s64 ); break;
-   case U8_TYPE:  printf(".u8  0x%02x\n", (unsigned) value.u8 );  break;
-   case U16_TYPE: printf(".u16 0x%04x\n", (unsigned) value.u16 ); break;
-   case U32_TYPE: printf(".u32 0x%08x\n", (unsigned) value.u32 ); break;
-   case U64_TYPE: printf(".u64 0x%llx\n", value.u64 ); break;
-   case F16_TYPE: printf(".f16 %f [0x%04x]\n",  value.f16, (unsigned) value.u16 ); break;
-   case F32_TYPE: printf(".f32 %.15lf [0x%08x]\n",  value.f32, value.u32 ); break;
-   case F64_TYPE: printf(".f64 %.15le [0x%016llx]\n", value.f64, value.u64 ); break;
-   case B8_TYPE:  printf(".b8  0x%02x\n",   (unsigned) value.u8 );  break;
-   case B16_TYPE: printf(".b16 0x%04x\n",   (unsigned) value.u16 ); break;
-   case B32_TYPE: printf(".b32 0x%08x\n", (unsigned) value.u32 ); break;
-   case B64_TYPE: printf(".b64 0x%llx\n",    (unsigned long long ) value.u64 ); break;
-   case PRED_TYPE: printf(".pred %u\n",     (unsigned) value.pred ); break;
+   case S8_TYPE:  fprintf(fp,".s8  %d\n", value.s8 );  break;
+   case S16_TYPE: fprintf(fp,".s16 %d\n", value.s16 ); break;
+   case S32_TYPE: fprintf(fp,".s32 %d\n", value.s32 ); break;
+   case S64_TYPE: fprintf(fp,".s64 %Ld\n", value.s64 ); break;
+   case U8_TYPE:  fprintf(fp,".u8  0x%02x\n", (unsigned) value.u8 );  break;
+   case U16_TYPE: fprintf(fp,".u16 0x%04x\n", (unsigned) value.u16 ); break;
+   case U32_TYPE: fprintf(fp,".u32 0x%08x\n", (unsigned) value.u32 ); break;
+   case U64_TYPE: fprintf(fp,".u64 0x%llx\n", value.u64 ); break;
+   case F16_TYPE: fprintf(fp,".f16 %f [0x%04x]\n",  value.f16, (unsigned) value.u16 ); break;
+   case F32_TYPE: fprintf(fp,".f32 %.15lf [0x%08x]\n",  value.f32, value.u32 ); break;
+   case F64_TYPE: fprintf(fp,".f64 %.15le [0x%016llx]\n", value.f64, value.u64 ); break;
+   case B8_TYPE:  fprintf(fp,".b8  0x%02x\n",   (unsigned) value.u8 );  break;
+   case B16_TYPE: fprintf(fp,".b16 0x%04x\n",   (unsigned) value.u16 ); break;
+   case B32_TYPE: fprintf(fp,".b32 0x%08x\n", (unsigned) value.u32 ); break;
+   case B64_TYPE: fprintf(fp,".b64 0x%llx\n",    (unsigned long long ) value.u64 ); break;
+   case PRED_TYPE: fprintf(fp,".pred %u\n",     (unsigned) value.pred ); break;
    default: 
-      printf( "non-scalar type\n" );
+      fprintf( fp, "non-scalar type\n" );
       break;
    }
 }
 
+static void print_reg( std::string name, ptx_reg_t value, symbol_table *symtab )
+{
+   print_reg(stdout,name,value,symtab);
+}
+
 void ptx_thread_info::callstack_push( unsigned pc, unsigned rpc, const symbol *return_var_src, const symbol *return_var_dst, unsigned call_uid )
 {
    m_RPC = -1;
@@ -438,7 +443,12 @@ const ptx_instruction *ptx_thread_info::get_inst( addr_t pc ) const
 
 void ptx_thread_info::dump_regs()
 {
-   printf("Register File Contents:\n");
+   dump_regs(stdout);
+}
+
+void ptx_thread_info::dump_regs( FILE *fp )
+{
+   fprintf(fp,"Register File Contents:\n");
    reg_map_t::const_iterator r;
    for ( r=m_regs.back().begin(); r != m_regs.back().end(); ++r ) {
       std::string name = r->first->name();
@@ -449,9 +459,14 @@ void ptx_thread_info::dump_regs()
 
 void ptx_thread_info::dump_modifiedregs()
 {
+   dump_modifiedregs(stdout);
+}
+
+void ptx_thread_info::dump_modifiedregs(FILE *fp)
+{
    if( m_debug_trace_regs_modified.empty() ) 
       return;
-   printf("Modified Registers:\n");
+   fprintf(fp,"Modified Registers:\n");
    reg_map_t::const_iterator r;
    for ( r=m_debug_trace_regs_modified.begin(); r != m_debug_trace_regs_modified.end(); ++r ) {
       std::string name = r->first->name();
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index d712275..8730f2b 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -414,7 +414,9 @@ public:
       m_PC = m_NPC;
    }
    void dump_regs();
+   void dump_regs(FILE *fp);
    void dump_modifiedregs();
+   void dump_modifiedregs(FILE *fp);
    void clear_modifiedregs() { m_debug_trace_regs_modified.clear();}
    function_info *get_finfo() { return m_func_info;   }
    const function_info *get_finfo() const { return m_func_info;   }
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index f86d968..9f76b96 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -86,6 +86,7 @@ dram_t* dram_create( unsigned int id, unsigned int nbk,
    unsigned i;
 
    dm = (dram_t*)calloc(1,sizeof(dram_t));
+   dm = new (dm) dram_t();
 
    dm->id = id;
 
@@ -577,17 +578,6 @@ void dram_print_stat( dram_t* dm, FILE* simFile )
 
 unsigned dram_busy( dram_t* dm) 
 {
-   unsigned busy = 0;
-
-   switch (dm->scheduler_type) {
-   case DRAM_FIFO:
-      busy = (dm->mrqq->length > 0);
-      break;
-   case DRAM_IDEAL_FAST:
-      busy = (fast_scheduler_queue_length(dm) > 0) || (dm->mrqq->length > 0);
-      break;
-   }
-
-   return busy;
+   return !dm->m_request_tracker.empty();
 }
 
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
index ee5fc77..34be319 100644
--- a/src/gpgpu-sim/dram.h
+++ b/src/gpgpu-sim/dram.h
@@ -71,6 +71,7 @@
 
 #include "delayqueue.h"
 #include "../cuda-sim/dram_callback.h"
+#include <set>
 
 #ifndef DRAM_H
 #define DRAM_H
@@ -124,6 +125,8 @@ typedef struct {
    unsigned int n_idle;
 } bank_t;
 
+struct mem_fetch;
+
 typedef struct dram_timing {
    unsigned int id;
 
@@ -198,6 +201,8 @@ typedef struct dram_timing {
    unsigned int bwutil_partial;
 
    void * req_hist;
+
+   std::set<mem_fetch*> m_request_tracker;
 } dram_t;
 
 
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index e0549e1..b5527e1 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -100,6 +100,9 @@
 
 #include <stdio.h>
 #include <string.h>
+
+#include <queue>
+
 #define MAX(a,b) (((a)>(b))?(a):(b))
 
 bool g_interactive_debugger_enabled=false;
@@ -224,14 +227,15 @@ unsigned int gpu_mem_n_bk;
 unsigned int gpu_n_mem_per_ctrlr;
 unsigned int gpu_n_shader;
 int gpu_concentration;
-int gpu_n_tpc = 8;
+int gpu_n_tpc;
 unsigned int gpu_n_mshr_per_shader;
-unsigned int gpu_n_thread_per_shader = 128;
+unsigned int gpu_n_thread_per_shader;
 unsigned int gpu_n_warp_per_shader;
 unsigned int gpu_n_mshr_per_thread = 1;
 bool gpgpu_reg_bankconflict;
 bool gpgpu_operand_collector;
 int gpgpu_operand_collector_num_units;
+int gpgpu_operand_collector_num_units_sfu;
 unsigned int gpgpu_pre_mem_stages;
 bool gpgpu_no_divg_load;
 char *gpgpu_dwf_hw_opt;
@@ -244,6 +248,7 @@ int *num_warps_issuable;
 int *num_warps_issuable_pershader;
 bool gpgpu_cuda_sim;
 bool gpgpu_spread_blocks_across_cores;
+bool gpgpu_stall_on_use;
 shader_core_ctx_t **sc;
 dram_t **dram;
 unsigned int common_clock = 0;
@@ -282,12 +287,24 @@ void print_shader_cycle_distro( FILE *fout ) ;
 void find_reconvergence_points();
 void dwf_process_reconv_pts();
 
+int   g_ptx_inst_debug_to_file;
+char* g_ptx_inst_debug_file;
+int   g_ptx_inst_debug_thread_uid;
+
 #define CREATELOG 111
 #define SAMPLELOG 222
 #define DUMPLOG 333
 void L2c_log(int task);
 void dram_log(int task);
 
+// DRAM delay queue and memory_fetch container
+// A delay queue for each mem - vector of queues
+struct dram_delay_t{
+	unsigned long long ready_cycle;
+	mem_fetch_t* mf;
+};
+std::vector< std::queue<dram_delay_t> > dram_delay_queues;
+
 void visualizer_options(option_parser_t opp);
 void gpu_reg_options(option_parser_t opp)
 {
@@ -466,6 +483,11 @@ void gpu_reg_options(option_parser_t opp)
                 "Spread block-issuing across all cores instead of filling up core by core (do NOT disable)", 
                 "1");
 
+   option_parser_register(opp, "-gpgpu_stall_on_use", OPT_BOOL,
+                &gpgpu_stall_on_use,
+                "Enable stall-on-use",
+                "1");
+
    option_parser_register(opp, "-gpgpu_cuda_sim", OPT_BOOL, &gpgpu_cuda_sim, 
                 "use PTX instruction set", 
                 "1");
@@ -507,12 +529,28 @@ void gpu_reg_options(option_parser_t opp)
    option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, 
                "Number of register banks (default = 8)", 
                "8");
+   option_parser_register(opp, "-gpgpu_reg_bank_use_warp_id", OPT_BOOL, &gpgpu_reg_bank_use_warp_id,
+            "Use warp ID in mapping registers to banks (default = off)",
+            "0");
+   option_parser_register(opp, "-gpgpu_ptx_inst_debug_to_file", OPT_BOOL, 
+                &g_ptx_inst_debug_to_file, 
+                "Dump executed instructions' debug information to file", 
+                "0");
+   option_parser_register(opp, "-gpgpu_ptx_inst_debug_file", OPT_CSTR, &g_ptx_inst_debug_file, 
+                  "Executed instructions' debug output file",
+                  "inst_debug.txt");
+   option_parser_register(opp, "-gpgpu_ptx_inst_debug_thread_uid", OPT_INT32, &g_ptx_inst_debug_thread_uid, 
+               "Thread UID for executed instructions' debug output", 
+               "1");
    option_parser_register(opp, "-gpgpu_operand_collector", OPT_BOOL, &gpgpu_operand_collector,
                "Enable operand collector model (default = off)",
                "0");
    option_parser_register(opp, "-gpgpu_operand_collector_num_units", OPT_INT32, &gpgpu_operand_collector_num_units,
                "number of collecture units (default = 4)", 
                "4");
+   option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu,
+               "number of collecture units (default = 4)", 
+               "4");
    option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, 
                            "Coalescing arch (default = 13, anything else is off for now)", 
                            "13");
@@ -578,8 +616,10 @@ void init_gpu ()
    gpgpu_no_divg_load = gpgpu_no_divg_load && (gpgpu_simd_model == DWF);
    // always use no diverge on load for PDOM and NAIVE
    gpgpu_no_divg_load = gpgpu_no_divg_load || (gpgpu_simd_model == POST_DOMINATOR || gpgpu_simd_model == NO_RECONVERGE);
-   if (gpgpu_no_divg_load)
-      init_warp_tracker();
+   if (gpgpu_no_divg_load) {
+      //init_warp_tracker();
+      printf("warp_tracker_pool size = %d\n", get_warp_tracker_pool().size());
+   }
 
    assert(gpu_n_shader % gpu_concentration == 0);
    gpu_n_tpc = gpu_n_shader / gpu_concentration;
@@ -747,6 +787,11 @@ unsigned int run_gpu_sim(int grid_num)
    if (g_network_mode) {
       icnt_init_grid(); 
    }
+
+   // Initialize dram delay queues
+   dram_delay_queues.resize(gpu_n_mem);
+
+
    last_gpu_sim_insn = 0;
    // add this condition as well? (gpgpu_n_processed_writes < gpgpu_n_sent_writes)
    while (not_completed || mem_busy || icnt2mem_busy) {
@@ -1230,6 +1275,7 @@ int mem_ctrl_full( int mc_id )
 //#define DEBUG_PARTIAL_WRITES
 void mem_ctrl_push( int mc_id, mem_fetch_t* mf ) 
 {
+   dram[mc_id]->m_request_tracker.insert(mf);
    if (gpgpu_cache_dl2_opt) {
       L2c_push(dram[mc_id], mf);
    } else {
@@ -1258,6 +1304,7 @@ void* mem_ctrl_pop( int mc_id )
          dram_callback_t* cb = &(mf->mshr->insts[0].callback);
          cb->function(cb->instruction, cb->thread);
       }
+      dram[mc_id]->m_request_tracker.erase(mf);
       return mf;
    } else {
       mf = static_cast<mem_fetch_t*> (dq_pop(dram[mc_id]->returnq)); //dram_pop(dram[mc_id]);
@@ -1266,6 +1313,7 @@ void* mem_ctrl_pop( int mc_id )
          dram_callback_t* cb = &(mf->mshr->insts[0].callback);
          cb->function(cb->instruction, cb->thread);
       }
+      dram[mc_id]->m_request_tracker.erase(mf);
       return mf;
    }
 }
@@ -1446,11 +1494,10 @@ void gpu_sim_loop( int grid_num )
 
    if (clock_mask & ICNT) {
       // pop memory request from ICNT and 
-      // push it to the proper memory controller (L2 or DRAM controller)
+      // push it to a dram delay queue
       for (unsigned i=0;i<gpu_n_mem;i++) {
-
+      	 // Push memory request to dram delay queue if mem_ctrl is not full
          if ( mem_ctrl_full(i) ) {
-            gpu_stall_dramfull++;
             continue;
          }
 
@@ -1458,15 +1505,35 @@ void gpu_sim_loop( int grid_num )
          mf = (mem_fetch_t*) icnt_pop( mem2device(i) );
 
          if (mf) {
-            if (mf->type==RD_REQ) {
-               time_vector_update(mf->mshr->insts[0].uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;             
-            } else {
-               time_vector_update(mf->request_uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;
-            }
-            memlatstat_icnt2mem_pop(mf);
-            mem_ctrl_push( i, mf );
+
+        	dram_delay_t dram_delay;
+        	dram_delay.mf = mf;
+        	dram_delay.ready_cycle = gpu_sim_cycle + gpu_tot_sim_cycle + 115; // Add 115*4=460 delay cycles
+        	dram_delay_queues[i].push(dram_delay);
          }
       }
+
+      // pop memory request from dram delay queue and
+      // push it to the proper memory controller (L2 or DRAM controller)
+      for (unsigned i=0;i<gpu_n_mem;i++) {
+    	  if(!dram_delay_queues[i].empty() && dram_delay_queues[i].front().ready_cycle <= gpu_sim_cycle + gpu_tot_sim_cycle) {
+    		  if ( mem_ctrl_full(i) ) {
+				gpu_stall_dramfull++;
+				continue;
+			  }
+    		  mem_fetch_t* mf = dram_delay_queues[i].front().mf;
+              if (mf->type==RD_REQ) {
+                 time_vector_update(mf->mshr->insts[0].uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;
+              } else {
+                 time_vector_update(mf->request_uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ;
+              }
+              memlatstat_icnt2mem_pop(mf);
+              mem_ctrl_push( i, mf );
+
+              dram_delay_queues[i].pop();
+    	  }
+      }
+
       icnt_transfer( );
    }
 
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index bad4adc..45d65e3 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -143,6 +143,7 @@ extern unsigned int gpu_stall_shd_mem;
 extern unsigned int gpu_stall_sh2icnt;
 extern bool gpgpu_operand_collector;
 extern int gpgpu_operand_collector_num_units;
+extern int gpgpu_operand_collector_num_units_sfu;
 extern int gpu_runtime_stat_flag;
 extern unsigned int *max_return_queue_length;
 extern int gpgpu_partial_write_mask;
@@ -167,5 +168,8 @@ extern unsigned int gpgpu_n_processed_writes;
 extern int gpgpu_simd_model;
 extern unsigned int gpu_mem_n_bk;
 extern unsigned g_next_mf_request_uid;
+extern int   g_ptx_inst_debug_to_file;
+extern char* g_ptx_inst_debug_file;
+extern int   g_ptx_inst_debug_thread_uid;
 
 #endif
diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc
index 625e54f..f225087 100644
--- a/src/gpgpu-sim/mem_latency_stat.cc
+++ b/src/gpgpu-sim/mem_latency_stat.cc
@@ -104,7 +104,6 @@ unsigned int **totalbankwrites; //bankwrites[dram chip id][bank id]
 unsigned int **totalbankreads; //bankreads[dram chip id][bank id]
 unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id]
 unsigned int *requests_by_warp;
-unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp
 unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache
 unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen 
 unsigned *mf_num_lat_pw_perwarp;
@@ -149,8 +148,7 @@ void memlatstat_init( )
    mf_max_lat_table = (unsigned **) calloc(gpu_n_mem, sizeof(unsigned *));
    bankreads = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**));
    bankwrites = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**));
-   MCB_accesses = (unsigned int*) calloc(gpu_n_mem*4, sizeof(unsigned int));
-   num_MCBs_accessed = (unsigned int*) calloc(gpu_n_mem*4+1, sizeof(unsigned int));
+   num_MCBs_accessed = (unsigned int*) calloc(gpu_n_mem*gpu_mem_n_bk, sizeof(unsigned int));
    if (gpgpu_dram_sched_queue_size) {
       position_of_mrq_chosen = (unsigned int*) calloc(gpgpu_dram_sched_queue_size, sizeof(unsigned int));
    } else
@@ -512,7 +510,7 @@ void memlatstat_print( )
       printf("\nNumber of Memory Banks Accessed per Memory Operation per Warp (from 0):\n");
       unsigned long long accum_MCBs_accessed = 0;
       unsigned long long tot_mem_ops_per_warp = 0;
-      for (i=0;i<= gpu_n_mem*4 ; i++ ) {
+      for (i=0;i<= gpu_n_mem*gpu_mem_n_bk ; i++ ) {
          accum_MCBs_accessed += i*num_MCBs_accessed[i];
          tot_mem_ops_per_warp += num_MCBs_accessed[i];
          printf("%d\t", num_MCBs_accessed[i]);
diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h
index 655cf04..fd057c4 100644
--- a/src/gpgpu-sim/mem_latency_stat.h
+++ b/src/gpgpu-sim/mem_latency_stat.h
@@ -93,7 +93,6 @@ extern unsigned int **totalbankwrites; //bankwrites[dram chip id][bank id]
 extern unsigned int **totalbankreads; //bankreads[dram chip id][bank id]
 extern unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id]
 extern unsigned int *requests_by_warp;
-extern unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp
 extern unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache
 extern unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen 
 extern unsigned *mf_num_lat_pw_perwarp;
diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc
new file mode 100644
index 0000000..4ebcee9
--- /dev/null
+++ b/src/gpgpu-sim/scoreboard.cc
@@ -0,0 +1,129 @@
+/*
+ * scoreboard.cc
+ *
+ *  Created on: Aug 10, 2010
+ *      Author: inder
+ */
+
+#include "scoreboard.h"
+#include "shader.h"
+#include "../cuda-sim/ptx_sim.h"
+
+
+//Constructor
+Scoreboard::Scoreboard( int sid, int n_warps )
+{
+	this->sid = sid;
+	//Initialize size of table
+	reg_table.resize(n_warps);
+}
+
+// Print scoreboard contents
+void Scoreboard::printContents() {
+	printf("scoreboard contents (sid=%d): \n", sid);
+	for(unsigned i=0; i<reg_table.size(); i++) {
+		if(reg_table[i].size() == 0 ) continue;
+		printf("  wid = %d: ", i);
+		std::set<int>::iterator it;
+		for ( it=reg_table[i].begin() ; it != reg_table[i].end(); it++ )
+			printf("%d ", *it);
+		printf("\n");
+	}
+}
+
+
+// Mark register as write-pending
+void Scoreboard::reserveRegister(int wid, int regnum) {
+	//printf("Reserving register sid=%d wid=%d regnum=%d\n", sid, wid, regnum);
+
+	if( !(reg_table[wid].find(regnum) == reg_table[wid].end()) ){
+		printf("Error: trying to reserve an already reserved register (sid=%d, wid=%d, regnum=%d).", sid, wid, regnum);
+		assert(reg_table[wid].find(regnum) == reg_table[wid].end());
+	}
+
+	reg_table[wid].insert(regnum);
+}
+
+
+// Unmark register as write-pending
+void Scoreboard::releaseRegister(int wid, int regnum) {
+	//printf("Releasing register sid=%d wid=%d regnum=%d\n", sid, wid, regnum);
+
+	if( !(reg_table[wid].find(regnum) != reg_table[wid].end()) ) {
+		printf("Error: trying to release an unreserved register (sid=%d, wid=%d, regnum=%d).", sid, wid, regnum);
+		assert(reg_table[wid].find(regnum) != reg_table[wid].end());
+	}
+
+	reg_table[wid].erase(regnum);
+}
+
+
+// Reserve registers for an instruction
+void Scoreboard::reserveRegisters(int wid, void* inst_void) {
+	inst_t *inst = (inst_t *) inst_void;
+
+	// Reserve registers
+	if(inst->out[0] > 0) reserveRegister(wid, inst->out[0]);
+	if(inst->out[1] > 0) reserveRegister(wid, inst->out[1]);
+	if(inst->out[2] > 0) reserveRegister(wid, inst->out[2]);
+	if(inst->out[3] > 0) reserveRegister(wid, inst->out[3]);
+}
+
+// Release registers for an instruction
+void Scoreboard::releaseRegisters(int wid, void *inst_void) {
+	inst_t *inst = (inst_t *) inst_void;
+
+	if(inst->out[0] > 0) releaseRegister(wid, inst->out[0]);
+	if(inst->out[1] > 0) releaseRegister(wid, inst->out[1]);
+	if(inst->out[2] > 0) releaseRegister(wid, inst->out[2]);
+	if(inst->out[3] > 0) releaseRegister(wid, inst->out[3]);
+}
+
+// Checks to see if registers used by an instruction are reserved in the scoreboard
+bool Scoreboard::checkCollision(int wid, void *inst_void) {
+	inst_t *inst = (inst_t *) inst_void;
+
+	// Get list of all input and output registers
+	std::set<int> inst_regs;
+
+	if(inst->out[0] > 0) inst_regs.insert(inst->out[0]);
+	if(inst->out[1] > 0) inst_regs.insert(inst->out[1]);
+	if(inst->out[2] > 0) inst_regs.insert(inst->out[2]);
+	if(inst->out[3] > 0) inst_regs.insert(inst->out[3]);
+	if(inst->in[0] > 0) inst_regs.insert(inst->in[0]);
+	if(inst->in[1] > 0) inst_regs.insert(inst->in[1]);
+	if(inst->in[2] > 0) inst_regs.insert(inst->in[2]);
+	if(inst->in[3] > 0) inst_regs.insert(inst->in[3]);
+	if(inst->pred > 0) inst_regs.insert(inst->pred);
+	if(inst->ar1 > 0) inst_regs.insert(inst->ar1);
+	if(inst->ar2 > 0) inst_regs.insert(inst->ar2);
+
+	/*
+	printf("Inst registers: ");
+	std::set<int>::iterator it;
+	for ( it=inst_regs.begin() ; it != inst_regs.end(); it++ )
+		printf("%d ", *it);
+	printf("\n");
+	*/
+
+	// Check for collision, get the intersection of reserved registers and instruction registers
+	//std::set<int> reg_intr;
+	std::set<int>::iterator it2;
+	for ( it2=inst_regs.begin() ; it2 != inst_regs.end(); it2++ )
+		if(reg_table[wid].find(*it2) != reg_table[wid].end()) {
+			//reg_intr.insert(*it2);
+			return true;
+		}
+
+	return false;
+
+	/*
+	printf("Intersection registers: ");
+	std::set<int>::iterator it3;
+	for ( it3=reg_intr.begin() ; it3 != reg_intr.end(); it3++ )
+		printf("%d ", *it3);
+	printf("\n");
+	*/
+
+
+}
diff --git a/src/gpgpu-sim/scoreboard.h b/src/gpgpu-sim/scoreboard.h
new file mode 100644
index 0000000..4139948
--- /dev/null
+++ b/src/gpgpu-sim/scoreboard.h
@@ -0,0 +1,43 @@
+/*
+ * scoreboard.h
+ *
+ *  Created on: Aug 10, 2010
+ *      Author: inder
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <vector>
+#include <set>
+#include "assert.h"
+
+#ifndef SCOREBOARD_H_
+#define SCOREBOARD_H_
+
+typedef unsigned op_type;
+
+class Scoreboard
+{
+	private:
+		int sid; // Shader id
+		// Table to keep track of write-pending registers
+		// Indexed by warp id (wid)
+		std::vector< std::set<int> > reg_table;
+
+		void reserveRegister(int wid, int regnum);
+		void releaseRegister(int wid, int regnum);
+
+	public:
+		Scoreboard( int sid, int n_warps );
+
+		void printContents();
+
+		void reserveRegisters(int wid, void *inst_void);
+		void releaseRegisters(int wid, void *inst_void);
+
+		bool checkCollision(int wid, void *inst_void);
+
+};
+
+
+#endif /* SCOREBOARD_H_ */
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 9373929..5f465ed 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -87,6 +87,7 @@
 #define PRIORITIZE_MSHR_OVER_WB 1
 #define MAX(a,b) (((a)>(b))?(a):(b))
 
+extern bool gpgpu_stall_on_use;
 enum mem_stage_access_type {
    C_MEM,
    T_MEM,
@@ -419,7 +420,9 @@ inst_t create_nop_inst() // just because C++ does not have designated initialize
    nop_inst.id_cycle = 0;
    nop_inst.ex_cycle = 0;
    nop_inst.mm_cycle = 0;
+   nop_inst.cache_miss = false;
    nop_inst.space = memory_space_t();
+   nop_inst.cycles = 0;
    return nop_inst;
 }
 
@@ -476,6 +479,24 @@ shader_core_ctx_t* shader_create( const char *name, int sid,
                                   unsigned int model )
 {
    shader_core_ctx_t *sc;
+   sc = (shader_core_ctx_t*)calloc(sizeof(shader_core_ctx_t),1);
+   sc = new (sc) shader_core_ctx(name,sid,n_threads,n_mshr,fq_push,fq_has_buffer,model,
+                                 gpu_n_warp_per_shader,gpgpu_shader_cta);
+   return sc;
+}
+
+shader_core_ctx::shader_core_ctx( const char *name, int sid, 
+                                  unsigned int n_threads, 
+                                  unsigned int n_mshr, 
+                                  fq_push_t fq_push, 
+                                  fq_has_buffer_t fq_has_buffer,
+                                  unsigned model,
+                                  unsigned max_warps_per_cta, unsigned max_cta_per_core )
+   : m_barriers( max_warps_per_cta, max_cta_per_core )
+{
+   shader_core_ctx *sc = this;
+   assert( !((model == DWF) && gpgpu_operand_collector) );
+
    int i;
    unsigned int shd_n_set;
    unsigned int shd_linesize;
@@ -504,9 +525,6 @@ shader_core_ctx_t* shader_create( const char *name, int sid,
       }
    }
 
-   sc = (shader_core_ctx_t*)calloc(sizeof(shader_core_ctx_t),1);
-   sc = new (sc) shader_core_ctx_t( gpu_n_warp_per_shader, gpgpu_shader_cta );
-
    sc->name = name;
    sc->sid = sid;
 
@@ -627,7 +645,14 @@ shader_core_ctx_t* shader_create( const char *name, int sid,
 
    sc->shader_memory_new_instruction_processed = false;
 
-   return sc;
+   // Initialize scoreboard
+   sc->scrb = new Scoreboard(sc->sid, n_warp);
+
+   if( gpgpu_operand_collector ) {
+      m_opndcoll_new.init( gpgpu_operand_collector_num_units, 
+                           gpgpu_operand_collector_num_units_sfu, 
+                           gpgpu_num_reg_banks, this );
+   }
 }
 
 
@@ -1021,6 +1046,9 @@ void shader_fetch_simd_postdominator(shader_core_ctx_t *shader, unsigned int sha
          w_pipe_c++;
       } else if ( shader->warp_waiting_at_barrier(shader->next_warp) ) {
          w_barr_c++;
+      } else if ( shader_warp_scoreboard_hazard(shader, shader->next_warp) ) {
+    	  // Do nothing - warp is filtered out
+    	  //printf("SCOREBOARD COLLISION - wid=%d\n", shader->next_warp);
       } else {
          // A valid warp is found at this point
          tmp_ready_warps[ready_warp_count] = shader->next_warp;
@@ -1060,81 +1088,10 @@ void shader_fetch_simd_postdominator(shader_core_ctx_t *shader, unsigned int sha
 
    warp_id = shader->next_warp;
    last_warp[shader->sid] = warp_id;
-   int wtid = warp_size*shader->next_warp;
+   int wtid = warp_size*warp_id;
 
    pdom_warp_ctx_t *scheduled_warp = &(shader->pdom_warp[warp_id]);
 
-   int stack_top = scheduled_warp->m_stack_top;
-
-   address_type top_pc = scheduled_warp->m_pc[stack_top];
-   unsigned int top_active_mask = scheduled_warp->m_active_mask[stack_top];
-   address_type top_recvg_pc = scheduled_warp->m_recvg_pc[stack_top];
-
-   assert(top_active_mask != 0);
-
-   const address_type null_pc = 0;
-   int warp_diverged = 0;
-   address_type new_recvg_pc = null_pc;
-   while (top_active_mask != 0) {
-
-      // extract a group of threads with the same next PC among the active threads in the warp
-      address_type tmp_next_pc = null_pc;
-      unsigned int tmp_active_mask = 0;
-      void *first_active_thread=NULL;
-      for (i = warp_size - 1; i >= 0; i--) {
-         unsigned int mask = (1 << i);
-         if ((top_active_mask & mask) == mask) { // is this thread active?
-            if (ptx_thread_done( shader->thread[wtid+i].ptx_thd_info )) {
-               top_active_mask &= ~mask; // remove completed thread from active mask
-            } else if (tmp_next_pc == null_pc) {
-               first_active_thread = shader->thread[wtid+i].ptx_thd_info;
-               tmp_next_pc = shader_thread_nextpc(shader, wtid+i);
-               tmp_active_mask |= mask;
-               top_active_mask &= ~mask;
-            } else if (tmp_next_pc == shader_thread_nextpc(shader, wtid+i)) {
-               tmp_active_mask |= mask;
-               top_active_mask &= ~mask;
-            }
-         }
-      }
-
-      // discard the new entry if its PC matches with reconvergence PC
-      // that automatically reconverges the entry 
-      if (tmp_next_pc == top_recvg_pc) continue;
-
-      // this new entry is not converging
-      // if this entry does not include thread from the warp, divergence occurs
-      if (top_active_mask != 0 && warp_diverged == 0) {
-         warp_diverged = 1;
-         // modify the existing top entry into a reconvergence entry in the pdom stack
-         new_recvg_pc = get_converge_point(top_pc,first_active_thread);
-         if (new_recvg_pc != top_recvg_pc) {
-            scheduled_warp->m_pc[stack_top] = new_recvg_pc;
-            scheduled_warp->m_branch_div_cycle[stack_top] = gpu_sim_cycle;
-            stack_top += 1;
-            scheduled_warp->m_branch_div_cycle[stack_top] = 0;
-         }
-      }
-
-      // discard the new entry if its PC matches with reconvergence PC
-      if (warp_diverged && tmp_next_pc == new_recvg_pc) continue;
-
-      // update the current top of pdom stack
-      scheduled_warp->m_pc[stack_top] = tmp_next_pc;
-      scheduled_warp->m_active_mask[stack_top] = tmp_active_mask;
-      if (warp_diverged) {
-         scheduled_warp->m_calldepth[stack_top] = 0;
-         scheduled_warp->m_recvg_pc[stack_top] = new_recvg_pc;
-      } else {
-         scheduled_warp->m_recvg_pc[stack_top] = top_recvg_pc;
-      }
-      stack_top += 1; // set top to next entry in the pdom stack
-   }
-   scheduled_warp->m_stack_top = stack_top - 1;
-
-   assert(scheduled_warp->m_stack_top >= 0);
-   assert(scheduled_warp->m_stack_top < (int)warp_size * 2);
-
    // schedule threads according to active mask on the top of pdom stack
    for (i = 0; i < (int)warp_size; i++) {
       unsigned int mask = (1 << i);
@@ -1157,6 +1114,124 @@ void shader_fetch_simd_postdominator(shader_core_ctx_t *shader, unsigned int sha
    }
 }
 
+bool shader_warp_scoreboard_hazard(shader_core_ctx_t *shader, int warp_id) {
+	static inst_t active_inst;
+	static op_type op = NO_OP;
+	static int i1, i2, i3, i4, o1, o2, o3, o4; //4 outputs needed for texture fetches in cuda-sim
+	static int vectorin, vectorout;
+	static int arch_reg[MAX_REG_OPERANDS] = { -1 };
+	static int pred;
+	static int ar1, ar2; // address registers for memory operands
+
+	// Get an active thread in the warp
+	int wtid = warp_size*warp_id;
+	pdom_warp_ctx_t *scheduled_warp = &(shader->pdom_warp[warp_id]);
+	thread_ctx_t *active_thread = NULL;
+	for (int i = 0; i < (int)warp_size; i++) {
+		unsigned int mask = (1 << i);
+		if ((scheduled_warp->m_active_mask[scheduled_warp->m_stack_top] & mask) == mask) {
+			active_thread = &(shader->thread[wtid+i]);
+		}
+	}
+	if(active_thread == NULL) return false;
+
+	// Decode instruction
+	ptx_decode_inst( active_thread->ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg, &pred, &ar1, &ar2);
+	active_inst.op = op;
+	active_inst.in[0] = i1;
+	active_inst.in[1] = i2;
+	active_inst.in[2] = i3;
+	active_inst.in[3] = i4;
+	active_inst.out[0] = o1;
+	active_inst.out[1] = o2;
+	active_inst.out[2] = o3;
+	active_inst.out[3] = o4;
+	active_inst.is_vectorin = vectorin;
+	active_inst.is_vectorout = vectorout;
+	active_inst.pred = pred;
+	active_inst.ar1 = ar1;
+	active_inst.ar2 = ar2;
+
+	return shader->scrb->checkCollision(warp_id, &active_inst);
+}
+
+void shader_pdom_update_warp_mask(shader_core_ctx_t *shader, int warp_id) {
+	int wtid = warp_size*warp_id;
+
+	pdom_warp_ctx_t *scheduled_warp = &(shader->pdom_warp[warp_id]);
+
+	int stack_top = scheduled_warp->m_stack_top;
+
+	address_type top_pc = scheduled_warp->m_pc[stack_top];
+	unsigned int top_active_mask = scheduled_warp->m_active_mask[stack_top];
+	address_type top_recvg_pc = scheduled_warp->m_recvg_pc[stack_top];
+
+	assert(top_active_mask != 0);
+
+	const address_type null_pc = 0;
+	int warp_diverged = 0;
+	address_type new_recvg_pc = null_pc;
+	while (top_active_mask != 0) {
+
+	  // extract a group of threads with the same next PC among the active threads in the warp
+	  address_type tmp_next_pc = null_pc;
+	  unsigned int tmp_active_mask = 0;
+	  void *first_active_thread=NULL;
+	  for (int i = warp_size - 1; i >= 0; i--) {
+		 unsigned int mask = (1 << i);
+		 if ((top_active_mask & mask) == mask) { // is this thread active?
+			if (ptx_thread_done( shader->thread[wtid+i].ptx_thd_info )) {
+			   top_active_mask &= ~mask; // remove completed thread from active mask
+			} else if (tmp_next_pc == null_pc) {
+			   first_active_thread = shader->thread[wtid+i].ptx_thd_info;
+			   tmp_next_pc = shader_thread_nextpc(shader, wtid+i);
+			   tmp_active_mask |= mask;
+			   top_active_mask &= ~mask;
+			} else if (tmp_next_pc == shader_thread_nextpc(shader, wtid+i)) {
+			   tmp_active_mask |= mask;
+			   top_active_mask &= ~mask;
+			}
+		 }
+	  }
+
+	  // discard the new entry if its PC matches with reconvergence PC
+	  // that automatically reconverges the entry
+	  if (tmp_next_pc == top_recvg_pc) continue;
+
+	  // this new entry is not converging
+	  // if this entry does not include thread from the warp, divergence occurs
+	  if (top_active_mask != 0 && warp_diverged == 0) {
+		 warp_diverged = 1;
+		 // modify the existing top entry into a reconvergence entry in the pdom stack
+		 new_recvg_pc = get_converge_point(top_pc,first_active_thread);
+		 if (new_recvg_pc != top_recvg_pc) {
+			scheduled_warp->m_pc[stack_top] = new_recvg_pc;
+			scheduled_warp->m_branch_div_cycle[stack_top] = gpu_sim_cycle;
+			stack_top += 1;
+			scheduled_warp->m_branch_div_cycle[stack_top] = 0;
+		 }
+	  }
+
+	  // discard the new entry if its PC matches with reconvergence PC
+	  if (warp_diverged && tmp_next_pc == new_recvg_pc) continue;
+
+	  // update the current top of pdom stack
+	  scheduled_warp->m_pc[stack_top] = tmp_next_pc;
+	  scheduled_warp->m_active_mask[stack_top] = tmp_active_mask;
+	  if (warp_diverged) {
+		 scheduled_warp->m_calldepth[stack_top] = 0;
+		 scheduled_warp->m_recvg_pc[stack_top] = new_recvg_pc;
+	  } else {
+		 scheduled_warp->m_recvg_pc[stack_top] = top_recvg_pc;
+	  }
+	  stack_top += 1; // set top to next entry in the pdom stack
+	}
+	scheduled_warp->m_stack_top = stack_top - 1;
+
+	assert(scheduled_warp->m_stack_top >= 0);
+	assert(scheduled_warp->m_stack_top < (int)warp_size * 2);
+}
+
 
 void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc )
 {
@@ -1432,24 +1507,6 @@ void shader_fetch( shader_core_ctx_t *shader, unsigned int shader_number, int gr
 
    if (shader->gpu_cycle % n_warp_parts == 0) {
 
-      if (shader->model == POST_DOMINATOR || shader->model == NO_RECONVERGE) {
-         int warpupdate_bw = 1; // number of warps to be unlocked per scheduler cycle
-         while (!dq_empty(shader->thd_commit_queue) && warpupdate_bw > 0) {
-            // grab a committed warp and unlock it here
-            int *tid_commit = (int*)dq_pop(shader->thd_commit_queue);
-            for (unsigned i = 0; i < warp_size; i++) {
-               if (tid_commit[i] >= 0) {
-                  shader->thread[tid_commit[i]].avail4fetch++;
-                  assert(shader->thread[tid_commit[i]].avail4fetch <= 1);
-                  assert( shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch < (unsigned)warp_size );
-                  shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch++;
-               }
-            }
-            warpupdate_bw -= 1;
-            free_commit_warp(tid_commit);
-         }
-      }
-
       switch (shader->model) {
       case NO_RECONVERGE:
          shader_fetch_simd_no_reconverge(shader, shader_number, grid_num );
@@ -1489,7 +1546,10 @@ void shader_fetch( shader_core_ctx_t *shader, unsigned int shader_number, int gr
                pc_out = shader->pipeline_reg[TS_IF][i].pc;
             }
          }
-         wpt_register_warp(tid_out, shader);
+
+         //wpt_register_warp(tid_out, shader);
+         get_warp_tracker_pool().wpt_register_warp(tid_out, shader, pc_out);
+
          if (gpu_runtime_stat_flag & GPU_RSTAT_DWF_MAP) {
             track_thread_pc( shader->sid, tid_out, pc_out );
          }
@@ -1654,6 +1714,7 @@ bool gpgpu_reg_bank_conflict_model;
 
 #define MAX_REG_BANKS 32
 unsigned int gpgpu_num_reg_banks; // this needs to be less than MAX_REG_BANKS
+bool gpgpu_reg_bank_use_warp_id;
 
 #define MAX_BANK_CONFLICT 8 /* tex can have four source and four destination regs */
 
@@ -1668,6 +1729,14 @@ public:
 	int rd_regs[4];
 }; 
 
+int register_bank(int regnum, int tid)
+{
+   int bank = regnum;
+   if (gpgpu_reg_bank_use_warp_id)
+      bank += tid >> 5/*log2(warp_size)*/;
+   return bank % gpgpu_num_reg_banks;
+}
+
 reg_bank_access g_reg_bank_access[MAX_REG_BANKS];
 
 // just to use as "shorthand" for clearing accesses each cycle
@@ -1678,8 +1747,7 @@ unsigned int gpu_reg_bank_conflict_stalls = 0;
 void shader_opnd_collect_read(shader_core_ctx_t* shader)
 {
    const int prevstage = ID_OC;
-   const int nextstage = shader->using_rrstage ? ID_RR : ID_EX;
-   shader->m_opndcoll_new.step(shader->pipeline_reg[prevstage],shader->pipeline_reg[nextstage]);
+   shader->m_opndcoll_new.step(shader->pipeline_reg[prevstage]);
 }
 
 void shader_opnd_collect_write(shader_core_ctx_t* shader)
@@ -1703,12 +1771,17 @@ void shader_decode( shader_core_ctx_t *shader,
    int warp_tid=0;
    unsigned data_size;
    memory_space_t space;
+   unsigned cycles;
    int vectorin, vectorout;
    int arch_reg[MAX_REG_OPERANDS] = { -1 };
+   int pred;
+   int ar1, ar2; // address registers for memory operands
    address_type regs_regs_PC = 0xDEADBEEF;
    address_type warp_current_pc = 0x600DBEEF;
    address_type warp_next_pc = 0x600DBEEF;
    int       warp_diverging = 0;
+   const int nextstage = (gpgpu_operand_collector) ? ID_OC : \
+      (shader->using_rrstage ? ID_RR : ID_EX);
    unsigned warp_id = -1;
    unsigned cta_id = -1;
 
@@ -1727,8 +1800,7 @@ void shader_decode( shader_core_ctx_t *shader,
    }
 
    for (i=0; i<pipe_simd_width;i++) {
-      int next_stage = (shader->using_rrstage)? ID_RR:ID_EX;
-      if (shader->pipeline_reg[next_stage][i].hw_thread_id != -1 ) {
+      if (shader->pipeline_reg[nextstage][i].hw_thread_id != -1 ) {
          return;  /* stalled */
       }
    }
@@ -1752,7 +1824,7 @@ void shader_decode( shader_core_ctx_t *shader,
       }
 
       if ( gpgpu_cuda_sim ) {
-         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg);
+         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg, &pred, &ar1, &ar2);
          shader->pipeline_reg[IF_ID][i].op = op;
          shader->pipeline_reg[IF_ID][i].pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
          shader->pipeline_reg[IF_ID][i].ptx_thd_info = shader->thread[tid].ptx_thd_info;
@@ -1823,15 +1895,27 @@ void shader_decode( shader_core_ctx_t *shader,
    }
 
    // execute the instruction functionally
+   short last_hw_thread_id = -1;
+   bool first_thread_in_warp = true;
    for (i=0; i<pipe_simd_width;i++) {
       if (shader->pipeline_reg[IF_ID][i].hw_thread_id == -1 )
          continue; /* bubble */
+
+      if(last_hw_thread_id > -1)
+    	  first_thread_in_warp = false;
+      last_hw_thread_id = shader->pipeline_reg[IF_ID][i].hw_thread_id;
+
       /* get the next instruction to execute from fetch stage */
       tid = shader->pipeline_reg[IF_ID][i].hw_thread_id;
       if ( gpgpu_cuda_sim ) {
          int arch_reg[MAX_REG_OPERANDS];
-         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg );
-         ptx_exec_inst( shader->thread[tid].ptx_thd_info, &addr, &space, &data_size, &callback, shader->pipeline_reg[IF_ID][i].warp_active_mask );
+
+         // Decode instruction
+         ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg, &pred, &ar1, &ar2 );
+
+         // Functionally execute instruction
+         ptx_exec_inst( shader->thread[tid].ptx_thd_info, &addr, &space, &data_size, &cycles, &callback, shader->pipeline_reg[IF_ID][i].warp_active_mask );
+
          shader->pipeline_reg[IF_ID][i].callback = callback;
          shader->pipeline_reg[IF_ID][i].space = space;
          if (is_local(space) && (is_load(op) || is_store(op))) {
@@ -1839,7 +1923,19 @@ void shader_decode( shader_core_ctx_t *shader,
          }
          shader->pipeline_reg[IF_ID][i].is_vectorin = vectorin;
          shader->pipeline_reg[IF_ID][i].is_vectorout = vectorout;
+         shader->pipeline_reg[IF_ID][i].pred = pred;
+         shader->pipeline_reg[IF_ID][i].ar1 = ar1;
+         shader->pipeline_reg[IF_ID][i].ar2 = ar2;
          shader->pipeline_reg[IF_ID][i].data_size = data_size;
+         shader->pipeline_reg[IF_ID][i].cycles = cycles;
+
+         // Mark destination registers as write-pending in scoreboard
+	     // Only do this for the first thread in warp
+	     if(first_thread_in_warp) {
+   		    shader->scrb->reserveRegisters(warp_id, &(shader->pipeline_reg[IF_ID][i]));
+			//shader->scrb->printContents();
+	     }
+
          warp_current_pc = shader->pipeline_reg[IF_ID][i].pc;
          memcpy( shader->pipeline_reg[IF_ID][i].arch_reg, arch_reg, sizeof(arch_reg) );
          regs_regs_PC = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info );
@@ -1909,9 +2005,8 @@ void shader_decode( shader_core_ctx_t *shader,
       }
 
       // direct the instruction to the appropriate next stage (config dependent)
-      int next_stage = (shader->using_rrstage)? ID_RR:ID_EX;
-      shader->pipeline_reg[next_stage][i] = shader->pipeline_reg[IF_ID][i];
-      shader->pipeline_reg[next_stage][i].id_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+      shader->pipeline_reg[nextstage][i] = shader->pipeline_reg[IF_ID][i];
+      shader->pipeline_reg[nextstage][i].id_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
       shader->pipeline_reg[IF_ID][i] = nop_inst;
    }
 
@@ -2008,36 +2103,54 @@ void shader_preexecute( shader_core_ctx_t *shader,
 }
 
 
-void shader_execute( shader_core_ctx_t *shader, 
-                     unsigned int shader_number ) {
-
+void shader_execute_pipe( shader_core_ctx_t *shader, unsigned int shader_number, unsigned pipeline, unsigned next_stage ) 
+{
    int i;
-
    for (i=0; i<pipe_simd_width; i++) {
       if (gpgpu_pre_mem_stages) {
          if (shader->pre_mem_pipeline[0][i].hw_thread_id != -1 ) {
-            //printf("stalled in shader_execute\n");
             return;  // stalled 
          }
       } else {
-         if (shader->pipeline_reg[EX_MM][i].hw_thread_id != -1 )
+         if (shader->pipeline_reg[next_stage][i].hw_thread_id != -1 )
             return;  // stalled 
       }
    }
 
    check_stage_pcs(shader,ID_EX);
 
+   // Check that all threads have the same delay cycles
+   unsigned cycles = -1;
    for (i=0; i<pipe_simd_width; i++) {
-      if (shader->pipeline_reg[ID_EX][i].hw_thread_id == -1 )
-         continue; // bubble 
+      if (shader->pipeline_reg[pipeline][i].hw_thread_id == -1 )
+         continue; // bubble
+      if(cycles == (unsigned)-1)
+         cycles = shader->pipeline_reg[pipeline][i].cycles;
+      else {
+         if( cycles != shader->pipeline_reg[pipeline][i].cycles ) {
+            printf("Shader %d: threads do not have the same delay cycles.\n", shader->sid);
+            assert(0);
+         }
+      }
+   }
+
+   for (i=0; i<pipe_simd_width; i++) {
+      if (shader->pipeline_reg[pipeline][i].hw_thread_id == -1 )
+         continue; // bubble
+
+      // Stall based on delay cycles
+      shader->pipeline_reg[pipeline][i].cycles--;
+      if( shader->pipeline_reg[pipeline][i].cycles > 0 )
+         continue;
+
       if (gpgpu_pre_mem_stages) {
-         shader->pre_mem_pipeline[0][i] = shader->pipeline_reg[ID_EX][i];
+         shader->pre_mem_pipeline[0][i] = shader->pipeline_reg[pipeline][i];
          shader->pre_mem_pipeline[0][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
       } else {
-         shader->pipeline_reg[EX_MM][i] = shader->pipeline_reg[ID_EX][i];
-         shader->pipeline_reg[EX_MM][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+         shader->pipeline_reg[next_stage][i] = shader->pipeline_reg[pipeline][i];
+         shader->pipeline_reg[next_stage][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
       }
-      shader->pipeline_reg[ID_EX][i] = nop_inst;
+      shader->pipeline_reg[pipeline][i] = nop_inst;
    }  
 
    if (!gpgpu_pre_mem_stages) {
@@ -2046,6 +2159,12 @@ void shader_execute( shader_core_ctx_t *shader,
    }
 }
 
+void shader_execute( shader_core_ctx_t *shader, unsigned int shader_number ) 
+{
+   shader_execute_pipe(shader,shader_number, OC_EX_SFU, EX_MM);
+   shader_execute_pipe(shader,shader_number, ID_EX, EX_MM);
+}
+
 void shader_pre_memory( shader_core_ctx_t *shader, 
                         unsigned int shader_number ) {
    int i,j;
@@ -2346,6 +2465,7 @@ void shader_memory_global_process_inst(shader_core_ctx_t * shader, unsigned char
             break;
          case 4:
          case 8:
+         case 16:
             line_size = 128;
             break;
          default:
@@ -2520,14 +2640,19 @@ mem_stage_stall_type send_mem_request(shader_core_ctx_t *shader, mem_access_t &a
 #endif   
       }
 
+      // Scoreboard addition: do not make cache miss instructions wait for memory,
+      //                      let the scoreboard handle stalling of instructions.
+      //                      Mark thread as a cache miss
+
       if (not access.iswrite) {
          // set the pipeline instructions in this request to noops, they all wait for memory;
          for (unsigned i = 0; i < access.warp_indices.size(); i++) {
             unsigned o = access.warp_indices[i];
-            shader->pipeline_reg[EX_MM][o] = nop_inst;
-            
+            //shader->pipeline_reg[EX_MM][o] = nop_inst;
+            shader->pipeline_reg[EX_MM][o].cache_miss = true;
          }
       }
+
    }
 
    return NO_RC_FAIL;
@@ -2830,7 +2955,7 @@ void shader_memory( shader_core_ctx_t *shader, unsigned int shader_number )
    }
 
    bool done = true;
-   mem_stage_access_type type; 
+   mem_stage_access_type type;
 
    done &= shader_memory_shared_cycle(shader, accessqs->shared, rc_fail, type);
    done &= shader_memory_constant_cycle(shader, accessqs->constant, rc_fail, type);
@@ -2859,10 +2984,10 @@ void shader_memory( shader_core_ctx_t *shader, unsigned int shader_number )
    // pipeline forward
 
    check_stage_pcs(shader,EX_MM);
-   // and pass instruction from EX_MM to MM_WB for cache hit
+   // and pass instruction from EX_MM to MM_WB
    for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) {
       if (shader->pipeline_reg[EX_MM][i].hw_thread_id == -1 )
-         continue; // bubble 
+         continue; // bubble
       shader->pipeline_reg[MM_WB][i] = shader->pipeline_reg[EX_MM][i];
       shader->pipeline_reg[MM_WB][i].mm_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
       shader->pipeline_reg[EX_MM][i] = nop_inst;
@@ -2898,12 +3023,6 @@ void register_cta_thread_exit(shader_core_ctx_t *shader, int tid )
    }
 }
 
-typedef struct {
-   unsigned pc;
-   unsigned long latency;
-   void *ptx_thd_info;
-} insn_latency_info;
-
 void obtain_insn_latency_info(insn_latency_info *latinfo, inst_t *insn)
 {
    latinfo->pc = insn->pc;
@@ -2911,15 +3030,17 @@ void obtain_insn_latency_info(insn_latency_info *latinfo, inst_t *insn)
    latinfo->ptx_thd_info = insn->ptx_thd_info;
 }
 
+int debug_tid = 0;
+
 unsigned   gpu_n_max_mshr_writeback=1;
 void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) 
 {
-   static int *unlock_tid = NULL;
-   static int *freed_warp = NULL;
+   std::vector<inst_t> done_insts;
+
    static int *mshr_tid = NULL;
    static int *pl_tid = NULL;
-   static int *done_tid = NULL;
-   insn_latency_info *unlock_lat_info = NULL;
+
+   std::vector<insn_latency_info> unlock_lat_infos;
    static insn_latency_info *mshr_lat_info = NULL;
    static insn_latency_info *pl_lat_info = NULL;
 
@@ -2932,30 +3053,23 @@ void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, in
    bool writeback_by_MSHR = false;
    bool w2rf = false;
 
-   if ( unlock_tid == NULL ) {
-      unlock_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
+   if ( mshr_tid == NULL ) {
       mshr_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
       pl_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
-      done_tid = (int*) malloc(sizeof(int)*pipe_simd_width);
-      freed_warp = (int *) malloc(sizeof(int)*pipe_simd_width);
       mshr_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width);
       pl_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width);
    }
-   memset(unlock_tid, -1, sizeof(int)*pipe_simd_width);
+
    memset(mshr_tid,   -1, sizeof(int)*pipe_simd_width);
    memset(pl_tid,     -1, sizeof(int)*pipe_simd_width);
-   memset(done_tid,   -1, sizeof(int)*pipe_simd_width);
-   memset(freed_warp,  0, sizeof(int)*pipe_simd_width);
-   unlock_lat_info = NULL;
+
 
    check_stage_pcs(shader,MM_WB);
 
-   /* Generate Condition for instruction writeback to register file.
-      A load miss *instruction* does not reach writeback until the data is fetched */
+   /* Generate Condition for instruction writeback to register file. */
    for (int i=0; i<pipe_simd_width; i++) {
-      tid = shader->pipeline_reg[MM_WB][i].hw_thread_id;
-      w2rf |= (tid >= 0); 
-      pl_tid[i] = tid;
+	  w2rf |= (shader->pipeline_reg[MM_WB][i].hw_thread_id >= 0);
+	  pl_tid[i] = shader->pipeline_reg[MM_WB][i].hw_thread_id;
    }
 
    //check mshrs for commit;
@@ -2973,14 +3087,17 @@ void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, in
             obtain_insn_latency_info(&mshr_lat_info[mshr_threads_unlocked], &(mshr_head->insts[j]));
             inflight_memory_insn_sub(shader, &mshr_head->insts[j]);
             assert (insn.hw_thread_id >= 0);
-            unlock_tid[mshr_threads_unlocked] = insn.hw_thread_id;
             shader->pending_mem_access--;
             // for ensuring that we don't unlock more than the code allows, needs to be fixed.
             mshr_threads_unlocked++;
          }
+         done_insts.insert(done_insts.end(), mshr_head->insts.begin(), mshr_head->insts.end());
+
          shader->mshr_unit->pop_return_head();
          writeback_by_MSHR = true;
-         unlock_lat_info = mshr_lat_info; 
+         unlock_lat_infos.resize(mshr_threads_unlocked);
+         std::copy(mshr_lat_info, mshr_lat_info + mshr_threads_unlocked, unlock_lat_infos.begin());
+
          if (w2rf) {
             stalled_by_MSHR = true;
          }
@@ -2992,6 +3109,8 @@ void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, in
    }
 
    if (!writeback_by_MSHR) { //!writeback_by_MSHR
+	  memory_space_t warp_space = undefined_space;
+
       for (int i=0; i<pipe_simd_width; i++) {
          op  = shader->pipeline_reg[MM_WB][i].op;
          tid = shader->pipeline_reg[MM_WB][i].hw_thread_id;
@@ -3000,76 +3119,62 @@ void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, in
          o3  = shader->pipeline_reg[MM_WB][i].out[2];
          o4  = shader->pipeline_reg[MM_WB][i].out[3];
 
-         unlock_tid[i] = pl_tid[i];
          obtain_insn_latency_info(&pl_lat_info[i], &shader->pipeline_reg[MM_WB][i]);
-      }
-      unlock_lat_info = pl_lat_info;
-   }
-   int thd_unlocked = 0;
-   for (int i=0; i<pipe_simd_width; i++) {
-      // NOTE: no need to check for next-stage stall at the last stage
-      if (unlock_tid[i] >= 0 ) { // not unlocking an invalid thread (ie. due to a bubble)
-         // thread completed if it is going to fetching beyond code boundry 
-         if ( gpgpu_cuda_sim && ptx_thread_done(shader->thread[unlock_tid[i]].ptx_thd_info) ) {
-
-            shader->not_completed -= 1;
-            gpu_completed_thread += 1;
-
-            int warp_id = wid_from_hw_tid(unlock_tid[i],warp_size);
-            if (!(shader->warp[warp_id].n_completed < (unsigned)warp_size)) {
-               printf("shader[%d]->warp[%d].n_completed = %d; warp_size = %d\n", 
-                      shader->sid,warp_id, shader->warp[warp_id].n_completed, warp_size);
-            }
-            assert( shader->warp[warp_id].n_completed < (unsigned)warp_size );
-            shader->warp[warp_id].n_completed++;
-            if ( shader->model == NO_RECONVERGE ) {
-               update_max_branch_priority(shader,warp_id,grid_num);
-            }
 
-            if (gpgpu_no_divg_load) {
-               int amask = wpt_signal_complete(unlock_tid[i], shader);
-               freed_warp[i] = (amask != 0)? 1 : 0;
-            } else {
-               register_cta_thread_exit(shader, unlock_tid[i] );
-            }
-         } else { //thread is not finished yet
-            // program is not finished yet, allow more fetch 
-            if (gpgpu_no_divg_load) {
-               freed_warp[i] = wpt_signal_avail(unlock_tid[i], shader);
-            } else {
-               shader->thread[unlock_tid[i]].avail4fetch++;
-               assert(shader->thread[unlock_tid[i]].avail4fetch <= 1);
-               assert( shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch < (unsigned)warp_size );
-               shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch++;
-               thd_unlocked = 1;
-            }
+         // Collect threads that are done
+         // Do not include cache misses for a writeback
+         if(!shader->pipeline_reg[MM_WB][i].cache_miss) {
+			 if(shader->pipeline_reg[MM_WB][i].hw_thread_id > -1) {
+				 done_insts.push_back(shader->pipeline_reg[MM_WB][i]);
+				 unlock_lat_infos.push_back(pl_lat_info[i]);
+			 }
          }
 
-         // At any rate, a real instruction is committed
-         // - don't count cache miss
-         if ( shader->pipeline_reg[MM_WB][i].inst_type != NO_OP_FLAG ) {
-            gpu_sim_insn++;
-            if ( !is_const(shader->pipeline_reg[MM_WB][i].space) ) 
-               gpu_sim_insn_no_ld_const++;
-            gpu_sim_insn_last_update = gpu_sim_cycle;
-            shader->num_sim_insn++;
-            shader->thread[unlock_tid[i]].n_insn++;
-            shader->thread[unlock_tid[i]].n_insn_ac++;
+         // All threads in the warp should have the same pc and space
+         if(pl_tid[i] > -1 ) {
+        	 warp_space = shader->pipeline_reg[MM_WB][i].space;
          }
 
-         if (enable_ptx_file_line_stats) {
-            unsigned pc = unlock_lat_info[i].pc;
-            unsigned long latency = unlock_lat_info[i].latency;
-            ptx_file_line_stats_add_latency(unlock_lat_info[i].ptx_thd_info, pc, latency);
+         if(tid > -1) {
+/*
+        	 if(!shader->pipeline_reg[MM_WB][i].cache_miss)
+        		 printf("CACHE HIT sid=%d tid=%d pc=%d \n", shader->sid, tid, shader->pipeline_reg[MM_WB][i].pc);
+        	 else
+        		 printf("CACHE MISS sid=%d tid=%d pc=%d \n", shader->sid, tid, shader->pipeline_reg[MM_WB][i].pc);
+*/
          }
       }
+
+      // Unlock the warp for re-fetching (put it in the fixed delay queue)
+      // Only need to unlock if warp is not empty
+      if(w2rf)
+    	  shader_queue_warp_unlocking(shader, pl_tid, warp_space, grid_num);
    }
-   if (shader->using_commit_queue && thd_unlocked) {
-      int *tid_unlocked = alloc_commit_warp();
-      memcpy(tid_unlocked, unlock_tid, sizeof(int)*pipe_simd_width); //NOTE: this maybe warp_size
-      dq_push(shader->thd_commit_queue,(void*)tid_unlocked);
-   }
 
+   // Mark threads as done in warp tracker
+   for (unsigned i=0; i<done_insts.size(); i++) {
+	   inst_t done_inst = done_insts[i];
+
+		shader_call_thread_done(shader, grid_num, done_inst);
+
+		// Statistics
+		// At any rate, a real instruction is committed
+		// - don't count cache miss
+		gpu_sim_insn++;
+		if ( !is_const(done_inst.space) )
+			gpu_sim_insn_no_ld_const++;
+		gpu_sim_insn_last_update = gpu_sim_cycle;
+		shader->num_sim_insn++;
+		shader->thread[done_inst.hw_thread_id].n_insn++;
+		shader->thread[done_inst.hw_thread_id].n_insn_ac++;
+
+		if (enable_ptx_file_line_stats) {
+		  unsigned pc = unlock_lat_infos[i].pc;
+		  unsigned long latency = unlock_lat_infos[i].latency;
+		  ptx_file_line_stats_add_latency(unlock_lat_infos[i].ptx_thd_info, pc, latency);
+		}
+
+  }
 
    /* The pipeline can be stalled by MSHR */
    if (!stalled_by_MSHR) {
@@ -3078,6 +3183,189 @@ void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, in
          shader->pipeline_reg[MM_WB][i] = nop_inst;
       }
    }
+
+   // Process the delay queue for current cycle
+   shader_process_delay_queue(shader);
+}
+
+/*
+ * Queues a warp into fixed delay queue for unlocking
+ *
+ * The amount of delay to add is determined by the instruction type.
+ *
+ * @param *shader Pointer to shader core
+ * @param *tid Array of tid in the warp to unlock
+ * @param pc Program counter for the current instruction in the warp
+ * @param space Address space for the current instruction in the warp
+ *
+ */
+void shader_queue_warp_unlocking(shader_core_ctx_t *shader, int *tids, memory_space_t space, int grid_num) {
+
+	// Create a delay queue object and add it to the queue
+	shader_core_ctx_t::fixeddelay_queue_warp_t fixeddelay_queue_warp;
+
+	fixeddelay_queue_warp.grid_num = grid_num;
+
+	// Set ready_cycle based on instruction space
+	switch(space.get_type()) {
+		case shared_space:
+			fixeddelay_queue_warp.ready_cycle = gpu_tot_sim_cycle + gpu_sim_cycle + 5; // Adds 5*4=20 cycles
+			break;
+		default:
+			fixeddelay_queue_warp.ready_cycle = gpu_tot_sim_cycle + gpu_sim_cycle;
+			break;
+	}
+
+	// Store threads in delay queue warp object
+	fixeddelay_queue_warp.tids.resize(warp_size);
+	std::copy(tids, tids+warp_size, fixeddelay_queue_warp.tids.begin());
+
+	shader->fixeddelay_queue.insert(fixeddelay_queue_warp);
+}
+
+/*
+ * Process a delay queue by unlocking warps ready this cycle
+ *
+ * @param *shader Pointer to shader core
+ *
+ */
+void shader_process_delay_queue(shader_core_ctx_t *shader) {
+	// Unlock warps in fixeddelay_queue_warp
+	std::multiset<shader_core_ctx_t::fixeddelay_queue_warp_t, shader_core_ctx_t::fixeddelay_queue_warp_comp>::iterator it;
+	std::multiset<shader_core_ctx_t::fixeddelay_queue_warp_t, shader_core_ctx_t::fixeddelay_queue_warp_comp>::iterator it_last;
+	for ( it=shader->fixeddelay_queue.begin() ;
+		  it != shader->fixeddelay_queue.end();
+		) {
+	   if(it->ready_cycle <= gpu_tot_sim_cycle + gpu_sim_cycle) {
+		   if(!gpgpu_stall_on_use) {
+			   // This disables stall-on-use
+			   // If thread is still in warp_tracker, do not unlock yet
+			   bool skip_unlock = false;
+			   for(unsigned i=0; i<warp_size; i++) {
+				   int tid = it->tids[i];
+				   if(tid < 0) continue;
+				   if(get_warp_tracker_pool().wpt_thread_in_wpt(shader,tid)) {
+					   skip_unlock = true;
+					   break;
+				   }
+			   }
+			   if(skip_unlock) {
+				   it_last = it++;
+				   continue;
+			   }
+		   }
+
+		   // Unlock warp
+		   shader_unlock_warp(shader,it->tids, it->grid_num);
+
+		   // Remove warp information from delay queue
+		   it_last = it++;
+		   shader->fixeddelay_queue.erase(it_last);
+	   } else {
+		   break;
+	   }
+	}
+}
+
+/*
+ * Unlock a warp
+ *
+ * @param *shd Pointer to shader core
+ * @param tids Vector of tid in the warp to unlock
+ *
+ */
+void shader_unlock_warp(shader_core_ctx_t *shd, std::vector<int> tids, int grid_num) {
+    int thd_unlocked = 0;
+    int thd_exited = 0;
+    int tid;
+    int valid_tid = -1;
+    // Unlock
+    for (unsigned i=0; i<warp_size; i++) {
+        tid = tids[i];
+		if (tid >= 0) {
+			valid_tid = tid;
+			// thread completed if it is going to fetching beyond code boundary
+			if ( gpgpu_cuda_sim && ptx_thread_done(shd->thread[tid].ptx_thd_info) ) {
+				shd->not_completed -= 1;
+				gpu_completed_thread += 1;
+
+				int warp_id = wid_from_hw_tid(tid,warp_size);
+				if (!(shd->warp[warp_id].n_completed < (unsigned)warp_size)) {
+				 printf("shader[%d]->warp[%d].n_completed = %d; warp_size = %d\n",
+						shd->sid,warp_id, shd->warp[warp_id].n_completed, warp_size);
+				}
+				assert( shd->warp[warp_id].n_completed < (unsigned)warp_size );
+				shd->warp[warp_id].n_completed++;
+				if ( shd->model == NO_RECONVERGE ) {
+				 update_max_branch_priority(shd,warp_id,grid_num);
+				}
+
+				register_cta_thread_exit(shd, tid );
+				thd_exited = 1;
+
+				//printf("THREAD EXIT sid=%d tid=%d \n", shd->sid, tid);
+
+			} else {
+				shd->thread[tid].avail4fetch++;
+				assert(shd->thread[tid].avail4fetch <= 1);
+				assert( shd->warp[tid/warp_size].n_avail4fetch < warp_size );
+				shd->warp[tid/warp_size].n_avail4fetch++;
+				thd_unlocked = 1;
+
+				//printf("THREAD UNLOCK sid=%d tid=%d \n", shd->sid, tid);
+			}
+		}
+    }
+
+    // Update warp was unlocked, update the warp active mask
+    if(thd_unlocked || thd_exited) {
+    	// Update the warp active mask
+		shader_pdom_update_warp_mask(shd, wid_from_hw_tid(valid_tid,warp_size));
+    }
+
+
+
+	if (shd->model == POST_DOMINATOR || shd->model == NO_RECONVERGE) {
+		// Do nothing
+	} else {
+		// For this case, submit to commit_queue
+		if (shd->using_commit_queue && thd_unlocked) {
+			int *tid_unlocked = alloc_commit_warp();
+			std::copy(tids.begin(), tids.end(), tid_unlocked);
+			dq_push(shd->thd_commit_queue,(void*)tid_unlocked);
+		}
+	}
+}
+
+
+/*
+ * Signals to the warp_tracker that a thread in a warp (for a given pc/instruction) is done
+ *
+ * @param *shd Pointer to shader core
+ * @param grid_num Grid number
+ * @param done_inst Completed instruction
+ *
+ */
+void shader_call_thread_done( shader_core_ctx_t *shader, int grid_num, inst_t &done_inst ) {
+
+	if (gpgpu_no_divg_load) {
+
+		//printf("THREAD RETURNED sid=%d tid=%d pc=%d \n", shader->sid, done_inst.hw_thread_id, done_inst.pc);
+
+		// Signal to unlock the thread. If all threads are done, deregister warp
+		if( get_warp_tracker_pool().wpt_signal_avail(done_inst.hw_thread_id, shader, done_inst.pc) == 1 ) {
+			// Entire warp has returned
+			//printf("WARP RETURNED sid=%d tid=%d pc=%d \n", shader->sid, done_inst.hw_thread_id, done_inst.pc);
+
+			// Deregister warp
+			get_warp_tracker_pool().wpt_deregister_warp(done_inst.hw_thread_id, shader, done_inst.pc);
+
+			// Signal scoreboard to release register
+			shader->scrb->releaseRegisters( wid_from_hw_tid(done_inst.hw_thread_id, warp_size), &done_inst );
+
+		}
+	}
+
 }
 
 
@@ -3454,34 +3742,21 @@ void shader_cycle( shader_core_ctx_t *shader,
                    unsigned int shader_number,
                    int grid_num ) 
 {
-
    if (gpgpu_operand_collector) 
       shader_opnd_collect_write(shader);
-
-   // last pipeline stage
    shader_writeback(shader, shader_number, grid_num);
-
-   // three parallel stages (only one does something on a given cycle)
-   //shader_const_memory   (shader, shader_number);
-   shader_memory   (shader, shader_number);
-   //shader_texture_memory   (shader, shader_number);
-
-   // empty stage
-   if (gpgpu_pre_mem_stages)
-      shader_pre_memory(shader, shader_number);
-
-   shader_execute  (shader, shader_number);
+   shader_memory(shader, shader_number);
+   if (gpgpu_pre_mem_stages) // for modeling deeper pipelines
+      shader_pre_memory(shader, shader_number); 
+   shader_execute(shader, shader_number);
    if (shader->using_rrstage) {
-      // model register bank conflicts 
-      // (see Fung et al. MICRO'07 paper or ACM TACO paper)
+      // Model register bank conflicts as in 
+      // Fung et al. MICRO'07 / ACM TACO'09 papers.
       shader_preexecute (shader, shader_number);
    }
-
    if (gpgpu_operand_collector) 
       shader_opnd_collect_read(shader);
-
    shader_decode   (shader, shader_number, grid_num);
-
    shader_fetch    (shader, shader_number, grid_num);
 }
 
@@ -3603,7 +3878,7 @@ std::list<opndcoll_rfu_t::op_t> opndcoll_rfu_t::arbiter_t::allocate_reads()
       _inmatch = new int[ _inputs ];
       _outmatch = new int[ _outputs ];
       _request = new int*[ _inputs ];
-      for(int i=0; i<_outputs;i++) 
+      for(int i=0; i<_inputs;i++) 
          _request[i] = new int[_outputs];
    }
 
@@ -3790,11 +4065,6 @@ void barrier_set_t::dump() const
    fflush(stdout); 
 }
 
-shader_core_ctx::shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core )
-   : m_barriers( max_warps_per_cta, max_cta_per_core ), m_opndcoll_new( gpgpu_operand_collector_num_units, gpgpu_num_reg_banks, this )
-{
-}
-
 void shader_core_ctx::set_at_barrier( unsigned cta_id, unsigned warp_id )
 {
    m_barriers.warp_reaches_barrier(cta_id,warp_id);
@@ -3819,3 +4089,37 @@ void shader_core_ctx::deallocate_barrier( unsigned cta_id )
 {
    m_barriers.deallocate_barrier(cta_id);
 }
+
+void opndcoll_rfu_t::init( unsigned num_collectors_alu, 
+                           unsigned num_collectors_sfu, 
+                           unsigned num_banks, 
+                           const shader_core_ctx *shader ) 
+{
+   unsigned num_alu_cu = gpgpu_operand_collector_num_units;
+   unsigned num_sfu_cu = gpgpu_operand_collector_num_units_sfu;
+   m_num_collectors = num_alu_cu+num_sfu_cu;
+    
+   m_shader=shader;
+   m_arbiter.init(m_num_collectors,num_banks);
+
+   m_alu_port = shader->pipeline_reg[ID_EX];
+   m_sfu_port = shader->pipeline_reg[OC_EX_SFU];
+
+   m_dispatch_units[ m_alu_port ].init( num_alu_cu );
+   m_dispatch_units[ m_sfu_port ].init( num_sfu_cu );
+
+   m_num_banks = num_banks;
+   m_cu = new collector_unit_t[m_num_collectors];
+
+   unsigned c=0;
+   for(; c<num_alu_cu; c++) {
+      m_cu[c].init(c,m_alu_port);
+      m_free_cu[m_alu_port].push_back(&m_cu[c]);
+      m_dispatch_units[m_alu_port].add_cu(&m_cu[c]);
+   }
+   for(; c<m_num_collectors; c++) {
+      m_cu[c].init(c,m_sfu_port);
+      m_free_cu[m_sfu_port].push_back(&m_cu[c]);
+      m_dispatch_units[m_sfu_port].add_cu(&m_cu[c]);
+   }
+}
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index ec1ae0b..1e62995 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -69,6 +69,8 @@
 #include <math.h>
 #include <assert.h>
 #include <map>
+#include <set>
+#include <vector>
 #include <list>
 
 #include "../cuda-sim/ptx.tab.h"
@@ -79,6 +81,7 @@
 #include "stack.h"
 #include "dram.h"
 #include "../abstract_hardware_model.h"
+#include "scoreboard.h"
 
 #ifndef SHADER_H
 #define SHADER_H
@@ -133,8 +136,11 @@ typedef struct {
    unsigned in[4];
    unsigned char is_vectorin;
    unsigned char is_vectorout;
+   int pred;
+   int ar1, ar2;
    int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation
    unsigned data_size; // what is the size of the word being operated on?
+   unsigned cycles; // number of cycles taken by current instruction
 
    int reg_bank_access_pending;
    int reg_bank_conflict_stall_checked; // flag to turn off register bank conflict checker to avoid double stalling
@@ -154,6 +160,8 @@ typedef struct {
    unsigned long long  ex_cycle;
    unsigned long long  mm_cycle;
 
+   bool cache_miss;
+
 } inst_t;
 
 typedef struct {
@@ -323,19 +331,19 @@ void shader_print_warp( const shader_core_ctx *shader, inst_t *warp, FILE *fout,
 class opndcoll_rfu_t{ // operand collector based register file unit
 public:
    // constructors
-   opndcoll_rfu_t( unsigned num_collectors, unsigned num_banks, const shader_core_ctx *shader ) 
-      : m_arbiter(num_collectors,num_banks)
+   opndcoll_rfu_t()
    {
-      m_num_collectors=num_collectors;
-      m_num_banks = num_banks;
-      m_cu = new collector_unit_t[num_collectors];
-      m_last_cu=0;
-      m_shader=shader;
-      for(unsigned c=0; c<num_collectors; c++) {
-         m_cu[c].set_cuid(c);
-         m_free_cu.push_back(&m_cu[c]);
-      }
+      m_num_collectors=0;
+      m_num_banks=0;
+      m_cu = NULL;
+      m_shader=NULL;
+      m_sfu_port=NULL;
+      m_alu_port=NULL;
    }
+   void init( unsigned num_collectors_alu, 
+              unsigned num_collectors_sfu, 
+              unsigned num_banks, 
+              const shader_core_ctx *shader );
 
    // modifiers
    void writeback( inst_t *warp )
@@ -353,9 +361,9 @@ public:
       }
    }
 
-   void step( inst_t *id_oc_reg, inst_t *oc_ex_reg ) 
+   void step( inst_t *id_oc_reg ) 
    {
-      dispatch_ready_cu(oc_ex_reg);   
+      dispatch_ready_cu();   
       allocate_reads();
       allocate_cu(id_oc_reg);
       process_banks();
@@ -379,17 +387,18 @@ private:
       m_arbiter.reset_alloction();
    }
 
-   void dispatch_ready_cu( inst_t *oc_ex_reg )
+   void dispatch_ready_cu()
    {
-      if( !pipeline_regster_empty(oc_ex_reg) ) 
-         return;
-      for( unsigned n=0; n < m_num_collectors; n++ ) {
-         unsigned c=(m_last_cu+n+1)%m_num_collectors;
-         if( m_cu[c].ready() ) {
-            m_cu[c].dispatch(oc_ex_reg);
-            m_free_cu.push_back(&m_cu[c]);
-            m_last_cu=c;
-            break;
+      port_to_du_t::iterator p;
+      for( p=m_dispatch_units.begin(); p!=m_dispatch_units.end(); ++p ) {
+         inst_t *port = p->first;
+         if( !pipeline_regster_empty(port) ) 
+            continue;
+         dispatch_unit_t &du = p->second;
+         collector_unit_t *cu = du.find_ready();
+         if( cu ) {
+            cu->dispatch();
+            m_free_cu[port].push_back(cu);
          }
       }
    }
@@ -397,11 +406,18 @@ private:
    void allocate_cu( inst_t *id_oc_reg )
    {
       inst_t *fvi = first_valid_thread(id_oc_reg);
-      if( fvi && !m_free_cu.empty() ) {
-         collector_unit_t *cu = m_free_cu.back();
-         m_free_cu.pop_back();
-         cu->allocate(id_oc_reg);
-         m_arbiter.add_read_requests(cu);
+      if( fvi ) {
+         inst_t *port = NULL;
+         if( fvi->op == SFU_OP ) 
+            port = m_sfu_port;
+         else
+            port = m_alu_port;
+         if( !m_free_cu[port].empty() ) {
+            collector_unit_t *cu = m_free_cu[port].back();
+            m_free_cu[port].pop_back();
+            cu->allocate(id_oc_reg);
+            m_arbiter.add_read_requests(cu);
+         }
       }
    }
 
@@ -521,7 +537,13 @@ private:
    class arbiter_t {
    public:
       // constructors
-      arbiter_t( unsigned num_cu, unsigned num_banks ) 
+      arbiter_t()
+      {
+         m_queue=NULL;
+         m_allocated_bank=NULL;
+         m_allocator_rr_head=NULL;
+      }
+      void init( unsigned num_cu, unsigned num_banks ) 
       { 
          m_num_collectors = num_cu;
          m_num_banks = num_banks;
@@ -607,7 +629,10 @@ private:
          m_warp_id = -1;
       }
       // accessors
-      bool ready() const { return (!m_free) && m_not_ready.none(); }
+      bool ready() const 
+      { 
+         return (!m_free) && m_not_ready.none() && pipeline_regster_empty(m_port); 
+      }
       const op_t *get_operands() const { return m_src_op; }
       void dump(FILE *fp, const shader_core_ctx *shader ) const
       {
@@ -629,7 +654,7 @@ private:
       unsigned get_id() const { return m_cuid; } // returns CU hw id
 
       // modifiers
-      void set_cuid(unsigned n) { m_cuid=n; }
+      void init(unsigned n, inst_t *port) { m_cuid=n; m_port=port; }
       void allocate( inst_t *pipeline_reg ) 
       {
          assert(m_free);
@@ -656,10 +681,10 @@ private:
          m_not_ready.reset(op);
       }
 
-      void dispatch( inst_t *pipeline_reg )
+      void dispatch()
       {
          assert( m_not_ready.none() );
-         move_warp(pipeline_reg,m_warp);
+         move_warp(m_port,m_warp);
          m_free=true;
          for( unsigned i=0; i<MAX_REG_OPERANDS;i++) 
             m_src_op[i].reset();
@@ -669,20 +694,69 @@ private:
       bool m_free;
       unsigned m_tid;
       unsigned m_cuid; // collector unit hw id
+      inst_t  *m_port; // pipeline register to issue to when ready
       unsigned m_warp_id;
-      inst_t *m_warp;
+      inst_t  *m_warp;
       op_t *m_src_op;
       std::bitset<MAX_REG_OPERANDS> m_not_ready;
    };
 
-   // data members
+   class dispatch_unit_t {
+   public:
+      dispatch_unit_t() 
+      { 
+         m_last_cu=0;
+         m_num_collectors=0;
+         m_collector_units=NULL;
+         m_next_cu=0;
+      }
+
+      void init( unsigned num_collectors )
+      { 
+         m_num_collectors = num_collectors;
+         m_collector_units = new collector_unit_t * [num_collectors];
+         m_next_cu=0;
+      }
+
+      void add_cu( collector_unit_t *cu )
+      {
+         assert(m_next_cu<m_num_collectors);
+         m_collector_units[m_next_cu] = cu;
+         m_next_cu++;
+      }
+
+      collector_unit_t *find_ready()
+      {
+         for( unsigned n=0; n < m_num_collectors; n++ ) {
+            unsigned c=(m_last_cu+n+1)%m_num_collectors;
+            if( m_collector_units[c]->ready() ) {
+               m_last_cu=c;
+               return m_collector_units[c];
+            }
+         }
+         return NULL;
+      }
+
+   private:
+      unsigned m_num_collectors;
+      collector_unit_t **m_collector_units;
+      unsigned m_last_cu; // dispatch ready cu's rr
+      unsigned m_next_cu;  // for initialization
+   };
+
+   // opndcoll_rfu_t data members
 
    unsigned                         m_num_collectors;
    unsigned                         m_num_banks;
    collector_unit_t                *m_cu;
-   unsigned                         m_last_cu; // dispatch ready cu's rr
    arbiter_t                        m_arbiter;
-   std::list<collector_unit_t*>     m_free_cu;
+
+   inst_t *m_alu_port;
+   inst_t *m_sfu_port;
+
+   typedef std::map<inst_t*/*port*/,dispatch_unit_t> port_to_du_t;
+   port_to_du_t                     m_dispatch_units;
+   std::map<inst_t*,std::list<collector_unit_t*> > m_free_cu;
    const shader_core_ctx           *m_shader;
 };
 
@@ -723,11 +797,19 @@ private:
 };
 
 class mshr_shader_unit;
-  
+class warp_tracker;
+class warp_tracker_pool;
+
 class shader_core_ctx : public core_t 
 {
 public:
-   shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core );
+   shader_core_ctx(   const char *name, int sid, 
+                      unsigned int n_threads, 
+                      unsigned int n_mshr, 
+                      fq_push_t fq_push, 
+                      fq_has_buffer_t fq_has_buffer,
+                      unsigned model,
+                      unsigned max_warps_per_cta, unsigned max_cta_per_core );
 
 	virtual void set_at_barrier( unsigned cta_id, unsigned warp_id );
    virtual void warp_exit( unsigned warp_id );
@@ -821,10 +903,39 @@ public:
    unsigned int n_cta;      //Limit on number of concurrent CTAs in shader core
 
    mshr_shader_unit *mshr_unit;
+
+   //
+   // Fixed-delay queue for locked warps
+   //
+
+   // Struct for storing warp information in fixeddelay_queue
+   struct fixeddelay_queue_warp_t {
+	   int grid_num;
+   	   unsigned long long ready_cycle;
+   	   std::vector<int> tids; // list of tid's in this warp (to unlock)
+   };
+   struct fixeddelay_queue_warp_comp {
+       inline bool operator()(const fixeddelay_queue_warp_t& left,const fixeddelay_queue_warp_t& right) const
+       {
+           return left.ready_cycle < right.ready_cycle;
+       }
+   };
+
+   // The queue
+   std::multiset<fixeddelay_queue_warp_t, fixeddelay_queue_warp_comp> fixeddelay_queue;
+
+   // Scoreboard
+   Scoreboard *scrb;
 };
 
 typedef shader_core_ctx shader_core_ctx_t;
 
+typedef struct {
+   unsigned pc;
+   unsigned long latency;
+   void *ptx_thd_info;
+} insn_latency_info;
+
 
 shader_core_ctx_t* shader_create( const char *name, int sid, unsigned int n_threads, 
                                   unsigned int n_mshr, fq_push_t fq_push, fq_has_buffer_t fq_has_buffer, unsigned int model);
@@ -853,6 +964,9 @@ void shader_writeback( shader_core_ctx_t *shader,
                        unsigned int shader_number,
                        int grid_num );
 
+bool shader_warp_scoreboard_hazard(shader_core_ctx_t *shader, int warp_id);
+void shader_pdom_update_warp_mask(shader_core_ctx_t *shader, int warp_id);
+
 void shader_display_pipeline(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask3bit );
 void shader_dump_thread_state(shader_core_ctx_t *shader, FILE *fout );
 void shader_cycle( shader_core_ctx_t *shader, 
@@ -875,6 +989,11 @@ void free_mshr_entry( mshr_entry * );
 void shader_clean(shader_core_ctx_t *sc, unsigned int n_threads);
 void shader_cache_flush(shader_core_ctx_t* sc);
 
+void shader_call_thread_done( shader_core_ctx_t *shader, int grid_num, inst_t &done_inst );
+void shader_queue_warp_unlocking(shader_core_ctx_t *shader, int *tids, memory_space_t space, int grid_num);
+void shader_process_delay_queue(shader_core_ctx_t *shader);
+void shader_unlock_warp(shader_core_ctx_t *shd, std::vector<int> tids, int grid_num);
+
 // print out the accumulative statistics for shaders (those that are not local to one shader)
 void shader_print_accstats( FILE* fout );
 void shader_print_runtime_stat( FILE *fout );
@@ -884,7 +1003,6 @@ void shader_print_l1_miss_stat( FILE *fout );
 //based on on the current kernel's CTA size and is 1 if mutiple CTA per block is not supported
 unsigned int max_cta_per_shader( shader_core_ctx_t *shader);
 
-#define N_PIPELINE_STAGES (gpgpu_operand_collector ? 8 : 7)
 #define TS_IF 0
 #define IF_ID 1
 #define ID_RR 2
@@ -893,7 +1011,11 @@ unsigned int max_cta_per_shader( shader_core_ctx_t *shader);
 #define EX_MM 4
 #define MM_WB 5
 #define WB_RT 6
-#define ID_OC 7
+
+#define ID_OC 7 
+#define OC_EX_SFU 8
+
+#define N_PIPELINE_STAGES 9
 
 extern shader_core_ctx_t **sc;
 extern unsigned int gpgpu_n_load_insn;
@@ -920,6 +1042,7 @@ extern int gpgpu_const_port_per_bank;
 extern int gpgpu_shmem_pipe_speedup;  
 extern bool gpgpu_reg_bank_conflict_model;
 extern unsigned int gpgpu_num_reg_banks;
+extern bool gpgpu_reg_bank_use_warp_id;
 extern unsigned int gpu_max_cta_per_shader;
 extern unsigned int gpu_padded_cta_size;
 extern int gpgpu_local_mem_map;
diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc
index d957e0c..c136d57 100644
--- a/src/gpgpu-sim/stat-tool.cc
+++ b/src/gpgpu-sim/stat-tool.cc
@@ -540,7 +540,7 @@ void destroy_thread_CFlogger( )
          remove_spill_log(thread_CFlogger[i]);
          delete thread_CFlogger[i];
       }
-      delete thread_CFlogger;
+      delete [] thread_CFlogger;
       thread_CFlogger = NULL;
    }
 }
diff --git a/src/gpgpu-sim/warp_tracker.cc b/src/gpgpu-sim/warp_tracker.cc
index 2096d93..d701c8f 100644
--- a/src/gpgpu-sim/warp_tracker.cc
+++ b/src/gpgpu-sim/warp_tracker.cc
@@ -1,5 +1,5 @@
 /* 
- * waro_tracker.cc
+ * warp_tracker.cc
  *
  * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda and the 
  * University of British Columbia
@@ -66,102 +66,242 @@
 #include "warp_tracker.h"
 #include "gpu-sim.h"
 #include "shader.h"
+#include <set>
 
 using namespace std;
 
-#include <set>
 
-class warp_tracker {
-public:
+void register_cta_thread_exit(shader_core_ctx_t *shader, int cta_num );
+
+/*
+ * Constructor for warp_tracker_pool.
+ *
+ * Resizes the warp_tracker map and pool and allocates empty warp_trackers.
+ *
+ * @param tid_in Array of thread id's corresponding to a warp
+ * @param *shd Pointer to the shader core
+ *
+ * @return Pointer to a warp_tracker
+ */
+warp_tracker_pool::warp_tracker_pool(unsigned gpu_n_shader, unsigned gpu_n_thread_per_shader) {
+	this->gpu_n_shader = gpu_n_shader;
+	this->gpu_n_thread_per_shader = gpu_n_thread_per_shader;
+
+	// Resize the warp tracker map
+	warp_tracker_map.resize(gpu_n_shader);
+	for(unsigned i=0; i<gpu_n_shader; i++) {
+		warp_tracker_map[i].resize(gpu_n_thread_per_shader);
+	}
+
+	// Create a pool of warp_trackers
+	warp_tracker_list.resize(gpu_n_shader * gpu_n_thread_per_shader);
+
+	// Add all warp_trackers to the list of free warp_trackers
+	std::list<warp_tracker>::iterator it;
+	for(it=warp_tracker_list.begin(); it!=warp_tracker_list.end(); it++)
+		warp_tracker_free_list.push_back(&(*it));
+
+}
+
+/*
+ * Fetch a free warp_tracker from the pool of warp_trackers. Assigns the warp_tracker to
+ * the input warp.
+ *
+ * If there are no free warp_trackers in the pool, the pool is extended by allocating more
+ * warp_trackers.
+ *
+ * @param tid_in Array of thread id's corresponding to a warp
+ * @param *shd Pointer to the shader core
+ *
+ * @return Pointer to a warp_tracker
+ */
+warp_tracker* warp_tracker_pool::alloc_warp_tracker( int *tid_in, shader_core_ctx_t *shd, address_type pc ) {
+	// If no free warp trackers are available, allocate some more
+	if(warp_tracker_free_list.empty()) {
+		printf("warp_tracker_list empty (size=%d) - allocating new warp_trackers\n", size());
+		fflush(stdout);
+		// Warp tracker list is empty, resize the list
+		unsigned previous_size = warp_tracker_list.size();
+		warp_tracker_list.resize( previous_size + this->gpu_n_thread_per_shader);
+
+		// Add newly allocated warp trackers to list of free warp trackers
+		std::list<warp_tracker>::iterator it = warp_tracker_list.begin();
+		for(unsigned i=0; i<previous_size; i++)
+			it++; // Increment iterator
+		for(; it!=warp_tracker_list.end(); it++)
+				warp_tracker_free_list.push_back(&(*it));
+	}
 
-   int *tid; // the threads in this warp
-   int n_thd; // total number of threads in this warp
-   int n_notavail; // number of threads still not available
-   shader_core_ctx_t *shd; // reference to shader core
+	assert(!warp_tracker_free_list.empty());
+	// Fetch a free warp_tracker
+	warp_tracker* wpt = warp_tracker_free_list.front();
+	warp_tracker_free_list.pop_front();
+	wpt->set_warp(tid_in, shd, pc);
+
+	return wpt;
+}
 
-   warp_tracker () {
-      tid = new int[warp_size];
-      memset(tid, -1, sizeof(int)*warp_size);
-      n_thd = 0;
-      n_notavail = 0;
-      shd = NULL;
+/*
+ * Free the warp_tracker.
+ *
+ * Puts the warp_tracker back into the pool of free warp_trackers.
+ *
+ * @param wpt Pointer to a warp_tracker
+ *
+ */
+void warp_tracker_pool::free_warp_tracker(warp_tracker* wpt) {
+	warp_tracker_free_list.push_back(wpt);
+}
+
+/*
+ * Register a new warp_tracker with a warp
+ *
+ * A warp_tracker is fetched from the pool of warp_trackers and assigned to
+ * track the input warp. A (sid,tid,pc) to warp_tracker mapping is also stored.
+ *
+ * @param tid_in Array of thread id's corresponding to a single warp (array of size warp_size)
+ * @param *shd Pointer to the shader core
+ *
+ */
+void warp_tracker_pool::wpt_register_warp( int *tid_in, shader_core_ctx_t *shd, address_type pc)
+{
+   int sid = shd->sid;
+   unsigned i;
+   int n_thd = 0;
+   for (i=0; i<warp_size; i++) {
+      if (tid_in[i] >= 0) n_thd++;
    }
-   warp_tracker( int *tid, shader_core_ctx_t * shd ) {
-      this->tid = new int[warp_size];
-      memcpy(this->tid, tid, sizeof(int)*warp_size);
-      this->n_thd = 0;
-      this->n_notavail = 0;
-      for (unsigned i=0; i<warp_size; i++) {
-         if (this->tid[i] >= 0) {
-            this->n_thd++;
-         }
+
+   if (!n_thd) return;
+
+   warp_tracker *wpt = this->alloc_warp_tracker(tid_in, shd, pc);
+
+   // assign the new warp_tracker to warp_tracker_map
+   for (i=0; i<warp_size; i++) {
+      if (tid_in[i] >= 0) {
+         assert( map_get_warp_tracker(sid,tid_in[i],pc) == NULL );
+         map_set_warp_tracker(sid, tid_in[i], pc, wpt);
       }
-      this->n_notavail = this->n_thd;
-      this->shd = shd;
    }
-   ~warp_tracker () {
-      delete[] tid;
+}
+
+/*
+ * Signal that the current thread has completed and ready to be unlocked.
+ *
+ * @param tid Thread that is exiting
+ * @param *shd Pointer to the shader core
+ *
+ * @return Returns true is all threads in the warp have completed.
+ */
+int warp_tracker_pool::wpt_signal_avail( int tid, shader_core_ctx_t *shd, address_type pc )
+{
+   int sid = shd->sid;
+   warp_tracker *wpt = map_get_warp_tracker(sid,tid,pc);
+   assert(wpt != NULL);
+
+
+   // signal the warp tracker
+   if (wpt->avail_thd()) {
+      return 1;
+   } else {
+      return 0;
    }
+}
+
+/*
+ * Unlock a warp
+ *
+ * Unlocks a warp for re-fetching. Sets avail4fetch = 1 for all threads and increments n_avail4fetch
+ * by number of active threads.
+ *
+ * @param tid Thread that is exiting
+ * @param *shd Pointer to the shader core
+ *
+ */
+void warp_tracker_pool::wpt_deregister_warp( int tid, shader_core_ctx_t *shd, address_type pc ) {
+   int sid = shd->sid;
+   warp_tracker *wpt = map_get_warp_tracker(sid,tid,pc);
+   assert(wpt != NULL);
+
+    // the warp is ready to be fetched again, remove this warp_tracker
+    for (unsigned i=0; i<warp_size; i++) {
+	   if (wpt->tid[i] >= 0) {
+		  map_clear_warp_tracker(sid,wpt->tid[i],pc);
+	   }
+    }
+
+    free_warp_tracker( wpt );
+}
+
 
-   // set the warp to be consist of the given threads
-   void set_warp ( int *tid, shader_core_ctx_t *shd) {
-      memcpy(this->tid, tid, sizeof(int)*warp_size);
-      this->n_thd = 0;
-      this->n_notavail = 0;
+/*
+ * Signal that the a thread is done and is exiting (exit instruction)
+ *
+ * Marks a thread as completed. If all threads in the warp have completed, call register_cta_thread_exit on all
+ * threads and removed the warp from warp tracker.
+ *
+ * @param tid Thread that is exiting
+ * @param *shd Pointer to the shader core
+ *
+ * @return The warp's mask of active threads.
+ */
+int warp_tracker_pool::wpt_signal_complete( int tid, shader_core_ctx_t *shd, address_type pc )
+{
+   int sid = shd->sid;
+   warp_tracker *wpt = map_get_warp_tracker(sid,tid,pc);
+   assert(wpt != NULL);
+
+   // signal the warp tracker
+   if (wpt->complete_thd(tid)) {
+      // if the warp has completed execution, remove this warp_tracker
+      int warp_mask = 0;
       for (unsigned i=0; i<warp_size; i++) {
-         if (this->tid[i] >= 0) {
-            this->n_thd++;
+         if (wpt->tid[i] >= 0) {
+            register_cta_thread_exit(shd, wpt->tid[i] );
+            map_clear_warp_tracker(sid,wpt->tid[i],pc);
+            warp_mask |= (1 << i);
          }
       }
-      this->n_notavail = this->n_thd;
-      this->shd = shd;
+
+      free_warp_tracker( wpt );
+
+      return warp_mask;
+   } else {
+      return 0;
    }
+}
 
-   // signal that this thread is available for fetch
-   // if all threads in the warp are available, change all their status
-   // and return true
-   bool avail_thd ( int tid_in ) {
-      n_notavail--;
-      if (n_notavail) {
-         return false;
-      } else {
+/*
+ * Check if this thread is being tracked by the warp tracker currently
+ *
+ * Checks if any pc of the given tid maps to a warp_tracker
+ *
+ * @param *shd Pointer to the shader core
+ * @param tid Thread to check
+ *
+ * @return True is thread is being tracked
+ */
+bool warp_tracker_pool::wpt_thread_in_wpt(shader_core_ctx *shd, int tid) {
+	int sid = shd->sid;
+	std::map<address_type, warp_tracker*>::iterator it;
+	for(it=warp_tracker_map[sid][tid].begin(); it!=warp_tracker_map[sid][tid].end(); it++)
+		if((*it).second != NULL)
+			return true;
 
-         int thd_unlocked = 0;
-         if (shd->model == POST_DOMINATOR || shd->model == NO_RECONVERGE) {
-            thd_unlocked = 1;
-         } else {
-            // unlock the threads here if scheduler is not PDOM or NO-RECONV
-            for (unsigned i=0; i<warp_size; i++) {
-               if (this->tid[i] >= 0) {
-                  shd->thread[tid[i]].avail4fetch++;
-                  assert(shd->thread[tid[i]].avail4fetch <= 1);
-                  assert( shd->warp[tid[i]/warp_size].n_avail4fetch < warp_size );
-                  shd->warp[tid[i]/warp_size].n_avail4fetch++;
-                  thd_unlocked = 1;
-               }
-            }
-         }
-         if (shd->using_commit_queue && thd_unlocked) {
-            int *tid_unlocked = alloc_commit_warp();
-            memcpy(tid_unlocked, this->tid, sizeof(int)*warp_size);
-            dq_push(shd->thd_commit_queue,(void*)tid_unlocked);
-         }
+	return false;
+}
 
-         return true;
-      }
-   }
 
-   // a bookkeeping method to allow a warp to be deallocated 
-   // when its threads have finished executing.
-   bool complete_thd ( int tid_in ) {
-      n_notavail--;
-      if (n_notavail) {
-         return false;
-      } else {
-         return true;
-      }
-   }
-};
+
+warp_tracker_pool& get_warp_tracker_pool(){
+	static warp_tracker_pool* wpt_pool = new warp_tracker_pool(gpu_n_shader, gpu_n_thread_per_shader);
+	return *wpt_pool;
+}
+
+
+//-------------------------------------------------------------------------------
+
+/*
 
 static warp_tracker ***warp_tracker_map;
 static unsigned **g_warp_tracker_map_setl_cycle;
@@ -236,25 +376,69 @@ int wpt_signal_avail( int tid, shader_core_ctx_t *shd )
 
 
    // signal the warp tracker
-   if (wpt->avail_thd(tid)) {
-      // if the warp is ready to be fetched again, remove this warp_tracker
-      for (unsigned i=0; i<warp_size; i++) {
-         if (wpt->tid[i] >= 0) {
-            warp_tracker_map[sid][wpt->tid[i]] = NULL;
-            g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle;
-         }
-      }
-
-      free_warp_tracker( wpt );
-
+   if (wpt->avail_thd()) {
       return 1;
    } else {
       return 0;
    }
 }
 
-void register_cta_thread_exit(shader_core_ctx_t *shader, int cta_num );
+// Unlock a warp
+void wpt_unlock_threads( int tid, shader_core_ctx_t *shd ) {
+   int sid = shd->sid;
+   warp_tracker *wpt = warp_tracker_map[sid][tid];
+   assert(wpt != NULL);
+
+    int thd_unlocked = 0;
+    // Unlock
+    for (unsigned i=0; i<warp_size; i++) {
+		   if (wpt->tid[i] >= 0) {
+			  shd->thread[wpt->tid[i]].avail4fetch++;
+			  assert(shd->thread[wpt->tid[i]].avail4fetch <= 1);
+			  assert( shd->warp[wpt->tid[i]/warp_size].n_avail4fetch < warp_size );
+			  shd->warp[wpt->tid[i]/warp_size].n_avail4fetch++;
+			   thd_unlocked = 1;
+		   }
+	   }
+
+    if (shd->model == POST_DOMINATOR || shd->model == NO_RECONVERGE) {
+   	 // Do nothing
+    } else {
+       // For this case, submit to commit_queue
+       if (shd->using_commit_queue && thd_unlocked) {
+          int *tid_unlocked = alloc_commit_warp();
+          memcpy(tid_unlocked, wpt->tid, sizeof(int)*warp_size);
+          dq_push(shd->thd_commit_queue,(void*)tid_unlocked);
+       }
+    }
+
+    // the warp is ready to be fetched again, remove this warp_tracker
+    for (unsigned i=0; i<warp_size; i++) {
+	   if (wpt->tid[i] >= 0) {
+		  warp_tracker_map[sid][wpt->tid[i]] = NULL;
+		  g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle;
+	   }
+    }
+
+    free_warp_tracker( wpt );
+}
+
+*/
+
+/*
+ * Signal that the a thread is done and is exiting (exit instruction)
+ *
+ * Marks a thread as completed. If all threads in the warp have completed, call register_cta_thread_exit on all
+ * threads and removed the warp from warp tracker.
+ *
+ * @param tid Thread that is exiting
+ * @param *shd Pointer to the shader core
+ *
+ * @return The warp's mask of active threads.
+ */
+//
 
+/*
 int wpt_signal_complete( int tid, shader_core_ctx_t *shd ) 
 {
    int sid = shd->sid;
@@ -281,6 +465,7 @@ int wpt_signal_complete( int tid, shader_core_ctx_t *shd )
       return 0;
    }
 }
+*/
 
 //------------------------------------------------------------------------------------
 
diff --git a/src/gpgpu-sim/warp_tracker.h b/src/gpgpu-sim/warp_tracker.h
index 5375543..1e2e5e0 100644
--- a/src/gpgpu-sim/warp_tracker.h
+++ b/src/gpgpu-sim/warp_tracker.h
@@ -63,6 +63,10 @@
  * Vancouver, BC V6T 1Z4
  */
 
+#include <vector>
+#include <map>
+#include <list>
+
 #ifndef warp_tracker_h_INCLUDED
 #define warp_tracker_h_INCLUDED
 
@@ -82,13 +86,19 @@
 #include "../abstract_hardware_model.h"
 #include "shader.h"
 
-void init_warp_tracker( ); 
+extern unsigned int warp_size;
+extern unsigned int gpu_n_shader;
+extern unsigned int gpu_n_thread_per_shader;
+
+//void init_warp_tracker( );
 
-void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd ); 
+//void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd );
 
-int wpt_signal_avail( int tid, shader_core_ctx_t *shd ); 
+//int wpt_signal_avail( int tid, shader_core_ctx_t *shd );
 
-int wpt_signal_complete( int tid, shader_core_ctx_t *shd ); 
+//void wpt_unlock_threads( int tid, shader_core_ctx_t *shd );
+
+//int wpt_signal_complete( int tid, shader_core_ctx_t *shd );
 
 void print_thread_pc_histogram( FILE *fout );
 void print_thread_pc( FILE *fout );
@@ -97,4 +107,132 @@ void track_thread_pc( int shader_id, int *tid, address_type pc );
 int* alloc_commit_warp( );
 void free_commit_warp( int *commit_warp );
 
+class warp_tracker {
+public:
+   std::vector<int> tid;
+
+   int n_thd; // total number of threads in this warp
+   int n_notavail; // number of threads still not available
+   shader_core_ctx_t *shd; // reference to shader core
+   address_type pc;
+
+   warp_tracker () {
+	  tid.resize(warp_size,-1);
+
+      n_thd = 0;
+      n_notavail = 0;
+      shd = NULL;
+   }
+
+   void print_info(){
+	   printf("sid=%d ", shd->sid);
+
+	   printf("tid=[");
+	   for(unsigned i=0; i<warp_size; i++)
+		   if(tid[i] > -1)
+			   printf("%d ", tid[i]);
+	   printf("]\n");
+   }
+
+   void copy_tid( int *tid ) {
+	   std::copy(tid, tid+warp_size, this->tid.begin());
+   }
+
+   // set the warp to be consist of the given threads
+   void set_warp ( int *tid, shader_core_ctx_t *shd, address_type pc) {
+	  copy_tid(tid);
+
+      this->n_thd = 0;
+      this->n_notavail = 0;
+      for (unsigned i=0; i<warp_size; i++) {
+         if (this->tid[i] >= 0) {
+            this->n_thd++;
+         }
+      }
+      this->n_notavail = this->n_thd;
+      this->shd = shd;
+      this->pc = pc;
+   }
+
+   // signal that this thread is available for fetch
+   // if all threads in the warp are available, change all their status
+   // and return true
+   bool avail_thd ( ) {
+      n_notavail--;
+      return (n_notavail==0);
+   }
+
+   // a bookkeeping method to allow a warp to be deallocated
+   // when its threads have finished executing.
+   bool complete_thd ( int tid_in ) {
+      n_notavail--;
+      if (n_notavail) {
+         return false;
+      } else {
+         return true;
+      }
+   }
+};
+
+//-------------------------------------------------------------------
+
+class warp_tracker_pool
+{
+	private:
+		unsigned gpu_n_shader;
+		unsigned gpu_n_thread_per_shader;
+
+		// Warp tracker map: a vector (index shader id) of vectors (index thread id) of maps (index pc)
+		std::vector< std::vector< std::map<address_type, warp_tracker*> > > warp_tracker_map;
+
+		// Pool of warp trackers
+		std::list<warp_tracker> warp_tracker_list;
+
+		// List to keep track of free warp trackers
+		std::list<warp_tracker*> warp_tracker_free_list;
+
+		warp_tracker* map_get_warp_tracker(int sid, int tid, address_type pc) {
+			// Return NULL pointer if no warp_tracker assigned
+			if(warp_tracker_map[sid][tid].find(pc) == warp_tracker_map[sid][tid].end())
+				return NULL;
+
+			return warp_tracker_map[sid][tid][pc];
+		}
+
+		void map_set_warp_tracker(int sid, int tid, address_type pc, warp_tracker* wpt) {
+			// Make sure that warp tracker is not already assigned here
+			if(warp_tracker_map[sid][tid].find(pc) != warp_tracker_map[sid][tid].end())
+				assert(warp_tracker_map[sid][tid][pc] == NULL);
+
+			warp_tracker_map[sid][tid][pc] = wpt;
+		}
+
+		void map_clear_warp_tracker(int sid, int tid, address_type pc) {
+			// Make sure that warp tracker was previously assigned
+			assert(warp_tracker_map[sid][tid].find(pc) != warp_tracker_map[sid][tid].end());
+			assert(warp_tracker_map[sid][tid][pc] != NULL);
+
+			warp_tracker_map[sid][tid][pc] = NULL;
+		}
+
+		warp_tracker* alloc_warp_tracker( int *tid_in, shader_core_ctx_t *shd, address_type pc );
+		void free_warp_tracker(warp_tracker* wpt);
+
+	public:
+		warp_tracker_pool( unsigned gpu_n_shader, unsigned gpu_n_thread_per_shader  );
+
+		void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd, address_type pc );
+		int wpt_signal_avail( int tid, shader_core_ctx_t *shd, address_type pc );
+		void wpt_deregister_warp( int tid, shader_core_ctx_t *shd, address_type pc );
+		int wpt_signal_complete( int tid, shader_core_ctx_t *shd, address_type pc );
+
+		unsigned size() { return warp_tracker_list.size(); }
+		unsigned free_size() { return warp_tracker_free_list.size(); }
+
+		bool wpt_thread_in_wpt(shader_core_ctx *shd, int tid);
+
+};
+
+warp_tracker_pool& get_warp_tracker_pool();
+
 #endif