1. refactoring cuda api code for loading PTX (removing external PTX loading entirely)

2. some bug fixes for warp_inst_t
3. creating a new class, gpgpu_t, which contains the functional "memory" state visible
   to all threads running on a GPU (doing this as part of my continuing effort to hunt 
   down and eradicate every global variable that is not the top level "the gpu")
4. other misc. changes

Almost passing CUDA 3.1 regression?  oclHistogram keeps failing under torque, but
does not fail when run on the command line from the same directory.

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 7827]

21 files changed, 226 insertions, 183 deletions

diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index f5149eb..a0e21f7 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -1,4 +1,5 @@
 #include "abstract_hardware_model.h"
+#include "cuda-sim/memory.h"
 
 void move_warp( warp_inst_t *&dst, warp_inst_t *&src )
 {
@@ -9,3 +10,12 @@ void move_warp( warp_inst_t *&dst, warp_inst_t *&src )
    src->clear();
 }
 
+gpgpu_t::gpgpu_t()
+{
+   g_global_mem = new memory_space_impl<8192>("global",64*1024);
+   g_param_mem = new memory_space_impl<8192>("param",64*1024);
+   g_tex_mem = new memory_space_impl<8192>("tex",64*1024);
+   g_surf_mem = new memory_space_impl<8192>("surf",64*1024);
+
+   g_dev_malloc=GLOBAL_HEAP_START; 
+}
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 15a7296..c90d56e 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -139,6 +139,44 @@ public:
    virtual class gpgpu_sim *get_gpu() = 0;
 };
 
+#define GLOBAL_HEAP_START 0x80000000
+   // start allocating from this address (lower values used for allocating globals in .ptx file)
+#define SHARED_MEM_SIZE_MAX (64*1024)
+#define LOCAL_MEM_SIZE_MAX (16*1024)
+#define MAX_STREAMING_MULTIPROCESSORS 64
+#define MAX_THREAD_PER_SM 1024
+#define TOTAL_LOCAL_MEM_PER_SM (MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX)
+#define TOTAL_SHARED_MEM (MAX_STREAMING_MULTIPROCESSORS*SHARED_MEM_SIZE_MAX)
+#define TOTAL_LOCAL_MEM (MAX_STREAMING_MULTIPROCESSORS*MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX)
+#define SHARED_GENERIC_START (GLOBAL_HEAP_START-TOTAL_SHARED_MEM)
+#define LOCAL_GENERIC_START (SHARED_GENERIC_START-TOTAL_LOCAL_MEM)
+#define STATIC_ALLOC_LIMIT (GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM+TOTAL_SHARED_MEM))
+
+class gpgpu_t {
+public:
+   gpgpu_t();
+   void* gpgpu_ptx_sim_malloc( size_t size );
+   void* gpgpu_ptx_sim_mallocarray( size_t count );
+   void  gpgpu_ptx_sim_memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count );
+   void  gpgpu_ptx_sim_memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count );
+   void  gpgpu_ptx_sim_memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count );
+   void  gpgpu_ptx_sim_memset( size_t dst_start_addr, int c, size_t count );
+
+   class memory_space *get_global_memory() { return g_global_mem; }
+   class memory_space *get_tex_memory() { return g_tex_mem; }
+   class memory_space *get_surf_memory() { return g_surf_mem; }
+   class memory_space *get_param_memory() { return g_param_mem; }
+
+protected:
+   // functional simulation state 
+   class memory_space *g_global_mem;
+   class memory_space *g_tex_mem;
+   class memory_space *g_surf_mem;
+   class memory_space *g_param_mem;
+
+   unsigned long long g_dev_malloc;
+};
+
 struct gpgpu_ptx_sim_kernel_info 
 {
    // Holds properties of the kernel (Kernel's resource use). 
@@ -306,6 +344,24 @@ public:
         m_per_scalar_thread[lane_id].callback.instruction = inst;
         m_per_scalar_thread[lane_id].callback.thread = thread;
     }
+    void set_active( std::vector<unsigned> &active ) 
+    {
+       warp_active_mask.reset();
+       for( std::vector<unsigned>::iterator i=active.begin(); i!=active.end(); ++i ) {
+          unsigned t = *i;
+          assert( t < m_warp_size );
+          warp_active_mask.set(t);
+       }
+       if( m_isatomic ) {
+          for( unsigned i=0; i < m_warp_size; i++ ) {
+             if( !warp_active_mask.test(i) ) {
+                 m_per_scalar_thread[i].callback.function = NULL;
+                 m_per_scalar_thread[i].callback.instruction = NULL;
+                 m_per_scalar_thread[i].callback.thread = NULL;
+             }
+          }
+       }
+    }
 
     // accessors
     virtual void print_insn(FILE *fp) const 
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index dcf1fd4..7f4eebd 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -309,20 +309,6 @@ void function_info::ptx_assemble()
    m_assembled = true;
 }
 
-
-
-void gpgpu_ptx_sim_init_memory()
-{
-   static bool initialized = false;
-   if ( !initialized ) {
-      g_global_mem = new memory_space_impl<8192>("global",64*1024);
-      g_param_mem = new memory_space_impl<8192>("param",64*1024);
-      g_tex_mem = new memory_space_impl<8192>("tex",64*1024);
-      g_surf_mem = new memory_space_impl<8192>("surf",64*1024);
-      initialized = true;
-   }
-}
-
 addr_t shared_to_generic( unsigned smid, addr_t addr )
 {
    assert( addr < SHARED_MEM_SIZE_MAX );
@@ -392,9 +378,7 @@ addr_t generic_to_global( addr_t addr )
 }
 
 
-unsigned long long g_dev_malloc=GLOBAL_HEAP_START; 
-
-void* gpgpu_ptx_sim_malloc( size_t size )
+void* gpgpu_t::gpgpu_ptx_sim_malloc( size_t size )
 {
    unsigned long long result = g_dev_malloc;
    printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, g_dev_malloc );
@@ -404,7 +388,7 @@ void* gpgpu_ptx_sim_malloc( size_t size )
    return(void*) result;
 }
 
-void* gpgpu_ptx_sim_mallocarray( size_t size )
+void* gpgpu_t::gpgpu_ptx_sim_mallocarray( size_t size )
 {
    unsigned long long result = g_dev_malloc;
    printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, g_dev_malloc );
@@ -415,7 +399,7 @@ void* gpgpu_ptx_sim_mallocarray( size_t size )
 }
 
 
-void gpgpu_ptx_sim_memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count )
+void gpgpu_t::gpgpu_ptx_sim_memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count )
 {
    printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr );
    fflush(stdout);
@@ -426,7 +410,7 @@ void gpgpu_ptx_sim_memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t
    fflush(stdout);
 }
 
-void gpgpu_ptx_sim_memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count )
+void gpgpu_t::gpgpu_ptx_sim_memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count )
 {
    printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst );
    fflush(stdout);
@@ -437,7 +421,7 @@ void gpgpu_ptx_sim_memcpy_from_gpu( void *dst, size_t src_start_addr, size_t cou
    fflush(stdout);
 }
 
-void gpgpu_ptx_sim_memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count )
+void gpgpu_t::gpgpu_ptx_sim_memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count )
 {
    printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, 
           (unsigned long long) src, (unsigned long long) dst );
@@ -451,7 +435,7 @@ void gpgpu_ptx_sim_memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count )
    fflush(stdout);
 }
 
-void gpgpu_ptx_sim_memset( size_t dst_start_addr, int c, size_t count )
+void gpgpu_t::gpgpu_ptx_sim_memset( size_t dst_start_addr, int c, size_t count )
 {
    printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", 
           count, (unsigned char) c, (unsigned long long) dst_start_addr );
@@ -890,6 +874,12 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id )
       }
    }
    if( !skip ) {
+      ptx_instruction *pJ = NULL;
+      if( pI->get_opcode() == VOTE_OP ) {
+         pJ = new ptx_instruction(*pI);
+         *((warp_inst_t*)pJ) = inst;
+         pI = pJ;
+      }
       switch ( pI->get_opcode() ) {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break;
 #include "opcodes.def"
@@ -899,6 +889,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id )
             printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() );
             break;
       }
+      delete pJ;
 
       // Run exit instruction if exit option included
       if(pI->is_exit())
@@ -1045,11 +1036,6 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id )
    }
 }
 
-std::list<ptx_thread_info *> g_active_threads;
-std::map<unsigned,memory_space*> g_shared_memory_lookup;
-std::map<unsigned,ptx_cta_info*> g_ptx_cta_lookup;
-std::map<unsigned,std::map<unsigned,memory_space*> > g_local_memory_lookup;
-
 void set_param_gpgpu_num_shaders(int num_shaders)
 {
    gpgpu_param_num_shaders = num_shaders;
@@ -1073,8 +1059,14 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
                               unsigned num_threads, 
                               core_t *core, 
                               unsigned hw_cta_id, 
-                              unsigned hw_warp_id )
+                              unsigned hw_warp_id,
+                              gpgpu_t *gpu )
 {
+   static std::list<ptx_thread_info *> active_threads;
+   static std::map<unsigned,memory_space*> shared_memory_lookup;
+   static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup;
+   static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup;
+
    if ( *thread_info != NULL ) {
       ptx_thread_info *thd = *thread_info;
       assert( thd->is_done() );
@@ -1091,16 +1083,12 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       *thread_info = NULL;
    }
 
-   if ( !g_active_threads.empty() ) { //if g_active_threads not empty...
-      assert( g_active_threads.size() <= threads_left );
-      ptx_thread_info *thd = g_active_threads.front(); 
-      g_active_threads.pop_front();
+   if ( !active_threads.empty() ) { //if g_active_threads not empty...
+      assert( active_threads.size() <= threads_left );
+      ptx_thread_info *thd = active_threads.front(); 
+      active_threads.pop_front();
       *thread_info = thd;
-      thd->set_hw_tid(tid);
-      thd->set_hw_wid(hw_warp_id);
-      thd->set_hw_ctaid(hw_cta_id);
-      thd->set_core(core);
-      thd->set_hw_sid(sid);
+      thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid );
       return 1;
    }
 
@@ -1127,7 +1115,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
 
    unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid;
 
-   if ( g_shared_memory_lookup.find(sm_idx) == g_shared_memory_lookup.end() ) {
+   if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) {
       if ( g_debug_execution >= 1 ) {
          printf("  <CTA alloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", 
                 sm_idx, sid, max_cta_per_sm );
@@ -1135,20 +1123,20 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       char buf[512];
       snprintf(buf,512,"shared_%u", sid);
       shared_mem = new memory_space_impl<16*1024>(buf,4);
-      g_shared_memory_lookup[sm_idx] = shared_mem;
+      shared_memory_lookup[sm_idx] = shared_mem;
       cta_info = new ptx_cta_info(sm_idx);
-      g_ptx_cta_lookup[sm_idx] = cta_info;
+      ptx_cta_lookup[sm_idx] = cta_info;
    } else {
       if ( g_debug_execution >= 1 ) {
          printf("  <CTA realloc> : sm_idx=%u sid=%u max_cta_per_sm=%u\n", 
                 sm_idx, sid, max_cta_per_sm );
       }
-      shared_mem = g_shared_memory_lookup[sm_idx];
-      cta_info = g_ptx_cta_lookup[sm_idx];
+      shared_mem = shared_memory_lookup[sm_idx];
+      cta_info = ptx_cta_lookup[sm_idx];
       cta_info->check_cta_thread_status_and_reset();
    }
 
-   std::map<unsigned,memory_space*> &local_mem_lookup = g_local_memory_lookup[sid];
+   std::map<unsigned,memory_space*> &local_mem_lookup = local_memory_lookup[sid];
    while( kernel.more_threads_in_cta() ) {
       dim3 ctaid3d = kernel.get_next_cta_id();
       unsigned new_tid = kernel.get_next_thread_id();
@@ -1174,11 +1162,6 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       thd->set_tid(tid3d);
       if( kernel.entry()->get_ptx_version().extensions() ) 
          thd->cpy_tid_to_reg(tid3d);
-      thd->set_hw_tid((unsigned)-1);
-      thd->set_hw_wid((unsigned)-1);
-      thd->set_hw_ctaid((unsigned)-1);
-      thd->set_core(NULL);
-      thd->set_hw_sid((unsigned)-1);
       thd->set_valid();
       thd->m_shared_mem = shared_mem;
       function_info *finfo = thd->func_info();
@@ -1192,7 +1175,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
                 ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd );
          fflush(stdout);
       }
-      g_active_threads.push_back(thd);
+      active_threads.push_back(thd);
    }
    if ( g_debug_execution==-1 ) {
       printf("GPGPU-Sim PTX simulator:  <-- FINISHING THREAD ALLOCATION\n");
@@ -1201,16 +1184,10 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
 
    kernel.increment_cta_id();
 
-   assert( g_active_threads.size() <= threads_left );
-
-   *thread_info = g_active_threads.front();
-   (*thread_info)->set_hw_tid(tid);
-   (*thread_info)->set_hw_wid(hw_warp_id);
-   (*thread_info)->set_hw_ctaid(hw_cta_id);
-   (*thread_info)->set_core(core);
-   (*thread_info)->set_hw_sid(sid);
-   g_active_threads.pop_front();
-
+   assert( active_threads.size() <= threads_left );
+   *thread_info = active_threads.front();
+   (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid );
+   active_threads.pop_front();
    return 1;
 }
 
@@ -1220,7 +1197,11 @@ size_t get_kernel_code_size( class function_info *entry )
 }
 
 
-kernel_info_t gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,gpgpu_ptx_sim_arg_list_t args, struct dim3 gridDim, struct dim3 blockDim )
+kernel_info_t gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
+                                             gpgpu_ptx_sim_arg_list_t args, 
+                                             struct dim3 gridDim,
+                                             struct dim3 blockDim,
+                                             gpgpu_t *gpu )
 {
    unsigned argcount=args.size();
    unsigned argn=1;
@@ -1228,7 +1209,7 @@ kernel_info_t gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,gpgpu_pt
       entry->add_param_data(argcount-argn,&(*a));
       argn++;
    }
-   entry->finalize(g_param_mem);
+   entry->finalize(gpu->get_param_memory());
    g_ptx_kernel_count++; 
    fflush(stdout);
 
@@ -1263,7 +1244,7 @@ void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceNam
    g_global_name_lookup[hostVar] = deviceName;
 }
 
-void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to )
+void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu )
 {
    printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar);
    bool found_sym = false;
@@ -1314,11 +1295,11 @@ void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t co
    unsigned dst = sym->get_address() + offset; 
    switch (mem_region.get_type()) {
    case const_space:
-      mem = g_global_mem;
+      mem = gpu->get_global_memory();
       mem_name = "global";
       break;
    case global_space:
-      mem = g_global_mem;
+      mem = gpu->get_global_memory();
       mem_name = "global";
       break;
    default:
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index 322e8a2..b72a157 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -13,7 +13,6 @@ class symbol_table;
 
 extern const char *g_gpgpusim_version_string;
 extern int g_ptx_sim_mode;
-extern memory_space *g_global_mem;
 extern int g_debug_execution;
 extern int g_debug_thread_uid;
 extern void ** g_inst_classification_stat;
@@ -25,20 +24,13 @@ extern FILE* ptx_inst_debug_file;
 extern class kernel_info_t gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
                                             gpgpu_ptx_sim_arg_list_t args, 
                                             struct dim3 gridDim, 
-                                            struct dim3 blockDim );
+                                            struct dim3 blockDim, 
+                                                          class gpgpu_t *gpu );
 extern void gpgpu_cuda_ptx_sim_main_func( kernel_info_t kernel, dim3 gridDim, dim3 blockDim, gpgpu_ptx_sim_arg_list_t args);
 extern void   print_splash();
-extern void*  gpgpu_ptx_sim_malloc( size_t count );
-extern void*  gpgpu_ptx_sim_mallocarray( size_t count );
-extern void   gpgpu_ptx_sim_memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count );
-extern void   gpgpu_ptx_sim_memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count );
-extern void   gpgpu_ptx_sim_memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count );
-extern void   gpgpu_ptx_sim_memset( size_t dst_start_addr, int c, size_t count );
-extern void   gpgpu_ptx_sim_init_memory();
-extern void   gpgpu_ptx_sim_register_kernel(void **fatCubinHandle,const char *hostFun, const char *deviceFun);
 extern void   gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size );
 extern void   gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size );
-extern void   gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to );
+extern void   gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu );
 
 extern void   gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array);
 extern void   gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref);
@@ -55,7 +47,8 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
                               unsigned num_threads, 
                               class core_t *core, 
                               unsigned hw_cta_id, 
-                              unsigned hw_warp_id );
+                              unsigned hw_warp_id,
+                              gpgpu_t *gpu );
 const warp_inst_t *ptx_fetch_inst( address_type pc );
 const struct gpgpu_ptx_sim_kernel_info* ptx_sim_kernel_info(class function_info *kernel);
 void ptx_print_insn( address_type pc, FILE *fp );
diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc
index 9edeb5e..8483dd9 100644
--- a/src/cuda-sim/cuda_device_printf.cc
+++ b/src/cuda-sim/cuda_device_printf.cc
@@ -65,7 +65,7 @@
 #include "cuda_device_printf.h"
 #include "ptx_ir.h"
 
-void decode_space( memory_space_t &space, const ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr);
+void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr);
 
 void my_cuda_printf(const char *fmtstr,const char *arg_list)
 {
@@ -104,7 +104,7 @@ void my_cuda_printf(const char *fmtstr,const char *arg_list)
    }
 }
 
-void gpgpusim_cuda_vprintf(const ptx_instruction * pI, const ptx_thread_info * thread, const function_info * target_func ) 
+void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) 
 {
       char *fmtstr = NULL;
       char *arg_list = NULL;
diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h
index 022b555..7b2a5ef 100644
--- a/src/cuda-sim/cuda_device_printf.h
+++ b/src/cuda-sim/cuda_device_printf.h
@@ -65,6 +65,6 @@
 #ifndef CUDA_DEVICE_PRINTF_INCLUDED
 #define CUDA_DEVICE_PRINTF_INCLUDED
 
-void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, const class ptx_thread_info * thread, const class function_info * target_func );
+void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func );
 
 #endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 23d6cc8..335ea59 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -249,7 +249,7 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
    //complete other cases for reading from memory, such as reading from other const memory
    if((op.get_addr_space() == 1)&&(derefFlag)) {
        // global memory - g[4], g[$r0]
-       mem = g_global_mem;
+       mem = thread->get_global_memory();
        type_info_key::type_decode(opType,size,t);
        mem->read(result.u32,size/8,&finalResult.u128);
        thread->m_last_effective_address = result.u32;
@@ -269,7 +269,7 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
          sign_extend(finalResult,size,dstInfo);
    } else if((op.get_addr_space() == 3)&&(derefFlag)) {
       // const memory - ce0c1[4], ce0c1[$r0]
-       mem = g_global_mem;
+       mem = thread->get_global_memory();
        type_info_key::type_decode(opType,size,t);
        mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128);
        thread->m_last_effective_address = result.u32;
@@ -606,7 +606,7 @@ void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_
    else if(dst.get_addr_space() == 1)
    {
        dstData = thread->get_operand_value(dst, dst, type, thread, 0);
-       mem = g_global_mem;
+       mem = thread->get_global_memory();
        type_info_key::type_decode(type,size,t);
 
        mem->write(dstData.u32,size/8,&data.u128,thread,pI);
@@ -863,7 +863,7 @@ void atom_callback( const inst_t* inst, ptx_thread_info* thread )
 
    // Copy value pointed to in operand 'a' into register 'd'
    // (i.e. copy src1_data to dst)
-   g_global_mem->read(src1_data.u32,size/8,&data.s64);
+   thread->get_global_memory()->read(src1_data.u32,size/8,&data.s64);
    thread->set_operand_value(dst, data, to_type, thread, pI);                         // Write value into register 'd'
 
    // Get the atomic operation to be performed
@@ -1086,7 +1086,7 @@ void atom_callback( const inst_t* inst, ptx_thread_info* thread )
 
    // Write operation result into global memory
    // (i.e. copy src1_data to dst)
-   g_global_mem->write(src1_data.u32,size/8,&op_result.s64,thread,pI);
+   thread->get_global_memory()->write(src1_data.u32,size/8,&op_result.s64,thread,pI);
    gpgpu_sim *gpu = thread->get_gpu();
    gpu->decrement_atomic_count(thread->get_hw_sid(),thread->get_hw_wid());
 }
@@ -1998,7 +1998,7 @@ void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread )
    thread->set_reg(dst.get_symbol(),p);
 }
 
-void decode_space( memory_space_t &space, const ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr)
+void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr)
 {
    unsigned smid = thread->get_hw_sid();
    unsigned hwtid = thread->get_hw_tid();
@@ -2018,23 +2018,23 @@ void decode_space( memory_space_t &space, const ptx_thread_info *thread, const o
       }
    }
    switch ( space.get_type() ) {
-   case global_space: mem = g_global_mem; break;
+   case global_space: mem = thread->get_global_memory(); break;
    case param_space_local:
    case local_space:
       mem = thread->m_local_mem; 
       addr += thread->get_local_mem_stack_pointer();
       break; 
-   case tex_space:    mem = g_tex_mem; break; 
-   case surf_space:   mem = g_surf_mem; break; 
-   case param_space_kernel:  mem = g_param_mem; break;
+   case tex_space:    mem = thread->get_tex_memory(); break; 
+   case surf_space:   mem = thread->get_surf_memory(); break; 
+   case param_space_kernel:  mem = thread->get_param_memory(); break;
    case shared_space:  mem = thread->m_shared_mem; break; 
-   case const_space:  mem = g_global_mem; break;
+   case const_space:  mem = thread->get_global_memory(); break;
    case generic_space:
       if( thread->get_ptx_version().ver() >= 2.0 ) {
          // convert generic address to memory space address
          space = whichspace(addr);
          switch ( space.get_type() ) {
-         case global_space: mem = g_global_mem; addr = generic_to_global(addr); break;
+         case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break;
          case local_space:  mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; 
          case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; 
          default: abort();
@@ -3555,7 +3555,7 @@ void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread )
 
    //assume always 2D f32 input
    //access array with src2 coordinates
-   memory_space *mem = g_global_mem;
+   memory_space *mem = thread->get_global_memory();
    float x_f32,  y_f32;
    size_t size;
    int t;
@@ -3799,12 +3799,10 @@ void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread )
       threads_in_warp.clear();
       and_all = true;
       or_all = false;
-      unsigned mask=0x80000000;
-      unsigned offset=31;
-      while( mask && !pI->active(mask) ) {
-         mask = mask>>1;
+      int offset=31;
+      while( (offset>=0) && !pI->active(offset) ) 
          offset--;
-      }
+      assert( offset >= 0 );
       last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset;
    }
 
@@ -3883,7 +3881,7 @@ ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_inf
    //complete other cases for reading from memory, such as reading from other const memory
    if(opInfo.get_addr_space() == 1)
    {
-       mem = g_global_mem;
+       mem = thread->get_global_memory();
        type_info_key::type_decode(type,size,t);
        mem->read(opData.u32,size/8,&result.u64);
        if( type == S16_TYPE || type == S32_TYPE ) 
@@ -3901,7 +3899,7 @@ ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_inf
    }
    else if(opInfo.get_addr_space() == 3)
    {
-       mem = g_global_mem;
+       mem = thread->get_global_memory();
        type_info_key::type_decode(type,size,t);
 
        mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64);
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index b7ec3ac..8172f80 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -1162,7 +1162,7 @@ void ptx_instruction::print_insn( FILE *fp ) const
    snprintf(buf,1024,"%s", m_source.c_str());
    p = strtok(buf,";");
    if( !is_label() ) 
-      fprintf(fp," PC=%3u [%3u] ", m_PC, m_instr_mem_index );
+      fprintf(fp," PC=0x%03x ", m_PC );
    else
       fprintf(fp,"                " );
    fprintf(fp,"(%s:%u) %s", m_source_file.c_str(), m_source_line, p );
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 6010caf..612326c 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -803,7 +803,6 @@ public:
                     const char *source,
                     unsigned warp_size );
 
-
    void print_insn() const;
    virtual void print_insn( FILE *fp ) const;
    unsigned inst_size() const { return m_inst_size; }
@@ -1404,20 +1403,6 @@ struct textureInfo {
 
 extern std::map<std::string,symbol_table*> g_sym_name_to_symbol_table;
 
-#define GLOBAL_HEAP_START 0x80000000
-   // start allocating from this address (lower values used for allocating globals in .ptx file)
-#define SHARED_MEM_SIZE_MAX (64*1024)
-#define LOCAL_MEM_SIZE_MAX (16*1024)
-#define MAX_STREAMING_MULTIPROCESSORS 64
-#define MAX_THREAD_PER_SM 1024
-#define TOTAL_LOCAL_MEM_PER_SM (MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX)
-#define TOTAL_SHARED_MEM (MAX_STREAMING_MULTIPROCESSORS*SHARED_MEM_SIZE_MAX)
-#define TOTAL_LOCAL_MEM (MAX_STREAMING_MULTIPROCESSORS*MAX_THREAD_PER_SM*LOCAL_MEM_SIZE_MAX)
-#define SHARED_GENERIC_START (GLOBAL_HEAP_START-TOTAL_SHARED_MEM)
-#define LOCAL_GENERIC_START (SHARED_GENERIC_START-TOTAL_LOCAL_MEM)
-#define STATIC_ALLOC_LIMIT (GLOBAL_HEAP_START - (TOTAL_LOCAL_MEM+TOTAL_SHARED_MEM))
-
-
 
 extern bool g_keep_intermediate_files;
 
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index d4f3822..2387528 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -87,9 +87,6 @@ extern "C" int ptxinfo_parse();
 extern "C" int ptxinfo_debug;
 extern "C" FILE *ptxinfo_in;
 
-extern int g_ptx_save_converted_ptxplus;
-
-
 static bool g_save_embedded_ptx;
 bool g_keep_intermediate_files;
 
@@ -129,7 +126,7 @@ void print_ptx_file( const char *p, unsigned source_num, const char *filename )
    fflush(stdout);
 }
 
-char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubin_str, unsigned source_num)
+char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubin_str, unsigned source_num, bool save_converted )
 {
 	printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n");
 
@@ -201,7 +198,7 @@ char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubi
 	strcpy(ptxplus_str, text.c_str());
 
 	// Save ptxplus to file if specified
-	if(g_ptx_save_converted_ptxplus) {
+	if(save_converted) {
 	    char fname_ptxplus_save[1024];
 	    snprintf(fname_ptxplus_save,1024,"_%u.ptxplus", source_num );
 		printf("GPGPU-Sim PTX: saving converted ptxplus to file \"%s\"\n", fname_ptxplus_save);
@@ -224,11 +221,10 @@ char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubi
 	printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n");
 
 	return ptxplus_str;
-
 }
 
 
-symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_for_info,  unsigned source_num )
+symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num )
 {
     char buf[1024];
     snprintf(buf,1024,"_%u.ptx", source_num );
@@ -242,7 +238,7 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_f
     int errors = ptx_parse ();
     if ( errors ) {
         char fname[1024];
-        snprintf(fname,1024,"_ptx_XXXXXX");
+        snprintf(fname,1024,"_ptx_errors_XXXXXX");
         int fd=mkstemp(fname); 
         close(fd);
         printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname);
@@ -257,7 +253,11 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_f
        print_ptx_file(p,source_num,buf);
 
     printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf);
+    return symtab;
+}
 
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num )
+{
     char fname[1024];
     snprintf(fname,1024,"_ptx_XXXXXX");
     int fd=mkstemp(fname); 
@@ -287,9 +287,11 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_f
     char commandline[1024];
     char extra_flags[1024];
     extra_flags[0]=0;
+
 #if CUDART_VERSION >= 3000
     snprintf(extra_flags,1024,"--gpu-name=sm_20");
 #endif
+
     snprintf(commandline,1024,"ptxas %s -v %s --output-file /dev/null 2> %s", 
              extra_flags, fname2, tempfile_ptxinfo);
     printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
@@ -310,6 +312,5 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_f
        printf("GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result);
        exit(1);
     }
-    return symtab;
 }
 
diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h
index e9efd9b..da0d10e 100644
--- a/src/cuda-sim/ptx_loader.h
+++ b/src/cuda-sim/ptx_loader.h
@@ -66,15 +66,10 @@
 #ifndef PTX_LOADER_H_INCLUDED
 #define PTX_LOADER_H_INCLUDED
 
-class memory_space;
-
-extern memory_space *g_global_mem;
-extern memory_space *g_tex_mem;
-extern memory_space *g_surf_mem;
-extern memory_space *g_param_mem;
 extern bool g_override_embedded_ptx;
  
-class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, const char *p_for_info, unsigned source_num );
-char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubin_str, unsigned source_num);
+class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num );
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num );
+char* gpgpu_ptx_sim_convert_ptx_to_ptxplus(const char *ptx_str, const char *cubin_str, unsigned source_num, bool save_converted );
 
 #endif
diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc
index 2cd0ba6..59f4522 100644
--- a/src/cuda-sim/ptx_sim.cc
+++ b/src/cuda-sim/ptx_sim.cc
@@ -245,6 +245,7 @@ ptx_thread_info::ptx_thread_info()
    m_last_was_call = false;
    m_enable_debug_trace = false;
    m_local_mem_stack_pointer = 0;
+   m_gpu = NULL;
 }
 
 const ptx_version &ptx_thread_info::get_ptx_version() const 
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index 80ccde3..298122e 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -276,6 +276,16 @@ public:
    ~ptx_thread_info();
    ptx_thread_info();
 
+   void init(gpgpu_t *gpu, core_t *core, unsigned sid, unsigned cta_id, unsigned wid, unsigned tid ) 
+   { 
+      m_gpu = gpu;
+      m_core = core; 
+      m_hw_sid=sid;
+      m_hw_ctaid=cta_id;
+      m_hw_wid=wid;
+      m_hw_tid=tid;
+   }
+
    void ptx_fetch_inst( inst_t &inst ) const;
    void ptx_exec_inst( warp_inst_t &inst, unsigned lane_id );
 
@@ -310,11 +320,6 @@ public:
    unsigned get_hw_ctaid() const { return m_hw_ctaid;}
    unsigned get_hw_wid() const { return m_hw_wid;}
    unsigned get_hw_sid() const { return m_hw_sid;}
-   void set_hw_tid(unsigned tid) { m_hw_tid=tid;}
-   void set_hw_wid(unsigned wid) { m_hw_wid=wid;}
-   void set_hw_sid(unsigned sid) { m_hw_sid=sid;}
-   void set_hw_ctaid(unsigned cta_id) { m_hw_ctaid=cta_id;}
-   void set_core(core_t *core) { m_core = core; }
    core_t *get_core() { return m_core; }
 
    unsigned get_icount() const { return m_icount;}
@@ -421,6 +426,11 @@ public:
    void enable_debug_trace() { m_enable_debug_trace = true; }
    unsigned get_local_mem_stack_pointer() const { return m_local_mem_stack_pointer; }
 
+   memory_space *get_global_memory() { return m_gpu->get_global_memory(); }
+   memory_space *get_tex_memory() { return m_gpu->get_tex_memory(); }
+   memory_space *get_surf_memory() { return m_gpu->get_surf_memory(); }
+   memory_space *get_param_memory() { return m_gpu->get_param_memory(); }
+
 public:
    addr_t         m_last_effective_address;
    bool        m_branch_taken;
@@ -436,6 +446,7 @@ private:
 
    unsigned m_uid;
    core_t *m_core;
+   gpgpu_t *m_gpu;
    bool   m_valid;
    dim3   m_ntid;
    dim3   m_tid;
diff --git a/src/debug.cc b/src/debug.cc
index 7bc72ac..c1aa3c6 100644
--- a/src/debug.cc
+++ b/src/debug.cc
@@ -61,7 +61,8 @@ void gpgpu_sim::gpgpu_debug()
       brk_pt &b=i->second;
       if( b.is_watchpoint() ) {
          unsigned addr = b.get_addr();
-         unsigned new_value = read_location(addr);
+         unsigned new_value; 
+         g_global_mem->read(addr,4,&new_value);
          if( new_value != b.get_value() || g_watchpoint_hits.find(num) != g_watchpoint_hits.end() ) {
             printf( "GPGPU-Sim PTX DBG: watch point %u triggered (old value=%x, new value=%x)\n",
                      num,b.get_value(),new_value );
@@ -155,7 +156,8 @@ void gpgpu_sim::gpgpu_debug()
          tok = strtok(NULL," \t\n");
          unsigned addr;
          sscanf(tok,"%x",&addr);
-         unsigned value = read_location(addr);
+         unsigned value; 
+         g_global_mem->read(addr,4,&value);
          g_global_mem->set_watch(addr,next_brkpt); 
          breakpoints[next_brkpt++] = brk_pt(addr,value);
       } else if( !strcmp(tok,"l") ) {
@@ -199,9 +201,3 @@ bool thread_at_brkpt( ptx_thread_info *thread, const struct brk_pt &b )
    return b.is_equal(thread->get_location(),thread->get_uid());
 }
 
-unsigned read_location( addr_t addr )
-{
-   unsigned result=0;
-   g_global_mem->read(addr,4,&result);
-   return result;
-}
diff --git a/src/debug.h b/src/debug.h
index 7c123a3..0884c99 100644
--- a/src/debug.h
+++ b/src/debug.h
@@ -61,7 +61,6 @@ extern int gpgpu_ptx_instruction_classification ;
 class ptx_thread_info;
 class ptx_instruction;
 bool thread_at_brkpt( ptx_thread_info *thd_info, const struct brk_pt &b );
-unsigned read_location( addr_t addr );
 void hit_watchpoint( unsigned watchpoint_num, ptx_thread_info *thd, const ptx_instruction *pI );
 
 #endif
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 96d5c62..77c6aae 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -174,10 +174,6 @@ int   g_ptx_inst_debug_to_file;
 char* g_ptx_inst_debug_file;
 int   g_ptx_inst_debug_thread_uid;
 
-int   g_ptx_convert_to_ptxplus;
-int   g_ptx_save_converted_ptxplus;
-unsigned g_ptx_force_max_capability;
-
 void visualizer_options(option_parser_t opp);
 
 void gpgpu_sim::reg_options(option_parser_t opp)
@@ -371,15 +367,15 @@ void gpgpu_sim::reg_options(option_parser_t opp)
                "Thread UID for executed instructions' debug output", 
                "1");
    option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL,
-                &g_ptx_convert_to_ptxplus,
+                &m_ptx_convert_to_ptxplus,
                 "Convert embedded ptx to ptxplus",
                 "0");
    option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL,
-                &g_ptx_save_converted_ptxplus,
+                &m_ptx_save_converted_ptxplus,
                 "Saved converted ptxplus to a file",
                 "0");
    option_parser_register(opp, "-gpgpu_ptx_force_max_capability", OPT_UINT32,
-                &g_ptx_force_max_capability,
+                &m_ptx_force_max_capability,
                 "Force maximum compute capability",
                 "0");
    option_parser_register(opp, "-gpgpu_operand_collector", OPT_BOOL, &m_shader_config->gpgpu_operand_collector,
@@ -1080,7 +1076,7 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     for (unsigned i = start_thread; i<end_thread; i++) {
         m_thread[i].m_cta_id = free_cta_hw_id;
         unsigned warp_id = i/m_config->warp_size;
-        nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i].m_functional_model_thread_state,m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id);
+        nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i].m_functional_model_thread_state,m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_gpu);
         warps.set( warp_id );
     }
     assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index ac03cbc..d690fab 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -178,7 +178,7 @@ extern int   g_ptx_inst_debug_thread_uid;
 
 
 
-class gpgpu_sim {
+class gpgpu_sim : public gpgpu_t {
 public:
    gpgpu_sim();
 
@@ -211,6 +211,10 @@ public:
    void gpu_print_stat() const;
    void dump_pipeline( int mask, int s, int m ) const;
 
+   unsigned get_forced_capability() const { return m_ptx_force_max_capability; }
+   bool convert_to_ptxplus() const { return m_ptx_convert_to_ptxplus; }
+   bool saved_converted_ptxplus() const { return m_ptx_save_converted_ptxplus; }
+
 private:
    // clocks
    void init_clock_domains(void);
@@ -271,6 +275,9 @@ private:
 
    // options
    bool gpu_deadlock_detect;
+   int   m_ptx_convert_to_ptxplus;
+   int   m_ptx_save_converted_ptxplus;
+   unsigned m_ptx_force_max_capability;
 
    // stats
    struct shader_core_stats  *m_shader_stats;
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 061952a..54e4ffa 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -593,7 +593,7 @@ void pdom_warp_ctx_t::print (FILE *fout) const
         }
         for (unsigned m=1,j=0; j<m_warp_size; j++, m<<=1)
             fprintf(fout, "%c", ((warp->m_active_mask[k] & m)?'1':'0') );
-        fprintf(fout, " pc: %4u", warp->m_pc[k] );
+        fprintf(fout, " pc: 0x%03x", warp->m_pc[k] );
         if ( warp->m_recvg_pc[k] == (unsigned)-1 ) {
             fprintf(fout," rp: ---- cd: %2u ", warp->m_calldepth[k] );
         } else {
@@ -639,13 +639,14 @@ void shader_core_ctx::fetch_new()
         // decode 1 or 2 instructions and place them into ibuffer
         address_type pc = m_inst_fetch_buffer.m_pc;
         const warp_inst_t* pI1 = ptx_fetch_inst(pc);
-        assert(pI1);
         m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(0,pI1);
         m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline();
-        const warp_inst_t* pI2 = ptx_fetch_inst(pc+pI1->isize);
-        if( pI2 ) {
-            m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(1,pI2);
-            m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline();
+        if( pI1 ) {
+           const warp_inst_t* pI2 = ptx_fetch_inst(pc+pI1->isize);
+           if( pI2 ) {
+               m_warp[m_inst_fetch_buffer.m_warp_id].ibuffer_fill(1,pI2);
+               m_warp[m_inst_fetch_buffer.m_warp_id].inc_inst_in_pipeline();
+           }
         }
         m_inst_fetch_buffer.m_valid = false;
     }
@@ -767,10 +768,12 @@ void shader_core_ctx::decode_new()
         unsigned issued=0;
         while( !m_warp[warp_id].waiting() && !m_warp[warp_id].ibuffer_empty() && (checked < 2) && (issued < 2) ) {
             unsigned active_mask = m_pdom_warp[warp_id]->get_active_mask();
-            const warp_inst_t *pI = m_warp[warp_id].ibuffer_next();
+            const warp_inst_t *pI = m_warp[warp_id].ibuffer_next_inst();
+            bool valid = m_warp[warp_id].ibuffer_next_valid();
             unsigned pc,rpc;
             m_pdom_warp[warp_id]->get_pdom_stack_top_info(&pc,&rpc);
             if( pI ) {
+                assert(valid);
                 if( pc != pI->pc ) {
                     // control hazard
                     m_warp[warp_id].set_next_pc(pc);
@@ -785,6 +788,10 @@ void shader_core_ctx::decode_new()
                         issued++;
                     } 
                 }
+            } else if( valid ) {
+               // this case can happen after a return instruction in diverged warp
+               m_warp[warp_id].set_next_pc(pc);
+               m_warp[warp_id].ibuffer_flush();
             }
             m_warp[warp_id].ibuffer_step();
             checked++;
@@ -792,7 +799,7 @@ void shader_core_ctx::decode_new()
         if ( issued ) {
             m_last_warp_issued=warp_id;
             break;
-        }
+        } 
     }
 }
 
@@ -852,7 +859,9 @@ mshr_entry* mshr_shader_unit::add_mshr(mem_access_t &access, warp_inst_t* warp)
     // creates an mshr based on the access struct information
     mshr_entry* mshr = alloc_free_mshr(access.space == tex_space);
     mshr->init(access.addr,access.iswrite,access.space,warp->warp_id());
-    mshr->add_inst(*warp);
+    warp_inst_t inst = *warp;
+    inst.set_active(access.warp_indices);
+    mshr->add_inst(inst);
     if( m_shader_config->gpgpu_interwarp_mshr_merge ) {
         mshr_entry* mergehit = m_mshr_lookup.shader_get_mergeable_mshr(mshr);
         if (mergehit) {
@@ -2025,8 +2034,10 @@ void shd_warp_t::print_ibuffer( FILE *fout ) const
 {
     fprintf(fout,"  ibuffer[%2u] : ", m_warp_id );
     for( unsigned i=0; i < IBUFFER_SIZE; i++) {
-        const inst_t *inst = m_ibuffer[i];
+        const inst_t *inst = m_ibuffer[i].m_inst;
         if( inst ) inst->print_insn(fout);
+        else if( m_ibuffer[i].m_valid ) 
+           fprintf(fout," <invalid instruction> ");
         else fprintf(fout," <empty> ");
     }
     fprintf(fout,"\n");
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index d9a8825..cdbaf37 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -146,8 +146,6 @@ public:
         m_done_exit=false;
         m_last_fetch=0;
         m_next=0;
-        for(unsigned i=0;i<IBUFFER_SIZE;i++) 
-            m_ibuffer[i]=NULL; 
     }
     void init( address_type start_pc, unsigned cta_id, unsigned wid, unsigned active )
     {
@@ -186,31 +184,32 @@ public:
     void ibuffer_fill( unsigned slot, const warp_inst_t *pI )
     {
        assert(slot < IBUFFER_SIZE );
-       m_ibuffer[slot]=pI;
+       m_ibuffer[slot].m_inst=pI;
+       m_ibuffer[slot].m_valid=true;
        m_next=0; 
     }
     bool ibuffer_empty() const
     {
         for( unsigned i=0; i < IBUFFER_SIZE; i++) 
-            if(m_ibuffer[i]) 
+            if(m_ibuffer[i].m_valid) 
                 return false;
         return true;
     }
     void ibuffer_flush()
     {
         for(unsigned i=0;i<IBUFFER_SIZE;i++) {
-            if( m_ibuffer[i] )
+            if( m_ibuffer[i].m_valid )
                 dec_inst_in_pipeline();
-            m_ibuffer[i]=NULL; 
+            m_ibuffer[i].m_inst=NULL; 
+            m_ibuffer[i].m_valid=false; 
         }
     }
-    const warp_inst_t *ibuffer_next()
-    {
-        return m_ibuffer[m_next];
-    }
+    const warp_inst_t *ibuffer_next_inst() { return m_ibuffer[m_next].m_inst; }
+    bool ibuffer_next_valid() { return m_ibuffer[m_next].m_valid; }
     void ibuffer_free()
     {
-        m_ibuffer[m_next] = NULL;
+        m_ibuffer[m_next].m_inst = NULL;
+        m_ibuffer[m_next].m_valid = false;
     }
     void ibuffer_step()
     {
@@ -252,8 +251,13 @@ private:
     unsigned n_completed;          // number of threads in warp completed
 
     class mshr_entry *m_imiss_pending;
-                                  
-    const warp_inst_t *m_ibuffer[IBUFFER_SIZE]; 
+    
+    struct ibuffer_entry {
+       ibuffer_entry() { m_valid = false; m_inst = NULL; }
+       const warp_inst_t *m_inst;
+       bool m_valid;
+    };
+    ibuffer_entry m_ibuffer[IBUFFER_SIZE]; 
     unsigned m_next;
                                    
     unsigned m_n_atomic;           // number of outstanding atomic operations 
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 7947a8f..6f13cd2 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -167,19 +167,18 @@ int gpgpu_cuda_ptx_sim_main_perf( kernel_info_t grid,
    return 0;
 }
 
-int gpgpu_opencl_ptx_sim_main_perf( class function_info *entry, 
+int gpgpu_opencl_ptx_sim_main_perf( kernel_info_t grid,
                                   struct dim3 gridDim, 
                                   struct dim3 blockDim, 
                                   gpgpu_ptx_sim_arg_list_t grid_params )
 {
-   kernel_info_t grid = gpgpu_opencl_ptx_sim_init_grid(entry,grid_params,gridDim,blockDim);
    g_the_gpu.launch(grid);
    sem_post(&g_sim_signal_start);
    sem_wait(&g_sim_signal_finish);
    return 0;
 }
 
-int gpgpu_opencl_ptx_sim_main_func( class function_info *entry, 
+int gpgpu_opencl_ptx_sim_main_func( kernel_info_t grid,
                                   struct dim3 gridDim, 
                                   struct dim3 blockDim, 
                                   gpgpu_ptx_sim_arg_list_t grid_params )
diff --git a/src/gpgpusim_entrypoint.h b/src/gpgpusim_entrypoint.h
index f77a906..689dcf9 100644
--- a/src/gpgpusim_entrypoint.h
+++ b/src/gpgpusim_entrypoint.h
@@ -77,12 +77,12 @@ int gpgpu_cuda_ptx_sim_main_perf( kernel_info_t grid,
                                    struct dim3 blockDim, 
                                    gpgpu_ptx_sim_arg_list_t grid_params );
 
-int gpgpu_opencl_ptx_sim_main_perf( class function_info *entry, 
+int gpgpu_opencl_ptx_sim_main_perf( kernel_info_t grid,
                                   struct dim3 gridDim, 
                                   struct dim3 blockDim, 
                                   gpgpu_ptx_sim_arg_list_t grid_params );
 
-int gpgpu_opencl_ptx_sim_main_func( class function_info *entry, 
+int gpgpu_opencl_ptx_sim_main_func( kernel_info_t grid,
                                   struct dim3 gridDim, 
                                   struct dim3 blockDim, 
                                   gpgpu_ptx_sim_arg_list_t grid_params );