Merged all work on the dev branch since the divergence point into the dnn branch, incorporating Dynamic Parallelism and many bug fixes.

15 files changed, 633 insertions, 23 deletions

diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile
index 166e256..999dad7 100644
--- a/src/cuda-sim/Makefile
+++ b/src/cuda-sim/Makefile
@@ -46,7 +46,7 @@ OPT	:=  -O3 -g3 -Wall -Wno-unused-function -Wno-sign-compare
 ifeq ($(DEBUG),1)
 	OPT := -g3 -Wall  -Wno-unused-function -Wno-sign-compare
 endif
-OPT += -I$(CUDA_INSTALL_PATH)/include  -I$(OUTPUT_DIR)/ -I.
+OPT += -I$(CUDA_INSTALL_PATH)/include  -I$(OUTPUT_DIR)/ -I. -I$(SIM_OBJ_FILES_DIR)
 OPT += -fPIC 
 
 ifeq ($(TRACE),1)
@@ -62,7 +62,7 @@ ifeq ($(GNUC_CPP0X),1)
 endif
 endif
 
-OBJS	:= $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o  $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o
+OBJS	:= $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o  $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o $(OUTPUT_DIR)/cuda_device_runtime.o
 
 
 OPT += -DCUDART_VERSION=$(CUDART_VERSION)
@@ -142,8 +142,9 @@ $(OUTPUT_DIR)/ptx_parser.o: $(OUTPUT_DIR)/ptx.tab.c $(OUTPUT_DIR)/ptx_parser_dec
 $(OUTPUT_DIR)/ptxinfo.tab.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/ptx-stats.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/ptx_sim.o: $(OUTPUT_DIR)/ptx.tab.c
-$(OUTPUT_DIR)/cuda-sim.o: $(OUTPUT_DIR)/ptx.tab.c
+$(OUTPUT_DIR)/cuda-sim.o: $(OUTPUT_DIR)/ptx.tab.c $(SIM_OBJ_FILES_DIR)/detailed_version
 $(OUTPUT_DIR)/lex.ptxinfo_.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/lex.ptx_.o: $(OUTPUT_DIR)/ptx.tab.c
+$(OUTPUT_DIR)/cuda_device_runtime.o: $(OUTPUT_DIR)/ptx.tab.c
 
 include $(OUTPUT_DIR)/Makefile.makedepend
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 4bae236..b5b79e7 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -48,6 +48,7 @@
 #include "../gpgpusim_entrypoint.h"
 #include "decuda_pred_table/decuda_pred_table.h"
 #include "../stream_manager.h"
+#include "cuda_device_runtime.h"
 
 int gpgpu_ptx_instruction_classification;
 void ** g_inst_classification_stat = NULL;
@@ -63,6 +64,8 @@ unsigned gpgpu_param_num_shaders = 0;
 
 char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp;
 char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp;
+char *cdp_latency_str;
+unsigned cdp_latency[5];
 
 void ptx_opcocde_latency_options (option_parser_t opp) {
 	option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int,
@@ -89,6 +92,12 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
 			"Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>"
 			"Default 8,8,8,8,130",
 			"8,8,8,8,130");
+	option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str,
+			"CDP API latency <cudaStreamCreateWithFlags, \
+cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \
+cudaLaunchDeviceV2_init_perWarp, cudaLaunchDevicV2_perKernel>"
+			"Default 7200,8000,100,12000,1600",
+			"7200,8000,100,12000,1600");
 }
 
 static address_type get_converge_point(address_type pc);
@@ -612,6 +621,9 @@ void ptx_instruction::set_opcode_and_latency()
 	sscanf(opcode_initiation_dp, "%u,%u,%u,%u,%u",
 			&dp_init[0],&dp_init[1],&dp_init[2],
 			&dp_init[3],&dp_init[4]);
+	sscanf(cdp_latency_str, "%u,%u,%u,%u,%u",
+			&cdp_latency[0],&cdp_latency[1],&cdp_latency[2], 
+            &cdp_latency[3],&cdp_latency[4]);
 
 	if(!m_operands.empty()){
 		std::vector<operand_info>::iterator it;
@@ -643,19 +655,21 @@ void ptx_instruction::set_opcode_and_latency()
    case MEMBAR_OP: op = MEMORY_BARRIER_OP; break;
    case CALL_OP:
    {
-       if(m_is_printf)
+       if(m_is_printf || m_is_cdp) {
            op = ALU_OP;
+       }
        else
            op = CALL_OPS;
        break;
    }
    case CALLP_OP:
    {
-       if(m_is_printf)
+       if(m_is_printf || m_is_cdp) {
                op = ALU_OP;
-           else
-               op = CALL_OPS;
-           break;
+       }
+       else
+           op = CALL_OPS;
+       break;
    }
    case RET_OP: case RETP_OP:  op = RET_OPS;break;
    case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP:
@@ -778,6 +792,10 @@ void ptx_instruction::set_opcode_and_latency()
 	   latency = int_latency[5];
 	   initiation_interval = int_init[5];
 	   break;
+   case SHFL_OP:
+	   latency = 32;
+	   initiation_interval = 15;
+	   break;
    default: 
        break;
    }
@@ -1076,6 +1094,32 @@ void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *arg
    } 
 }
 
+unsigned function_info::get_args_aligned_size() {
+ 
+   if(m_args_aligned_size >= 0)
+      return m_args_aligned_size;
+ 
+   unsigned param_address = 0;
+   unsigned int total_size = 0;
+   for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) {
+      param_info &p = i->second;
+      std::string name = p.get_name();
+      symbol *param = m_symtab->lookup(name.c_str());
+
+      size_t arg_size = p.get_size() / 8; // size of param in bytes
+      total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned
+      p.add_offset(total_size);
+      param->set_address(param_address + total_size);
+      total_size += arg_size;
+   }
+
+   m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word
+
+   return m_args_aligned_size;
+
+}
+
+
 void function_info::finalize( memory_space *param_mem ) 
 {
    unsigned param_address = 0;
@@ -1098,13 +1142,17 @@ void function_info::finalize( memory_space *param_mem )
          size = (size<(p.get_size()/8))?size:(p.get_size()/8);
       } 
       // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over
+      //Jin: copy parameter using aligned rules
       const size_t word_size = 4; 
+      param_address = (param_address + size - 1) / size * size; //aligned with size 
       for (size_t idx = 0; idx < size; idx += word_size) {
          const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic
          param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); 
       }
+      unsigned offset = p.get_offset();
+      assert(offset == param_address);
       param->set_address(param_address);
-      param_address += size; 
+      param_address += size;
    }
 }
 
@@ -1471,7 +1519,8 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
    unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 
    assert( max_cta_per_sm > 0 );
 
-   unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid;
+   //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid;
+   unsigned sm_idx = hw_cta_id*gpgpu_param_num_shaders + sid;
 
    if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) {
       if ( g_debug_execution >= 1 ) {
@@ -1506,7 +1555,6 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       kernel.increment_thread_id();
       new_tid += tid;
       ptx_thread_info *thd = new ptx_thread_info(kernel);
-   
       ptx_warp_info *warp_info = NULL;
       if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) {
     	  warp_info = new ptx_warp_info();
@@ -1591,14 +1639,13 @@ kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
 }
 
 #include "../../version"
+#include "detailed_version"
 
 void print_splash()
 {
    static int splash_printed=0;
    if ( !splash_printed ) {
-      unsigned build=0;
-      sscanf(g_gpgpusim_build_string, "$Change"": %u $", &build);
-      fprintf(stdout, "\n\n        *** %s [build %u] ***\n\n\n", g_gpgpusim_version_string, build );
+      fprintf(stdout, "\n\n        *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string );
       splash_printed=1;
    }
 }
@@ -1777,14 +1824,19 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL )
             g_the_gpu->getShaderCoreConfig()->warp_size
         );
         cta.execute();
+
+#if (CUDART_VERSION >= 5000)
+	launch_all_device_kernels();
+#endif
     }
     
    //registering this kernel as done      
-   extern stream_manager *g_stream_manager;
    
    //openCL kernel simulation calls don't register the kernel so we don't register its exit
-   if(!openCL)
-   g_stream_manager->register_finished_kernel(kernel.get_uid());
+   if(!openCL) {
+      extern stream_manager *g_stream_manager;
+      g_stream_manager->register_finished_kernel(kernel.get_uid());
+   }
 
    //******PRINTING*******
    printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn );
diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc
new file mode 100644
index 0000000..4a8ffe5
--- /dev/null
+++ b/src/cuda-sim/cuda_device_runtime.cc
@@ -0,0 +1,320 @@
+//Jin: cuda_device_runtime.cc
+//Defines CUDA device runtime APIs for CDP support
+
+
+#include <iostream>
+#include <map>
+
+unsigned long long g_total_param_size = 0;
+unsigned long long g_max_total_param_size = 0;
+
+
+#if (CUDART_VERSION >= 5000)
+#define __CUDA_RUNTIME_API_H__
+
+#include <builtin_types.h>
+#include <driver_types.h>
+#include "../gpgpu-sim/gpu-sim.h"
+#include "cuda-sim.h"
+#include "ptx_ir.h"
+#include "../stream_manager.h"
+#include "cuda_device_runtime.h"
+
+#define DEV_RUNTIME_REPORT(a) \
+   if( g_debug_execution ) { \
+      std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \
+      std::cout.flush(); \
+   }
+
+class device_launch_config_t {
+
+public:
+    device_launch_config_t() {}
+
+    device_launch_config_t(dim3 _grid_dim,
+        dim3 _block_dim,
+        unsigned int _shared_mem,
+        function_info * _entry):
+            grid_dim(_grid_dim),
+            block_dim(_block_dim),
+            shared_mem(_shared_mem),
+            entry(_entry) {}
+    
+    dim3 grid_dim;
+    dim3 block_dim;
+    unsigned int shared_mem;
+    function_info * entry;
+
+};
+
+class device_launch_operation_t {
+
+public:
+    device_launch_operation_t() {}
+    device_launch_operation_t(kernel_info_t *_grid,
+        CUstream_st * _stream) :
+            grid(_grid), stream(_stream) {}
+
+    kernel_info_t * grid; //a new child grid
+
+    CUstream_st * stream; 
+
+};
+
+
+std::map<void *, device_launch_config_t> g_cuda_device_launch_param_map;
+std::list<device_launch_operation_t> g_cuda_device_launch_op;
+extern stream_manager *g_stream_manager;
+
+//Handling device runtime api:
+//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize)
+void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func)
+{
+    DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2");
+      
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 4 );
+
+    function_info * child_kernel_entry;
+    struct dim3 grid_dim, block_dim;
+    unsigned int shared_mem;
+
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//function_info* for the child kernel
+            unsigned long long buf;
+            assert(size == sizeof(function_info *));
+            thread->m_local_mem->read(from_addr, size, &buf);
+            child_kernel_entry = (function_info *)buf;
+            assert(child_kernel_entry);
+            DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name());
+        }
+        else if(arg == 1) { //dim3 grid_dim for the child kernel
+            assert(size == sizeof(struct dim3));
+            thread->m_local_mem->read(from_addr, size, & grid_dim);
+            DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")");
+        }
+        else if(arg == 2) { //dim3 block_dim for the child kernel
+            assert(size == sizeof(struct dim3));
+            thread->m_local_mem->read(from_addr, size, & block_dim);
+            DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")");
+        }
+        else if(arg == 3) { //unsigned int shared_mem
+            assert(size == sizeof(unsigned int));
+            thread->m_local_mem->read(from_addr, size, & shared_mem);
+            DEV_RUNTIME_REPORT("shared memory " << shared_mem);
+        }
+    }
+
+    //get total child kernel argument size and malloc buffer in global memory
+    unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size();
+    void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size);
+    g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256);
+    DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer);
+    if(g_total_param_size > g_max_total_param_size)
+        g_max_total_param_size = g_total_param_size;
+
+    //store param buffer address and launch config
+    device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry);
+    assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end());
+    g_cuda_device_launch_param_map[param_buffer] = device_launch_config;
+
+    //copy the buffer address to retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //void *
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *));
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &param_buffer, NULL, NULL);
+
+}
+
+//Handling device runtime api:
+//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream)
+void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) {
+    DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2");
+
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 2 );
+
+    kernel_info_t * device_grid = NULL;
+    function_info * device_kernel_entry = NULL;
+    void * parameter_buffer;
+    struct CUstream_st * child_stream;
+    device_launch_config_t config;
+    device_launch_operation_t device_launch_op;
+
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//paramter buffer for child kernel (in global memory)
+            //get parameter_buffer from the cudaLaunchDeviceV2_param0
+            assert(size == sizeof(void *));
+            thread->m_local_mem->read(from_addr, size, &parameter_buffer);
+            assert((size_t)parameter_buffer >= GLOBAL_HEAP_START);
+            DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer);
+
+            //get child grid info through parameter_buffer address
+            assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end());
+            config = g_cuda_device_launch_param_map[parameter_buffer];
+            //device_grid = op.grid;
+            device_kernel_entry = config.entry;
+            DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name());
+
+            //copy data in parameter_buffer to device kernel param memory
+            unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size();
+            DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size);
+            memory_space *device_kernel_param_mem;
+
+            //create child kernel_info_t and index it with parameter_buffer address
+            device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry);
+            device_grid->launch_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
+            kernel_info_t & parent_grid = thread->get_kernel();
+            DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" <<
+                thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z <<
+                "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z <<
+                ")");
+            device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid());  
+            device_launch_op = device_launch_operation_t(device_grid, NULL);
+            device_kernel_param_mem = device_grid->get_param_memory(); //kernel param
+            size_t param_start_address = 0;
+            //copy in word
+            for(unsigned n = 0; n < device_kernel_arg_size; n += 4) {
+                unsigned int oneword;
+                thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword);
+                device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); 
+            }
+        }
+        else if(arg == 1) { //cudaStream for the child kernel
+
+            assert(size == sizeof(cudaStream_t));
+            thread->m_local_mem->read(from_addr, size, &child_stream);
+     
+            kernel_info_t & parent_kernel = thread->get_kernel();
+            if(child_stream == 0) { //default stream on device for current CTA
+                child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); 
+                DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << 
+                    " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream);
+            }
+            else {
+               assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); 
+                DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << 
+                " to stream " << child_stream->get_uid() << ": " << child_stream);
+            }
+    
+            device_launch_op.stream = child_stream;
+        }
+        
+    }
+
+  
+    //launch child kernel
+    g_cuda_device_launch_op.push_back(device_launch_op);
+    g_cuda_device_launch_param_map.erase(parameter_buffer);
+
+    //set retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //cudaError_t
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size 
+        && return_size == sizeof(cudaError_t));
+    cudaError_t error = cudaSuccess;
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL);
+
+}
+
+
+//Handling device runtime api:
+//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int  flags)
+//flags can only be cudaStreamNonBlocking
+void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) {
+    DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags");
+
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 2 );
+
+    size_t generic_pStream_addr;
+    addr_t pStream_addr;
+    unsigned int flags;
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t
+            assert(size == sizeof(cudaStream_t *));
+            thread->m_local_mem->read(from_addr, size, &generic_pStream_addr);
+            
+            //pStream should be non-zero address in local memory
+            pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr);
+
+            DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr);
+        }
+        else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking
+            assert(size == sizeof(unsigned int));
+            thread->m_local_mem->read(from_addr, size, &flags);
+            assert(flags == cudaStreamNonBlocking);
+        }
+    }
+
+    //create stream and write back to param0
+    CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid());
+    DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream);
+    thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL);
+ 
+    //set retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //cudaError_t
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size 
+        && return_size == sizeof(cudaError_t));
+    cudaError_t error = cudaSuccess;
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL);
+
+}
+
+
+void launch_one_device_kernel() {
+    if(!g_cuda_device_launch_op.empty()) {
+        device_launch_operation_t &op = g_cuda_device_launch_op.front();
+
+        stream_operation stream_op = stream_operation(op.grid, g_ptx_sim_mode, op.stream);
+        g_stream_manager->push(stream_op);
+        g_cuda_device_launch_op.pop_front();
+    }
+}
+
+void launch_all_device_kernels() {
+    while(!g_cuda_device_launch_op.empty()) {
+        launch_one_device_kernel();
+    }
+}
+#endif
diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h
new file mode 100644
index 0000000..6dbcd71
--- /dev/null
+++ b/src/cuda-sim/cuda_device_runtime.h
@@ -0,0 +1,11 @@
+//Jin: cuda_device_runtime.h
+//Defines CUDA device runtime APIs for CDP support
+#if (CUDART_VERSION >= 5000)
+#pragma once
+
+void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void launch_all_device_kernels();
+void launch_one_device_kernel();
+#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index bb15621..159fd4c 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -41,6 +41,9 @@
 #include "../gpgpu-sim/gpu-sim.h"
 #include "../gpgpu-sim/shader.h"
 
+//Jin: include device runtime for CDP
+#include "cuda_device_runtime.h"
+
 #include <stdarg.h>
 
 unsigned ptx_instruction::g_num_ptx_inst_uid=0;
@@ -154,6 +157,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
             result.u64 = op.get_symbol()->get_address();
          } else if ( op.is_local() ) {
             result.u64 = op.get_symbol()->get_address();
+         } else if ( op.is_function_address() ) {
+		 	result.u64 = (size_t)op.get_symbol()->get_pc();
          } else {
             const char *name = op.name().c_str();
             printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name );
@@ -1639,7 +1644,23 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread )
    if( fname == "vprintf" ) {
       gpgpusim_cuda_vprintf(pI, thread, target_func);
       return;
-   } 
+   }
+
+#if (CUDART_VERSION >= 5000)
+   //Jin: handle device runtime apis for CDP
+   else if(fname == "cudaGetParameterBufferV2") {
+      gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func);
+	  return;
+   }
+   else if(fname == "cudaLaunchDeviceV2") {
+      gpgpusim_cuda_launchDeviceV2(pI, thread, target_func);
+	  return;
+   }
+   else if(fname == "cudaStreamCreateWithFlags") {
+      gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func);
+	  return;
+   }
+#endif
 
    // read source arguements into register specified in declaration of function
    arg_buffer_list_t arg_values;
@@ -3674,6 +3695,81 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread )
 
 }
 
+void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
+{
+	unsigned i_type = pI->get_type();
+	int tid = inst.warp_id() * core->get_warp_size();
+	ptx_thread_info *thread = core->get_thread_info()[tid];
+	ptx_warp_info *warp_info = thread->m_warp_info;
+	int lane = warp_info->get_done_threads();
+	thread = core->get_thread_info()[tid + lane];
+
+	const operand_info &dst = pI->dst();
+	const operand_info &src1 = pI->src1();
+	const operand_info &src2 = pI->src2();
+	const operand_info &src3 = pI->src3();
+	int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32;
+	int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32;
+	int mask = cval >> 8;
+	bval &= 0x1F;
+	cval &= 0x1F;
+
+	int maxLane = (lane & mask) | (cval & ~mask);
+	int minLane = lane & mask;
+
+	int src_idx;
+	unsigned p;
+	switch(pI->shfl_op()) {
+	case UP_OPTION:
+		src_idx = lane - bval;
+		p = (src_idx >= maxLane);
+		break;
+	case DOWN_OPTION:
+		src_idx = lane + bval;
+		p = (src_idx <= maxLane);
+		break;
+	case BFLY_OPTION:
+		src_idx = lane ^ bval;
+		p = (src_idx <= maxLane);
+		break;
+	case IDX_OPTION:
+		src_idx = minLane | (bval & ~mask);
+		p = (src_idx <= maxLane);
+		break;
+	default:
+		printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n");
+		assert(0);
+		break;
+	}
+	// copy from own lane
+	if (!p) src_idx = lane;
+
+	// copy input from lane src_idx
+	ptx_reg_t data;
+	if (inst.active(src_idx)) {
+		ptx_thread_info *source = core->get_thread_info()[tid + src_idx];
+		data = source->get_operand_value(src1, dst, i_type, source, 1);
+	} else {
+		printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n");
+		data.u32 = 0;
+	}
+	thread->set_operand_value(dst, data, i_type, thread, pI);
+
+	/*
+	TODO: deal with predicates appropriately using the following pseudocode:
+	if (!isGuardPredicateTrue(src_idx)) {
+		printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n");
+	}
+	if (dest predicate selected) data.pred = p;
+	*/
+
+	// keep track of the number of threads that have executed in the warp
+	warp_info->inc_done_threads();
+	if (warp_info->get_done_threads() == inst.active_count()) {
+		warp_info->reset_done_threads();
+	}
+}
+
 void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) 
 {
    ptx_reg_t a, b, d;
diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def
index b363dca..41f2f22 100644
--- a/src/cuda-sim/opcodes.def
+++ b/src/cuda-sim/opcodes.def
@@ -99,6 +99,7 @@ OP_DEF(SAD_OP,sad_impl,"sad",1,1)
 OP_DEF(SELP_OP,selp_impl,"selp",1,1)
 OP_DEF(SETP_OP,setp_impl,"setp",1,1)
 OP_DEF(SET_OP,set_impl,"set",1,1)
+OP_W_DEF(SHFL_OP,shfl_impl,"shfl",1,10)
 OP_DEF(SHL_OP,shl_impl,"shl",1,1)
 OP_DEF(SHR_OP,shr_impl,"shr",1,1)
 OP_DEF(SIN_OP,sin_impl,"sin",1,4)
diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l
index 001ec04..7620134 100644
--- a/src/cuda-sim/ptx.l
+++ b/src/cuda-sim/ptx.l
@@ -116,6 +116,7 @@ sad     TC; ptx_lval.int_value = SAD_OP; return OPCODE;
 selp	TC; ptx_lval.int_value = SELP_OP; return OPCODE;
 setp    TC; ptx_lval.int_value = SETP_OP; return OPCODE;
 set	TC; ptx_lval.int_value = SET_OP; return OPCODE;
+shfl	TC; ptx_lval.int_value = SHFL_OP; return OPCODE;
 shl     TC; ptx_lval.int_value = SHL_OP; return OPCODE;
 shr     TC; ptx_lval.int_value = SHR_OP; return OPCODE;
 sin	TC; ptx_lval.int_value = SIN_OP; return OPCODE;
@@ -186,6 +187,7 @@ breakaddr  TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
 \.union  TC; return UNION_DIRECTIVE; /* not in PTX 2.1 */
 \.version TC; return VERSION_DIRECTIVE;
 \.visible TC; return VISIBLE_DIRECTIVE;
+\.weak TC; return WEAK_DIRECTIVE;
 \.address_size TC; return ADDRESS_SIZE_DIRECTIVE;
 \.weak	TC; return WEAK_DIRECTIVE;
 
@@ -334,6 +336,11 @@ breakaddr  TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
 
 \.nc	TC; return NC_OPTION;
 
+\.up	TC; return UP_OPTION;
+\.down	TC; return DOWN_OPTION;
+\.bfly	TC; return BFLY_OPTION;
+\.idx	TC; return IDX_OPTION;
+
 \.popc	TC; return ATOMIC_POPC;
 \.and	TC; return ATOMIC_AND;
 \.or	TC; return ATOMIC_OR;
diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y
index 97f4ff2..3360c55 100644
--- a/src/cuda-sim/ptx.y
+++ b/src/cuda-sim/ptx.y
@@ -73,6 +73,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  VERSION_DIRECTIVE
 %token  ADDRESS_SIZE_DIRECTIVE
 %token  VISIBLE_DIRECTIVE
+%token  WEAK_DIRECTIVE
 %token  <string_value> IDENTIFIER
 %token  <int_value> INT_OPERAND
 %token  <float_value> FLOAT_OPERAND
@@ -195,6 +196,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  WB_OPTION;
 %token  WT_OPTION;
 %token	NC_OPTION;
+%token	UP_OPTION;
+%token	DOWN_OPTION;
+%token	BFLY_OPTION;
+%token	IDX_OPTION;
 
 %type <int_value> function_decl_header
 %type <ptr_value> function_decl
@@ -244,10 +249,12 @@ function_ident_param: IDENTIFIER { add_function_name($1); } LEFT_PAREN {func_hea
 
 function_decl_header: ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
 	| VISIBLE_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
+	| WEAK_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
 	| FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| VISIBLE_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| EXTERN_DIRECTIVE FUNC_DIRECTIVE { $$ = 2; g_func_decl=1; func_header(".func"); }
+	| WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	;
 
 param_list: /*empty*/
@@ -273,8 +280,8 @@ statement_list: directive_statement { add_directive(); }
 	| instruction_statement { add_instruction(); }
 	| statement_list directive_statement { add_directive(); }
 	| statement_list instruction_statement { add_instruction(); }
-	| statement_list statement_block
-	| statement_block
+	| statement_list {start_inst_group();} statement_block {end_inst_group();}
+	| {start_inst_group();} statement_block {end_inst_group();}
 	;
 
 directive_statement: variable_declaration SEMI_COLON
@@ -326,6 +333,7 @@ var_spec: space_spec
 	| type_spec
 	| align_spec
 	| EXTERN_DIRECTIVE { add_extern_spec(); }
+    | WEAK_DIRECTIVE
 	;
 
 align_spec: ALIGN_DIRECTIVE INT_OPERAND { add_alignment_spec($2); }
@@ -453,6 +461,10 @@ option: type_spec
 	| WB_OPTION { add_option(WB_OPTION); }
 	| WT_OPTION { add_option(WT_OPTION); }
 	| NC_OPTION { add_option(NC_OPTION); }
+	| UP_OPTION { add_option(UP_OPTION); }
+	| DOWN_OPTION { add_option(DOWN_OPTION); }
+	| BFLY_OPTION { add_option(BFLY_OPTION); }
+	| IDX_OPTION { add_option(IDX_OPTION); }
 	;
 
 atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } 
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index 2eccabc..8ebdcf8 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -90,6 +90,11 @@ symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol
    m_const_next  = 0;
    m_global_next = 0x100;
    m_local_next  = 0;
+   m_tex_next = 0;
+
+   //Jin: handle instruction group for cdp
+   m_inst_group_id = 0;
+
    m_parent = parent;
    if ( m_parent ) {
       m_shared_next = m_parent->m_shared_next;
@@ -170,6 +175,41 @@ void symbol_table::add_function( function_info *func, const char *filename, unsi
    m_symbols[ func->get_name() ] = s;
 }
 
+//Jin: handle instruction group for cdp
+symbol_table* symbol_table::start_inst_group() {
+   char inst_group_name[1024];
+   snprintf(inst_group_name, 1024, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id);
+
+   //previous added
+   assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end());
+   symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this );
+ 
+   sym_table->m_global_next = m_global_next;
+   sym_table->m_shared_next = m_shared_next;
+   sym_table->m_local_next = m_local_next;
+   sym_table->m_reg_allocator = m_reg_allocator;
+   sym_table->m_tex_next = m_tex_next;
+   sym_table->m_const_next = m_const_next;
+
+   m_inst_group_symtab[std::string(inst_group_name)] = sym_table;
+
+   return sym_table;
+}
+
+symbol_table * symbol_table::end_inst_group() {
+   symbol_table * sym_table = m_parent;
+   
+   sym_table->m_global_next = m_global_next;
+   sym_table->m_shared_next = m_shared_next;
+   sym_table->m_local_next = m_local_next;
+   sym_table->m_reg_allocator = m_reg_allocator;
+   sym_table->m_tex_next = m_tex_next;
+   sym_table->m_const_next = m_const_next;
+   sym_table->m_inst_group_id++;
+
+   return sym_table;
+}
+
 void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl
 
 bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table )
@@ -458,7 +498,7 @@ void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of f
          if( pI->has_pred() ) {
             printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n");
             // if predicated, add link to next block
-            unsigned next_addr = pI->get_m_instr_mem_index() + 1;
+            unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size();
             if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) {
                basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb();
                (*bb_itr)->successor_ids.insert(next_bb->bb_id);
@@ -1171,6 +1211,12 @@ ptx_instruction::ptx_instruction( int opcode,
 		 break;
 	  case NC_OPTION:
 		 break;
+	  case UP_OPTION:
+	  case DOWN_OPTION:
+	  case BFLY_OPTION:
+	  case IDX_OPTION:
+		  m_shfl_op = last_ptx_inst_option;
+		  break;
       default:
          assert(0);
          break;
@@ -1205,6 +1251,12 @@ ptx_instruction::ptx_instruction( int opcode,
        if (fname =="vprintf"){
            m_is_printf = true;
        }
+       if(fname == "cudaStreamCreateWithFlags")
+           m_is_cdp = 1;
+       if(fname == "cudaGetParameterBufferV2")
+           m_is_cdp = 2;
+       if(fname == "cudaLaunchDeviceV2")
+           m_is_cdp = 4;
 
    }
 }
@@ -1252,6 +1304,7 @@ function_info::function_info(int entry_point )
    m_kernel_info.regs = 0;
    m_kernel_info.smem = 0;
    m_local_mem_framesize = 0;
+   m_args_aligned_size = -1;
 }
 
 unsigned function_info::print_insn( unsigned pc, FILE * fp ) const
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 7724443..4c10373 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -293,6 +293,7 @@ private:
 
    std::list<operand_info> m_initializer;
    static unsigned sm_next_uid;
+   
 };
 
 class symbol_table {
@@ -334,6 +335,11 @@ public:
    iterator const_iterator_end() { return m_consts.end();}
 
    void dump();
+   
+   //Jin: handle instruction group for cdp
+   symbol_table* start_inst_group();
+   symbol_table* end_inst_group();
+
 private:
    unsigned m_reg_allocator;
    unsigned m_shared_next;
@@ -352,6 +358,10 @@ private:
    std::list<symbol*> m_consts;
    std::map<std::string,function_info*> m_function_info_lookup;
    std::map<std::string,symbol_table*> m_function_symtab_lookup;
+   
+   //Jin: handle instruction group for cdp
+   unsigned m_inst_group_id;
+   std::map<std::string,symbol_table*> m_inst_group_symtab;
 };
 
 class operand_info {
@@ -695,7 +705,7 @@ public:
    }
 
    bool is_immediate_address() const {
-	   return   m_immediate_address;
+       return   m_immediate_address;
    }
 
    bool is_literal() const { return m_type == int_t ||
@@ -999,6 +1009,7 @@ public:
    unsigned saturation_mode() const { return m_saturation_mode;}
    unsigned dimension() const { return m_geom_spec;}
    unsigned barrier_op() const {return m_barrier_op;}
+   unsigned shfl_op() const {return m_shfl_op;}
    enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot };
    enum vote_mode_t vote_mode() const { return m_vote_mode; }
 
@@ -1064,6 +1075,7 @@ private:
    unsigned            m_compare_op;
    unsigned            m_saturation_mode;
    unsigned 		   m_barrier_op;
+   unsigned			   m_shfl_op;
 
    std::list<int>          m_scalar_type;
    memory_space_t m_space_spec;
@@ -1206,6 +1218,8 @@ public:
    {
       return m_args.size();
    }
+   unsigned get_args_aligned_size();
+
    const symbol* get_arg( unsigned n ) const
    {
       assert( n < m_args.size() );
@@ -1294,6 +1308,9 @@ private:
 
    static std::vector<ptx_instruction*> s_g_pc_to_insn; // a direct mapping from PC to instruction
    static unsigned sm_next_uid;
+
+   //parameter size for device kernels
+   int m_args_aligned_size;
 };
 
 class arg_buffer_t {
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 9bb5008..6c1b595 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -322,7 +322,11 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
 
 #if CUDART_VERSION >= 3000
     if (sm_version == 0) sm_version = 20;
-    snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+    extern bool g_cdp_enabled;
+    if(!g_cdp_enabled)
+        snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+    else
+        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
 #endif
 
     snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc
index a53a8fe..7fc54e9 100644
--- a/src/cuda-sim/ptx_parser.cc
+++ b/src/cuda-sim/ptx_parser.cc
@@ -120,6 +120,20 @@ symbol_table *init_parser( const char *ptx_filename )
 #define DEF(X,Y) g_ptx_token_decode[X] = Y;
 #include "ptx_parser_decode.def"
 #undef DEF
+   g_ptx_token_decode[undefined_space] = "undefined_space";
+   g_ptx_token_decode[undefined_space] = "undefined_space=0";
+   g_ptx_token_decode[reg_space] = "reg_space";
+   g_ptx_token_decode[local_space] = "local_space";
+   g_ptx_token_decode[shared_space] = "shared_space";
+   g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified";
+   g_ptx_token_decode[param_space_kernel] = "param_space_kernel";
+   g_ptx_token_decode[param_space_local] = "param_space_local";
+   g_ptx_token_decode[const_space] = "const_space";
+   g_ptx_token_decode[tex_space] = "tex_space";
+   g_ptx_token_decode[surf_space] = "surf_space";
+   g_ptx_token_decode[global_space] = "global_space";
+   g_ptx_token_decode[generic_space] = "generic_space";
+   g_ptx_token_decode[instruction_space] = "instruction_space";
 
    return g_global_symbol_table;
 }
@@ -187,6 +201,17 @@ void add_function_name( const char *name )
    g_global_symbol_table->add_function( g_func_info, g_filename, ptx_lineno );
 }
 
+//Jin: handle instruction group for cdp
+void start_inst_group() {
+   PTX_PARSE_DPRINTF("start_instruction_group");
+   g_current_symbol_table = g_current_symbol_table->start_inst_group();
+}
+
+void end_inst_group() {
+   PTX_PARSE_DPRINTF("end_instruction_group");
+   g_current_symbol_table = g_current_symbol_table->end_inst_group();
+}
+
 void add_directive() 
 {
    PTX_PARSE_DPRINTF("add_directive");
diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h
index fef7635..32f3903 100644
--- a/src/cuda-sim/ptx_parser.h
+++ b/src/cuda-sim/ptx_parser.h
@@ -94,6 +94,10 @@ void change_operand_neg( );
 void set_immediate_operand_type( );
 void version_header(double a);
 
+//Jin: handle instructino group for cdp
+void start_inst_group();
+void end_inst_group();
+
 #define NON_ARRAY_IDENTIFIER 1
 #define ARRAY_IDENTIFIER_NO_DIM 2
 #define ARRAY_IDENTIFIER 3
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index c62fa57..05acf20 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -433,6 +433,9 @@ public:
    void or_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->or_reduction(ctaid,barid,value);}
    void popc_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->popc_reduction(ctaid,barid,value);}
 
+   //Jin: get corresponding kernel grid for CDP purpose
+   kernel_info_t & get_kernel() { return m_kernel; }
+
 public:
    addr_t         m_last_effective_address;
    bool        m_branch_taken;
diff --git a/src/cuda-sim/ptxinfo.y b/src/cuda-sim/ptxinfo.y
index 37092f4..d241d8c 100644
--- a/src/cuda-sim/ptxinfo.y
+++ b/src/cuda-sim/ptxinfo.y
@@ -94,6 +94,7 @@ line: 	HEADER INFO COLON line_info
 
 line_info: function_name
 	| function_info { ptxinfo_addinfo(); }
+	| gmem_info
 	;
 
 function_name:	FUNC QUOTE IDENTIFIER QUOTE { ptxinfo_function($3); }
@@ -104,6 +105,9 @@ function_info: info
 	| function_info COMMA info
 	;
 
+gmem_info: INT_OPERAND BYTES GMEM
+	;
+
 info: 	  USED INT_OPERAND REGS { ptxinfo_regs($2); }
 	| tuple LMEM { ptxinfo_lmem(g_declared,g_system); }
 	| tuple SMEM { ptxinfo_smem(g_declared,g_system); }