Merge remote-tracking branch 'upstream/dev' into dev

46 files changed, 2193 insertions, 146 deletions

diff --git a/.gitignore b/.gitignore
index 53fadb5..887b605 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ libcuda/cuobjdump_parser.h
 libcuda/cuobjdump_parser.output
 
 lib/*
+doc/doxygen/html
 
 cuobjdump_to_ptxplus/elf_lexer.cc
 cuobjdump_to_ptxplus/elf_parser.cc
@@ -26,4 +27,5 @@ cuobjdump_to_ptxplus/sass_parser.cc
 cuobjdump_to_ptxplus/sass_parser.hh
 cuobjdump_to_ptxplus/sass_parser.output
 
-build/*
\ No newline at end of file
+build/*
+*.swp
diff --git a/.travis.yml b/.travis.yml
index 0564709..6ef6730 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,25 @@
-dist: trusty
+sudo: required
+
 before_install:
-        - sudo apt-get install -y libssl-dev libxml2-dev libboost-all-dev 
-        - git clone https://github.com/gpgpu-sim/setup_torque_on_ubuntu_14.04.git
-        - cd ./setup_torque_on_ubuntu_14.04; ./install_torque.sh; cd -
-        - wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/cuda-repo-ubuntu1404_7.5-18_amd64.deb		
-        - sudo dpkg -i cuda-repo-ubuntu1404_7.5-18_amd64.deb 
-        - sudo apt-get -qq update
-        - sudo apt-get install -y cuda
-        - sudo apt-get install -y build-essential xutils-dev bison zlib1g-dev flex libglu1-mesa-dev
-env: 
-        - CUDA_INSTALL_PATH=/usr/local/cuda-7.5/
+    - docker pull tgrogers/gpgpu-sim_regress:latest
+
 language: cpp
-script: source ./setup_environment && make
+
+
+matrix:
+  include:
+    - services: docker
+      env: CONFIG=configs.gtx480.yml
+# This config is just taking far too long...
+#    - services: docker
+#      env: CONFIG=configs.gtx750ti.yml
+    - services: docker
+      env: CONFIG=configs.quadro5600.yml
+    - services: docker
+      env: CONFIG=configs.quadro5800.yml
+    - services: docker
+      env: CONFIG=configs.teslac2050.yml
+    - services: docker
+      env: CONFIG=configs.gtx1080ti.yml
+
+script: docker run -v `pwd`:/home/runner/gpgpu-sim_distribution:rw tgrogers/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/$CONFIG -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress'"
diff --git a/CHANGES b/CHANGES
index e3b3177..dbf5b39 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,7 @@
 LOG:
 Version 3.2.3+edits (development branch) versus 3.2.3
+- Support for running regression tests using Travis
+- Support added for CUDA dynamic parallelism (courtesy of Jin Wang from Georgia Tech)
 - Added a parameter to the cache configuration to configure the set index 
   function. Added a hash set index function to the Fermi L1 data cache for 
   the two default cache sizes, 16KB/48KB with 32/64 sets.
diff --git a/Makefile b/Makefile
index 9e2c576..2d0466e 100644
--- a/Makefile
+++ b/Makefile
@@ -54,6 +54,9 @@ ifneq ($(shell which nvcc), "")
 	endif
 endif
 
+
+$(shell mkdir -p $(SIM_OBJ_FILES_DIR)/libcuda && echo "const char *g_gpgpusim_build_string=\"$(GPGPUSIM_BUILD)\";" > $(SIM_OBJ_FILES_DIR)/detailed_version)
+
 LIBS = cuda-sim gpgpu-sim_uarch $(INTERSIM) gpgpusimlib 
 
 
@@ -140,7 +143,7 @@ no_opencl_support:
 	@echo "Warning: gpgpu-sim is building without opencl support. Make sure NVOPENCL_LIBDIR and NVOPENCL_INCDIR are set"
 
 $(SIM_LIB_DIR)/libcudart.so: makedirs $(LIBS) cudalib
-	g++ -shared -Wl,-soname,libcudart.so \
+	g++ -shared -Wl,-soname,libcudart_$(GPGPUSIM_BUILD).so -Wl,--version-script=linux-so-version.txt\
 			$(SIM_OBJ_FILES_DIR)/libcuda/*.o \
 			$(SIM_OBJ_FILES_DIR)/cuda-sim/*.o \
 			$(SIM_OBJ_FILES_DIR)/cuda-sim/decuda_pred_table/*.o \
@@ -155,7 +158,12 @@ $(SIM_LIB_DIR)/libcudart.so: makedirs $(LIBS) cudalib
 	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.5.0 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.5.0; fi
 	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.5.5 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.5.5; fi
 	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.6.0 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.6.0; fi
+	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.6.5 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.6.5; fi
 	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.7.5 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.7.5; fi
+	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.8.0 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.8.0; fi
+	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.9.0 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.9.0; fi
+	if [ ! -f $(SIM_LIB_DIR)/libcudart.so.9.1 ]; then ln -s libcudart.so $(SIM_LIB_DIR)/libcudart.so.9.1; fi
+
 $(SIM_LIB_DIR)/libcudart.dylib: makedirs $(LIBS) cudalib
 	g++ -dynamiclib -Wl,-headerpad_max_install_names,-undefined,dynamic_lookup,-compatibility_version,1.1,-current_version,1.1\
 			$(SIM_OBJ_FILES_DIR)/libcuda/*.o \
@@ -168,7 +176,7 @@ $(SIM_LIB_DIR)/libcudart.dylib: makedirs $(LIBS) cudalib
 			-o $(SIM_LIB_DIR)/libcudart.dylib
 
 $(SIM_LIB_DIR)/libOpenCL.so: makedirs $(LIBS) opencllib
-	g++ -shared -Wl,-soname,libOpenCL.so \
+	g++ -shared -Wl,-soname,libOpenCL_$(GPGPUSIM_BUILD).so \
 			$(SIM_OBJ_FILES_DIR)/libopencl/*.o \
 			$(SIM_OBJ_FILES_DIR)/cuda-sim/*.o \
 			$(SIM_OBJ_FILES_DIR)/cuda-sim/decuda_pred_table/*.o \
diff --git a/README b/README
index 4883e93..6e2d734 100644
--- a/README
+++ b/README
@@ -5,7 +5,8 @@ AerialVision and a configurable and extensible energy model called GPUWattch.
 GPGPU-Sim and GPUWattch have been rigorously validated with performance and
 power measurements of real hardware GPUs. 
 
-This version of GPGPU-Sim has been tested with CUDA version 2.3, 3.1 and 4.0.
+This version of GPGPU-Sim has been tested with CUDA version 2.3, 3.1, 4.0, 
+5.0, 5.5, 6.0 and 7.5.
 
 Please see the copyright notice in the file COPYRIGHT distributed with this
 release in the same directory as this file.
@@ -24,6 +25,12 @@ Tor M. Aamodt, Vijay Janapa Reddi, GPUWattch: Enabling Energy Optimizations in
 GPGPUs, In proceedings of the ACM/IEEE International Symposium on Computer
 Architecture (ISCA 2013), Tel-Aviv, Israel, June 23-27, 2013.
 
+If you use the support for CUDA dynamic parallelism in your research, please cite:
+
+Jin Wang and Sudhakar Yalamanchili, Characterization and Analysis of Dynamic 
+Parallelism in Unstructured GPU Applications, 2014 IEEE International Symposium 
+on Workload Characterization (IISWC), November 2014.
+
 If you use figures plotted using AerialVision in your publications, please cite:
 
 Aaron Ariel, Wilson W. L. Fung, Andrew Turner, Tor M. Aamodt, Visualizing
diff --git a/aerialvision/guiclasses.py b/aerialvision/guiclasses.py
index 0a7013b..45fed26 100644
--- a/aerialvision/guiclasses.py
+++ b/aerialvision/guiclasses.py
@@ -69,7 +69,7 @@ import matplotlib
 matplotlib.use('TkAgg')
 from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
 from matplotlib.figure import Figure
-from matplotlib import mpl
+import matplotlib as mpl
 from matplotlib.colors import colorConverter
 from matplotlib import pyplot
 import Pmw
diff --git a/configs/GeForceGTX1080Ti/config_fermi_islip.icnt b/configs/GeForceGTX1080Ti/config_fermi_islip.icnt
new file mode 100644
index 0000000..2a69ddd
--- /dev/null
+++ b/configs/GeForceGTX1080Ti/config_fermi_islip.icnt
@@ -0,0 +1,70 @@
+//21*1 fly with 32 flits per packet under gpgpusim injection mode
+use_map = 0;
+flit_size = 32; 
+
+// currently we do not use this, see subnets below
+network_count = 2;
+
+// Topology
+topology = fly;
+k = 50;
+n = 1;
+
+// Routing
+
+routing_function = dest_tag;
+
+// Flow control
+
+num_vcs     = 1;
+vc_buf_size = 8;
+
+wait_for_tail_credit = 0;
+
+// Router architecture
+
+vc_allocator = islip; //separable_input_first;
+sw_allocator = islip; //separable_input_first;
+alloc_iters  = 1;
+
+credit_delay   = 0;
+routing_delay  = 0;
+vc_alloc_delay = 1;
+sw_alloc_delay = 1;
+
+input_speedup     = 2;
+output_speedup    = 1;
+internal_speedup  = 1.0;
+
+// Traffic, GPGPU-Sim does not use this
+
+traffic                = uniform;
+packet_size ={{1,2,3,4},{10,20}};
+packet_size_rate={{1,1,1,1},{2,1}};
+
+// Simulation - Don't change
+
+sim_type       = gpgpusim;
+//sim_type = latency;
+injection_rate = 0.1;
+
+subnets = 2;
+
+// Always use read and write no matter following line
+//use_read_write = 1;
+
+
+read_request_subnet = 0;
+read_reply_subnet = 1;
+write_request_subnet = 0;
+write_reply_subnet = 1;
+
+read_request_begin_vc = 0;
+read_request_end_vc = 0;
+write_request_begin_vc = 0;
+write_request_end_vc = 0;
+read_reply_begin_vc = 0;
+read_reply_end_vc = 0;
+write_reply_begin_vc = 0;
+write_reply_end_vc = 0;
+
diff --git a/configs/GeForceGTX1080Ti/gpgpusim.config b/configs/GeForceGTX1080Ti/gpgpusim.config
new file mode 100644
index 0000000..47c2b6a
--- /dev/null
+++ b/configs/GeForceGTX1080Ti/gpgpusim.config
@@ -0,0 +1,149 @@
+# This config models the Pascal GP102 (GeForceGTX 1080Ti)
+
+# functional simulator specification
+-gpgpu_ptx_instruction_classification 0
+-gpgpu_ptx_sim_mode 0
+-gpgpu_ptx_force_max_capability 20 
+
+# SASS execution (only supported with CUDA >= 4.0)
+-gpgpu_ptx_convert_to_ptxplus 0
+-gpgpu_ptx_save_converted_ptxplus 0
+
+# high level architecture configuration
+-gpgpu_n_clusters 28
+-gpgpu_n_cores_per_cluster 1
+-gpgpu_n_mem 11
+-gpgpu_n_sub_partition_per_mchannel 2 
+
+# Pascal clock domains
+#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
+# Pascal NVIDIA TITAN X clock domains are adopted from 
+# https://en.wikipedia.org/wiki/GeForce_10_series
+-gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0
+
+# shader core pipeline config
+-gpgpu_shader_registers 65536
+
+# This implies a maximum of 64 warps/SM
+-gpgpu_shader_core_pipeline 2048:32 
+-gpgpu_shader_cta 32
+-gpgpu_simd_model 1 
+
+# Pipeline widths and number of FUs
+# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
+## Pascal GP102 has 4 SP SIMD units and 1 SFU unit
+## we need to scale the number of pipeline registers to be equal to the number of SP units
+-gpgpu_pipeline_widths 4,1,1,4,1,1,6
+-gpgpu_num_sp_units 4
+-gpgpu_num_sfu_units 1
+
+# Instruction latencies and initiation intervals
+# "ADD,MAX,MUL,MAD,DIV"
+# SFU is 32-width in pascal, then dp units initiation is 1 cycle
+-ptx_opcode_latency_int 4,13,4,5,145
+-ptx_opcode_initiation_int 1,2,2,2,8
+-ptx_opcode_latency_fp 4,13,4,5,39
+-ptx_opcode_initiation_fp 1,2,1,1,4
+-ptx_opcode_latency_dp 8,19,8,8,330
+-ptx_opcode_initiation_dp 1,2,1,1,130
+
+# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
+# ** Optional parameter - Required when mshr_type==Texture Fifo
+# Note: Hashing set index function (H) only applies to a set size of 32 or 64. 
+# Pascal GP102 has 96KB Shared memory
+# Pascal GP102 has 64KB L1 cache
+# The default is to disable the L1 cache, unless cache modifieres is used
+-gpgpu_cache:dl1  64:128:6,L:L:m:N:H,A:128:8,8
+-gpgpu_shmem_size 98304
+-gmem_skip_L1D 1
+
+# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache
+-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4
+-gpgpu_cache:dl2_texture_only 0 
+
+# 4 KB Inst.
+-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4
+# 48 KB Tex 
+-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2
+# 12 KB Const
+-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4
+
+# enable operand collector 
+## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units
+-gpgpu_operand_collector_num_units_sp 20
+-gpgpu_operand_collector_num_units_sfu 4
+-gpgpu_operand_collector_num_units_mem 8
+-gpgpu_operand_collector_num_in_ports_sp 4
+-gpgpu_operand_collector_num_out_ports_sp 4
+-gpgpu_operand_collector_num_in_ports_sfu 1
+-gpgpu_operand_collector_num_out_ports_sfu 1
+-gpgpu_operand_collector_num_in_ports_mem 1
+-gpgpu_operand_collector_num_out_ports_mem 1
+# gpgpu_num_reg_banks should be increased to 32, but it gives an error!
+-gpgpu_num_reg_banks 32
+
+# shared memory bankconflict detection 
+-gpgpu_shmem_num_banks 32
+-gpgpu_shmem_limited_broadcast 0
+-gpgpu_shmem_warp_parts 1
+
+## In Pascal, a warp scheduler can issue 2 insts per cycle
+-gpgpu_max_insn_issue_per_warp 2
+
+# interconnection
+-network_mode 1 
+-inter_config_file config_fermi_islip.icnt
+
+# memory partition latency config 
+-rop_latency 120
+-dram_latency 100
+
+# dram model config
+-gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (100 core cycles).  I.e. 
+#   Total buffer space required = 100 x 924MHz / 700MHz = 132
+-gpgpu_frfcfs_dram_sched_queue_size 64
+-gpgpu_dram_return_queue_size 116
+
+# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits)
+# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition
+# the atom size of GDDR5X (the smallest read request) is 32 bytes 
+-gpgpu_n_mem_per_ctrlr 1
+-gpgpu_dram_buswidth 4
+-gpgpu_dram_burst_length 8
+-dram_data_command_freq_ratio 4  # GDDR5X is QDR
+-gpgpu_mem_address_mask 1
+-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS
+
+# Use the same GDDR5 timing from hynix H5GQ1H24AFR
+# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0
+-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40:
+                        CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0"
+
+# Pascal has four schedulers per core
+-gpgpu_num_sched_per_core 2
+# Two Level Scheduler with active and pending pools
+#-gpgpu_scheduler two_level_active:6:0:1
+# Loose round robbin scheduler
+#-gpgpu_scheduler lrr
+# Greedy then oldest scheduler
+-gpgpu_scheduler gto
+
+# stat collection
+-gpgpu_memlatency_stat 14 
+-gpgpu_runtime_stat 500
+-enable_ptx_file_line_stats 1
+-visualizer_enabled 0
+
+# power model configs
+-power_simulation_enabled 1
+-gpuwattch_xml_file gpuwattch_gtx1080Ti.xml
+
+# tracing functionality
+#-trace_enabled 1
+#-trace_components WARP_SCHEDULER,SCOREBOARD
+#-trace_sampling_core 0
+
diff --git a/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml b/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml
new file mode 100755
index 0000000..02619ff
--- /dev/null
+++ b/configs/GeForceGTX1080Ti/gpuwattch_gtx1080Ti.xml
@@ -0,0 +1,538 @@
+<?xml version="1.0" ?>
+<component id="root" name="root">
+	<component id="system" name="system">
+		<!--McPAT will skip the components if number is set to 0 -->
+		<param name="GPU_Architecture" value="1"/><!-- 0-G80; 1-Fermi; others not supported -->
+		<param name="number_of_cores" value="28"/>
+		<param name="architecture" value="1"/> <!-- fermi:1 quadro:2 other: undefined-->
+		<param name="number_of_L1Directories" value="0"/>
+		<param name="number_of_L2Directories" value="0"/>
+		<param name="number_of_L2s" value="1"/> <!-- This number means how many L2 clusters in each cluster there can be multiple banks/ports -->
+		<param name="number_of_L3s" value="0"/> <!-- This number means how many L3 clusters -->
+		<param name="number_of_NoCs" value="1"/> 
+		<param name="homogeneous_cores" value="1"/><!--1 means homo -->
+		<param name="homogeneous_L2s" value="1"/>
+		<param name="homogeneous_L1Directorys" value="1"/>
+		<param name="homogeneous_L2Directorys" value="1"/>
+		<param name="homogeneous_L3s" value="1"/>
+		<param name="homogeneous_ccs" value="1"/><!--cache coherece hardware -->
+		<param name="homogeneous_NoCs" value="1"/>
+		<param name="core_tech_node" value="23"/><!-- nm -->
+		<param name="target_core_clockrate" value="1481"/><!--MHz -->
+		<param name="temperature" value="380"/> <!-- Kelvin -->
+		<param name="number_cache_levels" value="2"/>
+		<param name="interconnect_projection_type" value="0"/><!--0: agressive wire technology; 1: conservative wire technology -->
+		<param name="device_type" value="0"/><!--0: HP(High Performance Type); 1: LSTP(Low standby power) 2: LOP (Low Operating Power)  -->
+		<param name="longer_channel_device" value="1"/><!-- 0 no use; 1 use when possible -->
+		<param name="machine_bits" value="32"/>
+		<param name="virtual_address_width" value="32"/>
+		<param name="physical_address_width" value="32"/>
+		<param name="virtual_memory_page_size" value="4096"/>
+		<param name="idle_core_power" value="1.59"/><!-- idle core power for GTX479 -->
+		<!--param name="scaling_coefficients" value="10,0.0884816,10,10,8,10,4.12782,10,2.48832,10,10,10,4.29982,0.387764,0.0714269,0.14302,0.01,0.546811,0.485351,0.806633,0.818073,1.9207,100,100,100,87.9303,100,10,4.3548,10"/-->
+		<param name="TOT_INST" value="10" />
+		<param name="FP_INT" value="10" />
+		<param name="IC_H" value="0.001" />
+		<param name="IC_M" value="10" />
+		<param name="DC_RH" value="1" />
+		<param name="DC_RM" value="1" />
+		<param name="DC_WH" value="1" />
+		<param name="DC_WM" value="1" />
+		<param name="TC_H" value="0.001" />
+		<param name="TC_M" value="10" />
+		<param name="CC_H" value="4.5071" />
+		<param name="CC_M" value="10" />
+		<param name="SHRD_ACC" value="10" />
+		<param name="REG_RD" value="1.6294" />
+		<param name="REG_WR" value="0.5031" />
+		<param name="NON_REG_OPs" value="0.01" />
+		<param name="SP_ACC" value="10" />
+		<param name="SFU_ACC" value="0.0082" />
+		<param name="FPU_ACC" value="0.4126" />
+		<param name="MEM_RD" value="0.1234" />
+		<param name="MEM_WR" value="0.001" />
+		<param name="MEM_PRE" value="0.001" />
+		<param name="L2_RH" value="100" />
+		<param name="L2_RM" value="100" />
+		<param name="L2_WH" value="100" />
+		<param name="L2_WM" value="42.6966" />
+		<param name="NOC_A" value="100" />
+		<param name="PIPE_A" value="44.8085" />
+		<param name="IDLE_CORE_N" value="2.0382"/>
+		<param name="CONST_DYNAMICN" value="5.0005" />
+		<stat name="num_idle_cores" value="0"/><!-- Average Number of idle cores during this period -->
+		<stat name="total_cycles" value="total_cycles_match_mcpat"/>
+		<stat name="idle_cycles" value="idle_cycles_match_mcpat"/>
+		<stat name="busy_cycles"  value="busy_cycles_match_mcpat"/>
+			<!--This page size(B) is complete different from the page size in Main memo secction. this page size is the size of 
+			virtual memory from OS/Archi perspective; the page size in Main memo secction is the actuall physical line in a DRAM bank  -->
+		<!-- *********************** cores ******************* -->
+		<component id="system.core0" name="core0">
+			<!-- Core property -->
+			<param name="clock_rate" value="1481"/>
+			<param name="instruction_length" value="32"/>
+			<param name="opcode_width" value="9"/>
+			<!-- address width determins the tag_width in Cache, LSQ and buffers in cache controller 
+			default value is machine_bits, if not set --> 
+			<param name="machine_type" value="1"/><!-- 1 inorder; 0 OOO-->
+			<!-- inorder/OoO -->
+			<param name="number_hardware_threads" value="32"/>
+			<!-- number_instruction_fetch_ports(icache ports) is always 1 in single-thread processor,
+			it only may be more than one in SMT processors. BTB ports always equals to fetch ports since 
+			branch information in consective branch instructions in the same fetch group can be read out from BTB once.--> 
+			<param name="fetch_width" value="1"/>
+			<!-- fetch_width determins the size of cachelines of L1 cache block -->
+			<param name="number_instruction_fetch_ports" value="1"/>
+			<param name="decode_width" value="1"/>
+			<!-- decode_width determins the number of ports of the 
+			renaming table (both RAM and CAM) scheme -->
+			<param name="issue_width" value="2"/>
+			<!-- issue_width determins the number of ports of Issue window and other logic 
+			as in the complexity effective proccessors paper; issue_width==dispatch_width -->
+			<param name="commit_width" value="2"/>
+			<!-- commit_width determins the number of ports of register files -->
+			<param name="fp_issue_width" value="1"/>
+			<param name="prediction_width" value="0"/> 
+			<!-- number of branch instructions can be predicted simultannouesl-->
+			<!-- Current version of McPAT does not distinguish int and floating point pipelines 
+			Theses parameters are reserved for future use.--> 
+			<param name="pipelines_per_core" value="1,1"/>
+			<!--integer_pipeline and floating_pipelines, if the floating_pipelines is 0, then the pipeline is shared-->
+			<param name="pipeline_depth" value="8,8"/>
+			<!-- pipeline depth of int and fp, if pipeline is shared, the second number is the average cycles of fp ops -->
+			<!-- issue and exe unit-->
+			<param name="ALU_per_core" value="32"/>
+			<!-- contains an adder, a shifter, and a logical unit -->
+			<param name="MUL_per_core" value="4"/>
+			<!-- For MUL and Div -->
+			<param name="FPU_per_core" value="32"/>		
+			<!-- buffer between IF and ID stage -->
+			<param name="instruction_buffer_size" value="1"/>
+			<!-- buffer between ID and sche/exe stage -->
+			<param name="decoded_stream_buffer_size" value="1"/>
+			<param name="instruction_window_scheme" value="0"/><!-- 0 PHYREG based, 1 RSBASED-->
+			<!-- McPAT support 2 types of OoO cores, RS based and physical reg based-->
+			<param name="instruction_window_size" value="1"/>
+			<param name="fp_instruction_window_size" value="1"/>
+			<!-- the instruction issue Q as in Alpha 21264; The RS as in Intel P6 -->
+			<param name="ROB_size" value="0"/>
+			<!-- each in-flight instruction has an entry in ROB -->
+			<!-- registers -->
+			<!-- SM parameters  Added by Syed Gilani -->
+			<param name="rf_banks" value="32"/>
+			<param name="simd_width" value="32"/>
+			<param name="collector_units" value="32"/>
+			<param name="core_clock_ratio" value="2"/>
+			<param name="warp_size" value="32"/>
+
+			<param name="archi_Regs_IRF_size" value="65536"/>			
+			<param name="archi_Regs_FRF_size" value="32"/>
+			<!--  if OoO processor, phy_reg number is needed for renaming logic, 
+			renaming logic is for both integer and floating point insts.  -->
+			<param name="phy_Regs_IRF_size" value="32"/>
+			<param name="phy_Regs_FRF_size" value="32"/>
+			<!-- rename logic -->
+			<param name="rename_scheme" value="0"/>
+			<!-- can be RAM based(0) or CAM based(1) rename scheme 
+			RAM-based scheme will have free list, status table;
+			CAM-based scheme have the valid bit in the data field of the CAM 
+			both RAM and CAM need RAM-based checkpoint table, checkpoint_depth=# of in_flight instructions;
+			Detailed RAT Implementation see TR -->
+			<param name="register_windows_size" value="0"/>
+			<!-- how many windows in the windowed register file, sun processors;
+			no register windowing is used when this number is 0 -->
+			<!-- In OoO cores, loads and stores can be issued whether inorder(Pentium Pro) or (OoO)out-of-order(Alpha),
+			They will always try to exeute out-of-order though. -->
+			<param name="LSU_order" value="inorder"/>
+			<param name="store_buffer_size" value="32"/>
+			<!-- By default, in-order cores do not have load buffers -->
+			<param name="load_buffer_size" value="32"/>	
+			<!-- number of ports refer to sustainable concurrent memory accesses --> 
+			<param name="memory_ports" value="2"/>	
+			<!-- max_allowed_in_flight_memo_instructions determins the # of ports of load and store buffer
+			as well as the ports of Dcache which is connected to LSU -->	
+			<!-- dual-pumped Dcache can be used to save the extra read/write ports -->
+			<param name="RAS_size" value="1"/>						
+			<!-- general stats, defines simulation periods;require total, idle, and busy cycles for senity check  -->
+			<!-- please note: if target architecture is X86, then all the instrucions refer to (fused) micro-ops -->
+			<stat name="total_instructions" value="total_instructions_match_mcpat"/>
+			<stat name="int_instructions" value="int_instruction_match_mcpat"/>
+			<stat name="fp_instructions" value="flt_instruction_match_mcpat"/>
+			<stat name="branch_instructions" value="branch_instruction_match_mcpat"/>
+			<stat name="branch_mispredictions" value="0"/>
+			<stat name="load_instructions" value="load_instruction_match_mcpat"/>
+			<stat name="store_instructions" value="store_instruction_match_mcpat"/>
+			<stat name="committed_instructions" value="total_instructions_match_mcpat"/>
+			<stat name="committed_int_instructions" value="int_instruction_match_mcpat"/>
+			<stat name="committed_fp_instructions" value="flt_instruction_match_mcpat"/>
+			<stat name="pipeline_duty_cycle" value="0.6"/><!--<=1, runtime_ipc/peak_ipc; averaged for all cores if homogenous -->
+			<!-- the following cycle stats are used for heterogeneouse cores only, 
+				please ignore them if homogeneouse cores -->
+			<stat name="total_cycles" value="total_cycles_match_mcpat"/>
+		    <stat name="idle_cycles" value="idle_cycles_match_mcpat"/>
+		    <stat name="busy_cycles"  value="busy_cycles_match_mcpat"/>
+			<!-- instruction buffer stats -->
+			<!-- ROB stats, both RS and Phy based OoOs have ROB
+			performance simulator should capture the difference on accesses,
+			otherwise, McPAT has to guess based on number of commited instructions. -->
+			<stat name="ROB_reads" value="263886"/>
+			<stat name="ROB_writes" value="263886"/>
+			<!-- RAT accesses -->
+			<stat name="rename_accesses" value="263886"/>
+			<stat name="fp_rename_accesses" value="263886"/>
+			<!-- decode and rename stage use this, should be total ic - nop -->
+			<!-- Inst window stats -->
+			<stat name="inst_window_reads" value="263886"/>
+			<stat name="inst_window_writes" value="263886"/>
+			<stat name="inst_window_wakeup_accesses" value="263886"/>
+			<stat name="fp_inst_window_reads" value="263886"/>
+			<stat name="fp_inst_window_writes" value="263886"/>
+			<stat name="fp_inst_window_wakeup_accesses" value="263886"/>
+			<!--  RF accesses -->
+			<stat name="int_regfile_reads" value="int_register_read_access_match_mcpat"/>
+			<stat name="float_regfile_reads" value="int_register_write_access_match_mcpat"/>
+			<stat name="int_regfile_writes" value="float_register_read_access_match_mcpat"/>
+			<stat name="float_regfile_writes" value="float_register_write_access_match_mcpat"/>
+			
+			<!-- The following stat is for operand collector power - Added by Syed -->
+			<stat name="non_rf_operands" value="0"/>
+
+			<!-- accesses to the working reg -->
+			<stat name="function_calls" value="0"/>
+			<stat name="context_switches" value="0"/> <!--not used in the McPAT -->
+			<!-- Number of Windowes switches (number of function calls and returns)-->
+			<!-- Alu stats by default, the processor has one FPU that includes the divider and 
+			 multiplier. The fpu accesses should include accesses to multiplier and divider  -->
+			<stat name="ialu_accesses" value="ialu_accesses_match_mcpat"/>			
+			<stat name="fpu_accesses" value="fpu_accesses_match_mcpat"/>
+			<stat name="mul_accesses" value="mul_accesses_match_mcpat"/>
+			<stat name="cdb_alu_accesses" value="0"/>
+			<stat name="cdb_mul_accesses" value="0"/>
+			<stat name="cdb_fpu_accesses" value="0"/>
+			<!-- multiple cycle accesses should be counted multiple times, 
+			otherwise, McPAT can use internal counter for different floating point instructions 
+			to get final accesses. But that needs detailed info for floating point inst mix -->
+			<!--  currently the performance simulator should 
+			make sure all the numbers are final numbers, 
+			including the explicit read/write accesses, 
+			and the implicite accesses such as replacements and etc.
+			Future versions of McPAT may be able to reason the implicite access
+			based on param and stats of last level cache
+			The same rule applies to all cache access stats too!  -->
+			<!-- following is AF for max power computation. 
+				Do not change them, unless you understand them-->
+			<stat name="IFU_duty_cycle" value="0.25"/>			
+			<stat name="LSU_duty_cycle" value="0.25"/>
+			<stat name="MemManU_I_duty_cycle" value="1"/>
+			<stat name="MemManU_D_duty_cycle" value="0.25"/>
+			<stat name="ALU_duty_cycle" value="0.9"/>
+			<stat name="MUL_duty_cycle" value="0.5"/>
+			<stat name="FPU_duty_cycle" value="1"/><!-- FPU numbers are already average -->
+			<stat name="ALU_cdb_duty_cycle" value="0.9"/>
+			<stat name="MUL_cdb_duty_cycle" value="0.5"/>
+			<stat name="FPU_cdb_duty_cycle" value="15"/>	
+			<component id="system.core0.predictor" name="PBT">
+				<!-- branch predictor; tournament predictor see Alpha implementation -->
+				<param name="local_predictor_size" value="10,3"/>
+				<param name="local_predictor_entries" value="1024"/>
+				<param name="global_predictor_entries" value="4096"/>
+				<param name="global_predictor_bits" value="2"/>
+				<param name="chooser_predictor_entries" value="4096"/>
+				<param name="chooser_predictor_bits" value="2"/>
+				<!-- These parameters can be combined like below in next version
+				<param name="load_predictor" value="10,3,1024"/>
+				<param name="global_predictor" value="4096,2"/>
+				<param name="predictor_chooser" value="4096,2"/>
+				-->
+			</component>
+			<component id="system.core0.itlb" name="itlb">
+				<param name="number_entries" value="1"/>
+				<stat name="total_accesses" value="0"/>
+				<stat name="total_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+				<!-- there is no write requests to itlb although writes happen to itlb after miss, 
+				which is actually a replacement -->
+			</component>
+			<component id="system.core0.icache" name="icache">
+				<!-- there is no write requests to itlb although writes happen to it after miss, 
+				which is actually a replacement -->
+				<param name="icache_config" value="16384,128,4,1,1,3,8,0"/>
+				<!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy -->
+				<!-- cache_policy;//0 no write or write-though with non-write allocate;1 write-back with write-allocate -->
+				<param name="buffer_sizes" value="16, 16, 16,0"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> 
+				<stat name="read_accesses" value="total_instructions_match_mcpat"/>
+				<stat name="read_misses" value="0"/>
+				<stat name="conflicts" value="0"/>				
+			</component>
+			<component id="system.core0.dtlb" name="dtlb">
+				<param name="number_entries" value="1"/>
+				<stat name="total_accesses" value="0"/>
+				<stat name="total_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+			</component>
+			<component id="system.core0.ccache" name="ccache">
+			        <!-- all the buffer related are optional -->
+				<param name="ccache_config" value="16384,64,2,1,1,3,8,0"/>
+				<param name="buffer_sizes" value="16, 16, 16, 0"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<stat name="read_accesses" value="ccache_read_accesses_match_mcpat"/>
+				<stat name="write_accesses" value="0"/>
+				<stat name="read_misses" value="ccache_read_misses_match_mcpat"/>
+				<stat name="write_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+			</component>
+			<component id="system.core0.tcache" name="tcache">
+			        <!-- all the buffer related are optional -->
+				<param name="tcache_config" value="49152,128,8,1,1,3,8,0"/>
+				<param name="buffer_sizes" value="16, 16, 16, 0"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<stat name="read_accesses" value="tcache_read_accesses_match_mcpat"/>
+				<stat name="write_accesses" value="0"/>
+				<stat name="read_misses" value="tcache_read_misses_match_mcpat"/>
+				<stat name="write_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+			</component>
+      <!--model the shared memory by mimicing dcache-->
+			<component id="system.core0.sharedmemory" name="sharedmemory">
+			        <!-- all the buffer related are optional -->
+				<param name="sharedmemory_config" value="98304,16,1,16,1,3,16,0"/>
+				<!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy -->
+				<param name="buffer_sizes" value="16, 16, 16, 16"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<stat name="read_accesses" value="sharedmemory_read_access_match_mcpat"/>
+				<stat name="write_accesses" value="sharedmemory_write_access_match_mcpat"/>
+				<stat name="read_misses" value="0"/>
+				<stat name="write_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+			</component>
+			<component id="system.core0.dcache" name="dcache">
+			        <!-- all the buffer related are optional -->
+				<param name="dcache_config" value="16384,32,4,1,1,3,8,0"/>
+				<param name="buffer_sizes" value="16, 16, 16, 0"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<stat name="read_accesses" value="dcache_read_access_match_mcpat"/>
+				<stat name="write_accesses" value="dcache_write_access_match_mcpat"/>
+				<stat name="read_misses" value="dcache_read_miss_match_mcpat"/>
+				<stat name="write_misses" value="dcache_write_miss_match_mcpat"/>
+				<stat name="conflicts" value="0"/>	
+			</component>
+			<component id="system.core0.BTB" name="BTB">
+			        <!-- all the buffer related are optional -->
+				<param name="BTB_config" value="8192,4,2,1, 1,3"/>
+				<!-- the parameters are capacity,block_width,associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,-->
+			</component>
+	</component>
+		<component id="system.L1Directory0" name="L1Directory0">
+				<param name="Directory_type" value="0"/>
+			    <!--0 cam based shadowed tag. 1 directory cache -->	
+				<param name="Dir_config" value="2048,1,0,1, 4, 4,8"/>
+				<!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,-->
+			    <param name="buffer_sizes" value="8, 8, 8, 8"/>	
+				<!-- all the buffer related are optional -->
+			    <param name="clockrate" value="1400"/>
+				<param name="ports" value="1,1,1"/>
+				<!-- number of r, w, and rw search ports -->
+				<param name="device_type" value="0"/>
+				<!-- altough there are multiple access types, 
+				Performance simulator needs to cast them into reads or writes
+				e.g. the invalidates can be considered as writes -->
+				<stat name="read_accesses" value="800000"/>
+				<stat name="write_accesses" value="27276"/>
+				<stat name="read_misses" value="1632"/>
+				<stat name="write_misses" value="183"/>
+				<stat name="conflicts" value="20"/>	
+				<stat name="duty_cycle" value="0.45"/>	
+		</component>
+		<component id="system.L2Directory0" name="L2Directory0">
+				<param name="Directory_type" value="1"/>
+			    <!--0 cam based shadowed tag. 1 directory cache -->	
+				<param name="Dir_config" value="1048576,16,16,1,2, 100"/>
+				<!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,-->
+			    <param name="buffer_sizes" value="8, 8, 8, 8"/>	
+				<!-- all the buffer related are optional -->
+			    <param name="clockrate" value="1400"/>
+				<param name="ports" value="1,1,1"/>
+				<!-- number of r, w, and rw search ports -->
+				<param name="device_type" value="0"/>
+				<!-- altough there are multiple access types, 
+				Performance simulator needs to cast them into reads or writes
+				e.g. the invalidates can be considered as writes -->
+				<stat name="read_accesses" value="0"/>
+				<stat name="write_accesses" value="0"/>
+				<stat name="read_misses" value="0"/>
+				<stat name="write_misses" value="0"/>
+				<stat name="conflicts" value="0"/>
+			    <stat name="duty_cycle" value="0.45"/>		
+		</component>
+		<component id="system.L20" name="L20">
+			<!-- all the buffer related are optional -->
+				<param name="L2_config" value="131072,128,16,1, 4,23, 64, 1"/>
+			    <!-- consider 4-way bank interleaving for Niagara 1 -->
+				<!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy -->
+				<param name="buffer_sizes" value="16, 16, 16, 16"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<param name="clockrate" value="2962"/>
+				<param name="ports" value="1,1,1"/>
+				<!-- number of r, w, and rw ports -->
+				<param name="device_type" value="0"/>
+				<stat name="read_accesses" value="200000"/>
+				<stat name="write_accesses" value="0"/>
+				<stat name="read_misses" value="0"/>
+				<stat name="write_misses" value="0"/>
+				<stat name="conflicts" value="0"/>	
+			    <stat name="duty_cycle" value="0.5"/>	
+		</component>
+		
+<!--**********************************************************************-->
+<component id="system.L30" name="L30">
+				<param name="L3_config" value="1048576,64,16,1, 2,100, 64,1"/>
+				<!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy -->
+				<param name="clockrate" value="3500"/>
+				<param name="ports" value="1,1,1"/>
+				<!-- number of r, w, and rw ports -->
+				<param name="device_type" value="0"/>
+				<param name="buffer_sizes" value="16, 16, 16, 16"/>
+				<!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size-->	
+				<stat name="read_accesses" value="58824"/>
+				<stat name="write_accesses" value="27276"/>
+				<stat name="read_misses" value="1632"/>
+				<stat name="write_misses" value="183"/>
+				<stat name="conflicts" value="0"/>	
+	            <stat name="duty_cycle" value="0.35"/>	
+		</component>
+
+
+<!--**********************************************************************-->
+		<component id="system.NoC0" name="noc0">
+			<param name="clockrate" value="700"/>
+			<param name="type" value="1"/>
+			<!-- 1 NoC, O bus -->
+			<param name="horizontal_nodes" value="2"/>
+			<param name="vertical_nodes" value="1"/>
+			<param name="has_global_link" value="0"/>
+			<!-- 1 has global link, 0 does not have global link -->
+			<param name="link_throughput" value="1"/><!--w.r.t clock -->
+			<param name="link_latency" value="1"/><!--w.r.t clock -->
+			<!-- througput >= latency -->
+			<!-- Router architecture -->
+			<param name="input_ports" value="6"/>
+			<param name="output_ports" value="6"/>
+			<param name="virtual_channel_per_port" value="1"/>
+			<!-- input buffer; in classic routers only input ports need buffers -->
+			<param name="flit_bits" value="32"/>
+			<param name="input_buffer_entries_per_vc" value="1"/><!--VCs within the same ports share input buffers whose size is propotional to the number of VCs-->
+			<param name="chip_coverage" value="1"/>
+			<!-- When multiple NOC present, one NOC will cover part of the whole chip. chip_coverage <=1 -->
+			<stat name="total_accesses" value="0"/>
+			<!-- This is the number of total accesses within the whole network not for each router -->
+			<stat name="duty_cycle" value="0.6"/>
+		</component>	
+<!--**********************************************************************-->
+<!--**********************************************************************-->
+
+		<component id="system.mem" name="mem">
+			<!-- Main memory property -->
+			<param name="mem_tech_node" value="23"/>
+			<param name="device_clock" value="200"/><!--MHz, this is clock rate of the actual memory device, not the FSB -->
+			<param name="peak_transfer_rate" value="3200"/><!--MB/S-->
+			<param name="internal_prefetch_of_DRAM_chip" value="4"/>
+			<!-- 2 for DDR, 4 for DDR2, 8 for DDR3...-->
+			<!-- the device clock, peak_transfer_rate, and the internal prefetch decide the DIMM property -->
+			<!-- above numbers can be easily found from Wikipedia -->
+			<param name="capacity_per_channel" value="4096"/> <!-- MB -->
+			<!-- capacity_per_Dram_chip=capacity_per_channel/number_of_dimms/number_ranks/Dram_chips_per_rank
+			Current McPAT assumes single DIMMs are used.--> 		
+			<param name="number_ranks" value="2"/>
+			<param name="num_banks_of_DRAM_chip" value="6"/>			
+			<param name="Block_width_of_DRAM_chip" value="64"/> <!-- B -->
+			<param name="output_width_of_DRAM_chip" value="8"/>
+			<!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip-->
+			<!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip-->
+			<param name="page_size_of_DRAM_chip" value="8"/> <!-- 8 or 16 -->
+			<param name="burstlength_of_DRAM_chip" value="8"/>
+			<stat name="memory_accesses" value="1052"/>
+			<stat name="memory_reads" value="1052"/>
+			<stat name="memory_writes" value="1052"/>									
+		</component>
+		<component id="system.mc" name="mc">
+			<!-- Memeory controllers are for DDR(2,3...) DIMMs -->
+			<!-- current version of McPAT uses published values for base parameters of memory controller
+			improvments on MC will be added in later versions. -->
+			<param name="type" value="0"/> <!-- 1: low power; 0 high performance -->
+			<param name="mc_clock" value="1848"/><!--DIMM IO bus clock rate MHz DDR2-400 for Niagara 1--> 
+			<param name="peak_transfer_rate" value="29568"/><!--MB/S  Syed: GTX 470 has 177.4GB/s mem transfer rate with 6 MCs -->
+			<param name="block_size" value="64"/><!--B-->
+			<param name="number_mcs" value="6"/><!-- 6 GDDR5 memory controllers  -->
+			<!-- current McPAT only supports homogeneous memory controllers -->
+			<param name="memory_channels_per_mc" value="2"/>
+			<param name="number_ranks" value="1"/>
+			<param name="withPHY" value="0"/>
+			<!-- # of ranks of each channel-->
+			<param name="req_window_size_per_channel" value="16"/>
+			<param name="IO_buffer_size_per_channel" value="16"/>
+			<param name="databus_width" value="32"/>
+			<param name="addressbus_width" value="32"/>
+			<param name="PRT_entries" value="32"/>
+			<!-- # of empirical DRAM model parameter -->
+			<param name="dram_cmd_coeff" value="0"/>
+			<param name="dram_act_coeff" value="0"/>
+			<param name="dram_nop_coeff" value="0"/>
+			<param name="dram_activity_coeff" value="0"/>
+			<param name="dram_pre_coeff" value="3.8475e-8f"/>
+			<param name="dram_rd_coeff" value="7.74707143e-8f"/>
+			<param name="dram_wr_coeff" value="3.54664286e-8f"/>
+			<param name="dram_req_coeff" value="0"/>
+			<param name="dram_const_coeff" value="0"/>
+
+			<!-- McPAT will add the control bus width to the addressbus width automatically -->
+			<stat name="memory_accesses" value="memory_accesses_match_mcpat"/>
+			<stat name="memory_reads" value="memory_reads_match_mcpat"/>
+			<stat name="memory_writes" value="memory_writes_match_mcpat"/>
+			<!-- McPAT does not track individual mc, instead, it takes the total accesses and calculate 
+			the average power per MC or per channel. This is sufficent for most application. 
+			Further trackdown can be easily added in later versions. -->  			
+		</component>
+<!--**********************************************************************-->
+		<component id="system.niu" name="niu">
+			<!-- On chip 10Gb Ethernet NIC, including XAUI Phy and MAC controller  -->
+			<!-- For a minimum IP packet size of 84B at 10Gb/s, a new packet arrives every 67.2ns. 
+				 the low bound of clock rate of a 10Gb MAC is 150Mhz -->
+			<param name="type" value="0"/> <!-- 1: low power; 0 high performance -->
+			<param name="clockrate" value="350"/>
+			<param name="number_units" value="0"/> <!-- unlike PCIe and memory controllers, each Ethernet controller only have one port -->
+			<stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 -->
+			<stat name="total_load_perc" value="0.7"/> <!-- ratio of total achived load to total achivable bandwidth  -->
+			<!-- McPAT does not track individual nic, instead, it takes the total accesses and calculate 
+			the average power per nic or per channel. This is sufficent for most application. -->  			
+		</component>
+<!--**********************************************************************-->
+		<component id="system.pcie" name="pcie">
+			<!-- On chip PCIe controller, including Phy-->
+			<!-- For a minimum PCIe packet size of 84B at 8Gb/s per lane (PCIe 3.0), a new packet arrives every 84ns. 
+				 the low bound of clock rate of a PCIe per lane logic is 120Mhz -->
+			<param name="type" value="0"/> <!-- 1: low power; 0 high performance -->
+			<param name="withPHY" value="1"/>
+			<param name="clockrate" value="350"/>
+			<param name="number_units" value="0"/>
+			<param name="num_channels" value="8"/> <!-- 2 ,4 ,8 ,16 ,32 -->
+			<stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 -->
+			<stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth  -->
+			<!-- McPAT does not track individual pcie controllers, instead, it takes the total accesses and calculate 
+			the average power per pcie controller or per channel. This is sufficent for most application. -->  			
+		</component>
+<!--**********************************************************************-->
+		<component id="system.flashc" name="flashc">
+		    <param name="number_flashcs" value="0"/>
+			<param name="type" value="1"/> <!-- 1: low power; 0 high performance -->
+            <param name="withPHY" value="1"/>
+			<param name="peak_transfer_rate" value="200"/><!--Per controller sustainable reak rate MB/S -->
+			<stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 -->
+			<stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth  -->
+			<!-- McPAT does not track individual flash controller, instead, it takes the total accesses and calculate 
+			the average power per fc or per channel. This is sufficent for most application -->  			
+		</component>
+<!--**********************************************************************-->
+
+		</component>
+</component>
diff --git a/cuobjdump_to_ptxplus/ptx_parser.h b/cuobjdump_to_ptxplus/ptx_parser.h
index 1c96b46..418a733 100644
--- a/cuobjdump_to_ptxplus/ptx_parser.h
+++ b/cuobjdump_to_ptxplus/ptx_parser.h
@@ -110,6 +110,11 @@ void add_alignment_spec( int ) {PTX_PARSE_DPRINTF(" ");}
 void add_pragma( const char *a ) {PTX_PARSE_DPRINTF(" ");}
 void add_constptr(const char* identifier1, const char* identifier2, int offset) {PTX_PARSE_DPRINTF(" ");}
 
+//Jin: handle instructino group for cdp
+void start_inst_group(){PTX_PARSE_DPRINTF(" ");};
+void end_inst_group(){PTX_PARSE_DPRINTF(" ");};
+
+
 /*non-dummy stuff below this point*/
 
 extern cuobjdumpInstList *g_headerList;
diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index e8a0e91..b7f25bf 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -126,7 +126,9 @@
 #include "host_defines.h"
 #include "builtin_types.h"
 #include "driver_types.h"
+#if (CUDART_VERSION < 8000)
 #include "__cudaFatFormat.h"
+#endif
 #include "../src/gpgpu-sim/gpu-sim.h"
 #include "../src/cuda-sim/ptx_loader.h"
 #include "../src/cuda-sim/cuda-sim.h"
@@ -262,10 +264,19 @@ struct CUctx_st {
 	{
 		if( m_code.find(fat_cubin_handle) != m_code.end() ) {
 			symbol *s = m_code[fat_cubin_handle]->lookup(deviceFun);
-			assert( s != NULL );
-			function_info *f = s->get_pc();
-			assert( f != NULL );
-			m_kernel_lookup[hostFun] = f;
+			if(s != NULL) {
+				function_info *f = s->get_pc();
+				assert( f != NULL );
+				m_kernel_lookup[hostFun] = f;
+			}
+			else {
+				printf("Warning: cannot find deviceFun %s\n", deviceFun);
+				m_kernel_lookup[hostFun] = NULL;
+			}
+	//		assert( s != NULL );
+	//		function_info *f = s->get_pc();
+	//		assert( f != NULL );
+	//		m_kernel_lookup[hostFun] = f;
 		} else {
 			m_kernel_lookup[hostFun] = NULL;
 		}
@@ -1353,7 +1364,12 @@ void extract_code_using_cuobjdump(){
 	printf("Running md5sum using \"%s\"\n", command);
 	system(command);
 	// Running cuobjdump using dynamic link to current process
-	snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname);
+	// Needs the option '-all' to extract PTX from CDP-enabled binary 
+	extern bool g_cdp_enabled;
+	if(!g_cdp_enabled)
+	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname);
+	else
+	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all %s > %s", app_binary.c_str(), fname);
 	bool parse_output = true; 
 	int result = system(command);
 	if(result) {
@@ -1390,7 +1406,7 @@ void extract_code_using_cuobjdump(){
 		cmd << "ldd " << app_binary << " | grep $CUDA_INSTALL_PATH | awk \'{print $3}\' > _tempfile_.txt";
 		int result = system(cmd.str().c_str());
 		if(result){
-			std::cout << "Failed to execute: " << cmd << std::endl;
+			std::cout << "Failed to execute: " << cmd.str() << std::endl;
 			exit(1);
 		}
 		std::ifstream libsf;
@@ -1422,7 +1438,7 @@ void extract_code_using_cuobjdump(){
 			if(result) {printf("ERROR: Failed to execute: %s\n", command); exit(1);}
 			std::cout << "Done" << std::endl;
 
-			std::cout << "Trying to parse " << libcodfn << std::endl;
+			std::cout << "Trying to parse " << libcodfn.str() << std::endl;
 			cuobjdump_in = fopen(libcodfn.str().c_str(), "r");
 			cuobjdump_parse();
 			fclose(cuobjdump_in);
@@ -1524,7 +1540,7 @@ std::list<cuobjdumpSection*> pruneSectionList(std::list<cuobjdumpSection*> cuobj
 
 //! Merge all PTX sections that have a specific identifier into one file
 std::list<cuobjdumpSection*> mergeMatchingSections(std::list<cuobjdumpSection*> cuobjdumpSectionList, std::string identifier){
-	char *ptxcode = "";
+	const char *ptxcode = "";
 	std::list<cuobjdumpSection*>::iterator old_iter;
 	cuobjdumpPTXSection* old_ptxsection = NULL;
 	cuobjdumpPTXSection* ptxsection;
@@ -1618,7 +1634,7 @@ cuobjdumpELFSection* findELFSection(const std::string identifier){
 	if (sec!=NULL)return sec;
 	sec = findELFSectionInList(libSectionList, identifier);
 	if (sec!=NULL)return sec;
-	std::cout << "Cound not find " << identifier << std::endl;
+	std::cout << "Could not find " << identifier << std::endl;
 	assert(0 && "Could not find the required ELF section");
 	return NULL;
 }
@@ -1634,6 +1650,14 @@ cuobjdumpPTXSection* findPTXSectionInList(std::list<cuobjdumpSection*> sectionli
 		if((ptxsection=dynamic_cast<cuobjdumpPTXSection*>(*iter)) != NULL){
 			if(ptxsection->getIdentifier() == identifier)
 				return ptxsection;
+			else {
+				extern bool g_cdp_enabled;
+				if(g_cdp_enabled) {
+					printf("Warning: __cudaRegisterFatBinary needs %s, but find PTX section with %s\n",
+						identifier.c_str(), ptxsection->getIdentifier().c_str());
+					return ptxsection;
+				}
+			}
 		}
 	}
 	return NULL;
@@ -1645,7 +1669,7 @@ cuobjdumpPTXSection* findPTXSection(const std::string identifier){
 	if (sec!=NULL)return sec;
 	sec = findPTXSectionInList(libSectionList, identifier);
 	if (sec!=NULL)return sec;
-	std::cout << "Cound not find " << identifier << std::endl;
+	std::cout << "Could not find " << identifier << std::endl;
 	assert(0 && "Could not find the required PTX section");
 	return NULL;
 }
@@ -1665,7 +1689,7 @@ std::map<int, bool>fatbin_registered;
 std::map<std::string, symbol_table*> name_symtab;
 
 //! Keep track of the association between filename and cubin handle
-void cuobjdumpRegisterFatBinary(unsigned int handle, char* filename){
+void cuobjdumpRegisterFatBinary(unsigned int handle, const char* filename){
 	fatbinmap[handle] = filename;
 }
 
@@ -1740,6 +1764,7 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		if (sizeof(void*) == 4) 
 			printf("GPGPU-Sim PTX: FatBin file name extraction has not been tested on 32-bit system.\n"); 
 
+		#if (CUDART_VERSION <= 6000)
 		// FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
 		typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
 		__fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
@@ -1748,12 +1773,11 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		// - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin).
 		// - An integer inside the fatbin structure contains the relative offset to the source code file name.
 		// - This offset differs among different CUDA and GCC versions. 
-		#if (CUDART_VERSION <= 6000)
 		char * pfatbin = (char*) fatDeviceText->d; 
 		int offset = *((int*)(pfatbin+48)); 
 		char * filename = (pfatbin+16+offset); 
 		#else
-		char * filename = "default";
+		const char * filename = "default";
 		#endif
 		// The extracted file name is associated with a fat_cubin_handle passed
 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
@@ -1773,7 +1797,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		cuobjdumpRegisterFatBinary(fat_cubin_handle, filename);
 
 		return (void**)fat_cubin_handle;
-	} else {
+	} 
+#if (CUDART_VERSION < 8000)
+	else {
 		static unsigned source_num=1;
 		unsigned long long fat_cubin_handle = next_fat_bin_handle++;
 		__cudaFatCudaBinary *info =   (__cudaFatCudaBinary *)fatCubin;
@@ -1830,6 +1856,11 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		}
 		return (void**)fat_cubin_handle;
 	}
+	#endif
+        else {
+		printf("ERROR **  __cudaRegisterFatBinary() needs to be updated\n");
+		abort();
+        }
 }
 
 void __cudaUnregisterFatBinary(void **fatCubinHandle)
@@ -2119,6 +2150,11 @@ __host__ cudaError_t CUDARTAPI cudaFuncSetCacheConfig(const char *func, enum cud
 	context->get_device()->get_gpgpu()->set_cache_config(context->get_kernel(func)->get_name(), (FuncCache)cacheConfig);
 	return g_last_cudaError = cudaSuccess;
 }
+
+//Jin: hack for cdp
+__host__ cudaError_t CUDARTAPI cudaDeviceSetLimit(enum cudaLimit limit, size_t value) {
+    return g_last_cudaError = cudaSuccess;
+}
 #endif
 
 #endif
diff --git a/libcuda/cuobjdump.l b/libcuda/cuobjdump.l
index f63ee73..0953ea1 100644
--- a/libcuda/cuobjdump.l
+++ b/libcuda/cuobjdump.l
@@ -159,8 +159,6 @@ newlines	{newline}+
 %%
 void cuobjdump_error(const char* message)
 {
-	printf("   "); printf(message); printf(" near \""); printf(yytext); printf("\"");
-	printf(" on line "); 
-	char line[5]; sprintf(line, "%i", yylineno); printf(line);
-	printf("\n");
+	printf("   %s near \"%s\"",message, yytext);
+	printf(" on line %i\n",yylineno); 
 }
diff --git a/libopencl/opencl_runtime_api.cc b/libopencl/opencl_runtime_api.cc
index 020384f..63f7c81 100644
--- a/libopencl/opencl_runtime_api.cc
+++ b/libopencl/opencl_runtime_api.cc
@@ -74,7 +74,9 @@
 #define __CUDA_RUNTIME_API_H__
 #include "host_defines.h"
 #include "builtin_types.h"
+#if (CUDART_VERSION < 8000)
 #include "__cudaFatFormat.h"
+#endif
 #include "../src/abstract_hardware_model.h"
 #include "../src/cuda-sim/cuda-sim.h"
 #include "../src/cuda-sim/ptx_loader.h"
diff --git a/linux-so-version.txt b/linux-so-version.txt
new file mode 100644
index 0000000..40f775d
--- /dev/null
+++ b/linux-so-version.txt
@@ -0,0 +1,2 @@
+libcudart.so.9.1{
+};
diff --git a/setup_environment b/setup_environment
index 2684514..96cc362 100644
--- a/setup_environment
+++ b/setup_environment
@@ -6,7 +6,11 @@ export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=
 export GPGPUSIM_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )"
 
 GPGPUSIM_VERSION_STRING=`cat $GPGPUSIM_ROOT/version | awk '/Version/ {print $8}'`
-GPGPUSIM_BUILD_STRING=`cat $GPGPUSIM_ROOT/version | awk '/Change/ {print $6}'`
+#Detect Git branch and commit #
+GIT_COMMIT=`git log -n 1 | head -1 | sed -re 's/commit (.*)/\1/'`
+GIT_FILES_CHANGED=`git diff --numstat --cached && git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1/'`
+GPGPUSIM_BUILD_STRING="gpgpu-sim_git-commit-$GIT_COMMIT-modified_$GIT_FILES_CHANGED"
+
 echo -n "GPGPU-Sim version $GPGPUSIM_VERSION_STRING (build $GPGPUSIM_BUILD_STRING) ";
 
 if [ ! -n "$CUDA_INSTALL_PATH" ]; then
@@ -43,11 +47,9 @@ CC_VERSION=`gcc --version | head -1 | awk '{for(i=1;i<=NF;i++){ if(match($i,/^[0
 
 CUDA_VERSION_STRING=`$CUDA_INSTALL_PATH/bin/nvcc --version | awk '/release/ {print $5;}' | sed 's/,//'`;
 CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'`
-if [ $CUDA_VERSION_NUMBER -gt 7500 -o $CUDA_VERSION_NUMBER -lt 2030  ]; then
+if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030  ]; then
 	echo "ERROR ** GPGPU-Sim version $GPGPUSIM_VERSION_STRING not tested with CUDA version $CUDA_VERSION_STRING (please see README)";
 	return
-elif [ $CUDA_VERSION_NUMBER -gt 4020 ]; then
-	echo "WARNING ** GPGPU-Sim version $GPGPUSIM_VERSION_STRING not fully tested with CUDA version $CUDA_VERSION_STRING (please see README)";
 fi
 
 if [ $# = '1' ] ;
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index 84d165c..fe6f8ab 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -550,6 +550,9 @@ void warp_inst_t::completed( unsigned long long cycle ) const
    ptx_file_line_stats_add_latency(pc, latency * active_count());  
 }
 
+//Jin: CDP support
+bool g_cdp_enabled;
+unsigned g_kernel_launch_latency;
 
 unsigned kernel_info_t::m_next_uid = 1;
 
@@ -565,11 +568,18 @@ kernel_info_t::kernel_info_t( dim3 gridDim, dim3 blockDim, class function_info *
     m_num_cores_running=0;
     m_uid = m_next_uid++;
     m_param_mem = new memory_space_impl<8192>("param",64*1024);
+
+    //Jin: parent and child kernel management for CDP
+    m_parent_kernel = NULL;
+   
+    //Jin: launch latency management
+    m_launch_latency = g_kernel_launch_latency;
 }
 
 kernel_info_t::~kernel_info_t()
 {
     assert( m_active_threads.empty() );
+    destroy_cta_streams();
     delete m_param_mem;
 }
 
@@ -578,6 +588,106 @@ std::string kernel_info_t::name() const
     return m_kernel_entry->get_name();
 }
 
+//Jin: parent and child kernel management for CDP
+void kernel_info_t::set_parent(kernel_info_t * parent, 
+    dim3 parent_ctaid, dim3 parent_tid) {
+    m_parent_kernel = parent;
+    m_parent_ctaid = parent_ctaid;
+    m_parent_tid = parent_tid;
+    parent->set_child(this);
+}
+
+void kernel_info_t::set_child(kernel_info_t * child) {
+    m_child_kernels.push_back(child);
+}
+
+void kernel_info_t::remove_child(kernel_info_t * child) {
+    assert(std::find(m_child_kernels.begin(), m_child_kernels.end(), child)
+        != m_child_kernels.end());
+    m_child_kernels.remove(child);
+}
+
+bool kernel_info_t::is_finished() {
+  if(done() && children_all_finished())
+     return true;
+  else
+     return false;
+}
+
+bool kernel_info_t::children_all_finished() {
+   if(!m_child_kernels.empty())
+         return false;
+   
+   return true;
+}
+
+void kernel_info_t::notify_parent_finished() {
+   if(m_parent_kernel) {
+       extern unsigned long long g_total_param_size;
+       g_total_param_size -= ((m_kernel_entry->get_args_aligned_size() + 255)/256*256);
+       m_parent_kernel->remove_child(this);
+       g_stream_manager->register_finished_kernel(m_parent_kernel->get_uid());
+   }
+}
+
+CUstream_st * kernel_info_t::create_stream_cta(dim3 ctaid) {
+    assert(get_default_stream_cta(ctaid));
+    CUstream_st * stream = new CUstream_st();
+    g_stream_manager->add_stream(stream);
+    assert(m_cta_streams.find(ctaid) != m_cta_streams.end());
+    assert(m_cta_streams[ctaid].size() >= 1); //must have default stream
+    m_cta_streams[ctaid].push_back(stream);
+
+    return stream;
+}
+
+CUstream_st * kernel_info_t::get_default_stream_cta(dim3 ctaid) {
+    if(m_cta_streams.find(ctaid) != m_cta_streams.end()) {
+       assert(m_cta_streams[ctaid].size() >= 1); //already created, must have default stream
+       return *(m_cta_streams[ctaid].begin());
+    }
+    else {
+      m_cta_streams[ctaid] = std::list<CUstream_st *>();
+      CUstream_st * stream = new CUstream_st();
+      g_stream_manager->add_stream(stream);
+      m_cta_streams[ctaid].push_back(stream);
+      return stream;
+    }
+}
+
+bool kernel_info_t::cta_has_stream(dim3 ctaid, CUstream_st* stream) {
+    if(m_cta_streams.find(ctaid) == m_cta_streams.end())
+       return false;
+
+    std::list<CUstream_st *> &stream_list = m_cta_streams[ctaid];
+    if(std::find(stream_list.begin(), stream_list.end(), stream) 
+         == stream_list.end())
+       return false;
+    else
+       return true;
+}
+
+void kernel_info_t::print_parent_info() {
+    if(m_parent_kernel) {
+        printf("Parent %d: \'%s\', Block (%d, %d, %d), Thread (%d, %d, %d)\n", 
+            m_parent_kernel->get_uid(), m_parent_kernel->name().c_str(), 
+            m_parent_ctaid.x, m_parent_ctaid.y, m_parent_ctaid.z,
+            m_parent_tid.x, m_parent_tid.y, m_parent_tid.z);
+    }
+}
+
+void kernel_info_t::destroy_cta_streams() {
+     printf("Destroy streams for kernel %d: ", get_uid()); size_t stream_size = 0;
+     for(auto s = m_cta_streams.begin(); s != m_cta_streams.end(); s++) {
+        stream_size += s->second.size();
+        for(auto ss = s->second.begin(); ss != s->second.end(); ss++)
+        g_stream_manager->destroy_stream(*ss);
+        s->second.clear();
+     }
+     printf("size %lu\n", stream_size);
+     m_cta_streams.clear();
+}
+
 simt_stack::simt_stack( unsigned wid, unsigned warpSize)
 {
     m_warp_id=wid;
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index b29f918..7125b6b 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -154,15 +154,35 @@ enum _memory_op_t {
 #include <stdlib.h>
 #include <map>
 #include <deque>
+#include <algorithm>
 
 #if !defined(__VECTOR_TYPES_H__)
 struct dim3 {
    unsigned int x, y, z;
 };
 #endif
+struct dim3comp {
+    bool operator() (const dim3 & a, const dim3 & b) const
+    {    
+        if(a.z < b.z)
+            return true;
+        else if(a.y < b.y)
+            return true;
+        else if (a.x < b.x)
+            return true;
+        else
+            return false;
+    }
+};
 
 void increment_x_then_y_then_z( dim3 &i, const dim3 &bound);
 
+//Jin: child kernel information for CDP
+#include "stream_manager.h"
+class stream_manager;
+struct CUstream_st;
+extern stream_manager * g_stream_manager;
+
 class kernel_info_t {
 public:
 //   kernel_info_t()
@@ -250,6 +270,35 @@ private:
 
    std::list<class ptx_thread_info *> m_active_threads;
    class memory_space *m_param_mem;
+
+public:
+   //Jin: parent and child kernel management for CDP
+   void set_parent(kernel_info_t * parent, dim3 parent_ctaid, dim3 parent_tid);
+   void set_child(kernel_info_t * child);
+   void remove_child(kernel_info_t * child);
+   bool is_finished();
+   bool children_all_finished();
+   void notify_parent_finished();
+   CUstream_st * create_stream_cta(dim3 ctaid);
+   CUstream_st * get_default_stream_cta(dim3 ctaid);
+   bool cta_has_stream(dim3 ctaid, CUstream_st* stream);
+   void destroy_cta_streams();
+   void print_parent_info();
+   kernel_info_t * get_parent() { return m_parent_kernel; }
+
+private:
+   kernel_info_t * m_parent_kernel;
+   dim3 m_parent_ctaid;
+   dim3 m_parent_tid;
+   std::list<kernel_info_t *> m_child_kernels; //child kernel launched
+   std::map< dim3, std::list<CUstream_st *>, dim3comp > m_cta_streams; //streams created in each CTA
+
+//Jin: kernel timing
+public:
+   unsigned long long launch_cycle;
+   unsigned long long start_cycle;
+   unsigned long long end_cycle;
+   unsigned m_launch_latency;
 };
 
 struct core_config {
@@ -334,7 +383,7 @@ protected:
     std::deque<simt_stack_entry> m_stack;
 };
 
-#define GLOBAL_HEAP_START 0x703E20000
+#define GLOBAL_HEAP_START 0xC0000000
    // start allocating from this address (lower values used for allocating globals in .ptx file)
 #define SHARED_MEM_SIZE_MAX (64*1024)
 #define LOCAL_MEM_SIZE_MAX (8*1024)
@@ -825,6 +874,7 @@ public:
         m_mem_accesses_created=false;
         m_cache_hit=false;
         m_is_printf=false;
+        m_is_cdp = 0;
     }
     virtual ~warp_inst_t(){
     }
@@ -997,6 +1047,11 @@ protected:
     std::list<mem_access_t> m_accessq;
 
     static unsigned sm_next_uid;
+
+    //Jin: cdp support
+public:
+    int m_is_cdp;
+    
 };
 
 void move_warp( warp_inst_t *&dst, warp_inst_t *&src );
@@ -1051,6 +1106,7 @@ class core_t {
         warp_inst_t getExecuteWarp(unsigned warpId);
         void get_pdom_stack_top_info( unsigned warpId, unsigned *pc, unsigned *rpc ) const;
         kernel_info_t * get_kernel_info(){ return m_kernel;}
+        class ptx_thread_info ** get_thread_info() { return m_thread; }
         unsigned get_warp_size() const { return m_warp_size; }
         void and_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] &= value; }
         void or_reduction(unsigned ctaid, unsigned barid, bool value) { reduction_storage[ctaid][barid] |= value; }
diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile
index 166e256..999dad7 100644
--- a/src/cuda-sim/Makefile
+++ b/src/cuda-sim/Makefile
@@ -46,7 +46,7 @@ OPT	:=  -O3 -g3 -Wall -Wno-unused-function -Wno-sign-compare
 ifeq ($(DEBUG),1)
 	OPT := -g3 -Wall  -Wno-unused-function -Wno-sign-compare
 endif
-OPT += -I$(CUDA_INSTALL_PATH)/include  -I$(OUTPUT_DIR)/ -I.
+OPT += -I$(CUDA_INSTALL_PATH)/include  -I$(OUTPUT_DIR)/ -I. -I$(SIM_OBJ_FILES_DIR)
 OPT += -fPIC 
 
 ifeq ($(TRACE),1)
@@ -62,7 +62,7 @@ ifeq ($(GNUC_CPP0X),1)
 endif
 endif
 
-OBJS	:= $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o  $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o
+OBJS	:= $(OUTPUT_DIR)/ptx_parser.o $(OUTPUT_DIR)/ptx_loader.o $(OUTPUT_DIR)/cuda_device_printf.o $(OUTPUT_DIR)/instructions.o $(OUTPUT_DIR)/cuda-sim.o $(OUTPUT_DIR)/ptx_ir.o $(OUTPUT_DIR)/ptx_sim.o  $(OUTPUT_DIR)/memory.o $(OUTPUT_DIR)/ptx-stats.o $(OUTPUT_DIR)/decuda_pred_table/decuda_pred_table.o $(OUTPUT_DIR)/ptx.tab.o $(OUTPUT_DIR)/lex.ptx_.o $(OUTPUT_DIR)/ptxinfo.tab.o $(OUTPUT_DIR)/lex.ptxinfo_.o $(OUTPUT_DIR)/cuda_device_runtime.o
 
 
 OPT += -DCUDART_VERSION=$(CUDART_VERSION)
@@ -142,8 +142,9 @@ $(OUTPUT_DIR)/ptx_parser.o: $(OUTPUT_DIR)/ptx.tab.c $(OUTPUT_DIR)/ptx_parser_dec
 $(OUTPUT_DIR)/ptxinfo.tab.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/ptx-stats.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/ptx_sim.o: $(OUTPUT_DIR)/ptx.tab.c
-$(OUTPUT_DIR)/cuda-sim.o: $(OUTPUT_DIR)/ptx.tab.c
+$(OUTPUT_DIR)/cuda-sim.o: $(OUTPUT_DIR)/ptx.tab.c $(SIM_OBJ_FILES_DIR)/detailed_version
 $(OUTPUT_DIR)/lex.ptxinfo_.o: $(OUTPUT_DIR)/ptx.tab.c
 $(OUTPUT_DIR)/lex.ptx_.o: $(OUTPUT_DIR)/ptx.tab.c
+$(OUTPUT_DIR)/cuda_device_runtime.o: $(OUTPUT_DIR)/ptx.tab.c
 
 include $(OUTPUT_DIR)/Makefile.makedepend
diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h
index 4721e8a..a3db0df 100644
--- a/src/cuda-sim/cuda-math.h
+++ b/src/cuda-sim/cuda-math.h
@@ -321,7 +321,7 @@ float __internal_accurate_fdividef(float a, float b)
 float __saturatef(float a)
 {
    float b; 
-   if (isnan(a)) b = 0.0f; 
+   if (std::isnan(a)) b = 0.0f; 
    else if (a >= 1.0f) b = 1.0f;
    else if (a <= 0.0f) b = 0.0f; 
    else b = a; 
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 09e9a81..d4ace76 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -48,6 +48,7 @@
 #include "../gpgpusim_entrypoint.h"
 #include "decuda_pred_table/decuda_pred_table.h"
 #include "../stream_manager.h"
+#include "cuda_device_runtime.h"
 
 int gpgpu_ptx_instruction_classification;
 void ** g_inst_classification_stat = NULL;
@@ -63,6 +64,8 @@ unsigned gpgpu_param_num_shaders = 0;
 
 char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp;
 char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp;
+char *cdp_latency_str;
+unsigned cdp_latency[5];
 
 void ptx_opcocde_latency_options (option_parser_t opp) {
 	option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int,
@@ -89,6 +92,12 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
 			"Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>"
 			"Default 8,8,8,8,130",
 			"8,8,8,8,130");
+	option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str,
+			"CDP API latency <cudaStreamCreateWithFlags, \
+cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \
+cudaLaunchDeviceV2_init_perWarp, cudaLaunchDevicV2_perKernel>"
+			"Default 7200,8000,100,12000,1600",
+			"7200,8000,100,12000,1600");
 }
 
 static address_type get_converge_point(address_type pc);
@@ -608,6 +617,9 @@ void ptx_instruction::set_opcode_and_latency()
 	sscanf(opcode_initiation_dp, "%u,%u,%u,%u,%u",
 			&dp_init[0],&dp_init[1],&dp_init[2],
 			&dp_init[3],&dp_init[4]);
+	sscanf(cdp_latency_str, "%u,%u,%u,%u,%u",
+			&cdp_latency[0],&cdp_latency[1],&cdp_latency[2], 
+            &cdp_latency[3],&cdp_latency[4]);
 
 	if(!m_operands.empty()){
 		std::vector<operand_info>::iterator it;
@@ -638,19 +650,21 @@ void ptx_instruction::set_opcode_and_latency()
    case MEMBAR_OP: op = MEMORY_BARRIER_OP; break;
    case CALL_OP:
    {
-       if(m_is_printf)
+       if(m_is_printf || m_is_cdp) {
            op = ALU_OP;
+       }
        else
            op = CALL_OPS;
        break;
    }
    case CALLP_OP:
    {
-       if(m_is_printf)
+       if(m_is_printf || m_is_cdp) {
                op = ALU_OP;
-           else
-               op = CALL_OPS;
-           break;
+       }
+       else
+           op = CALL_OPS;
+       break;
    }
    case RET_OP: case RETP_OP:  op = RET_OPS;break;
    case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP:
@@ -769,6 +783,10 @@ void ptx_instruction::set_opcode_and_latency()
 	  initiation_interval = dp_init[2];
       op = SFU_OP;
       break;
+   case SHFL_OP:
+	   latency = 32;
+	   initiation_interval = 15;
+	   break;
    default: 
        break;
    }
@@ -845,8 +863,10 @@ void ptx_instruction::pre_decode()
 
    switch ( get_opcode() ) {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break;
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break;
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
    default:
       printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() );
       break;
@@ -1065,6 +1085,32 @@ void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *arg
    } 
 }
 
+unsigned function_info::get_args_aligned_size() {
+ 
+   if(m_args_aligned_size >= 0)
+      return m_args_aligned_size;
+ 
+   unsigned param_address = 0;
+   unsigned int total_size = 0;
+   for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) {
+      param_info &p = i->second;
+      std::string name = p.get_name();
+      symbol *param = m_symtab->lookup(name.c_str());
+
+      size_t arg_size = p.get_size() / 8; // size of param in bytes
+      total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned
+      p.add_offset(total_size);
+      param->set_address(param_address + total_size);
+      total_size += arg_size;
+   }
+
+   m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word
+
+   return m_args_aligned_size;
+
+}
+
+
 void function_info::finalize( memory_space *param_mem ) 
 {
    unsigned param_address = 0;
@@ -1087,13 +1133,17 @@ void function_info::finalize( memory_space *param_mem )
          size = (size<(p.get_size()/8))?size:(p.get_size()/8);
       } 
       // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over
+      //Jin: copy parameter using aligned rules
       const size_t word_size = 4; 
+      param_address = (param_address + size - 1) / size * size; //aligned with size 
       for (size_t idx = 0; idx < size; idx += word_size) {
          const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic
          param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); 
       }
+      unsigned offset = p.get_offset();
+      assert(offset == param_address);
       param->set_address(param_address);
-      param_address += size; 
+      param_address += size;
    }
 }
 
@@ -1240,8 +1290,10 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
       }
       switch ( pI->get_opcode() ) {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break;
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break;
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
       default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break;
       }
       delete pJ;
@@ -1408,6 +1460,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
 
    static std::map<unsigned,memory_space*> shared_memory_lookup;
    static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup;
+   static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup;
    static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup;
 
    if ( *thread_info != NULL ) {
@@ -1455,7 +1508,8 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
    unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 
    assert( max_cta_per_sm > 0 );
 
-   unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid;
+   //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid;
+   unsigned sm_idx = hw_cta_id*gpgpu_param_num_shaders + sid;
 
    if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) {
       if ( g_debug_execution >= 1 ) {
@@ -1486,7 +1540,16 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       kernel.increment_thread_id();
       new_tid += tid;
       ptx_thread_info *thd = new ptx_thread_info(kernel);
-   
+
+      ptx_warp_info *warp_info = NULL;
+      if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) {
+    	  warp_info = new ptx_warp_info();
+    	  ptx_warp_lookup[hw_warp_id] = warp_info;
+      } else {
+    	  warp_info = ptx_warp_lookup[hw_warp_id];
+      }
+      thd->m_warp_info = warp_info;
+
       memory_space *local_mem = NULL;
       std::map<unsigned,memory_space*>::iterator l = local_mem_lookup.find(new_tid);
       if ( l != local_mem_lookup.end() ) {
@@ -1560,14 +1623,13 @@ kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry,
 }
 
 #include "../../version"
+#include "detailed_version"
 
 void print_splash()
 {
    static int splash_printed=0;
    if ( !splash_printed ) {
-      unsigned build=0;
-      sscanf(g_gpgpusim_build_string, "$Change"": %u $", &build);
-      fprintf(stdout, "\n\n        *** %s [build %u] ***\n\n\n", g_gpgpusim_version_string, build );
+      fprintf(stdout, "\n\n        *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string );
       splash_printed=1;
    }
 }
@@ -1746,14 +1808,19 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL )
             g_the_gpu->getShaderCoreConfig()->warp_size
         );
         cta.execute();
+
+#if (CUDART_VERSION >= 5000)
+	launch_all_device_kernels();
+#endif
     }
     
    //registering this kernel as done      
-   extern stream_manager *g_stream_manager;
    
    //openCL kernel simulation calls don't register the kernel so we don't register its exit
-   if(!openCL)
-   g_stream_manager->register_finished_kernel(kernel.get_uid());
+   if(!openCL) {
+      extern stream_manager *g_stream_manager;
+      g_stream_manager->register_finished_kernel(kernel.get_uid());
+   }
 
    //******PRINTING*******
    printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn );
diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc
new file mode 100644
index 0000000..4a8ffe5
--- /dev/null
+++ b/src/cuda-sim/cuda_device_runtime.cc
@@ -0,0 +1,320 @@
+//Jin: cuda_device_runtime.cc
+//Defines CUDA device runtime APIs for CDP support
+
+
+#include <iostream>
+#include <map>
+
+unsigned long long g_total_param_size = 0;
+unsigned long long g_max_total_param_size = 0;
+
+
+#if (CUDART_VERSION >= 5000)
+#define __CUDA_RUNTIME_API_H__
+
+#include <builtin_types.h>
+#include <driver_types.h>
+#include "../gpgpu-sim/gpu-sim.h"
+#include "cuda-sim.h"
+#include "ptx_ir.h"
+#include "../stream_manager.h"
+#include "cuda_device_runtime.h"
+
+#define DEV_RUNTIME_REPORT(a) \
+   if( g_debug_execution ) { \
+      std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \
+      std::cout.flush(); \
+   }
+
+class device_launch_config_t {
+
+public:
+    device_launch_config_t() {}
+
+    device_launch_config_t(dim3 _grid_dim,
+        dim3 _block_dim,
+        unsigned int _shared_mem,
+        function_info * _entry):
+            grid_dim(_grid_dim),
+            block_dim(_block_dim),
+            shared_mem(_shared_mem),
+            entry(_entry) {}
+    
+    dim3 grid_dim;
+    dim3 block_dim;
+    unsigned int shared_mem;
+    function_info * entry;
+
+};
+
+class device_launch_operation_t {
+
+public:
+    device_launch_operation_t() {}
+    device_launch_operation_t(kernel_info_t *_grid,
+        CUstream_st * _stream) :
+            grid(_grid), stream(_stream) {}
+
+    kernel_info_t * grid; //a new child grid
+
+    CUstream_st * stream; 
+
+};
+
+
+std::map<void *, device_launch_config_t> g_cuda_device_launch_param_map;
+std::list<device_launch_operation_t> g_cuda_device_launch_op;
+extern stream_manager *g_stream_manager;
+
+//Handling device runtime api:
+//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize)
+void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func)
+{
+    DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2");
+      
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 4 );
+
+    function_info * child_kernel_entry;
+    struct dim3 grid_dim, block_dim;
+    unsigned int shared_mem;
+
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//function_info* for the child kernel
+            unsigned long long buf;
+            assert(size == sizeof(function_info *));
+            thread->m_local_mem->read(from_addr, size, &buf);
+            child_kernel_entry = (function_info *)buf;
+            assert(child_kernel_entry);
+            DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name());
+        }
+        else if(arg == 1) { //dim3 grid_dim for the child kernel
+            assert(size == sizeof(struct dim3));
+            thread->m_local_mem->read(from_addr, size, & grid_dim);
+            DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")");
+        }
+        else if(arg == 2) { //dim3 block_dim for the child kernel
+            assert(size == sizeof(struct dim3));
+            thread->m_local_mem->read(from_addr, size, & block_dim);
+            DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")");
+        }
+        else if(arg == 3) { //unsigned int shared_mem
+            assert(size == sizeof(unsigned int));
+            thread->m_local_mem->read(from_addr, size, & shared_mem);
+            DEV_RUNTIME_REPORT("shared memory " << shared_mem);
+        }
+    }
+
+    //get total child kernel argument size and malloc buffer in global memory
+    unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size();
+    void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size);
+    g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256);
+    DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer);
+    if(g_total_param_size > g_max_total_param_size)
+        g_max_total_param_size = g_total_param_size;
+
+    //store param buffer address and launch config
+    device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry);
+    assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end());
+    g_cuda_device_launch_param_map[param_buffer] = device_launch_config;
+
+    //copy the buffer address to retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //void *
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *));
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &param_buffer, NULL, NULL);
+
+}
+
+//Handling device runtime api:
+//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream)
+void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) {
+    DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2");
+
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 2 );
+
+    kernel_info_t * device_grid = NULL;
+    function_info * device_kernel_entry = NULL;
+    void * parameter_buffer;
+    struct CUstream_st * child_stream;
+    device_launch_config_t config;
+    device_launch_operation_t device_launch_op;
+
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//paramter buffer for child kernel (in global memory)
+            //get parameter_buffer from the cudaLaunchDeviceV2_param0
+            assert(size == sizeof(void *));
+            thread->m_local_mem->read(from_addr, size, &parameter_buffer);
+            assert((size_t)parameter_buffer >= GLOBAL_HEAP_START);
+            DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer);
+
+            //get child grid info through parameter_buffer address
+            assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end());
+            config = g_cuda_device_launch_param_map[parameter_buffer];
+            //device_grid = op.grid;
+            device_kernel_entry = config.entry;
+            DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name());
+
+            //copy data in parameter_buffer to device kernel param memory
+            unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size();
+            DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size);
+            memory_space *device_kernel_param_mem;
+
+            //create child kernel_info_t and index it with parameter_buffer address
+            device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry);
+            device_grid->launch_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
+            kernel_info_t & parent_grid = thread->get_kernel();
+            DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" <<
+                thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z <<
+                "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z <<
+                ")");
+            device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid());  
+            device_launch_op = device_launch_operation_t(device_grid, NULL);
+            device_kernel_param_mem = device_grid->get_param_memory(); //kernel param
+            size_t param_start_address = 0;
+            //copy in word
+            for(unsigned n = 0; n < device_kernel_arg_size; n += 4) {
+                unsigned int oneword;
+                thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword);
+                device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); 
+            }
+        }
+        else if(arg == 1) { //cudaStream for the child kernel
+
+            assert(size == sizeof(cudaStream_t));
+            thread->m_local_mem->read(from_addr, size, &child_stream);
+     
+            kernel_info_t & parent_kernel = thread->get_kernel();
+            if(child_stream == 0) { //default stream on device for current CTA
+                child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); 
+                DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << 
+                    " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream);
+            }
+            else {
+               assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); 
+                DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << 
+                " to stream " << child_stream->get_uid() << ": " << child_stream);
+            }
+    
+            device_launch_op.stream = child_stream;
+        }
+        
+    }
+
+  
+    //launch child kernel
+    g_cuda_device_launch_op.push_back(device_launch_op);
+    g_cuda_device_launch_param_map.erase(parameter_buffer);
+
+    //set retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //cudaError_t
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size 
+        && return_size == sizeof(cudaError_t));
+    cudaError_t error = cudaSuccess;
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL);
+
+}
+
+
+//Handling device runtime api:
+//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int  flags)
+//flags can only be cudaStreamNonBlocking
+void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) {
+    DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags");
+
+    unsigned n_return = target_func->has_return();
+    assert(n_return);
+    unsigned n_args = target_func->num_args();
+    assert( n_args == 2 );
+
+    size_t generic_pStream_addr;
+    addr_t pStream_addr;
+    unsigned int flags;
+    for( unsigned arg=0; arg < n_args; arg ++ ) {
+        const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param#
+        const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_#
+        unsigned size=formal_param->get_size_in_bytes();
+        assert( formal_param->is_param_local() );
+        assert( actual_param_op.is_param_local() );
+        addr_t from_addr = actual_param_op.get_symbol()->get_address();
+
+        if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t
+            assert(size == sizeof(cudaStream_t *));
+            thread->m_local_mem->read(from_addr, size, &generic_pStream_addr);
+            
+            //pStream should be non-zero address in local memory
+            pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr);
+
+            DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr);
+        }
+        else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking
+            assert(size == sizeof(unsigned int));
+            thread->m_local_mem->read(from_addr, size, &flags);
+            assert(flags == cudaStreamNonBlocking);
+        }
+    }
+
+    //create stream and write back to param0
+    CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid());
+    DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream);
+    thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL);
+ 
+    //set retval0
+    const operand_info &actual_return_op = pI->operand_lookup(0); //retval0
+    const symbol *formal_return = target_func->get_return_var(); //cudaError_t
+    unsigned int return_size = formal_return->get_size_in_bytes();
+    DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size);
+    assert(actual_return_op.is_param_local());
+    assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size 
+        && return_size == sizeof(cudaError_t));
+    cudaError_t error = cudaSuccess;
+    addr_t ret_param_addr = actual_return_op.get_symbol()->get_address();
+    thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL);
+
+}
+
+
+void launch_one_device_kernel() {
+    if(!g_cuda_device_launch_op.empty()) {
+        device_launch_operation_t &op = g_cuda_device_launch_op.front();
+
+        stream_operation stream_op = stream_operation(op.grid, g_ptx_sim_mode, op.stream);
+        g_stream_manager->push(stream_op);
+        g_cuda_device_launch_op.pop_front();
+    }
+}
+
+void launch_all_device_kernels() {
+    while(!g_cuda_device_launch_op.empty()) {
+        launch_one_device_kernel();
+    }
+}
+#endif
diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h
new file mode 100644
index 0000000..6dbcd71
--- /dev/null
+++ b/src/cuda-sim/cuda_device_runtime.h
@@ -0,0 +1,11 @@
+//Jin: cuda_device_runtime.h
+//Defines CUDA device runtime APIs for CDP support
+#if (CUDART_VERSION >= 5000)
+#pragma once
+
+void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func);
+void launch_all_device_kernels();
+void launch_one_device_kernel();
+#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 7b0f4fa..71286c9 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -41,14 +41,19 @@
 #include "../gpgpu-sim/gpu-sim.h"
 #include "../gpgpu-sim/shader.h"
 
+//Jin: include device runtime for CDP
+#include "cuda_device_runtime.h"
+
 #include <stdarg.h>
 
 unsigned ptx_instruction::g_num_ptx_inst_uid=0;
 
 const char *g_opcode_string[NUM_OPCODES] = {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR,
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR,
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
 };
 
 void inst_not_implemented( const ptx_instruction * pI ) ;
@@ -148,6 +153,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in
             result.u64 = op.get_symbol()->get_address();
          } else if ( op.is_local() ) {
             result.u64 = op.get_symbol()->get_address();
+         } else if ( op.is_function_address() ) {
+		 	result.u64 = (size_t)op.get_symbol()->get_pc();
          } else {
             const char *name = op.name().c_str();
             printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name );
@@ -1482,7 +1489,23 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread )
    if( fname == "vprintf" ) {
       gpgpusim_cuda_vprintf(pI, thread, target_func);
       return;
-   } 
+   }
+
+#if (CUDART_VERSION >= 5000)
+   //Jin: handle device runtime apis for CDP
+   else if(fname == "cudaGetParameterBufferV2") {
+      gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func);
+	  return;
+   }
+   else if(fname == "cudaLaunchDeviceV2") {
+      gpgpusim_cuda_launchDeviceV2(pI, thread, target_func);
+	  return;
+   }
+   else if(fname == "cudaStreamCreateWithFlags") {
+      gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func);
+	  return;
+   }
+#endif
 
    // read source arguements into register specified in declaration of function
    arg_buffer_list_t arg_values;
@@ -1938,7 +1961,7 @@ ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign,
       y.f64 = x.f64;
       break; 
    }
-   if (isnan(y.f64)) {
+   if (std::isnan(y.f64)) {
       y.u64 = 0xfff8000000000000ull;
    } else if (saturation_mode) {
       y.f64 = cuda_math::__saturatef(y.f64); 
@@ -2063,7 +2086,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type)
       }
    }
    if ((type == F64_TYPE)||(type == FF64_TYPE)) {
-      if (isnan(data.f64)) {
+      if (std::isnan(data.f64)) {
          data.u64 = 0xfff8000000000000ull;
       }
    }
@@ -2625,12 +2648,12 @@ void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry
 
 bool isNaN(float x)
 {
-   return isnan(x);
+   return std::isnan(x);
 }
 
 bool isNaN(double x)
 {
-   return isnan(x);
+   return std::isnan(x);
 }
 
 void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) 
@@ -3516,6 +3539,81 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread )
 
 }
 
+void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
+{
+	unsigned i_type = pI->get_type();
+	int tid = inst.warp_id() * core->get_warp_size();
+	ptx_thread_info *thread = core->get_thread_info()[tid];
+	ptx_warp_info *warp_info = thread->m_warp_info;
+	int lane = warp_info->get_done_threads();
+	thread = core->get_thread_info()[tid + lane];
+
+	const operand_info &dst = pI->dst();
+	const operand_info &src1 = pI->src1();
+	const operand_info &src2 = pI->src2();
+	const operand_info &src3 = pI->src3();
+	int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32;
+	int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32;
+	int mask = cval >> 8;
+	bval &= 0x1F;
+	cval &= 0x1F;
+
+	int maxLane = (lane & mask) | (cval & ~mask);
+	int minLane = lane & mask;
+
+	int src_idx;
+	unsigned p;
+	switch(pI->shfl_op()) {
+	case UP_OPTION:
+		src_idx = lane - bval;
+		p = (src_idx >= maxLane);
+		break;
+	case DOWN_OPTION:
+		src_idx = lane + bval;
+		p = (src_idx <= maxLane);
+		break;
+	case BFLY_OPTION:
+		src_idx = lane ^ bval;
+		p = (src_idx <= maxLane);
+		break;
+	case IDX_OPTION:
+		src_idx = minLane | (bval & ~mask);
+		p = (src_idx <= maxLane);
+		break;
+	default:
+		printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n");
+		assert(0);
+		break;
+	}
+	// copy from own lane
+	if (!p) src_idx = lane;
+
+	// copy input from lane src_idx
+	ptx_reg_t data;
+	if (inst.active(src_idx)) {
+		ptx_thread_info *source = core->get_thread_info()[tid + src_idx];
+		data = source->get_operand_value(src1, dst, i_type, source, 1);
+	} else {
+		printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n");
+		data.u32 = 0;
+	}
+	thread->set_operand_value(dst, data, i_type, thread, pI);
+
+	/*
+	TODO: deal with predicates appropriately using the following pseudocode:
+	if (!isGuardPredicateTrue(src_idx)) {
+		printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n");
+	}
+	if (dest predicate selected) data.pred = p;
+	*/
+
+	// keep track of the number of threads that have executed in the warp
+	warp_info->inc_done_threads();
+	if (warp_info->get_done_threads() == inst.active_count()) {
+		warp_info->reset_done_threads();
+	}
+}
+
 void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) 
 {
    ptx_reg_t a, b, d;
diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def
index 2ee6976..e1b1422 100644
--- a/src/cuda-sim/opcodes.def
+++ b/src/cuda-sim/opcodes.def
@@ -98,6 +98,7 @@ OP_DEF(SAD_OP,sad_impl,"sad",1,1)
 OP_DEF(SELP_OP,selp_impl,"selp",1,1)
 OP_DEF(SETP_OP,setp_impl,"setp",1,1)
 OP_DEF(SET_OP,set_impl,"set",1,1)
+OP_W_DEF(SHFL_OP,shfl_impl,"shfl",1,10)
 OP_DEF(SHL_OP,shl_impl,"shl",1,1)
 OP_DEF(SHR_OP,shr_impl,"shr",1,1)
 OP_DEF(SIN_OP,sin_impl,"sin",1,4)
diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h
index 871091c..aa133da 100644
--- a/src/cuda-sim/opcodes.h
+++ b/src/cuda-sim/opcodes.h
@@ -30,9 +30,11 @@
 
 enum opcode_t {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP,
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP,
 #include "opcodes.def"
-   NUM_OPCODES
+	NUM_OPCODES
 #undef OP_DEF
+#undef OP_W_DEF
 };
 
 enum special_regs {
diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l
index 88ccf6a..5471d6f 100644
--- a/src/cuda-sim/ptx.l
+++ b/src/cuda-sim/ptx.l
@@ -115,6 +115,7 @@ sad     TC; ptx_lval.int_value = SAD_OP; return OPCODE;
 selp	TC; ptx_lval.int_value = SELP_OP; return OPCODE;
 setp    TC; ptx_lval.int_value = SETP_OP; return OPCODE;
 set	TC; ptx_lval.int_value = SET_OP; return OPCODE;
+shfl	TC; ptx_lval.int_value = SHFL_OP; return OPCODE;
 shl     TC; ptx_lval.int_value = SHL_OP; return OPCODE;
 shr     TC; ptx_lval.int_value = SHR_OP; return OPCODE;
 sin	TC; ptx_lval.int_value = SIN_OP; return OPCODE;
@@ -181,6 +182,7 @@ breakaddr  TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
 \.union  TC; return UNION_DIRECTIVE; /* not in PTX 2.1 */
 \.version TC; return VERSION_DIRECTIVE;
 \.visible TC; return VISIBLE_DIRECTIVE;
+\.weak TC; return WEAK_DIRECTIVE;
 \.address_size TC; return ADDRESS_SIZE_DIRECTIVE;
 \.weak	TC; return WEAK_DIRECTIVE;
 
@@ -329,6 +331,11 @@ breakaddr  TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
 
 \.nc	TC; return NC_OPTION;
 
+\.up	TC; return UP_OPTION;
+\.down	TC; return DOWN_OPTION;
+\.bfly	TC; return BFLY_OPTION;
+\.idx	TC; return IDX_OPTION;
+
 \.popc	TC; return ATOMIC_POPC;
 \.and	TC; return ATOMIC_AND;
 \.or	TC; return ATOMIC_OR;
diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y
index 4de39d1..4edae5d 100644
--- a/src/cuda-sim/ptx.y
+++ b/src/cuda-sim/ptx.y
@@ -47,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  PTR_DIRECTIVE
 %token  ENTRY_DIRECTIVE
 %token  EXTERN_DIRECTIVE
-%token  WEAK_DIRECTIVE
 %token  FILE_DIRECTIVE
 %token  FUNC_DIRECTIVE
 %token  GLOBAL_DIRECTIVE
@@ -72,6 +71,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  VERSION_DIRECTIVE
 %token  ADDRESS_SIZE_DIRECTIVE
 %token  VISIBLE_DIRECTIVE
+%token  WEAK_DIRECTIVE
 %token  <string_value> IDENTIFIER
 %token  <int_value> INT_OPERAND
 %token  <float_value> FLOAT_OPERAND
@@ -194,6 +194,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  WB_OPTION;
 %token  WT_OPTION;
 %token	NC_OPTION;
+%token	UP_OPTION;
+%token	DOWN_OPTION;
+%token	BFLY_OPTION;
+%token	IDX_OPTION;
 
 %type <int_value> function_decl_header
 %type <ptr_value> function_decl
@@ -243,10 +247,12 @@ function_ident_param: IDENTIFIER { add_function_name($1); } LEFT_PAREN {func_hea
 
 function_decl_header: ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
 	| VISIBLE_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
+	| WEAK_DIRECTIVE ENTRY_DIRECTIVE { $$ = 1; g_func_decl=1; func_header(".entry"); }
 	| FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| VISIBLE_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	| EXTERN_DIRECTIVE FUNC_DIRECTIVE { $$ = 2; g_func_decl=1; func_header(".func"); }
+	| WEAK_DIRECTIVE FUNC_DIRECTIVE { $$ = 0; g_func_decl=1; func_header(".func"); }
 	;
 
 param_list: /*empty*/
@@ -272,8 +278,8 @@ statement_list: directive_statement { add_directive(); }
 	| instruction_statement { add_instruction(); }
 	| statement_list directive_statement { add_directive(); }
 	| statement_list instruction_statement { add_instruction(); }
-	| statement_list statement_block
-	| statement_block
+	| statement_list {start_inst_group();} statement_block {end_inst_group();}
+	| {start_inst_group();} statement_block {end_inst_group();}
 	;
 
 directive_statement: variable_declaration SEMI_COLON
@@ -325,6 +331,7 @@ var_spec: space_spec
 	| type_spec
 	| align_spec
 	| EXTERN_DIRECTIVE { add_extern_spec(); }
+    | WEAK_DIRECTIVE
 	;
 
 align_spec: ALIGN_DIRECTIVE INT_OPERAND { add_alignment_spec($2); }
@@ -451,6 +458,10 @@ option: type_spec
 	| WB_OPTION { add_option(WB_OPTION); }
 	| WT_OPTION { add_option(WT_OPTION); }
 	| NC_OPTION { add_option(NC_OPTION); }
+	| UP_OPTION { add_option(UP_OPTION); }
+	| DOWN_OPTION { add_option(DOWN_OPTION); }
+	| BFLY_OPTION { add_option(BFLY_OPTION); }
+	| IDX_OPTION { add_option(IDX_OPTION); }
 	;
 
 atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } 
@@ -501,7 +512,8 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); }
 	| NAN_OPTION { add_option(NAN_OPTION); } 
 	;
 
-operand_list: operand
+operand_list: /* empty*/
+    | operand
 	| operand COMMA operand_list;
 
 operand: IDENTIFIER  { add_scalar_operand( $1 ); }
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index 2eccabc..8ebdcf8 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -90,6 +90,11 @@ symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol
    m_const_next  = 0;
    m_global_next = 0x100;
    m_local_next  = 0;
+   m_tex_next = 0;
+
+   //Jin: handle instruction group for cdp
+   m_inst_group_id = 0;
+
    m_parent = parent;
    if ( m_parent ) {
       m_shared_next = m_parent->m_shared_next;
@@ -170,6 +175,41 @@ void symbol_table::add_function( function_info *func, const char *filename, unsi
    m_symbols[ func->get_name() ] = s;
 }
 
+//Jin: handle instruction group for cdp
+symbol_table* symbol_table::start_inst_group() {
+   char inst_group_name[1024];
+   snprintf(inst_group_name, 1024, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id);
+
+   //previous added
+   assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end());
+   symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this );
+ 
+   sym_table->m_global_next = m_global_next;
+   sym_table->m_shared_next = m_shared_next;
+   sym_table->m_local_next = m_local_next;
+   sym_table->m_reg_allocator = m_reg_allocator;
+   sym_table->m_tex_next = m_tex_next;
+   sym_table->m_const_next = m_const_next;
+
+   m_inst_group_symtab[std::string(inst_group_name)] = sym_table;
+
+   return sym_table;
+}
+
+symbol_table * symbol_table::end_inst_group() {
+   symbol_table * sym_table = m_parent;
+   
+   sym_table->m_global_next = m_global_next;
+   sym_table->m_shared_next = m_shared_next;
+   sym_table->m_local_next = m_local_next;
+   sym_table->m_reg_allocator = m_reg_allocator;
+   sym_table->m_tex_next = m_tex_next;
+   sym_table->m_const_next = m_const_next;
+   sym_table->m_inst_group_id++;
+
+   return sym_table;
+}
+
 void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl
 
 bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table )
@@ -458,7 +498,7 @@ void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of f
          if( pI->has_pred() ) {
             printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n");
             // if predicated, add link to next block
-            unsigned next_addr = pI->get_m_instr_mem_index() + 1;
+            unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size();
             if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) {
                basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb();
                (*bb_itr)->successor_ids.insert(next_bb->bb_id);
@@ -1171,6 +1211,12 @@ ptx_instruction::ptx_instruction( int opcode,
 		 break;
 	  case NC_OPTION:
 		 break;
+	  case UP_OPTION:
+	  case DOWN_OPTION:
+	  case BFLY_OPTION:
+	  case IDX_OPTION:
+		  m_shfl_op = last_ptx_inst_option;
+		  break;
       default:
          assert(0);
          break;
@@ -1205,6 +1251,12 @@ ptx_instruction::ptx_instruction( int opcode,
        if (fname =="vprintf"){
            m_is_printf = true;
        }
+       if(fname == "cudaStreamCreateWithFlags")
+           m_is_cdp = 1;
+       if(fname == "cudaGetParameterBufferV2")
+           m_is_cdp = 2;
+       if(fname == "cudaLaunchDeviceV2")
+           m_is_cdp = 4;
 
    }
 }
@@ -1252,6 +1304,7 @@ function_info::function_info(int entry_point )
    m_kernel_info.regs = 0;
    m_kernel_info.smem = 0;
    m_local_mem_framesize = 0;
+   m_args_aligned_size = -1;
 }
 
 unsigned function_info::print_insn( unsigned pc, FILE * fp ) const
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 601a13d..9ad1571 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -291,6 +291,7 @@ private:
 
    std::list<operand_info> m_initializer;
    static unsigned sm_next_uid;
+   
 };
 
 class symbol_table {
@@ -330,6 +331,11 @@ public:
    iterator const_iterator_end() { return m_consts.end();}
 
    void dump();
+   
+   //Jin: handle instruction group for cdp
+   symbol_table* start_inst_group();
+   symbol_table* end_inst_group();
+
 private:
    unsigned m_reg_allocator;
    unsigned m_shared_next;
@@ -347,6 +353,10 @@ private:
    std::list<symbol*> m_consts;
    std::map<std::string,function_info*> m_function_info_lookup;
    std::map<std::string,symbol_table*> m_function_symtab_lookup;
+   
+   //Jin: handle instruction group for cdp
+   unsigned m_inst_group_id;
+   std::map<std::string,symbol_table*> m_inst_group_symtab;
 };
 
 class operand_info {
@@ -690,7 +700,7 @@ public:
    }
 
    bool is_immediate_address() const {
-	   return   m_immediate_address;
+       return   m_immediate_address;
    }
 
    bool is_literal() const { return m_type == int_t ||
@@ -993,6 +1003,7 @@ public:
    unsigned saturation_mode() const { return m_saturation_mode;}
    unsigned dimension() const { return m_geom_spec;}
    unsigned barrier_op() const {return m_barrier_op;}
+   unsigned shfl_op() const {return m_shfl_op;}
    enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot };
    enum vote_mode_t vote_mode() const { return m_vote_mode; }
 
@@ -1058,6 +1069,7 @@ private:
    unsigned            m_compare_op;
    unsigned            m_saturation_mode;
    unsigned 		   m_barrier_op;
+   unsigned			   m_shfl_op;
 
    std::list<int>          m_scalar_type;
    memory_space_t m_space_spec;
@@ -1200,6 +1212,8 @@ public:
    {
       return m_args.size();
    }
+   unsigned get_args_aligned_size();
+
    const symbol* get_arg( unsigned n ) const
    {
       assert( n < m_args.size() );
@@ -1288,6 +1302,9 @@ private:
 
    static std::vector<ptx_instruction*> s_g_pc_to_insn; // a direct mapping from PC to instruction
    static unsigned sm_next_uid;
+
+   //parameter size for device kernels
+   int m_args_aligned_size;
 };
 
 class arg_buffer_t {
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 9bb5008..6c1b595 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -322,7 +322,11 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
 
 #if CUDART_VERSION >= 3000
     if (sm_version == 0) sm_version = 20;
-    snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+    extern bool g_cdp_enabled;
+    if(!g_cdp_enabled)
+        snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+    else
+        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
 #endif
 
     snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc
index 824714a..a180da9 100644
--- a/src/cuda-sim/ptx_parser.cc
+++ b/src/cuda-sim/ptx_parser.cc
@@ -120,6 +120,20 @@ symbol_table *init_parser( const char *ptx_filename )
 #define DEF(X,Y) g_ptx_token_decode[X] = Y;
 #include "ptx_parser_decode.def"
 #undef DEF
+   g_ptx_token_decode[undefined_space] = "undefined_space";
+   g_ptx_token_decode[undefined_space] = "undefined_space=0";
+   g_ptx_token_decode[reg_space] = "reg_space";
+   g_ptx_token_decode[local_space] = "local_space";
+   g_ptx_token_decode[shared_space] = "shared_space";
+   g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified";
+   g_ptx_token_decode[param_space_kernel] = "param_space_kernel";
+   g_ptx_token_decode[param_space_local] = "param_space_local";
+   g_ptx_token_decode[const_space] = "const_space";
+   g_ptx_token_decode[tex_space] = "tex_space";
+   g_ptx_token_decode[surf_space] = "surf_space";
+   g_ptx_token_decode[global_space] = "global_space";
+   g_ptx_token_decode[generic_space] = "generic_space";
+   g_ptx_token_decode[instruction_space] = "instruction_space";
 
    return g_global_symbol_table;
 }
@@ -187,6 +201,17 @@ void add_function_name( const char *name )
    g_global_symbol_table->add_function( g_func_info, g_filename, ptx_lineno );
 }
 
+//Jin: handle instruction group for cdp
+void start_inst_group() {
+   PTX_PARSE_DPRINTF("start_instruction_group");
+   g_current_symbol_table = g_current_symbol_table->start_inst_group();
+}
+
+void end_inst_group() {
+   PTX_PARSE_DPRINTF("end_instruction_group");
+   g_current_symbol_table = g_current_symbol_table->end_inst_group();
+}
+
 void add_directive() 
 {
    PTX_PARSE_DPRINTF("add_directive");
@@ -408,7 +433,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
       assert( (num_bits%8) == 0  );
       addr = g_current_symbol_table->get_shared_next();
       addr_pad = pad_address(addr, num_bits/8, 128);
-      printf("from 0x%x to 0x%lx (shared memory space)\n",
+      printf("from 0x%llx to 0x%llx (shared memory space)\n",
               addr+addr_pad,
               addr+addr_pad + num_bits/8);
          fflush(stdout);
@@ -425,7 +450,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
          assert( (num_bits%8) == 0  ); 
          addr = g_current_symbol_table->get_global_next();
          addr_pad = pad_address(addr, num_bits/8, 128);
-         printf("from 0x%x to 0x%lx (global memory space) %u\n",
+         printf("from 0x%llx to 0x%llx (global memory space) %u\n",
               addr+addr_pad,
               addr+addr_pad + num_bits/8,
               g_const_alloc++);
@@ -446,7 +471,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
       assert( (num_bits%8) == 0  );
       addr = g_current_symbol_table->get_global_next();
       addr_pad = pad_address(addr, num_bits/8, 128);
-      printf("from 0x%x to 0x%lx (global memory space)\n",
+      printf("from 0x%llx to 0x%llx (global memory space)\n",
               addr+addr_pad,
               addr+addr_pad + num_bits/8);
       fflush(stdout);
@@ -463,7 +488,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
          assert( (num_bits%8) == 0  );
          addr = g_current_symbol_table->get_local_next();
          addr_pad = pad_address(addr, num_bits/8, 128);
-         printf("from 0x%x to 0x%lx (local memory space)\n",
+         printf("from 0x%llx to 0x%llx (local memory space)\n",
                  addr+addr_pad,
                  addr+addr_pad + num_bits/8);
          fflush(stdout);
@@ -476,7 +501,7 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident
         assert( (num_bits%8) == 0 );
         addr = g_current_symbol_table->get_local_next();
         addr_pad = pad_address(addr, num_bits/8, 128);
-        printf("from 0x%x to 0x%lx\n",
+        printf("from 0x%llx to 0x%llx\n",
                 addr+addr_pad,
                 addr+addr_pad + num_bits/8);
         fflush(stdout);
diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h
index fef7635..32f3903 100644
--- a/src/cuda-sim/ptx_parser.h
+++ b/src/cuda-sim/ptx_parser.h
@@ -94,6 +94,10 @@ void change_operand_neg( );
 void set_immediate_operand_type( );
 void version_header(double a);
 
+//Jin: handle instructino group for cdp
+void start_inst_group();
+void end_inst_group();
+
 #define NON_ARRAY_IDENTIFIER 1
 #define ARRAY_IDENTIFIER_NO_DIM 2
 #define ARRAY_IDENTIFIER 3
diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc
index 09844ae..a3e43aa 100644
--- a/src/cuda-sim/ptx_sim.cc
+++ b/src/cuda-sim/ptx_sim.cc
@@ -128,6 +128,26 @@ unsigned ptx_cta_info::get_sm_idx() const
    return m_sm_idx;
 }
 
+ptx_warp_info::ptx_warp_info()
+{
+	reset_done_threads();
+}
+
+unsigned ptx_warp_info::get_done_threads() const
+{
+	return m_done_threads;
+}
+
+void ptx_warp_info::inc_done_threads()
+{
+	m_done_threads++;
+}
+
+void ptx_warp_info::reset_done_threads()
+{
+	m_done_threads = 0;
+}
+
 unsigned g_ptx_thread_info_uid_next=1;
 unsigned g_ptx_thread_info_delete_count=0;
 
@@ -153,6 +173,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel )
    m_last_memory_space = undefined_space; 
    m_branch_taken = 0;
    m_shared_mem = NULL;
+   m_warp_info = NULL;
    m_cta_info = NULL;
    m_local_mem = NULL;
    m_symbol_table = NULL;
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index f926e6d..e6eb02e 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -167,6 +167,17 @@ private:
    std::set<ptx_thread_info*>  m_dangling_pointers;
 };
 
+class ptx_warp_info {
+public:
+	ptx_warp_info();
+	unsigned get_done_threads() const;
+	void inc_done_threads();
+	void reset_done_threads();
+
+private:
+	unsigned m_done_threads;
+};
+
 class symbol;
 
 struct stack_entry {
@@ -418,6 +429,9 @@ public:
    void or_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->or_reduction(ctaid,barid,value);}
    void popc_reduction(unsigned ctaid, unsigned barid, bool value) {m_core->popc_reduction(ctaid,barid,value);}
 
+   //Jin: get corresponding kernel grid for CDP purpose
+   kernel_info_t & get_kernel() { return m_kernel; }
+
 public:
    addr_t         m_last_effective_address;
    bool        m_branch_taken;
@@ -425,6 +439,7 @@ public:
    dram_callback_t   m_last_dram_callback; 
    memory_space   *m_shared_mem;
    memory_space   *m_local_mem;
+   ptx_warp_info  *m_warp_info;
    ptx_cta_info   *m_cta_info;
    ptx_reg_t m_last_set_operand_value;
 
diff --git a/src/cuda-sim/ptxinfo.y b/src/cuda-sim/ptxinfo.y
index 37092f4..d241d8c 100644
--- a/src/cuda-sim/ptxinfo.y
+++ b/src/cuda-sim/ptxinfo.y
@@ -94,6 +94,7 @@ line: 	HEADER INFO COLON line_info
 
 line_info: function_name
 	| function_info { ptxinfo_addinfo(); }
+	| gmem_info
 	;
 
 function_name:	FUNC QUOTE IDENTIFIER QUOTE { ptxinfo_function($3); }
@@ -104,6 +105,9 @@ function_info: info
 	| function_info COMMA info
 	;
 
+gmem_info: INT_OPERAND BYTES GMEM
+	;
+
 info: 	  USED INT_OPERAND REGS { ptxinfo_regs($2); }
 	| tuple LMEM { ptxinfo_lmem(g_declared,g_system); }
 	| tuple SMEM { ptxinfo_smem(g_declared,g_system); }
diff --git a/src/gpgpu-sim/Makefile b/src/gpgpu-sim/Makefile
index bead38a..f10a8a4 100644
--- a/src/gpgpu-sim/Makefile
+++ b/src/gpgpu-sim/Makefile
@@ -59,6 +59,7 @@ ifneq ($(GPGPUSIM_POWER_MODEL),)
 endif
 
 OPTFLAGS += -g3 -fPIC
+OPTFLAGS += -DCUDART_VERSION=$(CUDART_VERSION)
 
 CPP = g++ $(SNOW)
 OEXT = o
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index eafb909..3829861 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -32,6 +32,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
+#include <signal.h>
 #include "zlib.h"
 
 
@@ -61,6 +62,7 @@
 #include "power_stat.h"
 #include "visualizer.h"
 #include "stats.h"
+#include "../cuda-sim/cuda_device_runtime.h"
 
 #ifdef GPGPUSIM_POWER_MODEL
 #include "power_interface.h"
@@ -366,6 +368,10 @@ void shader_core_config::reg_options(class OptionParser * opp)
                                 "For complete list of prioritization values see shader.h enum scheduler_prioritization_type"
                                 "Default: gto",
                                  "gto");
+
+    option_parser_register(opp, "-gpgpu_concurrent_kernel_sm", OPT_BOOL, &gpgpu_concurrent_kernel_sm, 
+                "Support concurrent kernels on a SM (default = disabled)", 
+                "0");
 }
 
 void gpgpu_sim_config::reg_options(option_parser_t opp)
@@ -438,6 +444,16 @@ void gpgpu_sim_config::reg_options(option_parser_t opp)
                           &Trace::sampling_memory_partition, "The memory partition which is printed using MEMPART_DPRINTF. Default -1 (i.e. all)",
                           "-1");
    ptx_file_line_stats_options(opp);
+
+    //Jin: kernel launch latency
+    extern unsigned g_kernel_launch_latency;
+    option_parser_register(opp, "-gpgpu_kernel_launch_latency", OPT_INT32, 
+                          &g_kernel_launch_latency, "Kernel launch latency in cycles. Default: 0",
+                          "0");
+    extern bool g_cdp_enabled;
+    option_parser_register(opp, "-gpgpu_cdp_enabled", OPT_BOOL, 
+                          &g_cdp_enabled, "Turn on CDP",
+                          "0");
 }
 
 /////////////////////////////////////////////////////////////////////////////
@@ -518,16 +534,27 @@ bool gpgpu_sim::get_more_cta_left() const
 
 kernel_info_t *gpgpu_sim::select_kernel()
 {
+    if(m_running_kernels[m_last_issued_kernel] &&
+        !m_running_kernels[m_last_issued_kernel]->no_more_ctas_to_run()) {
+        unsigned launch_uid = m_running_kernels[m_last_issued_kernel]->get_uid(); 
+        if(std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), launch_uid) == m_executed_kernel_uids.end()) {
+            m_running_kernels[m_last_issued_kernel]->start_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
+            m_executed_kernel_uids.push_back(launch_uid); 
+            m_executed_kernel_names.push_back(m_running_kernels[m_last_issued_kernel]->name()); 
+        }
+        return m_running_kernels[m_last_issued_kernel];
+    }
+
     for(unsigned n=0; n < m_running_kernels.size(); n++ ) {
         unsigned idx = (n+m_last_issued_kernel+1)%m_config.max_concurrent_kernel;
         if( kernel_more_cta_left(m_running_kernels[idx]) ){
             m_last_issued_kernel=idx;
+            m_running_kernels[idx]->start_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
             // record this kernel for stat print if it is the first time this kernel is selected for execution  
             unsigned launch_uid = m_running_kernels[idx]->get_uid(); 
-            if (std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), launch_uid) == m_executed_kernel_uids.end()) {
-               m_executed_kernel_uids.push_back(launch_uid); 
-               m_executed_kernel_names.push_back(m_running_kernels[idx]->name()); 
-            }
+            assert(std::find(m_executed_kernel_uids.begin(), m_executed_kernel_uids.end(), launch_uid) == m_executed_kernel_uids.end());
+            m_executed_kernel_uids.push_back(launch_uid); 
+            m_executed_kernel_names.push_back(m_running_kernels[idx]->name()); 
 
             return m_running_kernels[idx];
         }
@@ -551,6 +578,7 @@ void gpgpu_sim::set_kernel_done( kernel_info_t *kernel )
     std::vector<kernel_info_t*>::iterator k;
     for( k=m_running_kernels.begin(); k!=m_running_kernels.end(); k++ ) {
         if( *k == kernel ) {
+            kernel->end_cycle = gpu_sim_cycle + gpu_tot_sim_cycle;
             *k = NULL;
             break;
         }
@@ -622,6 +650,10 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config )
     *active_sms=0;
 
     last_liveness_message_time = 0;
+   
+   //Jin: functional simulation for CDP
+   m_functional_sim = false;
+   m_functional_sim_kernel = NULL;
 }
 
 int gpgpu_sim::shared_mem_size() const
@@ -927,7 +959,8 @@ void gpgpu_sim::gpu_print_stat()
    printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn+gpu_sim_insn) / (gpu_tot_sim_cycle+gpu_sim_cycle));
    printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta + m_total_cta_launched);
 
-
+   extern unsigned long long g_max_total_param_size;
+   fprintf(statfout, "max_total_param_size = %llu\n", g_max_total_param_size);
 
    // performance counter for stalls due to congestion.
    printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull);
@@ -1070,7 +1103,119 @@ void shader_core_ctx::mem_instruction_stats(const warp_inst_t &inst)
         abort();
     }
 }
+bool shader_core_ctx::can_issue_1block(kernel_info_t & kernel) {
+
+   //Jin: concurrent kernels on one SM
+   if(m_config->gpgpu_concurrent_kernel_sm) {    
+      if(m_config->max_cta(kernel) < 1)
+           return false;
+
+      return occupy_shader_resource_1block(kernel, false);
+   }
+   else {
+      return (get_n_active_cta() < m_config->max_cta(kernel));
+   } 
+}
+
+int shader_core_ctx::find_available_hwtid(unsigned int cta_size, bool occupy) {
+   
+   unsigned int step;
+   for(step = 0; step < m_config->n_thread_per_shader; 
+        step += cta_size) {
+
+        unsigned int hw_tid;
+        for(hw_tid = step; hw_tid < step + cta_size;
+            hw_tid++) {
+            if(m_occupied_hwtid.test(hw_tid))
+                break;
+        }
+        if(hw_tid == step + cta_size) //consecutive non-active
+            break;
+   }
+   if(step >= m_config->n_thread_per_shader) //didn't find
+     return -1;
+   else {
+     if(occupy) {
+        for(unsigned hw_tid = step; hw_tid < step + cta_size;
+            hw_tid++)
+            m_occupied_hwtid.set(hw_tid);
+     }
+     return step;
+   }
+}
+
+bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t & k, bool occupy) {
+   unsigned threads_per_cta  = k.threads_per_cta();
+   const class function_info *kernel = k.entry();
+   unsigned int padded_cta_size = threads_per_cta;
+   unsigned int warp_size = m_config->warp_size; 
+   if (padded_cta_size%warp_size) 
+      padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+
+   if(m_occupied_n_threads + padded_cta_size > m_config->n_thread_per_shader)
+     return false;
 
+   if(find_available_hwtid(padded_cta_size, false) == -1)
+     return false;
+
+   const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel);
+
+   if(m_occupied_shmem + kernel_info->smem > m_config->gpgpu_shmem_size)
+     return false;
+
+   unsigned int used_regs = padded_cta_size * ((kernel_info->regs+3)&~3);
+   if(m_occupied_regs + used_regs > m_config->gpgpu_shader_registers)
+     return false;
+
+   if(m_occupied_ctas +1 > m_config->max_cta_per_core)
+     return false;
+   
+   if(occupy) {
+       m_occupied_n_threads += padded_cta_size;
+       m_occupied_shmem += kernel_info->smem;
+       m_occupied_regs += (padded_cta_size * ((kernel_info->regs+3)&~3));
+       m_occupied_ctas++;
+
+       printf("GPGPU-Sim uArch: Shader %d occupied %d threads, %d shared mem, %d registers, %d ctas\n",
+            m_sid, m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas);  
+   }
+
+   return true;
+}
+
+void shader_core_ctx::release_shader_resource_1block(unsigned hw_ctaid, kernel_info_t & k) {
+
+   if(m_config->gpgpu_concurrent_kernel_sm) {
+      unsigned threads_per_cta  = k.threads_per_cta();
+      const class function_info *kernel = k.entry();
+      unsigned int padded_cta_size = threads_per_cta;
+      unsigned int warp_size = m_config->warp_size; 
+      if (padded_cta_size%warp_size) 
+         padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size);
+   
+      assert(m_occupied_n_threads >= padded_cta_size);
+      m_occupied_n_threads -= padded_cta_size;
+   
+      int start_thread = m_occupied_cta_to_hwtid[hw_ctaid];
+   
+      for(unsigned hwtid = start_thread; hwtid < start_thread + padded_cta_size;
+       hwtid++)
+          m_occupied_hwtid.reset(hwtid);
+      m_occupied_cta_to_hwtid.erase(hw_ctaid);
+   
+      const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel);
+   
+      assert(m_occupied_shmem >= (unsigned int)kernel_info->smem);
+      m_occupied_shmem -= kernel_info->smem;
+   
+      unsigned int used_regs = padded_cta_size * ((kernel_info->regs+3)&~3);
+      assert(m_occupied_regs >= used_regs);
+      m_occupied_regs -= used_regs;
+   
+      assert(m_occupied_ctas >= 1);
+      m_occupied_ctas--;
+   }
+}
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -1083,11 +1228,23 @@ void shader_core_ctx::mem_instruction_stats(const warp_inst_t &inst)
 
 void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) 
 {
-    set_max_cta(kernel);
+
+    if(!m_config->gpgpu_concurrent_kernel_sm)
+        set_max_cta(kernel);
+    else
+        assert(occupy_shader_resource_1block(kernel, true));
+
+    kernel.inc_running();
 
     // find a free CTA context 
     unsigned free_cta_hw_id=(unsigned)-1;
-    for (unsigned i=0;i<kernel_max_cta_per_shader;i++ ) {
+
+    unsigned max_cta_per_core;
+    if(!m_config->gpgpu_concurrent_kernel_sm)
+        max_cta_per_core = kernel_max_cta_per_shader;
+    else
+        max_cta_per_core = m_config->max_cta_per_core;
+    for (unsigned i=0;i<max_cta_per_core;i++ ) {
       if( m_cta_status[i]==0 ) {
          free_cta_hw_id=i;
          break;
@@ -1104,8 +1261,20 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     int padded_cta_size = cta_size; 
     if (cta_size%m_config->warp_size)
       padded_cta_size = ((cta_size/m_config->warp_size)+1)*(m_config->warp_size);
-    unsigned start_thread = free_cta_hw_id * padded_cta_size;
-    unsigned end_thread  = start_thread +  cta_size;
+
+    unsigned int start_thread, end_thread;
+
+    if(!m_config->gpgpu_concurrent_kernel_sm) {
+        start_thread = free_cta_hw_id * padded_cta_size;
+        end_thread  = start_thread +  cta_size;
+    }
+    else {
+        start_thread = find_available_hwtid(padded_cta_size, true);
+        assert((int)start_thread != -1);
+        end_thread = start_thread + cta_size;
+        assert(m_occupied_cta_to_hwtid.find(free_cta_hw_id) == m_occupied_cta_to_hwtid.end());
+        m_occupied_cta_to_hwtid[free_cta_hw_id]= start_thread;
+    }
 
     // reset the microarchitecture state of the selected hardware thread and warp contexts
     reinit(start_thread, end_thread,false);
@@ -1133,7 +1302,9 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     m_n_active_cta++;
 
     shader_CTA_count_log(m_sid, 1);
-    printf("GPGPU-Sim uArch: core:%3d, cta:%2u initialized @(%lld,%lld)\n", m_sid, free_cta_hw_id, gpu_sim_cycle, gpu_tot_sim_cycle );
+    printf("GPGPU-Sim uArch: core:%3d, cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", 
+        m_sid, free_cta_hw_id, start_thread, end_thread, gpu_sim_cycle, gpu_tot_sim_cycle );
+
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////
@@ -1270,7 +1441,7 @@ void gpgpu_sim::cycle()
 
 
       if( g_single_step && ((gpu_sim_cycle+gpu_tot_sim_cycle) >= g_single_step) ) {
-          asm("int $03");
+          raise(SIGTRAP); // Debug breakpoint
       }
       gpu_sim_cycle++;
       if( g_interactive_debugger_enabled ) 
@@ -1366,6 +1537,11 @@ void gpgpu_sim::cycle()
       }
       try_snap_shot(gpu_sim_cycle);
       spill_log_to_file (stdout, 0, gpu_sim_cycle);
+
+#if (CUDART_VERSION >= 5000)
+      //launch device kernel
+      launch_one_device_kernel();
+#endif
    }
 }
 
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 33fffd3..7d92c66 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -195,7 +195,7 @@ struct memory_config {
       for (i=0; nbkt>0; i++) {
           nbkt = nbkt>>1;
       }
-      bk_tag_length = i;
+      bk_tag_length = i-1;
       assert(nbkgrp>0 && "Number of bank groups cannot be zero");
       tRCDWR = tRCD-(WL+1);
       tRTW = (CL+(BL/data_command_freq_ratio)+2-WL);
@@ -492,6 +492,7 @@ private:
    std::string executed_kernel_info_string(); //< format the kernel information into a string for stat printout
    void clear_executed_kernel_info(); //< clear the kernel information after stat printout
 
+
 public:
    unsigned long long  gpu_sim_insn;
    unsigned long long  gpu_tot_sim_insn;
@@ -504,6 +505,25 @@ public:
    void change_cache_config(FuncCache cache_config);
    void set_cache_config(std::string kernel_name);
 
+   //Jin: functional simulation for CDP
+private:
+   //set by stream operation every time a functoinal simulation is done
+   bool m_functional_sim;
+   kernel_info_t * m_functional_sim_kernel;
+
+public:
+   bool is_functional_sim() { return m_functional_sim; }
+   kernel_info_t * get_functional_kernel() { return m_functional_sim_kernel; }
+   void functional_launch(kernel_info_t * k) {
+     m_functional_sim = true;
+     m_functional_sim_kernel = k;
+   }
+   void finish_functional_sim(kernel_info_t * k) {
+     assert(m_functional_sim);
+     assert(m_functional_sim_kernel == k);
+     m_functional_sim = false;
+     m_functional_sim_kernel = NULL;
+   }
 };
 
 #endif
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index ff2fac7..d17e51d 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -296,6 +296,14 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
     
     m_last_inst_gpu_sim_cycle = 0;
     m_last_inst_gpu_tot_sim_cycle = 0;
+
+    //Jin: for concurrent kernels on a SM
+    m_occupied_n_threads = 0;
+    m_occupied_shmem = 0;
+    m_occupied_regs = 0;
+    m_occupied_ctas = 0;
+    m_occupied_hwtid.reset();
+    m_occupied_cta_to_hwtid.clear();
 }
 
 void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool reset_not_completed ) 
@@ -303,6 +311,15 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
    if( reset_not_completed ) {
        m_not_completed = 0;
        m_active_threads.reset();
+
+       //Jin: for concurrent kernels on a SM
+       m_occupied_n_threads = 0;
+       m_occupied_shmem = 0;
+       m_occupied_regs = 0;
+       m_occupied_ctas = 0;
+       m_occupied_hwtid.reset();
+       m_occupied_cta_to_hwtid.clear();
+
    }
    for (unsigned i = start_thread; i<end_thread; i++) {
       m_threadState[i].n_insn = 0;
@@ -620,7 +637,7 @@ void shader_core_ctx::fetch()
                         if( m_threadState[tid].m_active == true ) {
                             m_threadState[tid].m_active = false; 
                             unsigned cta_id = m_warp[warp_id].get_cta_id();
-                            register_cta_thread_exit(cta_id);
+                            register_cta_thread_exit(cta_id, &(m_thread[tid]->get_kernel()));
                             m_not_completed -= 1;
                             m_active_threads.reset(tid);
                             assert( m_thread[tid]!= NULL );
@@ -828,6 +845,13 @@ void scheduler_unit::cycle()
         unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp;
         while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) {
             const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst();
+            //Jin: handle cdp latency;
+            if(pI && pI->m_is_cdp && warp(warp_id).m_cdp_latency > 0) {
+                assert(warp(warp_id).m_cdp_dummy);
+                warp(warp_id).m_cdp_latency--;
+                break;
+            }
+
             bool valid = warp(warp_id).ibuffer_next_valid();
             bool warp_inst_issued = false;
             unsigned pc,rpc;
@@ -862,6 +886,25 @@ void scheduler_unit::cycle()
                             bool sp_pipe_avail = m_sp_out->has_free();
                             bool sfu_pipe_avail = m_sfu_out->has_free();
                             if( sp_pipe_avail && (pI->op != SFU_OP) ) {
+                                
+                                //Jin: special for CDP api
+                                if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) {
+                                    assert(warp(warp_id).m_cdp_latency == 0);
+                                    
+                                    extern unsigned cdp_latency[5];
+                                    if(pI->m_is_cdp == 1)
+                                        warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1];
+                                    else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2
+                                        warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1]
+                                            + cdp_latency[pI->m_is_cdp] * active_mask.count();
+                                    warp(warp_id).m_cdp_dummy = true;
+                                    break;
+                                }
+                                else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) {
+                                    assert(warp(warp_id).m_cdp_latency == 0);
+                                    warp(warp_id).m_cdp_dummy = false;
+                                }
+
                                 // always prefer SP pipe for operations that can use both SP and SFU pipelines
                                 m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id);
                                 issued++;
@@ -1917,7 +1960,7 @@ void ldst_unit::cycle()
    }
 }
 
-void shader_core_ctx::register_cta_thread_exit( unsigned cta_num )
+void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t * kernel)
 {
    assert( m_cta_status[cta_num] > 0 );
    m_cta_status[cta_num]--;
@@ -1925,22 +1968,35 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num )
       m_n_active_cta--;
       m_barriers.deallocate_barrier(cta_num);
       shader_CTA_count_unlog(m_sid, 1);
+
       printf("GPGPU-Sim uArch: Shader %d finished CTA #%d (%lld,%lld), %u CTAs running\n", m_sid, cta_num, gpu_sim_cycle, gpu_tot_sim_cycle,
              m_n_active_cta );
+
       if( m_n_active_cta == 0 ) {
-          assert( m_kernel != NULL );
-          m_kernel->dec_running();
-          printf("GPGPU-Sim uArch: Shader %u empty (release kernel %u \'%s\').\n", m_sid, m_kernel->get_uid(),
-                 m_kernel->name().c_str() );
-          if( !m_gpu->kernel_more_cta_left(m_kernel) ) {
-              if( !m_kernel->running() ) {
-                  printf("GPGPU-Sim uArch: GPU detected kernel \'%s\' finished on shader %u.\n", m_kernel->name().c_str(), m_sid );
-                  m_gpu->set_kernel_done( m_kernel );
-              }
-          }
-          m_kernel=NULL;
+          printf("GPGPU-Sim uArch: Shader %u empty (last released kernel %u \'%s\').\n", m_sid, kernel->get_uid(),
+                 kernel->name().c_str() );
           fflush(stdout);
+
+          //Shader can only be empty when no more cta are dispatched
+          if(kernel != m_kernel) {
+              assert(m_kernel == NULL || !m_gpu->kernel_more_cta_left(m_kernel));
+          }
+          m_kernel = NULL;
+      }
+
+      //Jin: for concurrent kernels on sm
+      release_shader_resource_1block(cta_num, *kernel);
+      kernel->dec_running();
+      if( !m_gpu->kernel_more_cta_left(kernel) ) {
+          if( !kernel->running() ) {
+              printf("GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(),
+                kernel->name().c_str(), m_sid );
+              if(m_kernel == kernel)
+                m_kernel = NULL;
+              m_gpu->set_kernel_done( kernel );
+          }
       }
+
    }
 }
 
@@ -3238,15 +3294,33 @@ unsigned simt_core_cluster::issue_block2core()
     unsigned num_blocks_issued=0;
     for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) {
         unsigned core = (i+m_cta_issue_next_core+1)%m_config->n_simt_cores_per_cluster;
-        if( m_core[core]->get_not_completed() == 0 ) {
-            if( m_core[core]->get_kernel() == NULL ) {
-                kernel_info_t *k = m_gpu->select_kernel();
-                if( k ) 
-                    m_core[core]->set_kernel(k);
+
+        kernel_info_t * kernel;
+         //Jin: fetch kernel according to concurrent kernel setting
+        if(m_config->gpgpu_concurrent_kernel_sm) {//concurrent kernel on sm 
+            //always select latest issued kernel
+            kernel_info_t *k = m_gpu->select_kernel();
+            kernel = k;
+        }
+        else {
+            //first select core kernel, if no more cta, get a new kernel
+            //only when core completes
+            kernel = m_core[core]->get_kernel();
+            if( !m_gpu->kernel_more_cta_left(kernel) ) {
+              //wait till current kernel finishes
+              if(m_core[core]->get_not_completed() == 0)
+              {
+                  kernel_info_t *k = m_gpu->select_kernel();
+                  if( k ) 
+                      m_core[core]->set_kernel(k);
+                  kernel = k;
+              }
             }
         }
-        kernel_info_t *kernel = m_core[core]->get_kernel();
-        if( m_gpu->kernel_more_cta_left(kernel) && (m_core[core]->get_n_active_cta() < m_config->max_cta(*kernel)) ) {
+
+        if( m_gpu->kernel_more_cta_left(kernel) && 
+//            (m_core[core]->get_n_active_cta() < m_config->max_cta(*kernel)) ) {
+            m_core[core]->can_issue_1block(*kernel)) {
             m_core[core]->issue_block2core(*kernel);
             num_blocks_issued++;
             m_cta_issue_next_core=core; 
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 38d09e9..bdd8dbe 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -108,6 +108,10 @@ public:
         m_last_fetch=0;
         m_next=0;
         m_inst_at_barrier=NULL;
+
+        //Jin: cdp support
+        m_cdp_latency = 0;
+        m_cdp_dummy = false;
     }
     void init( address_type start_pc,
                unsigned cta_id,
@@ -124,6 +128,10 @@ public:
         n_completed   -= active.count(); // active threads are not yet completed
         m_active_threads = active;
         m_done_exit=false;
+
+        //Jin: cdp support
+        m_cdp_latency = 0;
+        m_cdp_dummy = false;
     }
 
     bool functional_done() const;
@@ -260,6 +268,11 @@ private:
 
     unsigned m_stores_outstanding; // number of store requests sent but not yet acknowledged
     unsigned m_inst_in_pipeline;
+
+    //Jin: cdp support
+public:
+    unsigned int m_cdp_latency;
+    bool m_cdp_dummy;
 };
 
 
@@ -1327,6 +1340,9 @@ struct shader_core_config : public core_config
     int simt_core_sim_order; 
     
     unsigned mem2device(unsigned memid) const { return memid + n_simt_clusters; }
+
+    //Jin: concurrent kernel on sm
+    bool gpgpu_concurrent_kernel_sm;
 };
 
 struct shader_core_stats_pod {
@@ -1574,6 +1590,7 @@ public:
     void cycle();
     void reinit(unsigned start_thread, unsigned end_thread, bool reset_not_completed );
     void issue_block2core( class kernel_info_t &kernel );
+
     void cache_flush();
     void accept_fetch_response( mem_fetch *mf );
     void accept_ldst_unit_response( class mem_fetch * mf );
@@ -1582,7 +1599,7 @@ public:
     {
         assert(k);
         m_kernel=k; 
-        k->inc_running(); 
+//        k->inc_running(); 
         printf("GPGPU-Sim uArch: Shader %d bind to kernel %u \'%s\'\n", m_sid, m_kernel->get_uid(),
                  m_kernel->name().c_str() );
     }
@@ -1749,7 +1766,7 @@ public:
     virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid);
     address_type next_pc( int tid ) const;
     void fetch();
-    void register_cta_thread_exit( unsigned cta_num );
+    void register_cta_thread_exit(unsigned cta_num, kernel_info_t * kernel );
 
     void decode();
     
@@ -1831,6 +1848,22 @@ public:
     // is that the dynamic_warp_id is a running number unique to every warp
     // run on this shader, where the warp_id is the static warp slot.
     unsigned m_dynamic_warp_id;
+
+    //Jin: concurrent kernels on a sm
+public:
+    bool can_issue_1block(kernel_info_t & kernel);
+    bool occupy_shader_resource_1block(kernel_info_t & kernel, bool occupy);
+    void release_shader_resource_1block(unsigned hw_ctaid, kernel_info_t & kernel);
+    int find_available_hwtid(unsigned int cta_size, bool occupy);
+private:
+    unsigned int m_occupied_n_threads; 
+    unsigned int m_occupied_shmem; 
+    unsigned int m_occupied_regs;
+    unsigned int m_occupied_ctas;
+    std::bitset<MAX_THREAD_PER_SM> m_occupied_hwtid;
+    std::map<unsigned int, unsigned int> m_occupied_cta_to_hwtid; 
+
+
 };
 
 class simt_core_cluster {
@@ -1851,6 +1884,7 @@ public:
     bool icnt_injection_buffer_full(unsigned size, bool write);
     void icnt_inject_request_packet(class mem_fetch *mf);
 
+
     // for perfect memory interface
     bool response_queue_full() {
         return ( m_response_fifo.size() >= m_config->n_simt_ejection_buffer_size );
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 6ba38eb..ad4587a 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -91,6 +91,7 @@ void *gpgpu_sim_thread_sequential(void*)
 pthread_mutex_t g_sim_lock = PTHREAD_MUTEX_INITIALIZER;
 bool g_sim_active = false;
 bool g_sim_done = true;
+bool break_limit = false;
 
 void *gpgpu_sim_thread_concurrent(void*)
 {
@@ -100,7 +101,7 @@ void *gpgpu_sim_thread_concurrent(void*)
            printf("GPGPU-Sim: *** simulation thread starting and spinning waiting for work ***\n");
            fflush(stdout);
         }
-        while( g_stream_manager->empty_protected() && !g_sim_done )
+        while( g_stream_manager->empty() && !g_sim_done )
             ;
         if(g_debug_execution >= 3) {
            printf("GPGPU-Sim: ** START simulation thread (detected work) **\n");
@@ -127,6 +128,15 @@ void *gpgpu_sim_thread_concurrent(void*)
             if(g_stream_manager->operation(&sim_cycles) && !g_the_gpu->active())
                 break;
 
+            //functional simulation
+            if( g_the_gpu->is_functional_sim()) {
+                kernel_info_t * kernel = g_the_gpu->get_functional_kernel();
+                assert(kernel);
+                gpgpu_cuda_ptx_sim_main_func(*kernel);
+                g_the_gpu->finish_functional_sim(kernel);
+            }
+
+            //performance simulation
             if( g_the_gpu->active() ) {
                 g_the_gpu->cycle();
                 sim_cycles = true;
@@ -135,15 +145,18 @@ void *gpgpu_sim_thread_concurrent(void*)
                 if(g_the_gpu->cycle_insn_cta_max_hit()){
                     g_stream_manager->stop_all_running_kernels();
                     g_sim_done = true;
+                    break_limit = true;
                 }
             }
 
             active=g_the_gpu->active() || !g_stream_manager->empty_protected();
-        } while( active );
+
+        } while( active && !g_sim_done);
         if(g_debug_execution >= 3) {
            printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n");
            fflush(stdout);
         }
+        g_the_gpu->print_stats();
         if(sim_cycles) {
             g_the_gpu->update_stats();
             print_simulation_time();
@@ -156,6 +169,11 @@ void *gpgpu_sim_thread_concurrent(void*)
        printf("GPGPU-Sim: *** simulation thread exiting ***\n");
        fflush(stdout);
     }
+    if(break_limit) {
+    	printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n");
+    	exit(1);
+    }
+
     sem_post(&g_sim_signal_exit);
     return NULL;
 }
@@ -169,7 +187,7 @@ void synchronize()
     bool done = false;
     do {
         pthread_mutex_lock(&g_sim_lock);
-        done = g_stream_manager->empty() && !g_sim_active;
+        done = ( g_stream_manager->empty() && !g_sim_active ) || g_sim_done;
         pthread_mutex_unlock(&g_sim_lock);
     } while (!done);
     printf("GPGPU-Sim: detected inactive GPU simulation thread\n");
diff --git a/src/intersim2/Makefile b/src/intersim2/Makefile
index ef948d6..bd42000 100644
--- a/src/intersim2/Makefile
+++ b/src/intersim2/Makefile
@@ -125,6 +125,14 @@ endif
 
 # rules to compile simulator
 
+$(OBJDIR)/Makefile.makedepend: depend
+
+ALL_SRCS = $(CPP_SRCS)
+ALL_SRCS += $(shell ls **/*.cpp)
+
+depend:
+	touch $(OBJDIR)/Makefile.makedepend
+	makedepend -f$(OBJDIR)/Makefile.makedepend -I$(INCPATH) -p$(OBJDIR)/ $(ALL_SRCS) 2> /dev/null
 
 ${LEX_OBJS}: $(OBJDIR)/lex.yy.c $(OBJDIR)/y.tab.h
 	$(CC) $(CPPFLAGS) -c $< -o $@
@@ -173,3 +181,5 @@ $(OBJDIR)/y.tab.c $(OBJDIR)/y.tab.h: config.y
 
 $(OBJDIR)/lex.yy.c: config.l
 	$(LEX) -o$@ $<
+
+include $(OBJDIR)/Makefile.makedepend
diff --git a/src/stream_manager.cc b/src/stream_manager.cc
index dd42f0a..3b6cbd5 100644
--- a/src/stream_manager.cc
+++ b/src/stream_manager.cc
@@ -95,6 +95,15 @@ stream_operation CUstream_st::next()
     return result;
 }
 
+void CUstream_st::cancel_front()
+{
+    pthread_mutex_lock(&m_lock);
+    assert(m_pending);
+    m_pending = false;
+    pthread_mutex_unlock(&m_lock);
+
+}
+
 void CUstream_st::print(FILE *fp)
 {
     pthread_mutex_lock(&m_lock);
@@ -111,10 +120,10 @@ void CUstream_st::print(FILE *fp)
 }
 
 
-void stream_operation::do_operation( gpgpu_sim *gpu )
+bool stream_operation::do_operation( gpgpu_sim *gpu )
 {
     if( is_noop() ) 
-        return;
+        return true;
 
     assert(!m_done && m_stream);
     if(g_debug_execution >= 3)
@@ -151,17 +160,36 @@ void stream_operation::do_operation( gpgpu_sim *gpu )
         m_stream->record_next_done();
         break;
     case stream_kernel_launch:
-        if( gpu->can_start_kernel() ) {
-        	gpu->set_cache_config(m_kernel->name());
-        	printf("kernel \'%s\' transfer to GPU hardware scheduler\n", m_kernel->name().c_str() );
-            if( m_sim_mode )
-                gpgpu_cuda_ptx_sim_main_func( *m_kernel );
-            else
+        if( m_sim_mode ) { //Functional Sim
+            if(g_debug_execution >= 3) {
+                printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() );
+                m_kernel->print_parent_info();
+            }
+            gpu->set_cache_config(m_kernel->name());
+            gpu->functional_launch( m_kernel );
+        }
+        else { //Performance Sim
+            if( gpu->can_start_kernel() && m_kernel->m_launch_latency == 0) {
+                if(g_debug_execution >= 3) {
+                    printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() );
+                    m_kernel->print_parent_info();
+                }
+                gpu->set_cache_config(m_kernel->name());
                 gpu->launch( m_kernel );
+            }
+            else {
+                if(m_kernel->m_launch_latency)
+                    m_kernel->m_launch_latency--;
+                if(g_debug_execution >= 3)
+                    printf("kernel %d: \'%s\', latency %u not ready to transfer to GPU hardware scheduler\n", 
+                        m_kernel->get_uid(), m_kernel->name().c_str(), m_kernel->m_launch_latency);
+                return false;    
+            }
         }
         break;
     case stream_event: {
-        printf("event update\n");
+        if(g_debug_execution >= 3)
+            printf("event update\n");
         time_t wallclock = time((time_t *)NULL);
         m_event->update( gpu_tot_sim_cycle, wallclock );
         m_stream->record_next_done();
@@ -172,6 +200,7 @@ void stream_operation::do_operation( gpgpu_sim *gpu )
     }
     m_done=true;
     fflush(stdout);
+    return true;
 }
 
 void stream_operation::print( FILE *fp ) const
@@ -199,11 +228,20 @@ stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking )
 
 bool stream_manager::operation( bool * sim)
 {
-    pthread_mutex_lock(&m_lock);
     bool check=check_finished_kernel();
-    if(check)m_gpu->print_stats();
+    pthread_mutex_lock(&m_lock);
+//    if(check)m_gpu->print_stats();
     stream_operation op =front();
-    op.do_operation( m_gpu );
+    if(!op.do_operation( m_gpu )) //not ready to execute
+    {
+        //cancel operation
+        if( op.is_kernel() ) {
+            unsigned grid_uid = op.get_kernel()->get_uid();
+            m_grid_id_to_stream.erase(grid_uid);
+        }
+        op.get_stream()->cancel_front();
+
+    }
     pthread_mutex_unlock(&m_lock);
     //pthread_mutex_lock(&m_lock);
     // simulate a clock cycle on the GPU
@@ -212,11 +250,9 @@ bool stream_manager::operation( bool * sim)
 
 bool stream_manager::check_finished_kernel()
 {
-
-	unsigned grid_uid = m_gpu->finished_kernel();
-	bool check=register_finished_kernel(grid_uid);
-	return check;
-
+    unsigned grid_uid = m_gpu->finished_kernel();
+    bool check=register_finished_kernel(grid_uid);
+    return check;
 }
 
 bool stream_manager::register_finished_kernel(unsigned grid_uid)
@@ -226,13 +262,27 @@ bool stream_manager::register_finished_kernel(unsigned grid_uid)
         CUstream_st *stream = m_grid_id_to_stream[grid_uid];
         kernel_info_t *kernel = stream->front().get_kernel();
         assert( grid_uid == kernel->get_uid() );
-        stream->record_next_done();
-        m_grid_id_to_stream.erase(grid_uid);
-        delete kernel;
-        return true;
-    }else{
-    	return false;
+
+        //Jin: should check children kernels for CDP
+        if(kernel->is_finished()) {
+//            std::ofstream kernel_stat("kernel_stat.txt", std::ofstream::out | std::ofstream::app);
+//            kernel_stat<< " kernel " << grid_uid << ": " << kernel->name();
+//            if(kernel->get_parent())
+//                kernel_stat << ", parent " << kernel->get_parent()->get_uid() <<
+//                ", launch " << kernel->launch_cycle;
+//            kernel_stat<< ", start " << kernel->start_cycle <<
+//                ", end " << kernel->end_cycle << ", retire " << gpu_sim_cycle + gpu_tot_sim_cycle << "\n";
+//            printf("kernel %d finishes, retires from stream %d\n", grid_uid, stream->get_uid());
+//            kernel_stat.flush();
+//            kernel_stat.close();
+            stream->record_next_done();
+            m_grid_id_to_stream.erase(grid_uid);
+            kernel->notify_parent_finished();
+            delete kernel;
+            return true;
+        }
     }
+
     return false;
 }
 
@@ -259,21 +309,22 @@ stream_operation stream_manager::front()
 {
     // called by gpu simulation thread
     stream_operation result;
-    if( concurrent_streams_empty() )
-        m_service_stream_zero = true;
+//    if( concurrent_streams_empty() )
+    m_service_stream_zero = true;
     if( m_service_stream_zero ) {
-        if( !m_stream_zero.empty() ) {
-            if( !m_stream_zero.busy() ) {
+        if( !m_stream_zero.empty() && !m_stream_zero.busy() ) {
                 result = m_stream_zero.next();
                 if( result.is_kernel() ) {
                     unsigned grid_id = result.get_kernel()->get_uid();
                     m_grid_id_to_stream[grid_id] = &m_stream_zero;
                 }
-            }
         } else {
             m_service_stream_zero = false;
         }
-    } else {
+    }
+    
+    if(!m_service_stream_zero)
+    {
         std::list<struct CUstream_st*>::iterator s;
         for( s=m_streams.begin(); s != m_streams.end(); s++) {
             CUstream_st *stream = *s;
diff --git a/src/stream_manager.h b/src/stream_manager.h
index 701b33c..222a1b2 100644
--- a/src/stream_manager.h
+++ b/src/stream_manager.h
@@ -150,7 +150,7 @@ public:
     bool is_noop() const { return m_type == stream_no_op; }
     bool is_done() const { return m_done; }
     kernel_info_t *get_kernel() { return m_kernel; }
-    void do_operation( gpgpu_sim *gpu );
+    bool do_operation( gpgpu_sim *gpu );
     void print( FILE *fp ) const;
     struct CUstream_st *get_stream() { return m_stream; }
     void set_stream( CUstream_st *stream ) { m_stream = stream; }
@@ -218,6 +218,7 @@ public:
     void push( const stream_operation &op );
     void record_next_done();
     stream_operation next();
+    void cancel_front(); //front operation fails, cancle the pending status
     stream_operation &front() { return m_operations.front(); }
     void print( FILE *fp );
     unsigned get_uid() const { return m_uid; }
diff --git a/version b/version
index e565a98..c70c6ac 100644
--- a/version
+++ b/version
@@ -1,2 +1 @@
 const char *g_gpgpusim_version_string = "GPGPU-Sim Simulator Version 3.2.2 ";
-const char *g_gpgpusim_build_string   =  "$Change$"; 
diff --git a/version_detection.mk b/version_detection.mk
index 8796d5c..00a86c0 100644
--- a/version_detection.mk
+++ b/version_detection.mk
@@ -30,7 +30,11 @@
 ifeq ($(GPGPUSIM_ROOT),)
 else
 GPGPUSIM_VERSION=$(shell cat $(GPGPUSIM_ROOT)/version | awk '/Version/ {print $$8}' )
-GPGPUSIM_BUILD=$(shell cat $(GPGPUSIM_ROOT)/version | awk '/Change/ {print $$6}' )
+
+#Detect Git branch and commit #
+GIT_COMMIT := $(shell git log -n 1 | head -1 | sed -re 's/commit (.*)/\1/')
+GIT_FILES_CHANGED := $(shell git diff --numstat --cached && git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1/')
+GPGPUSIM_BUILD := "gpgpu-sim_git-commit-$(GIT_COMMIT)_modified_$(GIT_FILES_CHANGED)"
 endif
 
 # Detect CUDA Runtime Version 
@@ -42,4 +46,3 @@ CC_VERSION := $(shell gcc --version | head -1 | awk '{for(i=1;i<=NF;i++){ if(mat
 
 # Detect Support for C++11 (C++0x) from GCC Version 
 GNUC_CPP0X := $(shell gcc --version | perl -ne 'if (/gcc\s+\(.*\)\s+([0-9.]+)/){ if($$1 >= 4.3) {$$n=1} else {$$n=0;} } END { print $$n; }')
-