merging in the mainline

34 files changed, 2653 insertions, 503 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index 831e7d8..cf9d71e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -41,7 +41,7 @@ pipeline {
                 parallel "4.2-rodinia": {
                     sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/4.2_env_setup.sh &&\
                     source `pwd`/setup_environment &&\
-                    ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B rodinia_2.0-ft -C GTX480,GTX480-PTXPLUS,PASCALTITANX,PASCALTITANX-PTXPLUS -N regress-$$ && \
+                    ./gpgpu-sim_simulations/util/job_launching/run_simulations.py -B rodinia_2.0-ft -C GTX480,GTX480-PTXPLUS,PASCALTITANX,PASCALTITANX-PTXPLUS,TITANX-P102 -N regress-$$ && \
                     ./gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress-$$'
                 }, "9.1-rodinia": {
                     sh 'source /home/tgrogers-raid/a/common/gpgpu-sim-setup/9.1_env_setup.sh &&\
diff --git a/configs/GTX480/gpgpusim.config b/configs/GTX480/gpgpusim.config
index 436cb41..03fcda1 100644
--- a/configs/GTX480/gpgpusim.config
+++ b/configs/GTX480/gpgpusim.config
@@ -29,10 +29,12 @@
 -gpgpu_simd_model 1 
 
 # Pipeline widths and number of FUs
-# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
--gpgpu_pipeline_widths 2,1,1,2,1,1,2
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+#For Fermi, DP unit =0, DP inst is executed on SFU
+-gpgpu_pipeline_widths 2,0,1,1,2,0,1,1,2
 -gpgpu_num_sp_units 2
 -gpgpu_num_sfu_units 1
+-gpgpu_num_dp_units 0
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
@@ -48,20 +50,20 @@
 # <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
 # ** Optional parameter - Required when mshr_type==Texture Fifo
 # Note: Hashing set index function (H) only applies to a set size of 32 or 64. 
--gpgpu_cache:dl1  32:128:4,L:L:m:N:H,A:32:8,8
+-gpgpu_cache:dl1  N:32:128:4,L:L:m:N:H,A:32:8,8
 -gpgpu_shmem_size 49152
 
 # The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected
-#-gpgpu_cache:dl1  64:128:6,L:L:m:N:H,A:32:8,8
+#-gpgpu_cache:dl1  N:64:128:6,L:L:m:N:H,A:32:8,8
 #-gpgpu_shmem_size 16384
 
 # 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache
--gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32
+-gpgpu_cache:dl2 N:64:128:8,L:B:m:W:L,A:32:4,4:0,32
 -gpgpu_cache:dl2_texture_only 0 
 
--gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4
--gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2
--gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4
+-gpgpu_cache:il1 N:4:128:4,L:R:f:N:L,A:2:32,4
+-gpgpu_tex_cache:l1 N:4:128:24,L:R:m:N:L,F:128:4,128:2
+-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,A:2:32,4
 
 # enable operand collector 
 -gpgpu_operand_collector_num_units_sp 6
diff --git a/configs/GeForceGTX750Ti/gpgpusim.config b/configs/GeForceGTX750Ti/gpgpusim.config
index 8b030b6..9366f93 100644
--- a/configs/GeForceGTX750Ti/gpgpusim.config
+++ b/configs/GeForceGTX750Ti/gpgpusim.config
@@ -28,10 +28,11 @@
 -gpgpu_simd_model 1 
 
 # Pipeline widths and number of FUs
-# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
--gpgpu_pipeline_widths 2,1,1,2,1,1,2
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+-gpgpu_pipeline_widths 2,0,1,1,2,0,1,1,2
 -gpgpu_num_sp_units 8
 -gpgpu_num_sfu_units 32
+-gpgpu_num_dp_units 0
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
@@ -42,21 +43,21 @@
 -ptx_opcode_latency_dp 8,19,8,8,330
 -ptx_opcode_initiation_dp 8,16,8,8,130
 
--gpgpu_cache:dl1 32:128:4,L:L:m:N:H,A:32:8,8
+-gpgpu_cache:dl1 N:32:128:4,L:L:m:N:H,A:32:8,8
 -gmem_skip_L1D 1
 -gpgpu_shmem_size 65536
 
 # The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected
-#-gpgpu_cache:dl1  64:128:6,L:L:m:N:H,A:32:8,8
+#-gpgpu_cache:dl1  N:64:128:6,L:L:m:N:H,A:32:8,8
 #-gpgpu_shmem_size 16384
 
 # 1024 sets, each 128 bytes 8-way for each memory sub partition. This gives 2MB L2 cache
--gpgpu_cache:dl2 1024:128:8,L:B:m:W:L,A:32:4,4:0,32
+-gpgpu_cache:dl2 N:1024:128:8,L:B:m:W:L,A:32:4,4:0,32
 -gpgpu_cache:dl2_texture_only 0 
 
--gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4
--gpgpu_tex_cache:l1 16:128:32,L:R:m:N:L,F:128:4,128:2
--gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4
+-gpgpu_cache:il1 N:4:128:4,L:R:f:N:L,A:2:32,4
+-gpgpu_tex_cache:l1 N:16:128:32,L:R:m:N:L,F:128:4,128:2
+-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,A:2:32,4
 
 # enable operand collector 
 -gpgpu_operand_collector_num_units_sp 6
diff --git a/configs/Pascal-P100-HBM/config_fermi_islip.icnt b/configs/Pascal-P100-HBM/config_fermi_islip.icnt
new file mode 100644
index 0000000..e7c2c3b
--- /dev/null
+++ b/configs/Pascal-P100-HBM/config_fermi_islip.icnt
@@ -0,0 +1,73 @@
+//21*1 fly with 32 flits per packet under gpgpusim injection mode
+use_map = 0;
+flit_size = 40; 
+
+// currently we do not use this, see subnets below
+network_count = 2;
+
+// Topology
+topology = fly;
+k = 60;
+n = 1;
+
+// Routing
+
+routing_function = dest_tag;
+
+// Flow control
+
+num_vcs     = 1;
+vc_buf_size = 128;
+input_buffer_size = 256;
+ejection_buffer_size = 128;
+boundary_buffer_size = 128;
+
+wait_for_tail_credit = 0;
+
+// Router architecture
+
+vc_allocator = islip; //separable_input_first;
+sw_allocator = islip; //separable_input_first;
+alloc_iters  = 1;
+
+credit_delay   = 0;
+routing_delay  = 0;
+vc_alloc_delay = 1;
+sw_alloc_delay = 1;
+
+input_speedup     = 2;
+output_speedup    = 1;
+internal_speedup  = 1.0;
+
+// Traffic, GPGPU-Sim does not use this
+
+traffic                = uniform;
+packet_size ={{1,2,3,4},{10,20}};
+packet_size_rate={{1,1,1,1},{2,1}};
+
+// Simulation - Don't change
+
+sim_type       = gpgpusim;
+//sim_type = latency;
+injection_rate = 0.1;
+
+subnets = 2;
+
+// Always use read and write no matter following line
+//use_read_write = 1;
+
+
+read_request_subnet = 0;
+read_reply_subnet = 1;
+write_request_subnet = 0;
+write_reply_subnet = 1;
+
+read_request_begin_vc = 0;
+read_request_end_vc = 0;
+write_request_begin_vc = 0;
+write_request_end_vc = 0;
+read_reply_begin_vc = 0;
+read_reply_end_vc = 0;
+write_reply_begin_vc = 0;
+write_reply_end_vc = 0;
+
diff --git a/configs/Pascal-P100-HBM/gpgpusim.config b/configs/Pascal-P100-HBM/gpgpusim.config
new file mode 100644
index 0000000..533a865
--- /dev/null
+++ b/configs/Pascal-P100-HBM/gpgpusim.config
@@ -0,0 +1,171 @@
+# functional simulator specification
+-gpgpu_ptx_instruction_classification 0
+-gpgpu_ptx_sim_mode 0
+-gpgpu_ptx_force_max_capability 60 
+
+# SASS execution (only supported with CUDA >= 4.0)
+-gpgpu_ptx_convert_to_ptxplus 0
+-gpgpu_ptx_save_converted_ptxplus 0
+
+# high level architecture configuration
+-gpgpu_n_clusters 28
+-gpgpu_n_cores_per_cluster 2
+-gpgpu_n_mem 32
+-gpgpu_n_sub_partition_per_mchannel 1 
+
+# Pscal clock domains
+#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
+# Pascal NVIDIA GP100  clock domains are adopted from 
+# https://en.wikipedia.org/wiki/Nvidia_Tesla
+-gpgpu_clock_domains 1480.0:1480.0:1480.0:715.0
+
+# shader core pipeline config
+-gpgpu_shader_registers 65536
+
+# This implies a maximum of 64 warps/SM
+-gpgpu_shader_core_pipeline 2048:32 
+-gpgpu_shader_cta 32
+-gpgpu_simd_model 1 
+
+# Pipeline widths and number of FUs
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+## Pascal GP100 has 2 SP SIMD units, 2 SFU units, 2 DP units per core
+## we need to scale the number of pipeline registers to be equal to the number of SP units
+-gpgpu_pipeline_widths 2,2,2,1,2,2,2,1,6
+-gpgpu_num_sp_units 2
+-gpgpu_num_sfu_units 2
+-gpgpu_num_dp_units 2
+
+# Instruction latencies and initiation intervals
+# "ADD,MAX,MUL,MAD,DIV"
+# All Div operations are executed on SFU unit
+# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2
+-ptx_opcode_latency_int 4,13,4,5,145
+-ptx_opcode_initiation_int 1,1,1,1,4
+-ptx_opcode_latency_fp 4,13,4,5,39
+-ptx_opcode_initiation_fp 1,2,1,1,4
+-ptx_opcode_latency_dp 8,19,8,8,330
+-ptx_opcode_initiation_dp 2,2,2,2,130
+-ptx_opcode_latency_sfu 8
+-ptx_opcode_initiation_sfu 4
+
+
+# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
+# ** Optional parameter - Required when mshr_type==Texture Fifo
+# Note: Hashing set index function (H) only applies to a set size of 32 or 64. 
+# Pascal GP100 has 64KB Shared memory
+-gpgpu_cache:dl1  S:64:128:6,L:L:f:N:H,A:256:8,16:0,32
+-gpgpu_shmem_size 65536
+-gmem_skip_L1D 1
+-icnt_flit_size 40
+-gpgpu_n_cluster_ejection_buffer_size 32
+
+# 32 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives  4MB L2 cache
+-gpgpu_cache:dl2 S:64:128:16,L:B:m:W:L,A:256:4,32:0,32
+-gpgpu_cache:dl2_texture_only 0 
+-gpgpu_dram_partition_queues 64:64:64:64
+-perf_sim_memcpy 0
+
+# 4 KB Inst.
+-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4
+# 48 KB Tex 
+-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,F:128:4,128:2
+# 12 KB Const
+-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4
+
+# enable operand collector 
+-gpgpu_operand_collector_num_units_sp 12
+-gpgpu_operand_collector_num_units_sfu 6
+-gpgpu_operand_collector_num_units_mem 8
+-gpgpu_operand_collector_num_units_dp 6
+-gpgpu_operand_collector_num_in_ports_sp 4
+-gpgpu_operand_collector_num_out_ports_sp 4
+-gpgpu_operand_collector_num_in_ports_sfu 1
+-gpgpu_operand_collector_num_out_ports_sfu 1
+-gpgpu_operand_collector_num_in_ports_mem 1
+-gpgpu_operand_collector_num_out_ports_mem 1
+-gpgpu_operand_collector_num_in_ports_dp 1
+-gpgpu_operand_collector_num_out_ports_dp 1
+# gpgpu_num_reg_banks should be increased to 32
+-gpgpu_num_reg_banks 32
+
+# shared memory bankconflict detection 
+-gpgpu_shmem_num_banks 32
+-gpgpu_shmem_limited_broadcast 0
+-gpgpu_shmem_warp_parts 1
+-gpgpu_coalesce_arch 60
+
+## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units
+-gpgpu_max_insn_issue_per_warp 2
+-gpgpu_dual_issue_diff_exec_units 1
+
+# interconnection
+-network_mode 1 
+-inter_config_file config_fermi_islip.icnt
+
+# memory partition latency config 
+-rop_latency 120
+# DRAM latency should be lower compared to other configs, due to high-speed interposer connection
+-dram_latency 60
+
+# dram model config
+-gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (100 core cycles).  I.e. 
+#   Total buffer space required = 100 x 924MHz / 700MHz = 132
+-gpgpu_frfcfs_dram_sched_queue_size 64
+-gpgpu_dram_return_queue_size 192
+
+# for HBM, 32 channles, each (128 bits) 16 bytes width
+-gpgpu_n_mem_per_ctrlr 1
+-gpgpu_dram_buswidth 16
+-gpgpu_dram_burst_length 2
+-dram_data_command_freq_ratio 2  # HBM is DDR
+-gpgpu_mem_address_mask 1
+-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBBCCC.CCCSSSSS
+
+# HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf)
+# Timing for 1 GHZ
+# tRRDl and tWTR are missing, need to be added
+#-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=4:RCD=14:RAS=33:RP=14:RC=47:
+#                        CL=14:WL=2:CDLR=3:WR=12:nbkgrp=4:CCDL=2:RTPL=4"
+
+# Timing for 715 MHZ, Tesla Pascal P100 HBM runs at 715 MHZ
+-gpgpu_dram_timing_opt "nbk=16:CCD=1:RRD=3:RCD=10:RAS=24:RP=10:RC=34:
+                        CL=10:WL=2:CDLR=3:WR=9:nbkgrp=4:CCDL=2:RTPL=3"
+
+# HBM has dual bus interface, in which it can issue two col and row commands at a time
+-dual_bus_interface 1
+# select lower bits for bnkgrp to increase bnkgrp parallelism
+-dram_bnk_indexing_policy 0
+-dram_bnkgrp_indexing_policy 1
+
+#-Seperate_Write_Queue_Enable 1
+#-Write_Queue_Size 64:56:32
+
+# Pascal has two schedulers per core
+-gpgpu_num_sched_per_core 2
+# Two Level Scheduler with active and pending pools
+#-gpgpu_scheduler two_level_active:6:0:1
+# Loose round robbin scheduler
+#-gpgpu_scheduler lrr
+# Greedy then oldest scheduler
+-gpgpu_scheduler gto
+
+# stat collection
+-gpgpu_memlatency_stat 14 
+-gpgpu_runtime_stat 500
+-enable_ptx_file_line_stats 1
+-visualizer_enabled 0
+
+# power model configs, disable it untill we create a real energy model for Pascal 100
+-power_simulation_enabled 0
+-gpuwattch_xml_file gpuwattch_gtx480.xml
+
+# tracing functionality
+#-trace_enabled 1
+#-trace_components WARP_SCHEDULER,SCOREBOARD
+#-trace_sampling_core 0
+
diff --git a/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt b/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt
new file mode 100644
index 0000000..714d933
--- /dev/null
+++ b/configs/Pascal-P102-GDDR5X/config_fermi_islip.icnt
@@ -0,0 +1,73 @@
+//21*1 fly with 32 flits per packet under gpgpusim injection mode
+use_map = 0;
+flit_size = 40; 
+
+// currently we do not use this, see subnets below
+network_count = 2;
+
+// Topology
+topology = fly;
+k = 52;
+n = 1;
+
+// Routing
+
+routing_function = dest_tag;
+
+// Flow control
+
+num_vcs     = 1;
+vc_buf_size = 64;
+input_buffer_size = 256;
+ejection_buffer_size = 64;
+boundary_buffer_size = 64;
+
+wait_for_tail_credit = 0;
+
+// Router architecture
+
+vc_allocator = islip; //separable_input_first;
+sw_allocator = islip; //separable_input_first;
+alloc_iters  = 1;
+
+credit_delay   = 0;
+routing_delay  = 0;
+vc_alloc_delay = 1;
+sw_alloc_delay = 1;
+
+input_speedup     = 2;
+output_speedup    = 1;
+internal_speedup  = 1.0;
+
+// Traffic, GPGPU-Sim does not use this
+
+traffic                = uniform;
+packet_size ={{1,2,3,4},{10,20}};
+packet_size_rate={{1,1,1,1},{2,1}};
+
+// Simulation - Don't change
+
+sim_type       = gpgpusim;
+//sim_type = latency;
+injection_rate = 0.1;
+
+subnets = 2;
+
+// Always use read and write no matter following line
+//use_read_write = 1;
+
+
+read_request_subnet = 0;
+read_reply_subnet = 1;
+write_request_subnet = 0;
+write_reply_subnet = 1;
+
+read_request_begin_vc = 0;
+read_request_end_vc = 0;
+write_request_begin_vc = 0;
+write_request_end_vc = 0;
+read_reply_begin_vc = 0;
+read_reply_end_vc = 0;
+write_reply_begin_vc = 0;
+write_reply_end_vc = 0;
+
diff --git a/configs/Pascal-P102-GDDR5X/gpgpusim.config b/configs/Pascal-P102-GDDR5X/gpgpusim.config
new file mode 100644
index 0000000..0c6c126
--- /dev/null
+++ b/configs/Pascal-P102-GDDR5X/gpgpusim.config
@@ -0,0 +1,173 @@
+# This config models the Pascal GP102 (NVIDIA TITAN X)
+# For more info about this card, see Nvidia White paper
+# http://international.download.nvidia.com/geforce-com/international/pdfs/GeForce_GTX_1080_Whitepaper_FINAL.pdf
+
+# functional simulator specification
+-gpgpu_ptx_instruction_classification 0
+-gpgpu_ptx_sim_mode 0
+-gpgpu_ptx_force_max_capability 61 
+
+# SASS execution (only supported with CUDA >= 4.0)
+-gpgpu_ptx_convert_to_ptxplus 0
+-gpgpu_ptx_save_converted_ptxplus 0
+
+# high level architecture configuration
+# P102 has two semi-indp scheds per core, and two cores per cluster 
+-gpgpu_n_clusters 28
+-gpgpu_n_cores_per_cluster 2
+-gpgpu_n_mem 12
+-gpgpu_n_sub_partition_per_mchannel 2 
+
+# Pascal clock domains
+#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
+# Pascal NVIDIA TITAN X clock domains are adopted from 
+# https://en.wikipedia.org/wiki/GeForce_10_series
+-gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0
+
+# shader core pipeline config
+-gpgpu_shader_registers 32768
+
+# This implies a maximum of 32 warps/SM
+-gpgpu_shader_core_pipeline 1024:32 
+-gpgpu_shader_cta 16
+-gpgpu_simd_model 1 
+
+# Pipeline widths and number of FUs
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+## Pascal GP102 has 4 SP SIMD units and 4 SFU units
+## we need to scale the number of pipeline registers to be equal to the number of SP units
+-gpgpu_pipeline_widths 2,1,2,1,2,1,2,1,5
+-gpgpu_num_sp_units 2
+-gpgpu_num_sfu_units 2
+-gpgpu_num_dp_units 1
+
+
+# Instruction latencies and initiation intervals
+# "ADD,MAX,MUL,MAD,DIV"
+# All Div operations are executed on SFU unit
+# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2
+-ptx_opcode_latency_int 4,13,4,5,145
+-ptx_opcode_initiation_int 1,1,1,1,4
+-ptx_opcode_latency_fp 4,13,4,5,39
+-ptx_opcode_initiation_fp 1,2,1,1,4
+-ptx_opcode_latency_dp 8,19,8,8,330
+-ptx_opcode_initiation_dp 8,8,8,8,130
+-ptx_opcode_initiation_sfu 4
+-ptx_opcode_latency_sfu 8
+
+# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
+# ** Optional parameter - Required when mshr_type==Texture Fifo
+# Note: Hashing set index function (H) only applies to a set size of 32 or 64. 
+# Pascal GP102 has 96KB Shared memory
+# Pascal GP102 has 24KB L1 cache
+# The defulat is to disable the L1 cache, unless cache modifieres is used
+-gpgpu_cache:dl1  S:32:128:6,L:L:f:N:H,A:256:8,16:0,32
+-gpgpu_shmem_size 49152
+-gmem_skip_L1D 1
+-icnt_flit_size 40
+-gpgpu_n_cluster_ejection_buffer_size 32
+
+# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache
+-gpgpu_cache:dl2 S:64:128:16,L:B:m:F:L,A:128:4,16:0,32
+-gpgpu_cache:dl2_texture_only 0 
+-gpgpu_dram_partition_queues 32:32:32:32
+-perf_sim_memcpy 0
+
+# 4 KB Inst.
+-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4
+# 48 KB Tex 
+# this is unused
+-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,F:128:4,128:2
+# 12 KB Const
+-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4
+
+# enable operand collector 
+-gpgpu_operand_collector_num_units_sp 12
+-gpgpu_operand_collector_num_units_sfu 6
+-gpgpu_operand_collector_num_units_mem 8
+-gpgpu_operand_collector_num_units_dp 6
+-gpgpu_operand_collector_num_in_ports_sp 2
+-gpgpu_operand_collector_num_out_ports_sp 2
+-gpgpu_operand_collector_num_in_ports_sfu 2
+-gpgpu_operand_collector_num_out_ports_sfu 2
+-gpgpu_operand_collector_num_in_ports_mem 1
+-gpgpu_operand_collector_num_out_ports_mem 1
+-gpgpu_operand_collector_num_in_ports_dp 1
+-gpgpu_operand_collector_num_out_ports_dp 1
+# gpgpu_num_reg_banks should be increased to 32
+-gpgpu_num_reg_banks 32
+
+# shared memory bankconflict detection 
+-gpgpu_shmem_num_banks 32
+-gpgpu_shmem_limited_broadcast 0
+-gpgpu_shmem_warp_parts 1
+# Use Fermi Coalsce arhitetecture which is the same as Pascal
+-gpgpu_coalesce_arch 61
+
+## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units
+-gpgpu_max_insn_issue_per_warp 2
+-gpgpu_dual_issue_diff_exec_units 1
+
+# interconnection
+-network_mode 1 
+-inter_config_file config_fermi_islip.icnt
+
+# memory partition latency config 
+-rop_latency 100
+-dram_latency 100
+
+# dram model config
+-gpgpu_dram_scheduler 1
+# The DRAM return queue and the scheduler queue together should provide buffer
+# to sustain the memory level parallelism to tolerate DRAM latency 
+# To allow 100% DRAM utility, there should at least be enough buffer to sustain
+# the minimum DRAM latency (100 core cycles).  I.e. 
+#   Total buffer space required = 100 x 924MHz / 700MHz = 132
+-gpgpu_frfcfs_dram_sched_queue_size 16
+-gpgpu_dram_return_queue_size 240
+
+# for NVIDIA TITAN X, bus width is 384bits (12 DRAM chips x 32 bits)
+# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition
+# the atom size of GDDR5X (the smallest read request) is 32 bytes 
+-gpgpu_n_mem_per_ctrlr 1
+-gpgpu_dram_buswidth 4
+-gpgpu_dram_burst_length 8
+-dram_data_command_freq_ratio 4  # GDDR5X is QDR
+-gpgpu_mem_address_mask 1
+-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS
+
+# Use the same GDDR5 timing from hynix H5GQ1H24AFR
+# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0
+-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=8:RCD=16:RAS=37:RP=16:RC=52:
+                        CL=16:WL=6:CDLR=7:WR=16:nbkgrp=4:CCDL=4:RTPL=3"
+
+-dram_bnk_indexing_policy 0
+-dram_bnkgrp_indexing_policy 1
+
+#-Seperate_Write_Queue_Enable 1
+#-Write_Queue_Size 64:56:32
+
+# Pascal 102 has four schedulers per core
+-gpgpu_num_sched_per_core 2
+# Two Level Scheduler with active and pending pools
+#-gpgpu_scheduler two_level_active:6:0:1
+# Loose round robbin scheduler
+#-gpgpu_scheduler lrr
+# Greedy then oldest scheduler
+-gpgpu_scheduler gto
+
+# stat collection
+-gpgpu_memlatency_stat 14 
+-gpgpu_runtime_stat 500
+-enable_ptx_file_line_stats 1
+-visualizer_enabled 0
+
+# power model configs, disable it untill we create a real energy model for Pascal 102
+-power_simulation_enabled 0
+-gpuwattch_xml_file gpuwattch_gtx480.xml
+
+# tracing functionality
+#-trace_enabled 1
+#-trace_components WARP_SCHEDULER,SCOREBOARD
+#-trace_sampling_core 0
+
diff --git a/configs/PascalTitanX/gpgpu-sim-3.x/gpgpusim.config b/configs/PascalTitanX/gpgpu-sim-3.x/gpgpusim.config
index 782edf6..4407870 100644
--- a/configs/PascalTitanX/gpgpu-sim-3.x/gpgpusim.config
+++ b/configs/PascalTitanX/gpgpu-sim-3.x/gpgpusim.config
@@ -21,7 +21,7 @@
 #-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
 # Pascal NVIDIA TITAN X clock domains are adopted from 
 # https://en.wikipedia.org/wiki/GeForce_10_series
--gpgpu_clock_domains 1417.0:2834.0:1417.0:2500.0
+-gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0
 
 # shader core pipeline config
 -gpgpu_shader_registers 65536
@@ -32,21 +32,26 @@
 -gpgpu_simd_model 1 
 
 # Pipeline widths and number of FUs
-# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
 ## Pascal GP102 has 4 SP SIMD units and 4 SFU units
 ## we need to scale the number of pipeline registers to be equal to the number of SP units
--gpgpu_pipeline_widths 4,4,1,4,4,1,8
+-gpgpu_pipeline_widths 4,1,4,1,4,1,4,1,9
 -gpgpu_num_sp_units 4
 -gpgpu_num_sfu_units 4
+-gpgpu_num_dp_units 1
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
+# All Div operations are executed on SFU unit
+# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2
 -ptx_opcode_latency_int 4,13,4,5,145
 -ptx_opcode_initiation_int 1,1,1,1,4
 -ptx_opcode_latency_fp 4,13,4,5,39
 -ptx_opcode_initiation_fp 1,2,1,1,4
 -ptx_opcode_latency_dp 8,19,8,8,330
--ptx_opcode_initiation_dp 4,8,4,4,130
+-ptx_opcode_initiation_dp 8,8,8,8,130
+-ptx_opcode_initiation_sfu 4
+-ptx_opcode_latency_sfu 8
 
 # <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
 # ** Optional parameter - Required when mshr_type==Texture Fifo
@@ -54,33 +59,36 @@
 # Pascal GP102 has 96KB Shared memory
 # Pascal GP102 has 64KB L1 cache
 # The defulat is to disable the L1 cache, unless cache modifieres is used
--gpgpu_cache:dl1  64:128:6,L:L:m:N:H,A:128:8,16
+-gpgpu_cache:dl1  N:64:128:6,L:L:m:N:H,A:128:8,16
 -gpgpu_shmem_size 98304
 -gmem_skip_L1D 1
 
 # 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache
--gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:128:8,4:0,32
+-gpgpu_cache:dl2 N:64:128:16,L:B:m:W:L,A:128:8,4:0,32
 -gpgpu_cache:dl2_texture_only 0 
 -gpgpu_dram_partition_queues 32:32:32:32
 
 # 4 KB Inst.
--gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4
+-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,A:2:48,4
 # 48 KB Tex 
--gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2
+-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,F:128:4,128:2
 # 12 KB Const
--gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4
+-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,A:2:64,4
 
 # enable operand collector 
 ## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units
--gpgpu_operand_collector_num_units_sp 20
--gpgpu_operand_collector_num_units_sfu 4
+-gpgpu_operand_collector_num_units_sp 12
+-gpgpu_operand_collector_num_units_sfu 6
 -gpgpu_operand_collector_num_units_mem 8
+-gpgpu_operand_collector_num_units_dp 6
 -gpgpu_operand_collector_num_in_ports_sp 4
 -gpgpu_operand_collector_num_out_ports_sp 4
--gpgpu_operand_collector_num_in_ports_sfu 1
--gpgpu_operand_collector_num_out_ports_sfu 1
+-gpgpu_operand_collector_num_in_ports_sfu 4
+-gpgpu_operand_collector_num_out_ports_sfu 4
 -gpgpu_operand_collector_num_in_ports_mem 1
 -gpgpu_operand_collector_num_out_ports_mem 1
+-gpgpu_operand_collector_num_in_ports_dp 1
+-gpgpu_operand_collector_num_out_ports_dp 1
 # gpgpu_num_reg_banks should be increased to 32
 -gpgpu_num_reg_banks 32
 
@@ -97,7 +105,7 @@
 -inter_config_file config_fermi_islip.icnt
 
 # memory partition latency config 
--rop_latency 120
+-rop_latency 100
 -dram_latency 100
 
 # dram model config
@@ -122,8 +130,8 @@
 
 # Use the same GDDR5 timing from hynix H5GQ1H24AFR
 # disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0
--gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40:
-                        CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0"
+-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=8:RCD=16:RAS=37:RP=16:RC=52:
+                        CL=16:WL=6:CDLR=7:WR=16:nbkgrp=4:CCDL=4:RTPL=3"
 
 # Pascal GP102 has four schedulers per core
 -gpgpu_num_sched_per_core 4
diff --git a/configs/QuadroFX5600/gpgpusim.config b/configs/QuadroFX5600/gpgpusim.config
index cb87b65..e3cab18 100644
--- a/configs/QuadroFX5600/gpgpusim.config
+++ b/configs/QuadroFX5600/gpgpusim.config
@@ -17,10 +17,11 @@
 -gpgpu_simd_model 1
 
 # Pipeline widths and number of FUs
-# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
--gpgpu_pipeline_widths 1,1,1,1,1,1,1
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+-gpgpu_pipeline_widths 1,0,1,1,1,0,1,1,1
 -gpgpu_num_sp_units 1
 -gpgpu_num_sfu_units 1
+-gpgpu_num_dp_units 0
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
@@ -32,10 +33,10 @@
 -ptx_opcode_initiation_dp 8,8,8,8,130
 
 # memory stage behaviour
--gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4
--gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2
--gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4
--gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4
+-gpgpu_cache:il1 N:4:256:4,L:R:f:N:L,A:2:32,4
+-gpgpu_tex_cache:l1 N:8:128:5,L:R:m:N:L,F:128:4,128:2
+-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,A:2:32,4
+-gpgpu_cache:dl2 N:16:256:8,L:B:m:W:L,A:16:4,4
 -gpgpu_cache:dl2_texture_only 1
 
 # TLB parameters
diff --git a/configs/QuadroFX5800/gpgpusim.config b/configs/QuadroFX5800/gpgpusim.config
index 82243c2..56dbb17 100644
--- a/configs/QuadroFX5800/gpgpusim.config
+++ b/configs/QuadroFX5800/gpgpusim.config
@@ -16,10 +16,11 @@
 -gpgpu_simd_model 1
 
 # Pipeline widths and number of FUs
-# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
--gpgpu_pipeline_widths 1,1,1,1,1,1,1
+# ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB
+-gpgpu_pipeline_widths 1,0,1,1,1,0,1,1,1
 -gpgpu_num_sp_units 1
 -gpgpu_num_sfu_units 1
+-gpgpu_num_dp_units 0
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
@@ -33,10 +34,10 @@
 # memory stage behaviour
 # <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
 # ** Optional parameter - Required when mshr_type==Texture Fifo
--gpgpu_cache:il1 4:256:4,L:R:f:N:L,A:2:32,4
--gpgpu_tex_cache:l1 8:128:5,L:R:m:N:L,F:128:4,128:2
--gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4
--gpgpu_cache:dl2 16:256:8,L:B:m:W:L,A:16:4,4
+-gpgpu_cache:il1 N:4:256:4,L:R:f:N:L,A:2:32,4
+-gpgpu_tex_cache:l1 N:8:128:5,L:R:m:N:L,F:128:4,128:2
+-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,A:2:32,4
+-gpgpu_cache:dl2 N:16:256:8,L:B:m:W:L,A:16:4,4
 -gpgpu_cache:dl2_texture_only 1
 
 -gpgpu_shmem_warp_parts 2
diff --git a/configs/TeslaC2050/gpgpusim.config b/configs/TeslaC2050/gpgpusim.config
index 442ab8b..aa5f5f3 100644
--- a/configs/TeslaC2050/gpgpusim.config
+++ b/configs/TeslaC2050/gpgpusim.config
@@ -33,9 +33,10 @@
 
 # Pipeline widths and number of FUs
 # ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB
--gpgpu_pipeline_widths 2,1,1,2,1,1,2
+-gpgpu_pipeline_widths 2,0,1,1,2,0,1,1,2
 -gpgpu_num_sp_units 2
 -gpgpu_num_sfu_units 1
+-gpgpu_num_dp_units 0
 
 # Instruction latencies and initiation intervals
 # "ADD,MAX,MUL,MAD,DIV"
@@ -51,20 +52,20 @@
 # <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
 # ** Optional parameter - Required when mshr_type==Texture Fifo
 # Note: Hashing set index function (H) only applies to a set size of 32 or 64.
--gpgpu_cache:dl1  32:128:4,L:L:m:N:H,A:32:8,8
+-gpgpu_cache:dl1  N:32:128:4,L:L:m:N:H,A:32:8,8
 -gpgpu_shmem_size 49152
 
 # The alternative configuration for fermi in case cudaFuncCachePreferL1 is selected
-#-gpgpu_cache:dl1  64:128:6,L:L:m:N:H,A:32:8,8
+#-gpgpu_cache:dl1  N:64:128:6,L:L:m:N:H,A:32:8,8
 #-gpgpu_shmem_size 16384
 
 # 64 sets, each 128 bytes 8-way for each memory sub partition. This gives 786KB L2 cache
--gpgpu_cache:dl2 64:128:8,L:B:m:W:L,A:32:4,4:0,32
+-gpgpu_cache:dl2 N:64:128:8,L:B:m:W:L,A:32:4,4:0,32
 -gpgpu_cache:dl2_texture_only 0 
 
--gpgpu_cache:il1 4:128:4,L:R:f:N:L,A:2:32,4
--gpgpu_tex_cache:l1 4:128:24,L:R:m:N:L,F:128:4,128:2
--gpgpu_const_cache:l1 64:64:2,L:R:f:N:L,A:2:32,4
+-gpgpu_cache:il1 N:4:128:4,L:R:f:N:L,A:2:32,4
+-gpgpu_tex_cache:l1 N:4:128:24,L:R:m:N:L,F:128:4,128:2
+-gpgpu_const_cache:l1 N:64:64:2,L:R:f:N:L,A:2:32,4
 
 # enable operand collector 
 -gpgpu_operand_collector_num_units_sp 6
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index 64eb43c..d2a155c 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -314,7 +314,7 @@ void warp_inst_t::generate_mem_accesses()
         break;
 
     case global_space: case local_space: case param_space_local:
-    	 if( m_config->gpgpu_coalesce_arch == 13 || m_config->gpgpu_coalesce_arch == 20) {
+    	 if( m_config->gpgpu_coalesce_arch >= 13 && m_config->gpgpu_coalesce_arch <= 62) {
             if(isatomic())
                 memory_coalescing_arch_atomic(is_write, access_type);
             else
@@ -356,37 +356,28 @@ void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_
 {
     // see the CUDA manual where it discusses coalescing rules before reading this
     unsigned segment_size = 0;
-    unsigned warp_parts;
+    unsigned warp_parts = m_config->mem_warp_parts;
+    bool sector_segment_size = false;
 
-    //TO DO: need to double check how doubles are coalesced!
-    if(data_size == 1)
+    if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39)
     {
-    	//If it is byte data, then coalesce on the whole 32 threads, regardless the arch version
-    	warp_parts = 1;
-    }
-    else if(m_config->gpgpu_coalesce_arch == 13)
-    {
-    	//mem_warp_parts should equal 2 for arch=13
-    	//use the parameter mem_warp_parts for arch=13 to ensure it is backward compatibility with older gpgpu config files
-    	warp_parts = m_config->mem_warp_parts;
-    }
-    else if(m_config->gpgpu_coalesce_arch == 20)
-    {
-    	//It is expected that L1_warp_parts_non_cached = 4 and L1_warp_parts_cached = 1 for arch=20
-    	//non cached, coalesce on 8 threads to generate 32 bytes accesses
-    	//cached, coalesce on 32 threads to generate 128 bytes accesses
+    	//Fermi and Kepler, L1 is normal and L2 is sector
     	if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL)
-    		warp_parts = m_config->L1_warp_parts_non_cached;
+    		sector_segment_size = true;
     	else
-    		warp_parts = m_config->L1_warp_parts_cached;
+    		sector_segment_size = false;
+    }
+    else if(m_config->gpgpu_coalesce_arch >= 40)
+    {
+    	//Maxwell and Pascal, L1 and L2 are sectors
+    	//all requests should be 32 bytes
+    	sector_segment_size = true;
     }
-    else
-    	abort();
 
     switch( data_size ) {
     case 1: segment_size = 32; break;
-    case 2: segment_size = 64; break;
-    case 4: case 8: case 16: segment_size = 128; break;
+    case 2: segment_size = sector_segment_size? 32 : 64; break;
+    case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break;
     }
     unsigned subwarp_size = m_config->warp_size / warp_parts;
 
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 910a9ed..cec75f9 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -76,6 +76,7 @@ enum uarch_op_t {
    NO_OP=-1,
    ALU_OP=1,
    SFU_OP,
+   DP_OP,
    ALU_SFU_OP,
    LOAD_OP,
    STORE_OP,
@@ -131,6 +132,7 @@ typedef enum special_operations_t special_ops; // Required to identify for the p
 enum operation_pipeline_t {
     UNKOWN_OP,
     SP__OP,
+	DP__OP,
     SFU__OP,
     MEM__OP
 };
@@ -332,8 +334,6 @@ struct core_config {
     unsigned gpgpu_shmem_sizeDefault;
     unsigned gpgpu_shmem_sizePrefL1;
     unsigned gpgpu_shmem_sizePrefShared;
-    unsigned L1_warp_parts_cached;
-    unsigned L1_warp_parts_non_cached;
 
     // texture and constant cache line sizes (used to determine number of memory accesses)
     unsigned gpgpu_cache_texl1_linesize;
@@ -618,7 +618,8 @@ private:
 
 const unsigned MAX_MEMORY_ACCESS_SIZE = 128;
 typedef std::bitset<MAX_MEMORY_ACCESS_SIZE> mem_access_byte_mask_t;
-const unsigned SECTOR_CHUNCK_SIZE = 4;
+const unsigned SECTOR_CHUNCK_SIZE = 4;   //four sectors
+const unsigned SECTOR_SIZE = 32 ;        //sector is 32 bytes width
 typedef std::bitset<SECTOR_CHUNCK_SIZE> mem_access_sector_mask_t;
 #define NO_PARTIAL_WRITE (mem_access_byte_mask_t())
 
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index d4ace76..2f166aa 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -62,8 +62,8 @@ addr_t g_debug_pc = 0xBEEF1518;
 unsigned g_ptx_sim_num_insn = 0;
 unsigned gpgpu_param_num_shaders = 0;
 
-char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp;
-char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp;
+char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu;
+char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu;
 char *cdp_latency_str;
 unsigned cdp_latency[5];
 
@@ -80,6 +80,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
 			"Opcode latencies for double precision floating points <ADD,MAX,MUL,MAD,DIV>"
 			"Default 8,8,8,8,335",
 			"8,8,8,8,335");
+	option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu,
+			"Opcode latencies for SFU instructions"
+			"Default 8",
+			"8");
 	option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int,
 			"Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>"
 			"Default 1,1,4,4,32",
@@ -92,6 +96,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) {
 			"Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>"
 			"Default 8,8,8,8,130",
 			"8,8,8,8,130");
+	option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu,
+			"Opcode initiation intervals for sfu instructions"
+			"Default 8",
+			"8");
 	option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str,
 			"CDP API latency <cudaStreamCreateWithFlags, \
 cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \
@@ -393,6 +401,10 @@ void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t coun
    char *src_data = (char*)src;
    for (unsigned n=0; n < count; n ++ ) 
       m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL);
+
+   // Copy into the performance model.
+   extern gpgpu_sim* g_the_gpu; 
+   g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count);
    if(g_debug_execution >= 3) {
       printf( " done.\n");
       fflush(stdout);
@@ -408,6 +420,10 @@ void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count )
    unsigned char *dst_data = (unsigned char*)dst;
    for (unsigned n=0; n < count; n ++ ) 
       m_global_mem->read(src_start_addr+n,1,dst_data+n);
+
+   // Copy into the performance model.
+   extern gpgpu_sim* g_the_gpu; 
+   g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count);
    if(g_debug_execution >= 3) {
       printf( " done.\n");
       fflush(stdout);
@@ -589,9 +605,11 @@ void ptx_instruction::set_opcode_and_latency()
 	unsigned int_latency[5];
 	unsigned fp_latency[5];
 	unsigned dp_latency[5];
+	unsigned sfu_latency;
 	unsigned int_init[5];
 	unsigned fp_init[5];
 	unsigned dp_init[5];
+	unsigned sfu_init;
 	/*
 	 * [0] ADD,SUB
 	 * [1] MAX,Min
@@ -608,6 +626,8 @@ void ptx_instruction::set_opcode_and_latency()
 	sscanf(opcode_latency_dp, "%u,%u,%u,%u,%u",
 			&dp_latency[0],&dp_latency[1],&dp_latency[2],
 			&dp_latency[3],&dp_latency[4]);
+	sscanf(opcode_latency_sfu, "%u",
+			&sfu_latency);
 	sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u",
 			&int_init[0],&int_init[1],&int_init[2],
 			&int_init[3],&int_init[4]);
@@ -617,8 +637,10 @@ void ptx_instruction::set_opcode_and_latency()
 	sscanf(opcode_initiation_dp, "%u,%u,%u,%u,%u",
 			&dp_init[0],&dp_init[1],&dp_init[2],
 			&dp_init[3],&dp_init[4]);
+	sscanf(opcode_initiation_sfu, "%u",
+			&sfu_init);
 	sscanf(cdp_latency_str, "%u,%u,%u,%u,%u",
-			&cdp_latency[0],&cdp_latency[1],&cdp_latency[2], 
+			&cdp_latency[0],&cdp_latency[1],&cdp_latency[2],
             &cdp_latency[3],&cdp_latency[4]);
 
 	if(!m_operands.empty()){
@@ -678,6 +700,7 @@ void ptx_instruction::set_opcode_and_latency()
 	   case FF64_TYPE:
 		   latency = dp_latency[0];
 		   initiation_interval = dp_init[0];
+		   op = DP_OP;
 		   break;
 	   case B32_TYPE:
 	   case U32_TYPE:
@@ -699,6 +722,7 @@ void ptx_instruction::set_opcode_and_latency()
 	   case FF64_TYPE:
 		   latency = dp_latency[1];
 		   initiation_interval = dp_init[1];
+		   op = DP_OP;
 		   break;
 	   case B32_TYPE:
 	   case U32_TYPE:
@@ -715,13 +739,12 @@ void ptx_instruction::set_opcode_and_latency()
 	   case F32_TYPE:
 		   latency = fp_latency[2];
 		   initiation_interval = fp_init[2];
-		   op = ALU_SFU_OP;
 		   break;
 	   case F64_TYPE:
 	   case FF64_TYPE:
 		   latency = dp_latency[2];
 		   initiation_interval = dp_init[2];
-		   op = ALU_SFU_OP;
+		   op = DP_OP;
 		   break;
 	   case B32_TYPE:
 	   case U32_TYPE:
@@ -744,6 +767,7 @@ void ptx_instruction::set_opcode_and_latency()
 	   case FF64_TYPE:
 		   latency = dp_latency[3];
 		   initiation_interval = dp_init[3];
+		   op = DP_OP;
 		   break;
 	   case B32_TYPE:
 	   case U32_TYPE:
@@ -779,13 +803,13 @@ void ptx_instruction::set_opcode_and_latency()
 	   break;
    case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP:
 	   //Using double to approximate those
-	  latency = dp_latency[2];
-	  initiation_interval = dp_init[2];
+	  latency = sfu_latency;
+	  initiation_interval = sfu_init;
       op = SFU_OP;
       break;
    case SHFL_OP:
 	   latency = 32;
-	   initiation_interval = 15;
+	   initiation_interval = 4;
 	   break;
    default: 
        break;
diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h
index b25f143..0caa5d4 100644
--- a/src/gpgpu-sim/delayqueue.h
+++ b/src/gpgpu-sim/delayqueue.h
@@ -161,6 +161,7 @@ public:
    }
 
    bool full() const { return (m_max_len && m_length >= m_max_len); }
+   bool is_avilable_size(unsigned size) const { return (m_max_len && m_length+size-1 >= m_max_len); }
    bool empty() const { return m_head == NULL; }
    unsigned get_n_element() const { return m_n_element; }
    unsigned get_length() const { return m_length; }
diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc
index a0e024b..a57508c 100644
--- a/src/gpgpu-sim/dram.cc
+++ b/src/gpgpu-sim/dram.cc
@@ -49,11 +49,45 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m
    m_stats = stats;
    m_config = config;
 
+   //rowblp
+   access_num=0;
+   hits_num=0;
+   read_num=0;
+   write_num=0;
+   hits_read_num=0;
+   hits_write_num=0;
+   banks_1time=0;
+   banks_acess_total=0;
+   banks_acess_total_after=0;
+   banks_time_ready=0;
+   banks_access_ready_total=0;
+   issued_two=0;
+   issued_total=0;
+   issued_total_row=0;
+   issued_total_col=0;
+
    CCDc = 0;
    RRDc = 0;
    RTWc = 0;
    WTRc = 0;
 
+   wasted_bw_row=0;
+   wasted_bw_col=0;
+   util_bw=0;
+   idle_bw=0;
+   RCDc_limit=0;
+   CCDLc_limit=0;
+   CCDLc_limit_alone=0;
+   CCDc_limit=0;
+   WTRc_limit=0;
+   WTRc_limit_alone=0;
+   RCDWRc_limit=0;
+   RTWc_limit=0;
+   RTWc_limit_alone=0;
+   rwq_limit=0;
+   write_to_read_ratio_blp_rw_average=0;
+   bkgrp_parallsim_rw=0;
+
    rw = READ; //read mode is default
 
 	bkgrp = (bankgrp_t**) calloc(sizeof(bankgrp_t*), m_config->nbkgrp);
@@ -74,12 +108,13 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m
       bk[i]->state = BANK_IDLE;
       bk[i]->bkgrpindex = i/(m_config->nbk/m_config->nbkgrp);
    }
-   prio = 0;  
+   prio = 0;
+
    rwq = new fifo_pipeline<dram_req_t>("rwq",m_config->CL,m_config->CL+1);
    mrqq = new fifo_pipeline<dram_req_t>("mrqq",0,2);
    returnq = new fifo_pipeline<mem_fetch>("dramreturnq",0,m_config->gpgpu_dram_return_queue_size==0?1024:m_config->gpgpu_dram_return_queue_size); 
    m_frfcfs_scheduler = NULL;
-   if ( m_config->scheduler_type == DRAM_FRFCFS )
+   if ( m_config->scheduler_type == DRAM_FRFCFS)
       m_frfcfs_scheduler = new frfcfs_scheduler(m_config,this,stats);
    n_cmd = 0;
    n_activity = 0;
@@ -88,6 +123,8 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m
    n_pre = 0; 
    n_rd = 0;
    n_wr = 0;
+   n_wr_WB=0;
+   n_rd_L2_A=0;
    n_req = 0;
    max_mrqs_temp = 0;
    bwutil = 0;
@@ -113,13 +150,21 @@ dram_t::dram_t( unsigned int partition_id, const struct memory_config *config, m
       mrqq_Dist = StatCreate("mrqq_length",1, queue_limit());
    else //queue length is unlimited; 
       mrqq_Dist = StatCreate("mrqq_length",1,64); //track up to 64 entries
+
 }
 
-bool dram_t::full() const 
+bool dram_t::full(bool is_write) const
 {
-    if(m_config->scheduler_type == DRAM_FRFCFS ){
+    if(m_config->scheduler_type == DRAM_FRFCFS){
         if(m_config->gpgpu_frfcfs_dram_sched_queue_size == 0 ) return false;
-        return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
+        if(m_config->seperate_write_queue_enabled){
+        	if(is_write)
+        		return m_frfcfs_scheduler->num_write_pending() >= m_config->gpgpu_frfcfs_dram_write_queue_size;
+        	else
+        		return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
+        }
+        else
+        	return m_frfcfs_scheduler->num_pending() >= m_config->gpgpu_frfcfs_dram_sched_queue_size;
     }
    else return mrqq->full();
 }
@@ -127,7 +172,7 @@ bool dram_t::full() const
 unsigned dram_t::que_length() const
 {
    unsigned nreqs = 0;
-   if (m_config->scheduler_type == DRAM_FRFCFS ) {
+   if (m_config->scheduler_type == DRAM_FRFCFS) {
       nreqs = m_frfcfs_scheduler->num_pending();
    } else {
       nreqs = mrqq->get_length();
@@ -146,7 +191,7 @@ unsigned int dram_t::queue_limit() const
 }
 
 
-dram_req_t::dram_req_t( class mem_fetch *mf )
+dram_req_t::dram_req_t( class mem_fetch *mf, unsigned banks, unsigned dram_bnk_indexing_policy)
 {
    txbytes = 0;
    dqbytes = 0;
@@ -154,7 +199,16 @@ dram_req_t::dram_req_t( class mem_fetch *mf )
 
    const addrdec_t &tlx = mf->get_tlx_addr();
 
-   bk  = tlx.bk; 
+   if(dram_bnk_indexing_policy == 0) {
+	   bk = tlx.bk;
+   }
+    else if(dram_bnk_indexing_policy == 1) {
+    	 int lbank = log2(banks);
+    	 bk  = tlx.bk ^ (((1<<lbank)-1) & tlx.row);
+    }
+    else
+    	assert(1);
+
    row = tlx.row; 
    col = tlx.col; 
    nbytes = mf->get_data_size();
@@ -169,14 +223,15 @@ void dram_t::push( class mem_fetch *data )
 {
    assert(id == data->get_tlx_addr().chip); // Ensure request is in correct memory partition
 
-   dram_req_t *mrq = new dram_req_t(data);
+   dram_req_t *mrq = new dram_req_t(data,m_config->nbk,m_config->dram_bnk_indexing_policy);
+
    data->set_status(IN_PARTITION_MC_INTERFACE_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
-   mrqq->push(mrq);
+	   mrqq->push(mrq);
 
    // stats...
    n_req += 1;
    n_req_partial += 1;
-   if ( m_config->scheduler_type == DRAM_FRFCFS ) {
+   if ( m_config->scheduler_type == DRAM_FRFCFS) {
       unsigned nreqs = m_frfcfs_scheduler->num_pending();
       if ( nreqs > max_mrqs_temp)
          max_mrqs_temp = nreqs;
@@ -212,6 +267,7 @@ void dram_t::cycle()
            printf("\tDQ: BK%d Row:%03x Col:%03x", cmd->bk, cmd->row, cmd->col + cmd->dqbytes);
 #endif
            cmd->dqbytes += m_config->dram_atom_size; 
+
            if (cmd->dqbytes >= cmd->nbytes) {
               mem_fetch *data = cmd->data; 
               data->set_status(IN_PARTITION_MC_RETURNQ,gpu_sim_cycle+gpu_tot_sim_cycle); 
@@ -240,7 +296,7 @@ void dram_t::cycle()
 		printf("Error: Unknown DRAM scheduler type\n");
 		assert(0);
    }
-   if ( m_config->scheduler_type == DRAM_FRFCFS ) {
+   if ( m_config->scheduler_type == DRAM_FRFCFS) {
       unsigned nreqs = m_frfcfs_scheduler->num_pending();
       if ( nreqs > max_mrqs) {
          max_mrqs = nreqs;
@@ -258,130 +314,123 @@ void dram_t::cycle()
    unsigned k=m_config->nbk;
    bool issued = false;
 
-   // check if any bank is ready to issue a new read
+   //collect row buffer locality, BLP and other statistics
+   /////////////////////////////////////////////////////////////////////////
+   unsigned int memory_pending=0;
    for (unsigned i=0;i<m_config->nbk;i++) {
-      unsigned j = (i + prio) % m_config->nbk;
-	  unsigned grp = j>>m_config->bk_tag_length;
-      if (bk[j]->mrq) { //if currently servicing a memory request
-          bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle);
-         // correct row activated for a READ
-         if ( !issued && !CCDc && !bk[j]->RCDc &&
-              !(bkgrp[grp]->CCDLc) &&
-              (bk[j]->curr_row == bk[j]->mrq->row) && 
-              (bk[j]->mrq->rw == READ) && (WTRc == 0 )  &&
-              (bk[j]->state == BANK_ACTIVE) &&
-              !rwq->full() ) {
-            if (rw==WRITE) {
-               rw=READ;
-               rwq->set_min_length(m_config->CL);
-            }
-            rwq->push(bk[j]->mrq);
-            bk[j]->mrq->txbytes += m_config->dram_atom_size; 
-            CCDc = m_config->tCCD;
-            bkgrp[grp]->CCDLc = m_config->tCCDL;
-            RTWc = m_config->tRTW;
-            bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio;
-            bkgrp[grp]->RTPLc = m_config->tRTPL;
-            issued = true;
-            n_rd++;
-            bwutil += m_config->BL/m_config->data_command_freq_ratio;
-            bwutil_partial += m_config->BL/m_config->data_command_freq_ratio;
-            bk[j]->n_access++;
-#ifdef DRAM_VERIFY
-            PRINT_CYCLE=1;
-            printf("\tRD  Bk:%d Row:%03x Col:%03x \n",
-                   j, bk[j]->curr_row,
-                   bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size);
-#endif            
-            // transfer done
-            if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) {
-               bk[j]->mrq = NULL;
-            }
-         } else
-            // correct row activated for a WRITE
-            if ( !issued && !CCDc && !bk[j]->RCDWRc &&
-                 !(bkgrp[grp]->CCDLc) &&
-                 (bk[j]->curr_row == bk[j]->mrq->row)  && 
-                 (bk[j]->mrq->rw == WRITE) && (RTWc == 0 )  &&
-                 (bk[j]->state == BANK_ACTIVE) &&
-                 !rwq->full() ) {
-            if (rw==READ) {
-               rw=WRITE;
-               rwq->set_min_length(m_config->WL);
-            }
-            rwq->push(bk[j]->mrq);
+	   if (bk[i]->mrq)
+		   memory_pending++;
+   }
+   banks_1time += memory_pending;
+   if(memory_pending >0)
+	   banks_acess_total++;
 
-            bk[j]->mrq->txbytes += m_config->dram_atom_size; 
-            CCDc = m_config->tCCD;
-            bkgrp[grp]->CCDLc = m_config->tCCDL;
-            WTRc = m_config->tWTR; 
-            bk[j]->WTPc = m_config->tWTP; 
-            issued = true;
-            n_wr++;
-            bwutil += m_config->BL/m_config->data_command_freq_ratio;
-            bwutil_partial += m_config->BL/m_config->data_command_freq_ratio;
-#ifdef DRAM_VERIFY
-            PRINT_CYCLE=1;
-            printf("\tWR  Bk:%d Row:%03x Col:%03x \n",
-                   j, bk[j]->curr_row, 
-                   bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size);
-#endif  
-            // transfer done 
-            if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) {
-               bk[j]->mrq = NULL;
-            }
-         }
+   unsigned int memory_pending_rw=0;
+   unsigned read_blp_rw=0;
+   unsigned write_blp_rw=0;
+   std::bitset<8> bnkgrp_rw_found;  //assume max we have 8 bank groups
 
-         else
-            // bank is idle
-            if ( !issued && !RRDc && 
-                 (bk[j]->state == BANK_IDLE) &&
-                 !bk[j]->RPc && !bk[j]->RCc ) {
-#ifdef DRAM_VERIFY
-            PRINT_CYCLE=1;
-            printf("\tACT BK:%d NewRow:%03x From:%03x \n",
-                   j,bk[j]->mrq->row,bk[j]->curr_row);
-#endif
-            // activate the row with current memory request 
-            bk[j]->curr_row = bk[j]->mrq->row;
-            bk[j]->state = BANK_ACTIVE;
-            RRDc = m_config->tRRD;
-            bk[j]->RCDc = m_config->tRCD;
-            bk[j]->RCDWRc = m_config->tRCDWR;
-            bk[j]->RASc = m_config->tRAS;
-            bk[j]->RCc = m_config->tRC;
-            prio = (j + 1) % m_config->nbk;
-            issued = true;
-            n_act_partial++;
-            n_act++;
-         }
+   for (unsigned j=0;j<m_config->nbk;j++) {
+  	   unsigned grp = get_bankgrp_number(j);
+  	   if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) &&
+  		  (bk[j]->mrq->rw == READ)  &&
+  		  (bk[j]->state == BANK_ACTIVE))))
+  	   {
+  		    memory_pending_rw++;
+  		    read_blp_rw++;
+  		    bnkgrp_rw_found.set(grp);
+  	   }
+  	   else if
+  	        (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row)  &&
+  			 (bk[j]->mrq->rw == WRITE) &&
+  			 (bk[j]->state == BANK_ACTIVE))))
+  	   {
+  		     memory_pending_rw++;
+  		     write_blp_rw++;
+  		     bnkgrp_rw_found.set(grp);
+  	   }
+     }
+     banks_time_rw += memory_pending_rw;
+     bkgrp_parallsim_rw += bnkgrp_rw_found.count();
+     if(memory_pending_rw >0)
+     {
+    	 write_to_read_ratio_blp_rw_average += (double)write_blp_rw/(write_blp_rw+read_blp_rw);
+  	     banks_access_rw_total++;
+     }
 
-         else
-            // different row activated
-            if ( (!issued) && 
-                 (bk[j]->curr_row != bk[j]->mrq->row) &&
-                 (bk[j]->state == BANK_ACTIVE) && 
-                 (!bk[j]->RASc && !bk[j]->WTPc && 
-				  !bk[j]->RTPc &&
-				  !bkgrp[grp]->RTPLc) ) {
-            // make the bank idle again
-            bk[j]->state = BANK_IDLE;
-            bk[j]->RPc = m_config->tRP;
-            prio = (j + 1) % m_config->nbk;
-            issued = true;
-            n_pre++;
-            n_pre_partial++;
-#ifdef DRAM_VERIFY
-            PRINT_CYCLE=1;
-            printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row);
-#endif
-         }
-      } else {
-         if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc
-             && !bk[j]->RCc && !bk[j]->RPc  && !bk[j]->RCDWRc) k--;
-         bk[j]->n_idle++;
-      }
+   unsigned int memory_Pending_ready=0;
+   for (unsigned j=0;j<m_config->nbk;j++) {
+	   unsigned grp = get_bankgrp_number(j);
+	   if (bk[j]->mrq && ((!CCDc && !bk[j]->RCDc &&
+		  !(bkgrp[grp]->CCDLc) &&
+		  (bk[j]->curr_row == bk[j]->mrq->row) &&
+		  (bk[j]->mrq->rw == READ) && (WTRc == 0 )  &&
+		  (bk[j]->state == BANK_ACTIVE) &&
+		  !rwq->full())
+		   ||
+		   (!CCDc && !bk[j]->RCDWRc &&
+			 !(bkgrp[grp]->CCDLc) &&
+			 (bk[j]->curr_row == bk[j]->mrq->row)  &&
+			 (bk[j]->mrq->rw == WRITE) && (RTWc == 0 )  &&
+			 (bk[j]->state == BANK_ACTIVE) &&
+			 !rwq->full())))
+	   {
+		   memory_Pending_ready++;
+	   }
    }
+   banks_time_ready += memory_Pending_ready;
+   if(memory_Pending_ready >0)
+	   banks_access_ready_total++;
+   ///////////////////////////////////////////////////////////////////////////////////
+
+   bool issued_col_cmd = false;
+   bool issued_row_cmd = false;
+
+   if(m_config->dual_bus_interface)
+   {
+	   //dual bus interface
+	   //issue one row command and one column command
+	     for (unsigned i=0;i<m_config->nbk;i++) {
+	       unsigned j = (i + prio) % m_config->nbk;
+	  	   issued_col_cmd  = issue_col_command(j);
+	  	   if(issued_col_cmd) break;
+	     }
+	     for (unsigned i=0;i<m_config->nbk;i++) {
+	    	unsigned j = (i + prio) % m_config->nbk;
+	  	    issued_row_cmd = issue_row_command(j);
+	  	    if(issued_row_cmd) break;
+	     }
+	     for (unsigned i=0;i<m_config->nbk;i++) {
+	    	 unsigned j = (i + prio) % m_config->nbk;
+	    	 if(!bk[j]->mrq) {
+				  if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc
+					  && !bk[j]->RCc && !bk[j]->RPc  && !bk[j]->RCDWRc) k--;
+				  bk[j]->n_idle++;
+			}
+	     }
+   }
+   else
+   {
+	   //single bus interface
+	   //issue only one row/column command
+	   for (unsigned i=0;i<m_config->nbk;i++) {
+		   unsigned j = (i + prio) % m_config->nbk;
+		   if(!issued_col_cmd)
+		       issued_col_cmd  = issue_col_command(j);
+
+		   if(!issued_col_cmd && !issued_row_cmd)
+		       issued_row_cmd = issue_row_command(j);
+
+		   if(!bk[j]->mrq) {
+		          if (!CCDc && !RRDc && !RTWc && !WTRc && !bk[j]->RCDc && !bk[j]->RASc
+		              && !bk[j]->RCc && !bk[j]->RPc  && !bk[j]->RCDWRc) k--;
+		          bk[j]->n_idle++;
+		    }
+
+	   }
+   }
+
+   issued = issued_row_cmd || issued_col_cmd;
    if (!issued) {
       n_nop++;
       n_nop_partial++;
@@ -395,6 +444,85 @@ void dram_t::cycle()
    }
    n_cmd++;
    n_cmd_partial++;
+   if(issued)
+   {
+	   issued_total++;
+	   if(issued_col_cmd && issued_row_cmd)
+		   issued_two++;
+   }
+   if(issued_col_cmd)  issued_total_col++;
+   if(issued_row_cmd)  issued_total_row++;
+
+
+   //Collect some statistics
+   //check the limitation, see where BW is wasted?
+   /////////////////////////////////////////////////////////
+   unsigned int memory_pending_found=0;
+      for (unsigned i=0;i<m_config->nbk;i++) {
+   	   if (bk[i]->mrq)
+   		memory_pending_found++;
+      }
+      if(memory_pending_found>0)
+    	  banks_acess_total_after++;
+
+        bool memory_pending_rw_found=false;
+        for (unsigned j=0;j<m_config->nbk;j++) {
+     	   unsigned grp = get_bankgrp_number(j);
+     	   if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) &&
+     		  (bk[j]->mrq->rw == READ)  &&
+     		  (bk[j]->state == BANK_ACTIVE))
+     		   ||
+     		   (
+     			 (bk[j]->curr_row == bk[j]->mrq->row)  &&
+     			 (bk[j]->mrq->rw == WRITE) &&
+     			 (bk[j]->state == BANK_ACTIVE))))
+     		  memory_pending_rw_found=true;
+        }
+
+
+   if(issued_col_cmd  || CCDc)
+	   util_bw++;
+   else if (memory_pending_rw_found)
+   {
+	   wasted_bw_col++;
+	   for (unsigned j=0;j<m_config->nbk;j++) {
+		   unsigned grp = get_bankgrp_number(j);
+		   //read
+		   if (bk[j]->mrq && (((bk[j]->curr_row == bk[j]->mrq->row) &&
+			  (bk[j]->mrq->rw == READ)  &&
+			  (bk[j]->state == BANK_ACTIVE))))
+		   {
+			   if(bk[j]->RCDc) RCDc_limit++;
+			   if(bkgrp[grp]->CCDLc) CCDLc_limit++;
+			   if(WTRc) WTRc_limit++;
+			   if(CCDc) CCDc_limit++;
+			   if(rwq->full()) rwq_limit++;
+			   if(bkgrp[grp]->CCDLc && !WTRc) CCDLc_limit_alone++;
+			   if(!bkgrp[grp]->CCDLc && WTRc) WTRc_limit_alone++;
+		   }
+		   //write
+		   else if (bk[j]->mrq && ((bk[j]->curr_row == bk[j]->mrq->row)  &&
+				 (bk[j]->mrq->rw == WRITE) &&
+				 (bk[j]->state == BANK_ACTIVE)))
+		   {
+			   if(bk[j]->RCDWRc) RCDWRc_limit++;
+			   if(bkgrp[grp]->CCDLc) CCDLc_limit++;
+			   if(RTWc) RTWc_limit++;
+			   if(CCDc) CCDc_limit++;
+			   if(rwq->full()) rwq_limit++;
+			   if(bkgrp[grp]->CCDLc && !RTWc) CCDLc_limit_alone++;
+			   if(!bkgrp[grp]->CCDLc && RTWc) RTWc_limit_alone++;
+		   }
+	     }
+   }
+   else if (memory_pending_found)
+	   wasted_bw_row++;
+   else if (!memory_pending_found)
+  	   idle_bw++;
+   else
+	   assert(1);
+
+   /////////////////////////////////////////////////////////
 
    // decrements counters once for each time dram_issueCMD is called
    DEC2ZERO(RRDc);
@@ -420,26 +548,170 @@ void dram_t::cycle()
 #endif
 }
 
+bool dram_t::issue_col_command(int j)
+{
+	bool issued = false;
+	unsigned grp = get_bankgrp_number(j);
+    if (bk[j]->mrq) { //if currently servicing a memory request
+        bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle);
+       // correct row activated for a READ
+       if ( !issued && !CCDc && !bk[j]->RCDc &&
+            !(bkgrp[grp]->CCDLc) &&
+            (bk[j]->curr_row == bk[j]->mrq->row) &&
+            (bk[j]->mrq->rw == READ) && (WTRc == 0 )  &&
+            (bk[j]->state == BANK_ACTIVE) &&
+            !rwq->full() ) {
+          if (rw==WRITE) {
+             rw=READ;
+             rwq->set_min_length(m_config->CL);
+          }
+          rwq->push(bk[j]->mrq);
+          bk[j]->mrq->txbytes += m_config->dram_atom_size;
+          CCDc = m_config->tCCD;
+          bkgrp[grp]->CCDLc = m_config->tCCDL;
+          RTWc = m_config->tRTW;
+          bk[j]->RTPc = m_config->BL/m_config->data_command_freq_ratio;
+          bkgrp[grp]->RTPLc = m_config->tRTPL;
+          issued = true;
+          if(bk[j]->mrq->data->get_access_type() == L2_WR_ALLOC_R)
+        	  n_rd_L2_A++;
+          else
+                n_rd++;
+
+          bwutil += m_config->BL/m_config->data_command_freq_ratio;
+          bwutil_partial += m_config->BL/m_config->data_command_freq_ratio;
+          bk[j]->n_access++;
+
+#ifdef DRAM_VERIFY
+          PRINT_CYCLE=1;
+          printf("\tRD  Bk:%d Row:%03x Col:%03x \n",
+                 j, bk[j]->curr_row,
+                 bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size);
+#endif
+          // transfer done
+          if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) {
+             bk[j]->mrq = NULL;
+          }
+       } else
+          // correct row activated for a WRITE
+          if ( !issued && !CCDc && !bk[j]->RCDWRc &&
+               !(bkgrp[grp]->CCDLc) &&
+               (bk[j]->curr_row == bk[j]->mrq->row)  &&
+               (bk[j]->mrq->rw == WRITE) && (RTWc == 0 )  &&
+               (bk[j]->state == BANK_ACTIVE) &&
+               !rwq->full() ) {
+          if (rw==READ) {
+             rw=WRITE;
+             rwq->set_min_length(m_config->WL);
+          }
+          rwq->push(bk[j]->mrq);
+
+          bk[j]->mrq->txbytes += m_config->dram_atom_size;
+          CCDc = m_config->tCCD;
+          bkgrp[grp]->CCDLc = m_config->tCCDL;
+          WTRc = m_config->tWTR;
+          bk[j]->WTPc = m_config->tWTP;
+          issued = true;
+
+          if(bk[j]->mrq->data->get_access_type() == L2_WRBK_ACC)
+          	n_wr_WB++;
+          else
+          	 n_wr++;
+          bwutil += m_config->BL/m_config->data_command_freq_ratio;
+          bwutil_partial += m_config->BL/m_config->data_command_freq_ratio;
+#ifdef DRAM_VERIFY
+          PRINT_CYCLE=1;
+          printf("\tWR  Bk:%d Row:%03x Col:%03x \n",
+                 j, bk[j]->curr_row,
+                 bk[j]->mrq->col + bk[j]->mrq->txbytes - m_config->dram_atom_size);
+#endif
+          // transfer done
+          if ( !(bk[j]->mrq->txbytes < bk[j]->mrq->nbytes) ) {
+             bk[j]->mrq = NULL;
+          }
+       }
+
+    }
+
+    return issued;
+}
+
+bool dram_t::issue_row_command(int j)
+{
+	bool issued = false;
+	unsigned grp = get_bankgrp_number(j);
+    if (bk[j]->mrq) { //if currently servicing a memory request
+        bk[j]->mrq->data->set_status(IN_PARTITION_DRAM,gpu_sim_cycle+gpu_tot_sim_cycle);
+      //     bank is idle
+    //else
+  	  if ( !issued && !RRDc &&
+               (bk[j]->state == BANK_IDLE) &&
+               !bk[j]->RPc && !bk[j]->RCc) { //
+#ifdef DRAM_VERIFY
+          PRINT_CYCLE=1;
+          printf("\tACT BK:%d NewRow:%03x From:%03x \n",
+                 j,bk[j]->mrq->row,bk[j]->curr_row);
+#endif
+          // activate the row with current memory request
+          bk[j]->curr_row = bk[j]->mrq->row;
+          bk[j]->state = BANK_ACTIVE;
+          RRDc = m_config->tRRD;
+          bk[j]->RCDc = m_config->tRCD;
+          bk[j]->RCDWRc = m_config->tRCDWR;
+          bk[j]->RASc = m_config->tRAS;
+          bk[j]->RCc = m_config->tRC;
+          prio = (j + 1) % m_config->nbk;
+          issued = true;
+          n_act_partial++;
+          n_act++;
+       }
+
+       else
+          // different row activated
+          if ( (!issued) &&
+               (bk[j]->curr_row != bk[j]->mrq->row) &&
+               (bk[j]->state == BANK_ACTIVE) &&
+               (!bk[j]->RASc && !bk[j]->WTPc &&
+				  !bk[j]->RTPc &&
+				  !bkgrp[grp]->RTPLc) ) {
+          // make the bank idle again
+          bk[j]->state = BANK_IDLE;
+          bk[j]->RPc = m_config->tRP;
+          prio = (j + 1) % m_config->nbk;
+          issued = true;
+          n_pre++;
+          n_pre_partial++;
+#ifdef DRAM_VERIFY
+          PRINT_CYCLE=1;
+          printf("\tPRE BK:%d Row:%03x \n", j,bk[j]->curr_row);
+#endif
+       }
+    }
+    return issued;
+}
+
+
 //if mrq is being serviced by dram, gets popped after CL latency fulfilled
-class mem_fetch* dram_t::return_queue_pop() 
+class mem_fetch* dram_t::return_queue_pop()
 {
     return returnq->pop();
 }
 
-class mem_fetch* dram_t::return_queue_top() 
+class mem_fetch* dram_t::return_queue_top()
 {
     return returnq->top();
 }
 
+
 void dram_t::print( FILE* simFile) const
 {
    unsigned i;
    fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", 
            id, m_config->nbk, m_config->busW, m_config->BL, m_config->CL );
    fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n",
-           m_config->tCCD, m_config->tRRD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC );
-   fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g\n",
-           n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr,
+           m_config->tRRD, m_config->tCCD, m_config->tRCD, m_config->tRAS, m_config->tRP, m_config->tRC );
+   fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref_event=%d n_req=%d n_rd=%d n_rd_L2_A=%d n_write=%d n_wr_bk=%d bw_util=%.4g\n",
+           n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_rd_L2_A, n_wr, n_wr_WB,
            (float)bwutil/n_cmd);
    fprintf(simFile,"n_activity=%d dram_eff=%.4g\n",
            n_activity, (float)bwutil/n_activity);
@@ -447,12 +719,62 @@ void dram_t::print( FILE* simFile) const
       fprintf(simFile, "bk%d: %da %di ",i,bk[i]->n_access,bk[i]->n_idle);
    }
    fprintf(simFile, "\n");
+   fprintf(simFile, "\n------------------------------------------------------------------------\n");
+
+   printf("\nRow_Buffer_Locality = %.6f", (float)hits_num / access_num);
+   printf("\nRow_Buffer_Locality_read = %.6f", (float)hits_read_num / read_num);
+   printf("\nRow_Buffer_Locality_write = %.6f", (float)hits_write_num / write_num);
+   printf("\nBank_Level_Parallism = %.6f", (float)banks_1time / banks_acess_total);
+   printf("\nBank_Level_Parallism_Col = %.6f", (float)banks_time_rw / banks_access_rw_total);
+   printf("\nBank_Level_Parallism_Ready = %.6f", (float)banks_time_ready /banks_access_ready_total);
+   printf("\nwrite_to_read_ratio_blp_rw_average = %.6f", write_to_read_ratio_blp_rw_average /banks_access_rw_total);
+   printf("\nGrpLevelPara = %.6f \n", (float)bkgrp_parallsim_rw /banks_access_rw_total);
+
+   printf("\nbwutil = %.6f \n", (float)bwutil/n_cmd);
+   printf("total_CMD = %d \n", n_cmd);
+   printf("util_bw = %d \n", util_bw);
+   printf("Wasted_Col = %d \n", wasted_bw_col);
+   printf("Wasted_Row %d \n", wasted_bw_row);
+   printf("Idle = %d \n\n", idle_bw);
+
+   printf("RCDc_limit = %d \n", RCDc_limit);
+   printf("RCDWRc_limit = %d \n", RCDWRc_limit);
+   printf("WTRc_limit = %d \n", WTRc_limit);
+   printf("RTWc_limit = %d \n", RTWc_limit);
+   printf("CCDLc_limit = %d \n", CCDLc_limit);
+   printf("rwq = %d \n", rwq_limit);
+   printf("CCDLc_limit_alone = %d \n", CCDLc_limit_alone);
+   printf("WTRc_limit_alone = %d \n", WTRc_limit_alone);
+   printf("RTWc_limit_alone = %d \n", RTWc_limit_alone);
+
+   printf("total_CMD = %d \n", n_cmd);
+   printf("n_nop = %d \n", n_nop);
+   printf("Read = %d \n", n_rd);
+   printf("Write = %d \n",n_wr);
+   printf("L2_Alloc = %d \n", n_rd_L2_A);
+   printf("L2_WB = %d \n", n_wr_WB);
+   printf("n_act = %d \n", n_act);
+   printf("n_pre = %d \n", n_pre);
+   printf("n_ref = %d \n", n_ref);
+   printf("n_req = %d \n", n_req );
+   printf("total_req = %d \n\n", n_rd+n_wr+n_rd_L2_A+n_wr_WB);
+
+   printf("issued_total_row = %lu \n", issued_total_row);
+   printf("issued_total_col = %lu \n", issued_total_col);
+   printf("Row_Bus_Util =  %.6f \n", (float)issued_total_row / n_cmd);
+   printf("CoL_Bus_Util = %.6f \n", (float)issued_total_col / n_cmd);
+   printf("Either_Row_CoL_Bus_Util = %.6f \n",  (float)issued_total / n_cmd);
+   printf("Issued_on_Two_Bus_Simul_Util = %.6f \n",  (float)issued_two /n_cmd);
+   printf("issued_two_Eff = %.6f \n",  (float)issued_two /issued_total);
+   printf("queue_avg = %.6f \n\n", (float)ave_mrqs/n_cmd );
+
+   fprintf(simFile, "\n");
    fprintf(simFile, "dram_util_bins:");
    for (i=0;i<10;i++) fprintf(simFile, " %d", dram_util_bins[i]);
    fprintf(simFile, "\ndram_eff_bins:");
    for (i=0;i<10;i++) fprintf(simFile, " %d", dram_eff_bins[i]);
    fprintf(simFile, "\n");
-   if(m_config->scheduler_type== DRAM_FRFCFS) 
+   if(m_config->scheduler_type== DRAM_FRFCFS)
        fprintf(simFile, "mrqq: max=%d avg=%g\n", max_mrqs, (float)ave_mrqs/n_cmd);
 }
 
@@ -476,8 +798,8 @@ void dram_t::visualize() const
 
 void dram_t::print_stat( FILE* simFile ) 
 {
-   fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ",
-           id, n_cmd, n_nop, n_act, n_pre, n_req, n_rd, n_wr,
+   fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_ref=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ",
+           id, n_cmd, n_nop, n_act, n_pre, n_ref, n_req, n_rd, n_wr,
            (float)bwutil/n_cmd);
    fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", max_mrqs, (float)ave_mrqs/n_cmd, max_mrqs_temp);
    fprintf(simFile, "\n");
@@ -516,6 +838,7 @@ void dram_t::visualizer_print( gzFile visualizer_file )
    n_pre_partial = 0;
    n_req_partial = 0;
 
+
    // dram access type classification
    for (unsigned j = 0; j < m_config->nbk; j++) {
       gzprintf(visualizer_file,"dramglobal_acc_r: %u %u %u\n", id, j, 
@@ -553,3 +876,16 @@ void dram_t::set_dram_power_stats(	unsigned &cmd,
 	wr = n_wr;
 	req = n_req;
 }
+
+unsigned dram_t::get_bankgrp_number(unsigned i)
+{
+	if(m_config->dram_bnkgrp_indexing_policy == 0) { //higher bits
+		return i>>m_config->bk_tag_length;
+	}
+	else if (m_config->dram_bnkgrp_indexing_policy == 1) { //lower bits
+		return i&((m_config->nbkgrp-1));
+	}
+	else {
+		assert(1);
+	}
+}
diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h
index a8bff14..0d4c0e7 100644
--- a/src/gpgpu-sim/dram.h
+++ b/src/gpgpu-sim/dram.h
@@ -31,9 +31,15 @@
 
 #include "delayqueue.h"
 #include <set>
+#include <vector>
+#include <bitset>
+#include <sstream>
+#include <string>
+#include <fstream>
 #include <zlib.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include<iomanip>
 
 #define READ 'R'  //define read and write states
 #define WRITE 'W'
@@ -42,7 +48,7 @@
 
 class dram_req_t {
 public:
-   dram_req_t( class mem_fetch *data );
+   dram_req_t( class mem_fetch *data , unsigned banks, unsigned dram_bnk_indexing_policy);
 
    unsigned int row;
    unsigned int col;
@@ -95,7 +101,7 @@ public:
    dram_t( unsigned int parition_id, const struct memory_config *config, class memory_stats_t *stats, 
            class memory_partition_unit *mp );
 
-   bool full() const;
+   bool full(bool is_write) const;
    void print( FILE* simFile ) const;
    void visualize() const;
    void print_stat( FILE* simFile );
@@ -106,6 +112,7 @@ public:
 
    class mem_fetch* return_queue_pop();
    class mem_fetch* return_queue_top();
+
    void push( class mem_fetch *data );
    void cycle();
    void dram_log (int task);
@@ -123,17 +130,24 @@ public:
 								unsigned &wr,
 								unsigned &req) const;
 
-private:
-   void scheduler_fifo();
-   void scheduler_frfcfs();
+
 
    const struct memory_config *m_config;
 
+private:
    bankgrp_t **bkgrp;
 
    bank_t **bk;
    unsigned int prio;
 
+   unsigned get_bankgrp_number(unsigned i);
+
+   void scheduler_fifo();
+   void scheduler_frfcfs();
+
+   bool issue_col_command(int j);
+   bool issue_row_command(int j);
+
    unsigned int RRDc;
    unsigned int CCDc;
    unsigned int RTWc;   //read to write penalty applies across banks
@@ -146,7 +160,7 @@ private:
    fifo_pipeline<dram_req_t> *rwq;
    fifo_pipeline<dram_req_t> *mrqq;
    //buffer to hold packets when DRAM processing is over
-   //should be filled with dram clock and popped with l2or icnt clock 
+   //should be filled with dram clock and popped with l2or icnt clock
    fifo_pipeline<mem_fetch> *returnq;
 
    unsigned int dram_util_bins[10];
@@ -158,11 +172,51 @@ private:
    unsigned int n_nop;
    unsigned int n_act;
    unsigned int n_pre;
+   unsigned int n_ref;
    unsigned int n_rd;
+   unsigned int n_rd_L2_A;
    unsigned int n_wr;
+   unsigned int n_wr_WB;
    unsigned int n_req;
    unsigned int max_mrqs_temp;
 
+   //some statistics to collect to see where BW is wasted?
+   unsigned wasted_bw_row;
+   unsigned wasted_bw_col;
+   unsigned util_bw;
+   unsigned idle_bw;
+   unsigned RCDc_limit;
+   unsigned CCDLc_limit;
+   unsigned CCDLc_limit_alone;
+   unsigned CCDc_limit;
+   unsigned WTRc_limit;
+   unsigned WTRc_limit_alone;
+   unsigned RCDWRc_limit;
+   unsigned RTWc_limit;
+   unsigned RTWc_limit_alone;
+   unsigned rwq_limit;
+
+   //row locality, BLP and other statistics
+   unsigned long  access_num;
+   unsigned long  read_num;
+   unsigned long  write_num;
+   unsigned long long hits_num;
+   unsigned long long hits_read_num;
+   unsigned long long hits_write_num;
+   unsigned long long banks_1time;
+   unsigned long long banks_acess_total;
+   unsigned long long banks_acess_total_after;
+   unsigned long long banks_time_rw;
+   unsigned long long banks_access_rw_total;
+   unsigned long long banks_time_ready;
+   unsigned long long banks_access_ready_total;
+   unsigned long long issued_two;
+   unsigned long long issued_total;
+   unsigned long long issued_total_row;
+   unsigned long long issued_total_col;
+   double write_to_read_ratio_blp_rw_average;
+   unsigned long long bkgrp_parallsim_rw;
+
    unsigned int bwutil;
    unsigned int max_mrqs;
    unsigned int ave_mrqs;
diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc
index 8303e86..f754d36 100644
--- a/src/gpgpu-sim/dram_sched.cc
+++ b/src/gpgpu-sim/dram_sched.cc
@@ -36,6 +36,7 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem
    m_config = config;
    m_stats = stats;
    m_num_pending = 0;
+   m_num_write_pending = 0;
    m_dram = dm;
    m_queue = new std::list<dram_req_t*>[m_config->nbk];
    m_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ];
@@ -49,15 +50,36 @@ frfcfs_scheduler::frfcfs_scheduler( const memory_config *config, dram_t *dm, mem
       curr_row_service_time[i] = 0;
       row_service_timestamp[i] = 0;
    }
+   if(m_config->seperate_write_queue_enabled) {
+	   m_write_queue = new std::list<dram_req_t*>[m_config->nbk];
+	   m_write_bins = new std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >[ m_config->nbk ];
+	   m_last_write_row = new std::list<std::list<dram_req_t*>::iterator>*[ m_config->nbk ];
+
+	   for ( unsigned i=0; i < m_config->nbk; i++ ) {
+	         m_write_queue[i].clear();
+	         m_write_bins[i].clear();
+	         m_last_write_row[i] = NULL;
+	      }
+   }
+   m_mode = READ_MODE;
 
 }
 
 void frfcfs_scheduler::add_req( dram_req_t *req )
 {
-   m_num_pending++;
-   m_queue[req->bk].push_front(req);
-   std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin();
-   m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  if(m_config->seperate_write_queue_enabled && req->data->is_write()) {
+	  assert(m_num_write_pending < m_config->gpgpu_frfcfs_dram_write_queue_size);
+	  m_num_write_pending++;
+	  m_write_queue[req->bk].push_front(req);
+	  std::list<dram_req_t*>::iterator ptr = m_write_queue[req->bk].begin();
+	  m_write_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  } else {
+	   assert(m_num_pending < m_config->gpgpu_frfcfs_dram_sched_queue_size);
+	   m_num_pending++;
+	   m_queue[req->bk].push_front(req);
+	   std::list<dram_req_t*>::iterator ptr = m_queue[req->bk].begin();
+	   m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front
+  }
 }
 
 void frfcfs_scheduler::data_collection(unsigned int bank)
@@ -78,41 +100,90 @@ void frfcfs_scheduler::data_collection(unsigned int bank)
 
 dram_req_t *frfcfs_scheduler::schedule( unsigned bank, unsigned curr_row )
 {
-   if ( m_last_row[bank] == NULL ) {
-      if ( m_queue[bank].empty() )
+   //row
+   bool rowhit = true;
+   std::list<dram_req_t*> *m_current_queue = m_queue;
+   std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> > *m_current_bins = m_bins ;
+   std::list<std::list<dram_req_t*>::iterator> **m_current_last_row = m_last_row;
+
+   if(m_config->seperate_write_queue_enabled) {
+	   if(m_mode == READ_MODE &&
+			  ((m_num_write_pending >= m_config->write_high_watermark )
+			   || (m_queue[bank].empty() && !m_write_queue[bank].empty()))) {
+		   m_mode = WRITE_MODE;
+	   }
+	   else if(m_mode == WRITE_MODE &&
+				  (( m_num_write_pending < m_config->write_low_watermark )
+				   || (!m_queue[bank].empty() && m_write_queue[bank].empty()))){
+		   m_mode = READ_MODE;
+	   }
+   }
+
+   if(m_mode == WRITE_MODE) {
+	   m_current_queue = m_write_queue;
+	   m_current_bins = m_write_bins ;
+	   m_current_last_row = m_last_write_row;
+   }
+
+   if ( m_current_last_row[bank] == NULL ) {
+      if ( m_current_queue[bank].empty() )
          return NULL;
 
-      std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row );
-      if ( bin_ptr == m_bins[bank].end()) {
-         dram_req_t *req = m_queue[bank].back();
-         bin_ptr = m_bins[bank].find( req->row );
-         assert( bin_ptr != m_bins[bank].end() ); // where did the request go???
-         m_last_row[bank] = &(bin_ptr->second);
+      std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >::iterator bin_ptr = m_current_bins[bank].find( curr_row );
+      if ( bin_ptr == m_current_bins[bank].end()) {
+         dram_req_t *req = m_current_queue[bank].back();
+         bin_ptr = m_current_bins[bank].find( req->row );
+         assert( bin_ptr != m_current_bins[bank].end() ); // where did the request go???
+         m_current_last_row[bank] = &(bin_ptr->second);
          data_collection(bank);
+         rowhit = false;
       } else {
-         m_last_row[bank] = &(bin_ptr->second);
-
+    	  m_current_last_row[bank] = &(bin_ptr->second);
+         rowhit = true;
       }
    }
-   std::list<dram_req_t*>::iterator next = m_last_row[bank]->back();
+   std::list<dram_req_t*>::iterator next = m_current_last_row[bank]->back();
    dram_req_t *req = (*next);
 
+   //rowblp stats
+    m_dram->access_num++;
+    bool is_write = req->data->is_write();
+    if(is_write)
+  	  m_dram->write_num++;
+    else
+  	  m_dram->read_num++;
+
+    if(rowhit) {
+     m_dram->hits_num++;
+     if(is_write)
+    	  m_dram->hits_write_num++;
+      else
+    	  m_dram->hits_read_num++;
+    }
+
    m_stats->concurrent_row_access[m_dram->id][bank]++;
    m_stats->row_access[m_dram->id][bank]++;
-   m_last_row[bank]->pop_back();
+   m_current_last_row[bank]->pop_back();
 
-   m_queue[bank].erase(next);
-   if ( m_last_row[bank]->empty() ) {
-      m_bins[bank].erase( req->row );
-      m_last_row[bank] = NULL;
+   m_current_queue[bank].erase(next);
+   if ( m_current_last_row[bank]->empty() ) {
+	   m_current_bins[bank].erase( req->row );
+	   m_current_last_row[bank] = NULL;
    }
 #ifdef DEBUG_FAST_IDEAL_SCHED
    if ( req )
       printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", 
              (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row );
 #endif
-   assert( req != NULL && m_num_pending != 0 ); 
-   m_num_pending--;
+
+   if(m_config->seperate_write_queue_enabled && req->data->is_write()) {
+	   assert( req != NULL && m_num_write_pending != 0 );
+	   m_num_write_pending--;
+   }
+   else {
+	   assert( req != NULL && m_num_pending != 0 );
+	   m_num_pending--;
+   }
 
    return req;
 }
@@ -129,7 +200,7 @@ void dram_t::scheduler_frfcfs()
 {
    unsigned mrq_latency;
    frfcfs_scheduler *sched = m_frfcfs_scheduler;
-   while ( !mrqq->empty() && (!m_config->gpgpu_frfcfs_dram_sched_queue_size || sched->num_pending() < m_config->gpgpu_frfcfs_dram_sched_queue_size)) {
+   while ( !mrqq->empty() ) {
       dram_req_t *req = mrqq->pop();
 
       // Power stats
@@ -160,6 +231,8 @@ void dram_t::scheduler_frfcfs()
             bk[b]->mrq = req;
             if (m_config->gpgpu_memlatency_stat) {
                mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - bk[b]->mrq->timestamp;
+               m_stats->tot_mrq_latency += mrq_latency;
+               m_stats->tot_mrq_num++;
                bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle;
                m_stats->mrq_lat_table[LOGB2(mrq_latency)]++;
                if (mrq_latency > m_stats->max_mrq_latency) {
diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h
index 3860f5b..63f5831 100644
--- a/src/gpgpu-sim/dram_sched.h
+++ b/src/gpgpu-sim/dram_sched.h
@@ -35,6 +35,11 @@
 #include <list>
 #include <map>
 
+enum memory_mode {
+   READ_MODE = 0,
+   WRITE_MODE
+};
+
 class frfcfs_scheduler {
 public:
    frfcfs_scheduler( const memory_config *config, dram_t *dm, memory_stats_t *stats );
@@ -43,17 +48,24 @@ public:
    dram_req_t *schedule( unsigned bank, unsigned curr_row );
    void print( FILE *fp );
    unsigned num_pending() const { return m_num_pending;}
+   unsigned num_write_pending() const { return m_num_write_pending;}
 
 private:
    const memory_config *m_config;
    dram_t *m_dram;
    unsigned m_num_pending;
+   unsigned m_num_write_pending;
    std::list<dram_req_t*>                                    *m_queue;
    std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >    *m_bins;
    std::list<std::list<dram_req_t*>::iterator>                 **m_last_row;
    unsigned *curr_row_service_time; //one set of variables for each bank.
    unsigned *row_service_timestamp; //tracks when scheduler began servicing current row
 
+   std::list<dram_req_t*>                                    *m_write_queue;
+   std::map<unsigned,std::list<std::list<dram_req_t*>::iterator> >    *m_write_bins;
+   std::list<std::list<dram_req_t*>::iterator>                 **m_last_write_row;
+
+   enum memory_mode m_mode;
    memory_stats_t *m_stats;
 };
 
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index 7af7db0..0602e20 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -38,7 +38,8 @@ const char * cache_request_status_str(enum cache_request_status status)
       "HIT",
       "HIT_RESERVED",
       "MISS",
-      "RESERVATION_FAIL"
+      "RESERVATION_FAIL",
+	  "SECTOR_MISS"
    }; 
 
    assert(sizeof(static_cache_request_status_str) / sizeof(const char*) == NUM_CACHE_REQUEST_STATUS); 
@@ -47,6 +48,22 @@ const char * cache_request_status_str(enum cache_request_status status)
    return static_cache_request_status_str[status]; 
 }
 
+const char * cache_fail_status_str(enum cache_reservation_fail_reason status)
+{
+   static const char * static_cache_reservation_fail_reason_str[] = {
+	  "LINE_ALLOC_FAIL",
+	  "MISS_QUEUE_FULL",
+	  "MSHR_ENRTY_FAIL",
+      "MSHR_MERGE_ENRTY_FAIL",
+      "MSHR_RW_PENDING"
+   };
+
+   assert(sizeof(static_cache_reservation_fail_reason_str) / sizeof(const char*) == NUM_CACHE_RESERVATION_FAIL_STATUS);
+   assert(status < NUM_CACHE_RESERVATION_FAIL_STATUS);
+
+   return static_cache_reservation_fail_reason_str[status];
+}
+
 unsigned l1d_cache_config::set_index(new_addr_type addr) const{
     unsigned set_index = m_nset; // Default to linear set index function
     unsigned lower_xor = 0;
@@ -113,13 +130,16 @@ unsigned l2_cache_config::set_index(new_addr_type addr) const{
 
 tag_array::~tag_array() 
 {
+	unsigned cache_lines_num = MAX_DEFAULT_CACHE_SIZE_MULTIBLIER*m_config.get_num_lines();
+	for(unsigned i=0; i<cache_lines_num; ++i)
+		delete m_lines[i];
     delete[] m_lines;
 }
 
 tag_array::tag_array( cache_config &config,
                       int core_id,
                       int type_id,
-                      cache_block_t* new_lines)
+                      cache_block_t** new_lines)
     : m_config( config ),
       m_lines( new_lines )
 {
@@ -137,7 +157,21 @@ tag_array::tag_array( cache_config &config,
     : m_config( config )
 {
     //assert( m_config.m_write_policy == READ_ONLY ); Old assert
-    m_lines = new cache_block_t[MAX_DEFAULT_CACHE_SIZE_MULTIBLIER*config.get_num_lines()];
+	unsigned cache_lines_num = MAX_DEFAULT_CACHE_SIZE_MULTIBLIER*config.get_num_lines();
+	m_lines = new cache_block_t*[cache_lines_num];
+	if(config.m_cache_type == NORMAL)
+	{
+		for(unsigned i=0; i<cache_lines_num; ++i)
+			m_lines[i] = new line_cache_block();
+	}
+	else if(config.m_cache_type == SECTOR)
+	{
+		for(unsigned i=0; i<cache_lines_num; ++i)
+			m_lines[i] = new sector_cache_block();
+	}
+	else
+		assert(0);
+
     init( core_id, type_id );
 }
 
@@ -147,6 +181,7 @@ void tag_array::init( int core_id, int type_id )
     m_miss = 0;
     m_pending_hit = 0;
     m_res_fail = 0;
+    m_sector_miss = 0;
     // initialize snapshot counters for visualizer
     m_prev_snapshot_access = 0;
     m_prev_snapshot_miss = 0;
@@ -155,7 +190,14 @@ void tag_array::init( int core_id, int type_id )
     m_type_id = type_id;
 }
 
-enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx ) const {
+
+enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_fetch* mf) const {
+    mem_access_sector_mask_t mask = mf->get_access_sector_mask();
+    return probe(addr, idx, mask);
+}
+
+
+enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask) const {
     //assert( m_config.m_write_policy == READ_ONLY );
     unsigned set_index = m_config.set_index(addr);
     new_addr_type tag = m_config.tag(addr);
@@ -169,35 +211,45 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx )
     // check for hit or pending hit
     for (unsigned way=0; way<m_config.m_assoc; way++) {
         unsigned index = set_index*m_config.m_assoc+way;
-        cache_block_t *line = &m_lines[index];
+        cache_block_t *line = m_lines[index];
         if (line->m_tag == tag) {
-            if ( line->m_status == RESERVED ) {
+            if ( line->get_status(mask) == RESERVED ) {
                 idx = index;
                 return HIT_RESERVED;
-            } else if ( line->m_status == VALID ) {
+            } else if ( line->get_status(mask) == VALID ) {
                 idx = index;
                 return HIT;
-            } else if ( line->m_status == MODIFIED ) {
+            } else if ( line->get_status(mask) == MODIFIED) {
+            	if(line->is_readable(mask)) {
+					idx = index;
+					return HIT;
+            	}
+            	else {
+            		idx = index;
+            		return SECTOR_MISS;
+            	}
+
+            } else if ( line->is_valid_line() && line->get_status(mask) == INVALID ) {
                 idx = index;
-                return HIT;
-            } else {
-                assert( line->m_status == INVALID );
+                return SECTOR_MISS;
+            }else {
+                assert( line->get_status(mask) == INVALID );
             }
         }
-        if (line->m_status != RESERVED) {
+        if (!line->is_reserved_line()) {
             all_reserved = false;
-            if (line->m_status == INVALID) {
+            if (line->is_invalid_line()) {
                 invalid_line = index;
             } else {
                 // valid line : keep track of most appropriate replacement candidate
                 if ( m_config.m_replacement_policy == LRU ) {
-                    if ( line->m_last_access_time < valid_timestamp ) {
-                        valid_timestamp = line->m_last_access_time;
+                    if ( line->get_last_access_time() < valid_timestamp ) {
+                        valid_timestamp = line->get_last_access_time();
                         valid_line = index;
                     }
                 } else if ( m_config.m_replacement_policy == FIFO ) {
-                    if ( line->m_alloc_time < valid_timestamp ) {
-                        valid_timestamp = line->m_alloc_time;
+                    if ( line->get_alloc_time() < valid_timestamp ) {
+                        valid_timestamp = line->get_alloc_time();
                         valid_line = index;
                     }
                 }
@@ -218,37 +270,45 @@ enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx )
     return MISS;
 }
 
-enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx )
+enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf)
 {
     bool wb=false;
-    cache_block_t evicted;
-    enum cache_request_status result = access(addr,time,idx,wb,evicted);
+    evicted_block_info evicted;
+    enum cache_request_status result = access(addr,time,idx,wb,evicted,mf);
     assert(!wb);
     return result;
 }
 
-enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, cache_block_t &evicted ) 
+enum cache_request_status tag_array::access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf )
 {
     m_access++;
     shader_cache_access_log(m_core_id, m_type_id, 0); // log accesses to cache
-    enum cache_request_status status = probe(addr,idx);
+    enum cache_request_status status = probe(addr,idx,mf);
     switch (status) {
     case HIT_RESERVED: 
         m_pending_hit++;
     case HIT: 
-        m_lines[idx].m_last_access_time=time; 
+        m_lines[idx]->set_last_access_time(time, mf->get_access_sector_mask());
         break;
     case MISS:
         m_miss++;
         shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses
         if ( m_config.m_alloc_policy == ON_MISS ) {
-            if( m_lines[idx].m_status == MODIFIED ) {
+            if( m_lines[idx]->is_modified_line()) {
                 wb = true;
-                evicted = m_lines[idx];
+                evicted.set_info(m_lines[idx]->m_block_addr, m_lines[idx]->get_modified_size());
             }
-            m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time );
+            m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mf->get_access_sector_mask());
         }
         break;
+    case SECTOR_MISS:
+    	assert(m_config.m_cache_type == SECTOR);
+    	m_sector_miss++;
+		shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses
+		if ( m_config.m_alloc_policy == ON_MISS ) {
+			((sector_cache_block*)m_lines[idx])->allocate_sector( time, mf->get_access_sector_mask() );
+		}
+		break;
     case RESERVATION_FAIL:
         m_res_fail++;
         shader_cache_access_log(m_core_id, m_type_id, 1); // log cache misses
@@ -261,37 +321,51 @@ enum cache_request_status tag_array::access( new_addr_type addr, unsigned time,
     return status;
 }
 
-void tag_array::fill( new_addr_type addr, unsigned time )
+void tag_array::fill( new_addr_type addr, unsigned time, mem_fetch* mf)
 {
-    assert( m_config.m_alloc_policy == ON_FILL );
+    fill(addr, time, mf->get_access_sector_mask());
+}
+
+void tag_array::fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask )
+{
+    //assert( m_config.m_alloc_policy == ON_FILL );
     unsigned idx;
-    enum cache_request_status status = probe(addr,idx);
-    assert(status==MISS); // MSHR should have prevented redundant memory request
-    m_lines[idx].allocate( m_config.tag(addr), m_config.block_addr(addr), time );
-    m_lines[idx].fill(time);
+    enum cache_request_status status = probe(addr,idx,mask);
+    //assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request
+    if(status==MISS)
+    	m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mask );
+    else if (status==SECTOR_MISS) {
+    	assert(m_config.m_cache_type == SECTOR);
+    	((sector_cache_block*)m_lines[idx])->allocate_sector( time, mask );
+    }
+
+    m_lines[idx]->fill(time, mask);
 }
 
-void tag_array::fill( unsigned index, unsigned time ) 
+void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf)
 {
     assert( m_config.m_alloc_policy == ON_MISS );
-    m_lines[index].fill(time);
+    m_lines[index]->fill(time, mf->get_access_sector_mask());
 }
 
 void tag_array::flush() 
 {
     for (unsigned i=0; i < m_config.get_num_lines(); i++)
-        m_lines[i].m_status = INVALID;
+    	if(m_lines[i]->is_modified_line()) {
+    	for(unsigned j=0; j < SECTOR_CHUNCK_SIZE; j++)
+    		m_lines[i]->set_status(INVALID, mem_access_sector_mask_t().set(j)) ;
+    	}
 }
 
 float tag_array::windowed_miss_rate( ) const
 {
     unsigned n_access    = m_access - m_prev_snapshot_access;
-    unsigned n_miss      = m_miss - m_prev_snapshot_miss;
+    unsigned n_miss      = (m_miss+m_sector_miss) - m_prev_snapshot_miss;
     // unsigned n_pending_hit = m_pending_hit - m_prev_snapshot_pending_hit;
 
     float missrate = 0.0f;
     if (n_access != 0)
-        missrate = (float) n_miss / n_access;
+        missrate = (float) (n_miss+m_sector_miss) / n_access;
     return missrate;
 }
 
@@ -299,23 +373,24 @@ void tag_array::new_window()
 {
     m_prev_snapshot_access = m_access;
     m_prev_snapshot_miss = m_miss;
+    m_prev_snapshot_miss = m_miss + m_sector_miss;
     m_prev_snapshot_pending_hit = m_pending_hit;
 }
 
 void tag_array::print( FILE *stream, unsigned &total_access, unsigned &total_misses ) const
 {
     m_config.print(stream);
-    fprintf( stream, "\t\tAccess = %d, Miss = %d (%.3g), PendingHit = %d (%.3g)\n", 
-             m_access, m_miss, (float) m_miss / m_access, 
+    fprintf( stream, "\t\tAccess = %d, Miss = %d, Sector_Miss = %d, Total_Miss = %d (%.3g), PendingHit = %d (%.3g)\n",
+             m_access, m_miss, m_sector_miss, (m_miss+m_sector_miss), (float) (m_miss+m_sector_miss) / m_access,
              m_pending_hit, (float) m_pending_hit / m_access);
-    total_misses+=m_miss;
+    total_misses+=(m_miss+m_sector_miss);
     total_access+=m_access;
 }
 
 void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsigned &total_hit_res, unsigned &total_res_fail) const{
     // Update statistics from the tag array
     total_access    = m_access;
-    total_misses    = m_miss;
+    total_misses    = (m_miss+m_sector_miss);
     total_hit_res   = m_pending_hit;
     total_res_fail  = m_res_fail;
 }
@@ -324,16 +399,17 @@ void tag_array::get_stats(unsigned &total_access, unsigned &total_misses, unsign
 bool was_write_sent( const std::list<cache_event> &events )
 {
     for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) {
-        if( *e == WRITE_REQUEST_SENT ) 
+        if( (*e).m_cache_event_type == WRITE_REQUEST_SENT )
             return true;
     }
     return false;
 }
 
-bool was_writeback_sent( const std::list<cache_event> &events )
+bool was_writeback_sent( const std::list<cache_event> &events, cache_event& wb_event)
 {
     for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) {
-        if( *e == WRITE_BACK_REQUEST_SENT ) 
+        if( (*e).m_cache_event_type == WRITE_BACK_REQUEST_SENT )
+        	wb_event = *e;
             return true;
     }
     return false;
@@ -342,7 +418,16 @@ bool was_writeback_sent( const std::list<cache_event> &events )
 bool was_read_sent( const std::list<cache_event> &events )
 {
     for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) {
-        if( *e == READ_REQUEST_SENT ) 
+        if( (*e).m_cache_event_type == READ_REQUEST_SENT )
+            return true;
+    }
+    return false;
+}
+
+bool was_writeallocate_sent( const std::list<cache_event> &events )
+{
+    for( std::list<cache_event>::const_iterator e=events.begin(); e!=events.end(); e++ ) {
+        if( (*e).m_cache_event_type == WRITE_ALLOCATE_SENT )
             return true;
     }
     return false;
@@ -375,11 +460,27 @@ void mshr_table::add( new_addr_type block_addr, mem_fetch *mf ){
 	}
 }
 
+/// check is_read_after_write_pending
+bool mshr_table::is_read_after_write_pending( new_addr_type block_addr){
+	std::list<mem_fetch*> my_list = m_data[block_addr].m_list;
+	bool write_found = false;
+	for (std::list<mem_fetch*>::iterator it=my_list.begin(); it != my_list.end(); ++it)
+	{
+		if((*it)->is_write()) //Pending Write Request
+			write_found = true;
+		else if(write_found)   //Pending Read Request and we found previous Write
+				return true;
+	}
+
+	return false;
+
+}
+
 /// Accept a new cache fill response: mark entry ready for processing
 void mshr_table::mark_ready( new_addr_type block_addr, bool &has_atomic ){
     assert( !busy() );
     table::iterator a = m_data.find(block_addr);
-    assert( a != m_data.end() ); // don't remove same request twice
+    assert( a != m_data.end() );
     m_current_response.push_back( block_addr );
     has_atomic = a->second.m_has_atomic;
     assert( m_current_response.size() <= m_data.size() );
@@ -417,9 +518,11 @@ void mshr_table::display( FILE *fp ) const{
 /***************************************************************** Caches *****************************************************************/
 cache_stats::cache_stats(){
     m_stats.resize(NUM_MEM_ACCESS_TYPE);
+    m_fail_stats.resize(NUM_MEM_ACCESS_TYPE);
     for(unsigned i=0; i<NUM_MEM_ACCESS_TYPE; ++i){
         m_stats[i].resize(NUM_CACHE_REQUEST_STATUS, 0);
-    }
+		m_fail_stats[i].resize(NUM_CACHE_RESERVATION_FAIL_STATUS, 0);
+	}
     m_cache_port_available_cycles = 0; 
     m_cache_data_port_busy_cycles = 0; 
     m_cache_fill_port_busy_cycles = 0; 
@@ -431,7 +534,8 @@ void cache_stats::clear(){
     ///
     for(unsigned i=0; i<NUM_MEM_ACCESS_TYPE; ++i){
         std::fill(m_stats[i].begin(), m_stats[i].end(), 0);
-    }
+		std::fill(m_fail_stats[i].begin(), m_fail_stats[i].end(), 0);
+	}
     m_cache_port_available_cycles = 0; 
     m_cache_data_port_busy_cycles = 0; 
     m_cache_fill_port_busy_cycles = 0; 
@@ -446,6 +550,14 @@ void cache_stats::inc_stats(int access_type, int access_outcome){
 
     m_stats[access_type][access_outcome]++;
 }
+void cache_stats::inc_fail_stats(int access_type, int fail_outcome){
+
+	if(!check_fail_valid(access_type, fail_outcome))
+		 assert(0 && "Unknown cache access type or access fail");
+
+	m_fail_stats[access_type][fail_outcome]++;
+}
+
 
 enum cache_request_status cache_stats::select_stats_status(enum cache_request_status probe, enum cache_request_status access) const {
 	///
@@ -454,29 +566,47 @@ enum cache_request_status cache_stats::select_stats_status(enum cache_request_st
 	///
 	if(probe == HIT_RESERVED && access != RESERVATION_FAIL)
 		return probe;
+	else if(probe == SECTOR_MISS && access == MISS)
+		return probe;
 	else
 		return access;
 }
 
-unsigned &cache_stats::operator()(int access_type, int access_outcome){
+unsigned &cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome){
     ///
     /// Simple method to read/modify the stat corresponding to (access_type, access_outcome)
     /// Used overloaded () to avoid the need for separate read/write member functions
     ///
-    if(!check_valid(access_type, access_outcome))
-        assert(0 && "Unknown cache access type or access outcome");
+	if(fail_outcome) {
+		if(!check_fail_valid(access_type, access_outcome))
+			assert(0 && "Unknown cache access type or fail outcome");
+
+		return m_fail_stats[access_type][access_outcome];
+	}
+	else {
+		if(!check_valid(access_type, access_outcome))
+			assert(0 && "Unknown cache access type or access outcome");
 
-    return m_stats[access_type][access_outcome];
+		return m_stats[access_type][access_outcome];
+	}
 }
 
-unsigned cache_stats::operator()(int access_type, int access_outcome) const{
+unsigned cache_stats::operator()(int access_type, int access_outcome, bool fail_outcome) const{
     ///
     /// Const accessor into m_stats.
     ///
-    if(!check_valid(access_type, access_outcome))
-        assert(0 && "Unknown cache access type or access outcome");
+	if(fail_outcome) {
+		if(!check_fail_valid(access_type, access_outcome))
+			assert(0 && "Unknown cache access type or fail outcome");
+
+		return m_fail_stats[access_type][access_outcome];
+	}
+	else {
+		if(!check_valid(access_type, access_outcome))
+			assert(0 && "Unknown cache access type or access outcome");
 
-    return m_stats[access_type][access_outcome];
+		return m_stats[access_type][access_outcome];
+	}
 }
 
 cache_stats cache_stats::operator+(const cache_stats &cs){
@@ -486,9 +616,12 @@ cache_stats cache_stats::operator+(const cache_stats &cs){
     cache_stats ret;
     for(unsigned type=0; type<NUM_MEM_ACCESS_TYPE; ++type){
         for(unsigned status=0; status<NUM_CACHE_REQUEST_STATUS; ++status){
-            ret(type, status) = m_stats[type][status] + cs(type, status);
+            ret(type, status, false) = m_stats[type][status] + cs(type, status, false);
+        }
+		for(unsigned status=0; status<NUM_CACHE_RESERVATION_FAIL_STATUS; ++status){
+			   ret(type, status, true) = m_fail_stats[type][status] + cs(type, status, true);
+		   }
         }
-    }
     ret.m_cache_port_available_cycles = m_cache_port_available_cycles + cs.m_cache_port_available_cycles; 
     ret.m_cache_data_port_busy_cycles = m_cache_data_port_busy_cycles + cs.m_cache_data_port_busy_cycles; 
     ret.m_cache_fill_port_busy_cycles = m_cache_fill_port_busy_cycles + cs.m_cache_fill_port_busy_cycles; 
@@ -501,8 +634,11 @@ cache_stats &cache_stats::operator+=(const cache_stats &cs){
     ///
     for(unsigned type=0; type<NUM_MEM_ACCESS_TYPE; ++type){
         for(unsigned status=0; status<NUM_CACHE_REQUEST_STATUS; ++status){
-            m_stats[type][status] += cs(type, status);
+            m_stats[type][status] += cs(type, status, false);
         }
+        for(unsigned status=0; status<NUM_CACHE_RESERVATION_FAIL_STATUS; ++status){
+            m_fail_stats[type][status] += cs(type, status, true);
+	   }
     }
     m_cache_port_available_cycles += cs.m_cache_port_available_cycles; 
     m_cache_data_port_busy_cycles += cs.m_cache_data_port_busy_cycles; 
@@ -517,6 +653,8 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
     /// the provided name is used.
     /// The printed format is "<cache_name>[<request_type>][<request_status>] = <stat_value>"
     ///
+	std::vector< unsigned > total_access;
+	total_access.resize(NUM_MEM_ACCESS_TYPE, 0);
     std::string m_cache_name = cache_name;
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
         for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) {
@@ -526,9 +664,34 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
                     mem_access_type_str((enum mem_access_type)type),
                     cache_request_status_str((enum cache_request_status)status),
                     m_stats[type][status]);
+                if(status != RESERVATION_FAIL)
+                	 total_access[type]+= m_stats[type][status];
             }
         }
     }
+    for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
+    	 if(total_access[type] > 0)
+    	  fprintf(fout, "\t%s[%s][%s] = %u\n",
+				m_cache_name.c_str(),
+				mem_access_type_str((enum mem_access_type)type),
+				"TOTAL_ACCESS",
+				total_access[type]);
+    }
+}
+
+void cache_stats::print_fail_stats(FILE *fout, const char *cache_name) const{
+	std::string m_cache_name = cache_name;
+	    for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
+	        for (unsigned fail = 0; fail < NUM_CACHE_RESERVATION_FAIL_STATUS; ++fail) {
+	            if(m_fail_stats[type][fail] > 0){
+	                fprintf(fout, "\t%s[%s][%s] = %u\n",
+	                    m_cache_name.c_str(),
+	                    mem_access_type_str((enum mem_access_type)type),
+						cache_fail_status_str((enum cache_reservation_fail_reason)fail),
+						m_fail_stats[type][fail]);
+	            }
+	        }
+	    }
 }
 
 void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const
@@ -570,10 +733,10 @@ void cache_stats::get_sub_stats(struct cache_sub_stats &css) const{
 
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
         for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) {
-            if(status == HIT || status == MISS || status == HIT_RESERVED)
+            if(status == HIT || status == MISS || status == SECTOR_MISS || status == HIT_RESERVED)
                 t_css.accesses += m_stats[type][status];
 
-            if(status == MISS)
+            if(status == MISS || status == SECTOR_MISS)
                 t_css.misses += m_stats[type][status];
 
             if(status == HIT_RESERVED)
@@ -601,6 +764,16 @@ bool cache_stats::check_valid(int type, int status) const{
         return false;
 }
 
+bool cache_stats::check_fail_valid(int type, int fail) const{
+    ///
+    /// Verify a valid access_type/access_status
+    ///
+    if((type >= 0) && (type < NUM_MEM_ACCESS_TYPE) && (fail >= 0) && (fail < NUM_CACHE_RESERVATION_FAIL_STATUS))
+        return true;
+    else
+        return false;
+}
+
 void cache_stats::sample_cache_port_utility(bool data_port_busy, bool fill_port_busy) 
 {
     m_cache_port_available_cycles += 1; 
@@ -629,15 +802,18 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac
         unsigned data_cycles = data_size / port_width + ((data_size % port_width > 0)? 1 : 0); 
         m_data_port_occupied_cycles += data_cycles; 
         } break; 
-    case HIT_RESERVED: 
+    case HIT_RESERVED:
     case MISS: {
         // the data array is accessed to read out the entire line for write-back 
-        if (was_writeback_sent(events)) {
-            unsigned data_cycles = m_config.m_line_sz / port_width; 
+    	// in case of sector cache we need to write bank only the modified sectors
+    	cache_event ev(WRITE_BACK_REQUEST_SENT);
+        if (was_writeback_sent(events, ev)) {
+            unsigned data_cycles = ev.m_evicted_block.m_modified_size / port_width;
             m_data_port_occupied_cycles += data_cycles; 
         }
         } break; 
-    case RESERVATION_FAIL: 
+    case SECTOR_MISS:
+    case RESERVATION_FAIL:
         // Does not consume any port bandwidth 
         break; 
     default: 
@@ -650,7 +826,7 @@ void baseline_cache::bandwidth_management::use_data_port(mem_fetch *mf, enum cac
 void baseline_cache::bandwidth_management::use_fill_port(mem_fetch *mf)
 {
     // assume filling the entire line with the returned request 
-    unsigned fill_cycles = m_config.m_line_sz / m_config.m_data_port_width; 
+    unsigned fill_cycles = m_config.get_atom_sz() / m_config.m_data_port_width;
     m_fill_port_occupied_cycles += fill_cycles; 
 }
 
@@ -697,21 +873,40 @@ void baseline_cache::cycle(){
 
 /// Interface for response from lower memory level (model bandwidth restictions in caller)
 void baseline_cache::fill(mem_fetch *mf, unsigned time){
+
+	if(m_config.m_mshr_type == SECTOR_ASSOC) {
+	assert(mf->original_mf);
+	extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf->original_mf);
+    assert( e != m_extra_mf_fields.end() );
+    e->second.pending_read--;
+
+    if(e->second.pending_read > 0) {
+    	//wait for the other requests to come back
+    	delete mf;
+    	return;
+      } else {
+    	mem_fetch *temp = mf;
+    	mf = mf->original_mf;
+    	delete temp;
+      }
+	}
+
     extra_mf_fields_lookup::iterator e = m_extra_mf_fields.find(mf);
     assert( e != m_extra_mf_fields.end() );
     assert( e->second.m_valid );
     mf->set_data_size( e->second.m_data_size );
+    mf->set_addr( e->second.m_addr );
     if ( m_config.m_alloc_policy == ON_MISS )
-        m_tag_array->fill(e->second.m_cache_index,time);
+        m_tag_array->fill(e->second.m_cache_index,time,mf);
     else if ( m_config.m_alloc_policy == ON_FILL )
-        m_tag_array->fill(e->second.m_block_addr,time);
+        m_tag_array->fill(e->second.m_block_addr,time,mf);
     else abort();
     bool has_atomic = false;
     m_mshrs.mark_ready(e->second.m_block_addr, has_atomic);
     if (has_atomic) {
         assert(m_config.m_alloc_policy == ON_MISS);
-        cache_block_t &block = m_tag_array->get_block(e->second.m_cache_index);
-        block.m_status = MODIFIED; // mark line as dirty for atomic operation
+        cache_block_t* block = m_tag_array->get_block(e->second.m_cache_index);
+        block->set_status(MODIFIED, mf->get_access_sector_mask()); // mark line as dirty for atomic operation
     }
     m_extra_mf_fields.erase(mf);
     m_bandwidth_management.use_fill_port(mf); 
@@ -739,45 +934,55 @@ void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_a
 		unsigned time, bool &do_miss, std::list<cache_event> &events, bool read_only, bool wa){
 
 	bool wb=false;
-	cache_block_t e;
+	evicted_block_info e;
 	send_read_request(addr, block_addr, cache_index, mf, time, do_miss, wb, e, events, read_only, wa);
 }
 
 /// Read miss handler. Check MSHR hit or MSHR available
 void baseline_cache::send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf,
-		unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list<cache_event> &events, bool read_only, bool wa){
+		unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list<cache_event> &events, bool read_only, bool wa){
 
-    bool mshr_hit = m_mshrs.probe(block_addr);
-    bool mshr_avail = !m_mshrs.full(block_addr);
+	new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr());
+    bool mshr_hit = m_mshrs.probe(mshr_addr);
+    bool mshr_avail = !m_mshrs.full(mshr_addr);
     if ( mshr_hit && mshr_avail ) {
     	if(read_only)
-    		m_tag_array->access(block_addr,time,cache_index);
+    		m_tag_array->access(block_addr,time,cache_index,mf);
     	else
-    		m_tag_array->access(block_addr,time,cache_index,wb,evicted);
+    		m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf);
 
-        m_mshrs.add(block_addr,mf);
+        m_mshrs.add(mshr_addr,mf);
         do_miss = true;
     } else if ( !mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size) ) {
     	if(read_only)
-    		m_tag_array->access(block_addr,time,cache_index);
+    		m_tag_array->access(block_addr,time,cache_index,mf);
     	else
-    		m_tag_array->access(block_addr,time,cache_index,wb,evicted);
+    		m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf);
 
-        m_mshrs.add(block_addr,mf);
-        m_extra_mf_fields[mf] = extra_mf_fields(block_addr,cache_index, mf->get_data_size());
-        mf->set_data_size( m_config.get_line_sz() );
+        m_mshrs.add(mshr_addr,mf);
+        m_extra_mf_fields[mf] = extra_mf_fields(mshr_addr,mf->get_addr(),cache_index, mf->get_data_size(), m_config);
+        mf->set_data_size( m_config.get_atom_sz() );
+        mf->set_addr( block_addr );
         m_miss_queue.push_back(mf);
         mf->set_status(m_miss_queue_status,time);
         if(!wa)
-        	events.push_back(READ_REQUEST_SENT);
+        	events.push_back(cache_event(READ_REQUEST_SENT));
+
         do_miss = true;
     }
+    else if(mshr_hit && !mshr_avail)
+        m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL);
+    else if (!mshr_hit && !mshr_avail)
+    	 m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL);
+    else
+    	assert(0);
 }
 
 
 /// Sends write request to lower level memory (write or writeback)
 void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned time, std::list<cache_event> &events){
-    events.push_back(request);
+
+	events.push_back(request);
     m_miss_queue.push_back(mf);
     mf->set_status(m_miss_queue_status,time);
 }
@@ -788,40 +993,44 @@ void data_cache::send_write_request(mem_fetch *mf, cache_event request, unsigned
 /// Write-back hit: Mark block as modified
 cache_request_status data_cache::wr_hit_wb(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){
 	new_addr_type block_addr = m_config.block_addr(addr);
-	m_tag_array->access(block_addr,time,cache_index); // update LRU state
-	cache_block_t &block = m_tag_array->get_block(cache_index);
-	block.m_status = MODIFIED;
+	m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state
+	cache_block_t* block = m_tag_array->get_block(cache_index);
+	block->set_status(MODIFIED, mf->get_access_sector_mask());
 
 	return HIT;
 }
 
 /// Write-through hit: Directly send request to lower level memory
 cache_request_status data_cache::wr_hit_wt(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){
-	if(miss_queue_full(0))
+	if(miss_queue_full(0)) {
+		m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
 		return RESERVATION_FAIL; // cannot handle request this cycle
+	}
 
 	new_addr_type block_addr = m_config.block_addr(addr);
-	m_tag_array->access(block_addr,time,cache_index); // update LRU state
-	cache_block_t &block = m_tag_array->get_block(cache_index);
-	block.m_status = MODIFIED;
+	m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state
+	cache_block_t* block = m_tag_array->get_block(cache_index);
+	block->set_status(MODIFIED, mf->get_access_sector_mask());
 
 	// generate a write-through
-	send_write_request(mf, WRITE_REQUEST_SENT, time, events);
+	send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
 
 	return HIT;
 }
 
 /// Write-evict hit: Send request to lower level memory and invalidate corresponding block
 cache_request_status data_cache::wr_hit_we(new_addr_type addr, unsigned cache_index, mem_fetch *mf, unsigned time, std::list<cache_event> &events, enum cache_request_status status ){
-	if(miss_queue_full(0))
+	if(miss_queue_full(0)) {
+		m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
 		return RESERVATION_FAIL; // cannot handle request this cycle
+	}
 
 	// generate a write-through/evict
-	cache_block_t &block = m_tag_array->get_block(cache_index);
-	send_write_request(mf, WRITE_REQUEST_SENT, time, events);
+	cache_block_t* block = m_tag_array->get_block(cache_index);
+	send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
 
 	// Invalidate block
-	block.m_status = INVALID;
+	block->set_status(INVALID, mf->get_access_sector_mask());
 
 	return HIT;
 }
@@ -840,35 +1049,46 @@ enum cache_request_status data_cache::wr_hit_global_we_local_wb(new_addr_type ad
 /// Write-allocate miss: Send write request to lower level memory
 // and send a read request for the same block
 enum cache_request_status
-data_cache::wr_miss_wa( new_addr_type addr,
+data_cache::wr_miss_wa_naive( new_addr_type addr,
                         unsigned cache_index, mem_fetch *mf,
                         unsigned time, std::list<cache_event> &events,
                         enum cache_request_status status )
 {
     new_addr_type block_addr = m_config.block_addr(addr);
+    new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr());
 
     // Write allocate, maximum 3 requests (write miss, read request, write back request)
     // Conservatively ensure the worst-case request can be handled this cycle
-    bool mshr_hit = m_mshrs.probe(block_addr);
-    bool mshr_avail = !m_mshrs.full(block_addr);
+    bool mshr_hit = m_mshrs.probe(mshr_addr);
+    bool mshr_avail = !m_mshrs.full(mshr_addr);
     if(miss_queue_full(2) 
         || (!(mshr_hit && mshr_avail) 
-        && !(!mshr_hit && mshr_avail 
-        && (m_miss_queue.size() < m_config.m_miss_queue_size))))
+        && !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) {
+    	//check what is the exactly the failure reason
+    	 if(miss_queue_full(2) )
+    		 m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+         else if(mshr_hit && !mshr_avail)
+    	      m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL);
+    	 else if (!mshr_hit && !mshr_avail)
+    	    m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL);
+    	 else
+    		 assert(0);
+
         return RESERVATION_FAIL;
+    }
 
-    send_write_request(mf, WRITE_REQUEST_SENT, time, events);
+    send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
     // Tries to send write allocate request, returns true on success and false on failure
     //if(!send_write_allocate(mf, addr, block_addr, cache_index, time, events))
     //    return RESERVATION_FAIL;
 
     const mem_access_t *ma = new  mem_access_t( m_wr_alloc_type,
                         mf->get_addr(),
-                        mf->get_data_size(),
+						m_config.get_atom_sz(),
                         false, // Now performing a read
                         mf->get_access_warp_mask(),
                         mf->get_access_byte_mask(),
-		        mf->get_access_sector_mask());
+		                mf->get_access_sector_mask());
 
     mem_fetch *n_mf = new mem_fetch( *ma,
                     NULL,
@@ -880,20 +1100,22 @@ data_cache::wr_miss_wa( new_addr_type addr,
 
     bool do_miss = false;
     bool wb = false;
-    cache_block_t evicted;
+    evicted_block_info evicted;
 
     // Send read request resulting from write miss
     send_read_request(addr, block_addr, cache_index, n_mf, time, do_miss, wb,
         evicted, events, false, true);
 
+    events.push_back(cache_event(WRITE_ALLOCATE_SENT));
+
     if( do_miss ){
         // If evicted block is modified and not a write-through
         // (already modified lower level)
         if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) { 
+        	assert(status == MISS);   //SECTOR_MISS and HIT_RESERVED should not send write back
             mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr,
-                m_wrbk_type,m_config.get_line_sz(),true);
-            m_miss_queue.push_back(wb);
-            wb->set_status(m_miss_queue_status,time);
+                m_wrbk_type,evicted.m_modified_size,true);
+            send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events);
         }
         return MISS;
     }
@@ -901,6 +1123,170 @@ data_cache::wr_miss_wa( new_addr_type addr,
     return RESERVATION_FAIL;
 }
 
+
+enum cache_request_status
+data_cache::wr_miss_wa_fetch_on_write( new_addr_type addr,
+                        unsigned cache_index, mem_fetch *mf,
+                        unsigned time, std::list<cache_event> &events,
+                        enum cache_request_status status )
+{
+
+	new_addr_type block_addr = m_config.block_addr(addr);
+	    new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr());
+
+
+			//if the request writes to the whole cache line/sector, then, write and set cache line Modified.
+			//and no need to send read request to memory or reserve mshr
+
+			if(miss_queue_full(0)) {
+				m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+				return RESERVATION_FAIL; // cannot handle request this cycle
+			}
+
+			bool wb = false;
+			evicted_block_info evicted;
+
+			cache_request_status m_status =  m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf);
+			assert(m_status != HIT);
+			cache_block_t* block = m_tag_array->get_block(cache_index);
+			block->set_status(MODIFIED, mf->get_access_sector_mask());
+			if(m_status == HIT_RESERVED) {
+				block->set_ignore_on_fill(true, mf->get_access_sector_mask());
+				block->set_modified_on_fill(true, mf->get_access_sector_mask());
+			}
+
+			if(mf->get_access_byte_mask().count() == m_config.get_atom_sz())
+			{
+				block->set_m_readable(true, mf->get_access_sector_mask());
+			} else
+			{
+				block->set_m_readable(false, mf->get_access_sector_mask());
+			}
+
+			if( m_status != RESERVATION_FAIL ){
+				   // If evicted block is modified and not a write-through
+				   // (already modified lower level)
+				   if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) {
+					   mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr,
+						   m_wrbk_type,evicted.m_modified_size,true);
+					   send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events);
+				   }
+				   return MISS;
+			   }
+			return RESERVATION_FAIL;
+
+	/*new_addr_type block_addr = m_config.block_addr(addr);
+    new_addr_type mshr_addr = m_config.mshr_addr(mf->get_addr());
+
+	if(mf->get_access_byte_mask().count() == m_config.get_atom_sz())
+	{
+		//if the request writes to the whole cache line/sector, then, write and set cache line Modified.
+		//and no need to send read request to memory or reserve mshr
+
+		if(miss_queue_full(0)) {
+			m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+			return RESERVATION_FAIL; // cannot handle request this cycle
+		}
+
+		bool wb = false;
+		evicted_block_info evicted;
+
+		cache_request_status status =  m_tag_array->access(block_addr,time,cache_index,wb,evicted,mf);
+		assert(status != HIT);
+		cache_block_t* block = m_tag_array->get_block(cache_index);
+		block->set_status(MODIFIED, mf->get_access_sector_mask());
+		if(status == HIT_RESERVED)
+			block->set_ignore_on_fill(true, mf->get_access_sector_mask());
+
+		if( status != RESERVATION_FAIL ){
+			   // If evicted block is modified and not a write-through
+			   // (already modified lower level)
+			   if( wb && (m_config.m_write_policy != WRITE_THROUGH) ) {
+				   mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr,
+					   m_wrbk_type,evicted.m_modified_size,true);
+				   send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events);
+			   }
+			   return MISS;
+		   }
+		return RESERVATION_FAIL;
+	}
+	else
+	{
+		bool mshr_hit = m_mshrs.probe(mshr_addr);
+		bool mshr_avail = !m_mshrs.full(mshr_addr);
+		if(miss_queue_full(1)
+			|| (!(mshr_hit && mshr_avail)
+			&& !(!mshr_hit && mshr_avail && (m_miss_queue.size() < m_config.m_miss_queue_size)))) {
+			//check what is the exactly the failure reason
+			 if(miss_queue_full(1) )
+				 m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+			 else if(mshr_hit && !mshr_avail)
+				  m_stats.inc_fail_stats(mf->get_access_type(), MSHR_MERGE_ENRTY_FAIL);
+			 else if (!mshr_hit && !mshr_avail)
+				m_stats.inc_fail_stats(mf->get_access_type(), MSHR_ENRTY_FAIL);
+			 else
+				 assert(0);
+
+			return RESERVATION_FAIL;
+		}
+
+
+		  //prevent Write - Read - Write in pending mshr
+		  //allowing another write will override the value of the first write, and the pending read request will read incorrect result from the second write
+		  if(m_mshrs.probe(mshr_addr) && m_mshrs.is_read_after_write_pending(mshr_addr) && mf->is_write())
+		  {
+			  //assert(0);
+			  m_stats.inc_fail_stats(mf->get_access_type(), MSHR_RW_PENDING);
+			  return RESERVATION_FAIL;
+		  }
+
+		  const mem_access_t *ma = new  mem_access_t( m_wr_alloc_type,
+									mf->get_addr(),
+									m_config.get_atom_sz(),
+									false, // Now performing a read
+									mf->get_access_warp_mask(),
+									mf->get_access_byte_mask(),
+									mf->get_access_sector_mask());
+
+		  mem_fetch *n_mf = new mem_fetch( *ma,
+								NULL,
+								mf->get_ctrl_size(),
+								mf->get_wid(),
+								mf->get_sid(),
+								mf->get_tpc(),
+								mf->get_mem_config(),
+								NULL,
+								mf);
+
+
+			new_addr_type block_addr = m_config.block_addr(addr);
+			bool do_miss = false;
+			bool wb = false;
+			evicted_block_info evicted;
+			send_read_request( addr,
+							   block_addr,
+							   cache_index,
+							   n_mf, time, do_miss, wb, evicted, events, false, true);
+
+			cache_block_t* block = m_tag_array->get_block(cache_index);
+			block->set_modified_on_fill(true, mf->get_access_sector_mask());
+
+			events.push_back(cache_event(WRITE_ALLOCATE_SENT));
+
+			if( do_miss ){
+				// If evicted block is modified and not a write-through
+				// (already modified lower level)
+				if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){
+					mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr,
+						m_wrbk_type,evicted.m_modified_size,true);
+					send_write_request(wb, cache_event(WRITE_BACK_REQUEST_SENT, evicted), time, events);
+			}
+				return MISS;
+			}
+	   return RESERVATION_FAIL;
+	}*/
+}
+
 /// No write-allocate miss: Simply send write request to lower level memory
 enum cache_request_status
 data_cache::wr_miss_no_wa( new_addr_type addr,
@@ -910,11 +1296,14 @@ data_cache::wr_miss_no_wa( new_addr_type addr,
                            std::list<cache_event> &events,
                            enum cache_request_status status )
 {
-    if(miss_queue_full(0))
-        return RESERVATION_FAIL; // cannot handle request this cycle
+    if(miss_queue_full(0)) {
+    	m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
+    	return RESERVATION_FAIL; // cannot handle request this cycle
+    }
+
 
     // on miss, generate write through (no write buffering -- too many threads for that)
-    send_write_request(mf, WRITE_REQUEST_SENT, time, events);
+    send_write_request(mf, cache_event(WRITE_REQUEST_SENT), time, events);
 
     return MISS;
 }
@@ -932,13 +1321,13 @@ data_cache::rd_hit_base( new_addr_type addr,
                          enum cache_request_status status )
 {
     new_addr_type block_addr = m_config.block_addr(addr);
-    m_tag_array->access(block_addr,time,cache_index);
+    m_tag_array->access(block_addr,time,cache_index,mf);
     // Atomics treated as global read/write requests - Perform read, mark line as
     // MODIFIED
     if(mf->isatomic()){ 
         assert(mf->get_access_type() == GLOBAL_ACC_R);
-        cache_block_t &block = m_tag_array->get_block(cache_index);
-        block.m_status = MODIFIED;  // mark line as dirty
+        cache_block_t* block = m_tag_array->get_block(cache_index);
+        block->set_status(MODIFIED, mf->get_access_sector_mask()) ;  // mark line as dirty
     }
     return HIT;
 }
@@ -954,15 +1343,17 @@ data_cache::rd_miss_base( new_addr_type addr,
                           unsigned time,
                           std::list<cache_event> &events,
                           enum cache_request_status status ){
-    if(miss_queue_full(1))
+    if(miss_queue_full(1)) {
         // cannot handle request this cycle
         // (might need to generate two requests)
+    	m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
         return RESERVATION_FAIL; 
+    }
 
     new_addr_type block_addr = m_config.block_addr(addr);
     bool do_miss = false;
     bool wb = false;
-    cache_block_t evicted;
+    evicted_block_info evicted;
     send_read_request( addr,
                        block_addr,
                        cache_index,
@@ -973,12 +1364,12 @@ data_cache::rd_miss_base( new_addr_type addr,
         // (already modified lower level)
         if(wb && (m_config.m_write_policy != WRITE_THROUGH) ){ 
             mem_fetch *wb = m_memfetch_creator->alloc(evicted.m_block_addr,
-                m_wrbk_type,m_config.get_line_sz(),true);
+                m_wrbk_type,evicted.m_modified_size,true);
         send_write_request(wb, WRITE_BACK_REQUEST_SENT, time, events);
     }
         return MISS;
     }
-    return RESERVATION_FAIL;
+        return RESERVATION_FAIL;
 }
 
 /// Access cache for read_only_cache: returns RESERVATION_FAIL if
@@ -989,16 +1380,16 @@ read_only_cache::access( new_addr_type addr,
                          unsigned time,
                          std::list<cache_event> &events )
 {
-    assert( mf->get_data_size() <= m_config.get_line_sz());
+    assert( mf->get_data_size() <= m_config.get_atom_sz());
     assert(m_config.m_write_policy == READ_ONLY);
     assert(!mf->get_is_write());
     new_addr_type block_addr = m_config.block_addr(addr);
     unsigned cache_index = (unsigned)-1;
-    enum cache_request_status status = m_tag_array->probe(block_addr,cache_index);
+    enum cache_request_status status = m_tag_array->probe(block_addr,cache_index,mf);
     enum cache_request_status cache_status = RESERVATION_FAIL;
 
     if ( status == HIT ) {
-        cache_status = m_tag_array->access(block_addr,time,cache_index); // update LRU state
+        cache_status = m_tag_array->access(block_addr,time,cache_index,mf); // update LRU state
     }else if ( status != RESERVATION_FAIL ) {
         if(!miss_queue_full(0)){
             bool do_miss=false;
@@ -1009,7 +1400,10 @@ read_only_cache::access( new_addr_type addr,
                 cache_status = RESERVATION_FAIL;
         }else{
             cache_status = RESERVATION_FAIL;
+            m_stats.inc_fail_stats(mf->get_access_type(), MISS_QUEUE_FULL);
         }
+    }else {
+    	m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL);
     }
 
     m_stats.inc_stats(mf->get_access_type(), m_stats.select_stats_status(status, cache_status));
@@ -1042,6 +1436,9 @@ data_cache::process_tag_probe( bool wr,
             access_status = (this->*m_wr_miss)( addr,
                                        cache_index,
                                        mf, time, events, probe_status );
+        }else {
+        	//the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved)
+        	m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL);
         }
     }else{ // Read
         if(probe_status == HIT){
@@ -1052,6 +1449,9 @@ data_cache::process_tag_probe( bool wr,
             access_status = (this->*m_rd_miss)( addr,
                                        cache_index,
                                        mf, time, events, probe_status );
+        }else {
+        	//the only reason for reservation fail here is LINE_ALLOC_FAIL (i.e all lines are reserved)
+        	m_stats.inc_fail_stats(mf->get_access_type(), LINE_ALLOC_FAIL);
         }
     }
 
@@ -1071,12 +1471,12 @@ data_cache::access( new_addr_type addr,
                     std::list<cache_event> &events )
 {
 
-    assert( mf->get_data_size() <= m_config.get_line_sz());
+    assert( mf->get_data_size() <= m_config.get_atom_sz());
     bool wr = mf->get_is_write();
     new_addr_type block_addr = m_config.block_addr(addr);
     unsigned cache_index = (unsigned)-1;
     enum cache_request_status probe_status
-        = m_tag_array->probe( block_addr, cache_index );
+        = m_tag_array->probe( block_addr, cache_index, mf );
     enum cache_request_status access_status
         = process_tag_probe( wr, probe_status, addr, cache_index, mf, time, events );
     m_stats.inc_stats(mf->get_access_type(),
@@ -1125,7 +1525,7 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf,
     // at this point, we will accept the request : access tags and immediately allocate line
     new_addr_type block_addr = m_config.block_addr(addr);
     unsigned cache_index = (unsigned)-1;
-    enum cache_request_status status = m_tags.access(block_addr,time,cache_index);
+    enum cache_request_status status = m_tags.access(block_addr,time,cache_index,mf);
     enum cache_request_status cache_status = RESERVATION_FAIL;
     assert( status != RESERVATION_FAIL );
     assert( status != HIT_RESERVED ); // as far as tags are concerned: HIT or MISS
@@ -1135,10 +1535,10 @@ enum cache_request_status tex_cache::access( new_addr_type addr, mem_fetch *mf,
         unsigned rob_index = m_rob.push( rob_entry(cache_index, mf, block_addr) );
         m_extra_mf_fields[mf] = extra_mf_fields(rob_index);
         mf->set_data_size(m_config.get_line_sz());
-        m_tags.fill(cache_index,time); // mark block as valid
+        m_tags.fill(cache_index,time,mf); // mark block as valid
         m_request_fifo.push(mf);
         mf->set_status(m_request_queue_status,time);
-        events.push_back(READ_REQUEST_SENT);
+        events.push_back(cache_event(READ_REQUEST_SENT));
         cache_status = MISS;
     } else {
         // the value *will* *be* in the cache already
@@ -1165,7 +1565,7 @@ void tex_cache::cycle(){
             unsigned rob_index = m_rob.next_pop_index();
             const rob_entry &r = m_rob.peek(rob_index);
             assert( r.m_request == e.m_request );
-            assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) );
+            //assert( r.m_block_addr == m_config.block_addr(e.m_request->get_addr()) );
             if ( r.m_ready ) {
                 assert( r.m_index == e.m_cache_index );
                 m_cache[r.m_index].m_valid = true;
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index 7535a1d..0d07878 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -36,6 +36,7 @@
 #include "../tr1_hash_map.h"
 
 #include "addrdec.h"
+#include <iostream>
 
 enum cache_block_state {
     INVALID,
@@ -49,13 +50,51 @@ enum cache_request_status {
     HIT_RESERVED,
     MISS,
     RESERVATION_FAIL, 
+	SECTOR_MISS,
     NUM_CACHE_REQUEST_STATUS
 };
 
-enum cache_event {
+enum cache_reservation_fail_reason {
+	LINE_ALLOC_FAIL= 0,// all line are reserved
+	MISS_QUEUE_FULL,   // MISS queue (i.e. interconnect or DRAM) is full
+	MSHR_ENRTY_FAIL,
+	MSHR_MERGE_ENRTY_FAIL,
+	MSHR_RW_PENDING,
+    NUM_CACHE_RESERVATION_FAIL_STATUS
+};
+
+enum cache_event_type {
     WRITE_BACK_REQUEST_SENT,
     READ_REQUEST_SENT,
-    WRITE_REQUEST_SENT
+    WRITE_REQUEST_SENT,
+	WRITE_ALLOCATE_SENT
+};
+
+struct evicted_block_info {
+	new_addr_type m_block_addr;
+	unsigned m_modified_size;
+	evicted_block_info() {
+		m_block_addr = 0;
+		m_modified_size = 0;
+	}
+	void set_info(new_addr_type block_addr, unsigned modified_size){
+		m_block_addr = block_addr;
+		m_modified_size = modified_size;
+	}
+};
+
+struct cache_event {
+	enum cache_event_type m_cache_event_type;
+	evicted_block_info m_evicted_block; //if it was write_back event, fill the the evicted block info
+
+	cache_event(enum cache_event_type m_cache_event){
+		m_cache_event_type = m_cache_event;
+	}
+
+	cache_event(enum cache_event_type cache_event, evicted_block_info evicted_block){
+	m_cache_event_type = cache_event;
+	m_evicted_block = evicted_block;
+	}
 };
 
 const char * cache_request_status_str(enum cache_request_status status); 
@@ -65,33 +104,323 @@ struct cache_block_t {
     {
         m_tag=0;
         m_block_addr=0;
-        m_alloc_time=0;
-        m_fill_time=0;
-        m_last_access_time=0;
-        m_status=INVALID;
     }
-    void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time )
+
+    virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask) = 0;
+    virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask) = 0;
+
+    virtual bool is_invalid_line() = 0;
+    virtual bool is_valid_line() = 0;
+    virtual bool is_reserved_line() = 0;
+    virtual bool is_modified_line() = 0;
+
+    virtual enum cache_block_state get_status( mem_access_sector_mask_t sector_mask) = 0;
+    virtual void set_status(enum cache_block_state m_status, mem_access_sector_mask_t sector_mask) = 0;
+
+    virtual unsigned get_last_access_time() = 0;
+    virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask) = 0;
+    virtual unsigned get_alloc_time() = 0;
+    virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask) = 0;
+    virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask) = 0;
+    virtual unsigned get_modified_size() = 0;
+    virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)=0;
+    virtual bool is_readable(mem_access_sector_mask_t sector_mask)=0;
+    virtual ~cache_block_t() {}
+
+
+    new_addr_type    m_tag;
+    new_addr_type    m_block_addr;
+
+};
+
+struct line_cache_block: public cache_block_t  {
+	line_cache_block()
+	    {
+	        m_alloc_time=0;
+	        m_fill_time=0;
+	        m_last_access_time=0;
+	        m_status=INVALID;
+	        m_ignore_on_fill_status = false;
+	        m_set_modified_on_fill = false;
+	        m_readable = true;
+	    }
+	    void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask )
+	    {
+	        m_tag=tag;
+	        m_block_addr=block_addr;
+	        m_alloc_time=time;
+	        m_last_access_time=time;
+	        m_fill_time=0;
+	        m_status=RESERVED;
+	        m_ignore_on_fill_status = false;
+	        m_set_modified_on_fill = false;
+	    }
+		void fill( unsigned time, mem_access_sector_mask_t sector_mask )
+	    {
+	    	//if(!m_ignore_on_fill_status)
+	    	//	assert( m_status == RESERVED );
+
+	    	m_status = m_set_modified_on_fill? MODIFIED : VALID;
+
+	        m_fill_time=time;
+	    }
+		virtual bool is_invalid_line()
+	    {
+	    	return m_status == INVALID;
+	    }
+		virtual bool is_valid_line()
+	    {
+	    	 return m_status == VALID;
+	    }
+		virtual bool is_reserved_line()
+	    {
+	    	 return m_status == RESERVED;
+	    }
+		virtual bool is_modified_line()
+	    {
+	    	return m_status == MODIFIED;
+	    }
+
+		virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask)
+	    {
+	    	return m_status;
+	    }
+		virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask)
+	    {
+	    	m_status = status;
+	    }
+		virtual unsigned get_last_access_time()
+		{
+			return m_last_access_time;
+		}
+		virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask)
+	    {
+	    	m_last_access_time = time;
+	    }
+		virtual unsigned get_alloc_time()
+	    {
+	    	return m_alloc_time;
+	    }
+		virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask)
+		{
+			m_ignore_on_fill_status = m_ignore;
+		}
+		virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask)
+		{
+	    	m_set_modified_on_fill = m_modified;
+		}
+		virtual unsigned  get_modified_size()
+		{
+			return SECTOR_CHUNCK_SIZE * SECTOR_SIZE;   //i.e. cache line size
+		}
+		virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)
+		{
+			m_readable = readable;
+		}
+		virtual bool is_readable(mem_access_sector_mask_t sector_mask) {
+			return m_readable;
+		}
+
+
+private:
+	    unsigned         m_alloc_time;
+	    unsigned         m_last_access_time;
+	    unsigned         m_fill_time;
+	    cache_block_state    m_status;
+	    bool m_ignore_on_fill_status;
+	    bool m_set_modified_on_fill;
+	    bool m_readable;
+};
+
+struct sector_cache_block : public cache_block_t {
+	sector_cache_block()
     {
-        m_tag=tag;
-        m_block_addr=block_addr;
-        m_alloc_time=time;
-        m_last_access_time=time;
-        m_fill_time=0;
-        m_status=RESERVED;
+		init();
     }
-    void fill( unsigned time )
+
+	void init() {
+		for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+			m_sector_alloc_time[i]= 0;
+			m_sector_fill_time[i]= 0;
+			m_last_sector_access_time[i]= 0;
+			m_status[i]= INVALID;
+			m_ignore_on_fill_status[i] = false;
+			m_set_modified_on_fill[i] = false;
+			m_readable[i] = true;
+			}
+			m_line_alloc_time=0;
+			m_line_last_access_time=0;
+			m_line_fill_time=0;
+	}
+
+	virtual void allocate( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask )
     {
-        assert( m_status == RESERVED );
-        m_status=VALID;
-        m_fill_time=time;
+    	allocate_line( tag,  block_addr,  time, sector_mask );
     }
 
-    new_addr_type    m_tag;
-    new_addr_type    m_block_addr;
-    unsigned         m_alloc_time;
-    unsigned         m_last_access_time;
-    unsigned         m_fill_time;
-    cache_block_state    m_status;
+    void allocate_line( new_addr_type tag, new_addr_type block_addr, unsigned time, mem_access_sector_mask_t sector_mask )
+	{
+		//allocate a new line
+		//assert(m_block_addr != 0 && m_block_addr != block_addr);
+		init();
+		m_tag=tag;
+		m_block_addr=block_addr;
+
+		unsigned sidx = get_sector_index(sector_mask);
+
+		//set sector stats
+		m_sector_alloc_time[sidx]=time;
+		m_last_sector_access_time[sidx]=time;
+		m_sector_fill_time[sidx]=0;
+		m_status[sidx]=RESERVED;
+		m_ignore_on_fill_status[sidx] = false;
+		m_set_modified_on_fill[sidx] = false;
+
+		//set line stats
+		m_line_alloc_time=time;   //only set this for the first allocated sector
+		m_line_last_access_time=time;
+		m_line_fill_time=0;
+	}
+
+    void allocate_sector(unsigned time, mem_access_sector_mask_t sector_mask )
+	{
+    	//allocate invalid sector of this allocated valid line
+    	assert(is_valid_line());
+		unsigned sidx = get_sector_index(sector_mask);
+
+		//set sector stats
+		m_sector_alloc_time[sidx]=time;
+		m_last_sector_access_time[sidx]=time;
+		m_sector_fill_time[sidx]=0;
+		if(m_status[sidx]==MODIFIED)    //this should be the case only for fetch-on-write policy //TO DO
+			m_set_modified_on_fill[sidx] = true;
+		else
+			m_set_modified_on_fill[sidx] = false;
+
+		m_status[sidx]=RESERVED;
+		m_ignore_on_fill_status[sidx] = false;
+		//m_set_modified_on_fill[sidx] = false;
+		m_readable[sidx] = true;
+
+		//set line stats
+		m_line_last_access_time=time;
+		m_line_fill_time=0;
+	}
+
+    virtual void fill( unsigned time, mem_access_sector_mask_t sector_mask)
+    {
+    	unsigned sidx = get_sector_index(sector_mask);
+
+    //	if(!m_ignore_on_fill_status[sidx])
+    //	         assert( m_status[sidx] == RESERVED );
+
+    	m_status[sidx] = m_set_modified_on_fill[sidx]? MODIFIED : VALID;
+
+        m_sector_fill_time[sidx]=time;
+        m_line_fill_time=time;
+    }
+    virtual bool is_invalid_line() {
+    	//all the sectors should be invalid
+    	for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+    		if (m_status[i] != INVALID)
+    			return false;
+    	}
+    	return true;
+    }
+    virtual bool is_valid_line() { return  !(is_invalid_line()); }
+    virtual bool is_reserved_line() {
+    	//if any of the sector is reserved, then the line is reserved
+		for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+			if (m_status[i] == RESERVED)
+				return true;
+		}
+		return false;
+    }
+    virtual bool is_modified_line() {
+    	//if any of the sector is modified, then the line is modified
+    	for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+			if (m_status[i] == MODIFIED)
+				return true;
+		}
+		return false;
+    }
+
+    virtual enum cache_block_state get_status(mem_access_sector_mask_t sector_mask)
+	{
+    	unsigned sidx = get_sector_index(sector_mask);
+
+		return m_status[sidx];
+	}
+    virtual void set_status(enum cache_block_state status, mem_access_sector_mask_t sector_mask)
+	{
+		unsigned sidx = get_sector_index(sector_mask);
+		m_status[sidx] = status;
+	}
+    virtual unsigned get_last_access_time()
+	{
+		return m_line_last_access_time;
+	}
+    virtual void set_last_access_time(unsigned time, mem_access_sector_mask_t sector_mask)
+	{
+		unsigned sidx = get_sector_index(sector_mask);
+
+		m_last_sector_access_time[sidx] = time;
+		m_line_last_access_time = time;
+	}
+    virtual unsigned get_alloc_time()
+	{
+		return m_line_alloc_time;
+	}
+    virtual void set_ignore_on_fill(bool m_ignore, mem_access_sector_mask_t sector_mask)
+	{
+		unsigned sidx = get_sector_index(sector_mask);
+		m_ignore_on_fill_status[sidx] = m_ignore;
+	}
+    virtual void set_modified_on_fill(bool m_modified, mem_access_sector_mask_t sector_mask)
+	{
+		unsigned sidx = get_sector_index(sector_mask);
+		m_set_modified_on_fill[sidx] = m_modified;
+	}
+    virtual void set_m_readable(bool readable, mem_access_sector_mask_t sector_mask)
+    {
+    	unsigned sidx = get_sector_index(sector_mask);
+    	m_readable[sidx] = readable;
+    }
+    virtual bool is_readable(mem_access_sector_mask_t sector_mask) {
+    	unsigned sidx = get_sector_index(sector_mask);
+    	return m_readable[sidx];
+	}
+
+    virtual unsigned  get_modified_size()
+	{
+		unsigned modified=0;
+		for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+			if (m_status[i] == MODIFIED)
+				modified++;
+		}
+		return modified * SECTOR_SIZE;
+	}
+
+private:
+    unsigned m_sector_alloc_time[SECTOR_CHUNCK_SIZE];
+    unsigned m_last_sector_access_time[SECTOR_CHUNCK_SIZE];
+    unsigned m_sector_fill_time[SECTOR_CHUNCK_SIZE];
+    unsigned m_line_alloc_time;
+    unsigned m_line_last_access_time;
+    unsigned m_line_fill_time;
+    cache_block_state    m_status[SECTOR_CHUNCK_SIZE];
+    bool m_ignore_on_fill_status[SECTOR_CHUNCK_SIZE];
+    bool m_set_modified_on_fill[SECTOR_CHUNCK_SIZE];
+    bool m_readable[SECTOR_CHUNCK_SIZE];
+
+    unsigned get_sector_index(mem_access_sector_mask_t sector_mask)
+    {
+    	assert(sector_mask.count() == 1);
+    	for(unsigned i =0; i< SECTOR_CHUNCK_SIZE; ++i) {
+    		if(sector_mask.to_ulong() & (1<<i))
+    			return i;
+    	}
+    }
 };
 
 enum replacement_policy_t {
@@ -115,12 +444,14 @@ enum allocation_policy_t {
 
 enum write_allocate_policy_t {
 	NO_WRITE_ALLOCATE,
-	WRITE_ALLOCATE
+	WRITE_ALLOCATE,
+	FETCH_ON_WRITE
 };
 
 enum mshr_config_t {
     TEX_FIFO,
-    ASSOC // normal cache 
+    ASSOC, // normal cache
+	SECTOR_ASSOC // normal cache sends requests to high-level sector cache
 };
 
 enum set_index_function{
@@ -129,6 +460,11 @@ enum set_index_function{
     CUSTOM_SET_FUNCTION
 };
 
+enum cache_type{
+    NORMAL = 0,
+    SECTOR
+};
+
 class cache_config {
 public:
     cache_config() 
@@ -145,22 +481,33 @@ public:
     {
     	cache_status= status;
         assert( config );
-        char rp, wp, ap, mshr_type, wap, sif;
+        char ct, rp, wp, ap, mshr_type, wap, sif;
 
 
-        int ntok = sscanf(config,"%u:%u:%u,%c:%c:%c:%c:%c,%c:%u:%u,%u:%u,%u",
-                          &m_nset, &m_line_sz, &m_assoc, &rp, &wp, &ap, &wap,
+        int ntok = sscanf(config,"%c:%u:%u:%u,%c:%c:%c:%c:%c,%c:%u:%u,%u:%u,%u",
+                          &ct, &m_nset, &m_line_sz, &m_assoc, &rp, &wp, &ap, &wap,
                           &sif,&mshr_type,&m_mshr_entries,&m_mshr_max_merge,
                           &m_miss_queue_size, &m_result_fifo_entries,
                           &m_data_port_width);
 
-        if ( ntok < 11 ) {
+        if ( ntok < 12 ) {
             if ( !strcmp(config,"none") ) {
                 m_disabled = true;
                 return;
             }
             exit_parse_error();
         }
+
+        switch (ct) {
+			   case 'N': m_cache_type = NORMAL; break;
+			   case 'S': m_cache_type = SECTOR; break;
+			   default: exit_parse_error();
+        }
+        switch (rp) {
+               case 'L': m_replacement_policy = LRU; break;
+               case 'F': m_replacement_policy = FIFO; break;
+               default: exit_parse_error();
+        }
         switch (rp) {
         case 'L': m_replacement_policy = LRU; break;
         case 'F': m_replacement_policy = FIFO; break;
@@ -180,18 +527,24 @@ public:
         default: exit_parse_error();
         }
         switch (mshr_type) {
-        case 'F': m_mshr_type = TEX_FIFO; assert(ntok==13); break;
+        case 'F': m_mshr_type = TEX_FIFO; assert(ntok==14); break;
         case 'A': m_mshr_type = ASSOC; break;
+        case 'S' : m_mshr_type = SECTOR_ASSOC; break;
         default: exit_parse_error();
         }
         m_line_sz_log2 = LOGB2(m_line_sz);
         m_nset_log2 = LOGB2(m_nset);
         m_valid = true;
+        m_atom_sz = (m_cache_type == SECTOR)? SECTOR_SIZE : m_line_sz;
 
+        //For more details about difference between FETCH_ON_WRITE and WRITE VALIDAE policies
+        //Read: Jouppi, Norman P. "Cache write policies and performance". ISCA 93.
+        //WRITE_ALLOCATE is the old write policy in GPGPU-sim 3.x, that send WRITE and READ for every write request
         switch(wap){
-        case 'W': m_write_alloc_policy = WRITE_ALLOCATE; break;
         case 'N': m_write_alloc_policy = NO_WRITE_ALLOCATE; break;
-        default: exit_parse_error();
+        case 'W': m_write_alloc_policy = WRITE_ALLOCATE; break;
+        case 'F': m_write_alloc_policy = FETCH_ON_WRITE; break;
+		default: exit_parse_error();
         }
 
         // detect invalid configuration 
@@ -208,6 +561,15 @@ public:
             assert(0 && "Invalid cache configuration: Writeback cache cannot allocate new line on fill. "); 
         }
 
+        if(m_write_alloc_policy == FETCH_ON_WRITE && m_alloc_policy == ON_FILL)
+		{
+			assert(0 && "Invalid cache configuration: FETCH_ON_WRITE cannot work properly with ON_FILL policy. Cache must be ON_MISS. ");
+		}
+        if(m_cache_type == SECTOR)
+		{
+			assert(m_line_sz / SECTOR_SIZE == SECTOR_CHUNCK_SIZE && m_line_sz % SECTOR_SIZE == 0);
+		}
+
         // default: port to data array width and granularity = line size 
         if (m_data_port_width == 0) {
             m_data_port_width = m_line_sz; 
@@ -227,6 +589,11 @@ public:
         assert( m_valid );
         return m_line_sz;
     }
+    unsigned get_atom_sz() const
+	{
+		assert( m_valid );
+		return m_atom_sz;
+	}
     unsigned get_num_lines() const
     {
         assert( m_valid );
@@ -265,6 +632,14 @@ public:
     {
         return addr & ~(m_line_sz-1);
     }
+    new_addr_type mshr_addr( new_addr_type addr ) const
+	{
+    	return addr & ~(m_atom_sz-1);
+	}
+    enum mshr_config_t get_mshr_type() const
+	{
+    	return m_mshr_type;
+	}
     FuncCache get_cache_status() {return cache_status;}
     char *m_config_string;
     char *m_config_stringPrefL1;
@@ -285,11 +660,13 @@ protected:
     unsigned m_nset;
     unsigned m_nset_log2;
     unsigned m_assoc;
+    unsigned m_atom_sz;
 
     enum replacement_policy_t m_replacement_policy; // 'L' = LRU, 'F' = FIFO
     enum write_policy_t m_write_policy;             // 'T' = write through, 'B' = write back, 'R' = read only
     enum allocation_policy_t m_alloc_policy;        // 'm' = allocate on miss, 'f' = allocate on fill
     enum mshr_config_t m_mshr_type;
+    enum cache_type m_cache_type;
 
     write_allocate_policy_t m_write_alloc_policy;	// 'W' = Write allocate, 'N' = No write allocate
 
@@ -316,6 +693,7 @@ protected:
     friend class data_cache;
     friend class l1_cache;
     friend class l2_cache;
+    friend class memory_sub_partition;
 };
 
 class l1d_cache_config : public cache_config{
@@ -340,15 +718,17 @@ public:
     tag_array(cache_config &config, int core_id, int type_id );
     ~tag_array();
 
-    enum cache_request_status probe( new_addr_type addr, unsigned &idx ) const;
-    enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx );
-    enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, cache_block_t &evicted );
+    enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_fetch* mf ) const;
+    enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask ) const;
+    enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf );
+    enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf );
 
-    void fill( new_addr_type addr, unsigned time );
-    void fill( unsigned idx, unsigned time );
+    void fill( new_addr_type addr, unsigned time, mem_fetch* mf );
+    void fill( unsigned idx, unsigned time, mem_fetch* mf );
+    void fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask );
 
     unsigned size() const { return m_config.get_num_lines();}
-    cache_block_t &get_block(unsigned idx) { return m_lines[idx];}
+    cache_block_t* get_block(unsigned idx) { return m_lines[idx];}
 
     void flush(); // flash invalidate all entries
     void new_window();
@@ -365,19 +745,20 @@ protected:
     tag_array( cache_config &config,
                int core_id,
                int type_id,
-               cache_block_t* new_lines );
+               cache_block_t** new_lines );
     void init( int core_id, int type_id );
 
 protected:
 
     cache_config &m_config;
 
-    cache_block_t *m_lines; /* nbanks x nset x assoc lines in total */
+    cache_block_t **m_lines; /* nbanks x nset x assoc lines in total */
 
     unsigned m_access;
     unsigned m_miss;
     unsigned m_pending_hit; // number of cache miss that hit a line that is allocated but not filled
     unsigned m_res_fail;
+    unsigned m_sector_miss;
 
     // performance counters for calculating the amount of misses within a time window
     unsigned m_prev_snapshot_access;
@@ -390,7 +771,7 @@ protected:
 
 class mshr_table {
 public:
-    mshr_table( unsigned num_entries, unsigned max_merged )
+    mshr_table( unsigned num_entries, unsigned max_merged)
     : m_num_entries(num_entries),
     m_max_merged(max_merged)
 #if (tr1_hash_map_ismap == 0)
@@ -414,6 +795,8 @@ public:
     /// Returns next ready access
     mem_fetch *next_access();
     void display( FILE *fp ) const;
+    // Returns true if there is a pending read after write
+    bool is_read_after_write_pending(new_addr_type block_addr);
 
     void check_mshr_parameters( unsigned num_entries, unsigned max_merged )
     {
@@ -510,12 +893,14 @@ public:
     cache_stats();
     void clear();
     void inc_stats(int access_type, int access_outcome);
+    void inc_fail_stats(int access_type, int fail_outcome);
     enum cache_request_status select_stats_status(enum cache_request_status probe, enum cache_request_status access) const;
-    unsigned &operator()(int access_type, int access_outcome);
-    unsigned operator()(int access_type, int access_outcome) const;
+    unsigned &operator()(int access_type, int access_outcome, bool fail_outcome);
+    unsigned operator()(int access_type, int access_outcome, bool fail_outcome) const;
     cache_stats operator+(const cache_stats &cs);
     cache_stats &operator+=(const cache_stats &cs);
     void print_stats(FILE *fout, const char *cache_name = "Cache_stats") const;
+    void print_fail_stats(FILE *fout, const char *cache_name = "Cache_fail_stats") const;
 
     unsigned get_stats(enum mem_access_type *access_type, unsigned num_access_type, enum cache_request_status *access_status, unsigned num_access_status)  const;
     void get_sub_stats(struct cache_sub_stats &css) const;
@@ -523,8 +908,10 @@ public:
     void sample_cache_port_utility(bool data_port_busy, bool fill_port_busy); 
 private:
     bool check_valid(int type, int status) const;
+    bool check_fail_valid(int type, int fail) const;
 
     std::vector< std::vector<unsigned> > m_stats;
+    std::vector< std::vector<unsigned> > m_fail_stats;
 
     unsigned long long m_cache_port_available_cycles; 
     unsigned long long m_cache_data_port_busy_cycles; 
@@ -543,6 +930,7 @@ public:
 
 bool was_write_sent( const std::list<cache_event> &events );
 bool was_read_sent( const std::list<cache_event> &events );
+bool was_writeallocate_sent( const std::list<cache_event> &events );
 
 /// Baseline cache
 /// Implements common functions for read_only_cache and data_cache
@@ -552,7 +940,7 @@ public:
     baseline_cache( const char *name, cache_config &config, int core_id, int type_id, mem_fetch_interface *memport,
                      enum mem_fetch_status status )
     : m_config(config), m_tag_array(new tag_array(config,core_id,type_id)), 
-      m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge), 
+      m_mshrs(config.m_mshr_entries,config.m_mshr_max_merge),
       m_bandwidth_management(config) 
     {
         init( name, config, memport, status );
@@ -564,7 +952,7 @@ public:
                enum mem_fetch_status status )
     {
         m_name = name;
-        assert(config.m_mshr_type == ASSOC);
+        assert(config.m_mshr_type == ASSOC || config.m_mshr_type == SECTOR_ASSOC);
         m_memport=memport;
         m_miss_queue_status = status;
     }
@@ -612,6 +1000,15 @@ public:
     bool data_port_free() const { return m_bandwidth_management.data_port_free(); } 
     bool fill_port_free() const { return m_bandwidth_management.fill_port_free(); } 
 
+    // This is a gapping hole we are poking in the system to quickly handle
+    // filling the cache on cudamemcopies. We don't care about anything other than
+    // L2 state after the memcopy - so just force the tag array to act as though
+    // something is read or written without doing anything else.
+    void force_tag_access( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask )
+    {
+        m_tag_array->fill( addr, time, mask );
+    }
+
 protected:
     // Constructor that can be used by derived classes with custom tag arrays
     baseline_cache( const char *name,
@@ -640,17 +1037,24 @@ protected:
 
     struct extra_mf_fields {
         extra_mf_fields()  { m_valid = false;}
-        extra_mf_fields( new_addr_type a, unsigned i, unsigned d ) 
+        extra_mf_fields( new_addr_type a, new_addr_type ad, unsigned i, unsigned d, const cache_config& m_config)
         {
             m_valid = true;
             m_block_addr = a;
+            m_addr = ad;
             m_cache_index = i;
             m_data_size = d;
+        	pending_read = m_config.m_mshr_type == SECTOR_ASSOC? m_config.m_line_sz/SECTOR_SIZE : 0;
+
         }
         bool m_valid;
         new_addr_type m_block_addr;
+        new_addr_type m_addr;
         unsigned m_cache_index;
         unsigned m_data_size;
+        //this variable is used when a load request generates multiple load transactions
+        //For example, a read request from non-sector L1 request sends a request to sector L2
+        unsigned pending_read;
     };
 
     typedef std::map<mem_fetch*,extra_mf_fields> extra_mf_fields_lookup;
@@ -668,7 +1072,7 @@ protected:
     		unsigned time, bool &do_miss, std::list<cache_event> &events, bool read_only, bool wa);
     /// Read miss handler. Check MSHR hit or MSHR available
     void send_read_request(new_addr_type addr, new_addr_type block_addr, unsigned cache_index, mem_fetch *mf,
-    		unsigned time, bool &do_miss, bool &wb, cache_block_t &evicted, std::list<cache_event> &events, bool read_only, bool wa);
+    		unsigned time, bool &do_miss, bool &wb, evicted_block_info &evicted, std::list<cache_event> &events, bool read_only, bool wa);
 
     /// Sub-class containing all metadata for port bandwidth management 
     class bandwidth_management 
@@ -760,8 +1164,9 @@ public:
 
         // Set write miss function
         switch(m_config.m_write_alloc_policy){
-        case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa; break;
         case NO_WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_no_wa; break;
+		case WRITE_ALLOCATE: m_wr_miss = &data_cache::wr_miss_wa_naive; break;
+		case FETCH_ON_WRITE: m_wr_miss = &data_cache::wr_miss_wa_fetch_on_write; break;
         default:
             assert(0 && "Error: Must set valid cache write miss policy\n");
             break; // Need to set a write miss function
@@ -870,12 +1275,26 @@ protected:
     /// Sends read request, and possible write-back request,
     //  to lower level memory for a write miss with write-allocate
     enum cache_request_status
-        wr_miss_wa( new_addr_type addr,
-                    unsigned cache_index,
-                    mem_fetch *mf,
-                    unsigned time,
-                    std::list<cache_event> &events,
-                    enum cache_request_status status ); // write-allocate
+    	wr_miss_wa_naive( new_addr_type addr,
+                        unsigned cache_index,
+                        mem_fetch *mf,
+                        unsigned time,
+                        std::list<cache_event> &events,
+                        enum cache_request_status status ); // write-allocate-send-write-and-read-request
+	enum cache_request_status
+			   wr_miss_wa_fetch_on_write( new_addr_type addr,
+							unsigned cache_index,
+							mem_fetch *mf,
+							unsigned time,
+							std::list<cache_event> &events,
+							enum cache_request_status status ); // write-allocate with read-fetch-only
+	enum cache_request_status
+				wr_miss_wa_write_validate( new_addr_type addr,
+							unsigned cache_index,
+							mem_fetch *mf,
+							unsigned time,
+							std::list<cache_event> &events,
+							enum cache_request_status status ); // write-allocate that writes with no read fetch
     enum cache_request_status
         wr_miss_no_wa( new_addr_type addr,
                        unsigned cache_index,
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 483a7b6..17f1714 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -140,6 +140,8 @@ void power_config::reg_options(class OptionParser * opp)
 
 void memory_config::reg_options(class OptionParser * opp)
 {
+    option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, 
+                                "Fill the L2 cache on memcpy", "1");
     option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, 
                                 "0 = fifo, 1 = FR-FCFS (defaul)", "1");
     option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, 
@@ -192,7 +194,27 @@ void memory_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-dram_latency", OPT_UINT32, &dram_latency,
                      "DRAM latency (default 30)",
                      "30");
-
+    option_parser_register(opp, "-dual_bus_interface", OPT_UINT32, &dual_bus_interface,
+                                        "dual_bus_interface (default = 0) ",
+                                        "0");
+    option_parser_register(opp, "-dram_bnk_indexing_policy", OPT_UINT32, &dram_bnk_indexing_policy,
+                                            "dram_bnk_indexing_policy (0 = normal indexing, 1 = Xoring with the higher bits) (Default = 0)",
+                                            "0");
+    option_parser_register(opp, "-dram_bnkgrp_indexing_policy", OPT_UINT32, &dram_bnkgrp_indexing_policy,
+                                            "dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)",
+                                            "0");
+    option_parser_register(opp, "-Seperate_Write_Queue_Enable", OPT_BOOL, &seperate_write_queue_enabled,
+                           "Seperate_Write_Queue_Enable",
+                           "0");
+    option_parser_register(opp, "-Write_Queue_Size", OPT_CSTR, &write_queue_size_opt,
+                                  "Write_Queue_Size",
+                                  "32:28:16");
+    option_parser_register(opp, "-Elimnate_rw_turnaround", OPT_BOOL, &elimnate_rw_turnaround,
+                               "elimnate_rw_turnaround i.e set tWTR and tRTW = 0",
+                               "0");
+    option_parser_register(opp, "-icnt_flit_size", OPT_UINT32, &icnt_flit_size,
+                               "icnt_flit_size",
+                               "32");
     m_address_mapping.addrdec_setoption(opp);
 }
 
@@ -285,12 +307,6 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts,  
                  "Number of portions a warp is divided into for shared memory bank conflict check ",
                  "2");
-    option_parser_register(opp, "-gpgpu_L1_warp_parts_cached", OPT_INT32, &L1_warp_parts_cached,
-				 "Number of portions a warp is divided into when the request is cached",
-				 "2");
-    option_parser_register(opp, "-gpgpu_L1_warp_parts_cached", OPT_INT32, &L1_warp_parts_non_cached,
-				 "Number of portions a warp is divided into when the request is not cached",
-				 "4");
     option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts,
 				 "Number of portions a warp is divided into for shared memory bank conflict check ",
 				 "2");
@@ -312,6 +328,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", OPT_INT32, &gpgpu_operand_collector_num_units_sp,
                 "number of collector units (default = 4)", 
                 "4");
+    option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", OPT_INT32, &gpgpu_operand_collector_num_units_dp,
+                   "number of collector units (default = 0)",
+                   "0");
     option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu,
                 "number of collector units (default = 4)", 
                 "4");
@@ -324,6 +343,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sp,
                            "number of collector unit in ports (default = 1)", 
                            "1");
+    option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_in_ports_dp,
+                           "number of collector unit in ports (default = 0)",
+                           "0");
     option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu,
                            "number of collector unit in ports (default = 1)", 
                            "1");
@@ -336,6 +358,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sp,
                            "number of collector unit in ports (default = 1)", 
                            "1");
+    option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_dp", OPT_INT32, &gpgpu_operand_collector_num_out_ports_dp,
+                           "number of collector unit in ports (default = 0)",
+                           "0");
     option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu,
                            "number of collector unit in ports (default = 1)", 
                            "1");
@@ -355,18 +380,21 @@ void shader_core_config::reg_options(class OptionParser * opp)
     		            "Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)",
 			    "2");
     option_parser_register(opp, "-gpgpu_dual_issue_diff_exec_units", OPT_BOOL, &gpgpu_dual_issue_diff_exec_units,
-			    "should dual issue use two different execution unit resources",
+			    "should dual issue use two different execution unit resources (Default = 1)",
 			    "1");
     option_parser_register(opp, "-gpgpu_simt_core_sim_order", OPT_INT32, &simt_core_sim_order,
                             "Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)",
                             "1");
     option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string,
                             "Pipeline widths "
-                            "ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB",
-                            "1,1,1,1,1,1,1" );
+                            "ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB",
+                            "1,1,1,1,1,1,1,1,1" );
     option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units,
                             "Number of SP units (default=1)",
                             "1");
+    option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, &gpgpu_num_dp_units,
+                            "Number of DP units (default=0)",
+                            "0");
     option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, &gpgpu_num_sfu_units,
                             "Number of SF units (default=1)",
                             "1");
@@ -412,7 +440,6 @@ void gpgpu_sim_config::reg_options(option_parser_t opp)
    option_parser_register(opp, "-gpgpu_flush_l2_cache", OPT_BOOL, &gpgpu_flush_l2_cache,
                    "Flush L2 cache at the end of each kernel call",
                    "0");
-
    option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, 
                 "Stop the simulation at deadlock (1=on (default), 0=off)", 
                 "1");
@@ -656,7 +683,7 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config )
 
     m_running_kernels.resize( config.max_concurrent_kernel, NULL );
     m_last_issued_kernel = 0;
-    m_last_cluster_issue = 0;
+    m_last_cluster_issue = m_shader_config->n_simt_clusters-1; // this causes first launch to use simt cluster 0
     *average_pipeline_duty_cycle=0;
     *active_sms=0;
 
@@ -992,6 +1019,8 @@ void gpgpu_sim::gpu_print_stat()
    }
    printf("\nTotal_core_cache_stats:\n");
    core_cache_stats.print_stats(stdout, "Total_core_cache_stats_breakdown");
+   printf("\nTotal_core_cache_fail_stats:\n");
+   core_cache_stats.print_fail_stats(stdout, "Total_core_cache_fail_stats_breakdown");
    shader_print_scheduler_stat( stdout, false );
 
    m_shader_stats->print(stdout);
@@ -1036,6 +1065,8 @@ void gpgpu_sim::gpu_print_stat()
           printf("L2_total_cache_reservation_fails = %u\n", total_l2_css.res_fails);
           printf("L2_total_cache_breakdown:\n");
           l2_stats.print_stats(stdout, "L2_cache_stats_breakdown");
+          printf("L2_total_cache_reservation_fail_breakdown:\n");
+          l2_stats.print_fail_stats(stdout, "L2_cache_stats_fail_breakdown");
           total_l2_css.print_port_stats(stdout, "L2_cache");
        }
    }
@@ -1385,7 +1416,7 @@ void gpgpu_sim::cycle()
             if (mf) {
                 unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size();
                 if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) {
-                    if (!mf->get_is_write()) 
+                    //if (!mf->get_is_write())
                        mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle);
                     mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle);
                     ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size );
@@ -1415,7 +1446,8 @@ void gpgpu_sim::cycle()
       for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) {
           //move memory request from interconnect into memory partition (if not backed up)
           //Note:This needs to be called in DRAM clock domain if there is no L2 cache in the system
-          if ( m_memory_sub_partition[i]->full() ) {
+    	  //In the worst case, we may need to push SECTOR_CHUNCK_SIZE requests, so ensure you have enough buffer for them
+          if ( m_memory_sub_partition[i]->full(SECTOR_CHUNCK_SIZE) ) {
              gpu_stall_dramfull++;
           } else {
               mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) );
@@ -1565,6 +1597,24 @@ void shader_core_ctx::dump_warp_state( FILE *fout ) const
        m_warp[w].print(fout);
 }
 
+
+void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count )
+{
+    if (m_memory_config->m_perf_sim_memcpy) {
+       assert (dst_start_addr % 32 == 0);
+
+       for ( unsigned counter = 0; counter < count; counter += 32 ) {
+           const unsigned wr_addr = dst_start_addr + counter;
+           addrdec_t raw_addr;
+           mem_access_sector_mask_t mask;
+           mask.set(wr_addr % 128 / 32);
+           m_memory_config->m_address_mapping.addrdec_tlx( wr_addr, &raw_addr );
+           const unsigned partition_id = raw_addr.sub_partition / m_memory_config->m_n_sub_partition_per_memory_channel;
+           m_memory_partition_unit[ partition_id ]->handle_memcpy_to_gpu( wr_addr, raw_addr.sub_partition, mask );
+       }
+    }
+}
+
 void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const
 {
 /*
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 7d92c66..1778008 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -198,8 +198,14 @@ struct memory_config {
       bk_tag_length = i-1;
       assert(nbkgrp>0 && "Number of bank groups cannot be zero");
       tRCDWR = tRCD-(WL+1);
+      if(elimnate_rw_turnaround)
+      {
+    	  tRTW = 0;
+    	  tWTR = 0;
+      } else {
       tRTW = (CL+(BL/data_command_freq_ratio)+2-WL);
-      tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); 
+      tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR);
+      }
       tWTP = (WL+(BL/data_command_freq_ratio)+tWR);
       dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition 
 
@@ -213,7 +219,9 @@ struct memory_config {
       m_L2_config.init(&m_address_mapping);
 
       m_valid = true;
-      icnt_flit_size = 32; // Default 32
+
+      sscanf(write_queue_size_opt,"%d:%d:%d",
+                     &gpgpu_frfcfs_dram_write_queue_size,&write_high_watermark,&write_low_watermark);
    }
    void reg_options(class OptionParser * opp);
 
@@ -264,12 +272,25 @@ struct memory_config {
 
    unsigned nbk;
 
+   bool elimnate_rw_turnaround;
+
    unsigned data_command_freq_ratio; // frequency ratio between DRAM data bus and command bus (2 for GDDR3, 4 for GDDR5)
    unsigned dram_atom_size; // number of bytes transferred per read or write command 
 
    linear_to_raw_address_translation m_address_mapping;
 
    unsigned icnt_flit_size;
+
+   unsigned dram_bnk_indexing_policy;
+   unsigned dram_bnkgrp_indexing_policy;
+   bool dual_bus_interface;
+
+   bool seperate_write_queue_enabled;
+   char *write_queue_size_opt;
+   unsigned gpgpu_frfcfs_dram_write_queue_size;
+   unsigned write_high_watermark;
+   unsigned write_low_watermark;
+   bool m_perf_sim_memcpy;
 };
 
 // global counters and flags (please try not to add to this list!!!)
@@ -405,6 +426,8 @@ public:
    void gpu_print_stat();
    void dump_pipeline( int mask, int s, int m ) const;
 
+    void perf_memcpy_to_gpu( size_t dst_start_addr, size_t count );
+
    //The next three functions added to be used by the functional simulation function
    
    //! Get shader core configuration
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index c5fc44e..b1465a8 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -74,6 +74,15 @@ memory_partition_unit::memory_partition_unit( unsigned partition_id,
     }
 }
 
+void memory_partition_unit::handle_memcpy_to_gpu( size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask )
+{
+    unsigned p = global_sub_partition_id_to_local_id(global_subpart_id);
+    std::string mystring =
+        mask.to_string<char,std::string::traits_type,std::string::allocator_type>();
+    MEMPART_DPRINTF("Copy Engine Request Received For Address=%llx, local_subpart=%u, global_subpart=%u, sector_mask=%s \n", addr, p, global_subpart_id, mystring.c_str()); 
+    m_sub_partition[p]->force_l2_tag_update(addr,gpu_sim_cycle+gpu_tot_sim_cycle, mask);
+}
+
 memory_partition_unit::~memory_partition_unit() 
 {
     delete m_dram; 
@@ -93,7 +102,9 @@ memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct m
     m_private_credit_limit = 1; 
     m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size 
                             + config->gpgpu_dram_return_queue_size 
-                            - (config->m_n_sub_partition_per_memory_channel - 1); 
+                            - (config->m_n_sub_partition_per_memory_channel - 1);
+    if(config->seperate_write_queue_enabled )
+    	m_shared_credit_limit += config->gpgpu_frfcfs_dram_write_queue_size;
     if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 
         or config->gpgpu_dram_return_queue_size == 0) 
     {
@@ -220,7 +231,8 @@ void memory_partition_unit::dram_cycle()
     m_dram->cycle(); 
     m_dram->dram_log(SAMPLELOG);   
 
-    if( !m_dram->full() ) {
+   // mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+    //if( !m_dram->full(mf->is_write()) ) {
         // L2->DRAM queue to DRAM latency queue
         // Arbitrate among multiple L2 subpartitions 
         int last_issued_partition = m_arbitration_metadata.last_borrower(); 
@@ -228,6 +240,9 @@ void memory_partition_unit::dram_cycle()
             int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; 
             if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) {
                 mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top();
+                if(m_dram->full(mf->is_write()) )
+                	break;
+
                 m_sub_partition[spid]->L2_dram_queue_pop();
                 MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); 
                 dram_delay_t d;
@@ -239,12 +254,13 @@ void memory_partition_unit::dram_cycle()
                 break;  // the DRAM should only accept one request per cycle 
             }
         }
-    }
+    //}
 
     // DRAM latency queue
-    if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) {
-        mem_fetch* mf = m_dram_latency_queue.front().req;
-        m_dram_latency_queue.pop_front();
+
+    if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full(m_dram_latency_queue.front().req->is_write()) ) {
+    	mem_fetch* mf = m_dram_latency_queue.front().req;
+    	m_dram_latency_queue.pop_front();
         m_dram->push(mf);
     }
 }
@@ -299,6 +315,7 @@ memory_sub_partition::memory_sub_partition( unsigned sub_partition_id,
     m_id = sub_partition_id;
     m_config=config;
     m_stats=stats;
+    m_memcpy_cycle_offset = 0;
 
     assert(m_id < m_config->m_n_mem_sub_partition); 
 
@@ -343,6 +360,13 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
 				mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
 				m_L2_icnt_queue->push(mf);
            }else{
+        	    if(m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE)
+        	    {
+			 		assert(mf->original_wr_mf);
+			 		mf->original_wr_mf->set_reply();
+			 		mf->original_wr_mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
+			 		m_L2_icnt_queue->push(mf->original_wr_mf);
+        	    }
 				m_request_tracker.erase(mf);
 				delete mf;
            }
@@ -355,10 +379,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
         if ( !m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf) ) {
             if (m_L2cache->fill_port_free()) {
                 mf->set_status(IN_PARTITION_L2_FILL_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
-                m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle);
+                m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset);
                 m_dram_L2_queue->pop();
             }
         } else if ( !m_L2_icnt_queue->full() ) {
+        	if(mf->is_write() && mf->get_type() == WRITE_ACK)
             mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
             m_L2_icnt_queue->push(mf);
             m_dram_L2_queue->pop();
@@ -380,9 +405,10 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
             bool port_free = m_L2cache->data_port_free(); 
             if ( !output_full && port_free ) {
                 std::list<cache_event> events;
-                enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events);
+                enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset,events);
                 bool write_sent = was_write_sent(events);
                 bool read_sent = was_read_sent(events);
+                MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", mf->get_addr(), status); 
 
                 if ( status == HIT ) {
                     if( !write_sent ) {
@@ -402,6 +428,11 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
                         m_icnt_L2_queue->pop();
                     }
                 } else if ( status != RESERVATION_FAIL ) {
+                	if(mf->is_write() && m_config->m_L2_config.m_write_alloc_policy == FETCH_ON_WRITE && !was_writeallocate_sent(events)) {
+                		mf->set_reply();
+                		mf->set_status(IN_PARTITION_L2_TO_ICNT_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
+                		m_L2_icnt_queue->push(mf);
+                	}
                     // L2 cache accepted request
                     m_icnt_L2_queue->pop();
                 } else {
@@ -432,6 +463,11 @@ bool memory_sub_partition::full() const
     return m_icnt_L2_queue->full();
 }
 
+bool memory_sub_partition::full(unsigned size) const
+{
+    return m_icnt_L2_queue->is_avilable_size(size);
+}
+
 bool memory_sub_partition::L2_dram_queue_empty() const
 {
    return m_L2_dram_queue->empty(); 
@@ -540,21 +576,90 @@ bool memory_sub_partition::busy() const
     return !m_request_tracker.empty();
 }
 
-void memory_sub_partition::push( mem_fetch* req, unsigned long long cycle ) 
+std::vector<mem_fetch*> memory_sub_partition::breakdown_request_to_sector_requests(mem_fetch* mf)
 {
-    if (req) {
-        m_request_tracker.insert(req);
-        m_stats->memlatstat_icnt2mem_pop(req);
-        if( req->istexture() ) {
-            m_icnt_L2_queue->push(req);
-            req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
-        } else {
-            rop_delay_t r;
-            r.req = req;
-            r.ready_cycle = cycle + m_config->rop_latency;
-            m_rop.push(r);
-            req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle);
-        }
+	std::vector<mem_fetch*> result;
+
+	if(mf->get_data_size() == SECTOR_SIZE && mf->get_access_sector_mask().count() == 1) {
+		result.push_back(mf);
+	} else if (mf->get_data_size() == 128 || mf->get_data_size() == 64) {
+        //We only accept 32, 64 and 128 bytes reqs
+		unsigned start=0, end=0;
+		if(mf->get_data_size() == 128) {
+			start=0; end=3;
+		} else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "1100") {
+			start=2; end=3;
+		} else if (mf->get_data_size() == 64 && mf->get_access_sector_mask().to_string() == "0011") {
+			start=0; end=1;
+		} else if (mf->get_data_size() == 64 && (mf->get_access_sector_mask().to_string() == "1111" || mf->get_access_sector_mask().to_string() == "0000")) {
+			if(mf->get_addr() % 128 == 0) {
+				start=0; end=1;
+			} else {
+				start=2; end=3;
+			}
+		} else
+			{
+			    printf("Invalid sector received, address = 0x%06x, sector mask = %s, data size = %d",
+			    		mf->get_addr(), mf->get_access_sector_mask(), mf->get_data_size(), mf->get_data_size());
+				assert(0 && "Undefined sector mask is received");
+			}
+
+		std::bitset<SECTOR_SIZE*SECTOR_CHUNCK_SIZE> byte_sector_mask;
+		byte_sector_mask.reset();
+		for(unsigned  k=start*SECTOR_SIZE; k< SECTOR_SIZE; ++k)
+			byte_sector_mask.set(k);
+
+		for(unsigned j=start, i=0; j<= end ; ++j, ++i){
+
+			const mem_access_t *ma = new  mem_access_t( mf->get_access_type(),
+									mf->get_addr() + SECTOR_SIZE*i,
+									SECTOR_SIZE,
+									mf->is_write(),
+									mf->get_access_warp_mask(),
+									mf->get_access_byte_mask() & byte_sector_mask,
+									std::bitset<SECTOR_CHUNCK_SIZE>().set(j));
+
+			 mem_fetch *n_mf = new mem_fetch( *ma,
+								NULL,
+								mf->get_ctrl_size(),
+								mf->get_wid(),
+								mf->get_sid(),
+								mf->get_tpc(),
+								mf->get_mem_config(),
+								mf);
+
+			 result.push_back(n_mf);
+			 byte_sector_mask <<= SECTOR_SIZE;
+		}
+	} else assert(0 && "Undefined data size is received");
+
+	return result;
+}
+
+void memory_sub_partition::push( mem_fetch* m_req, unsigned long long cycle )
+{
+    if (m_req) {
+    	m_stats->memlatstat_icnt2mem_pop(m_req);
+    	std::vector<mem_fetch*> reqs;
+    	if(m_config->m_L2_config.m_cache_type == SECTOR)
+    		reqs = breakdown_request_to_sector_requests(m_req);
+    	else
+    		reqs.push_back(m_req);
+
+    	for(unsigned i=0; i<reqs.size(); ++i) {
+    		mem_fetch* req = reqs[i];
+			m_request_tracker.insert(req);
+			if( req->istexture() ) {
+				m_icnt_L2_queue->push(req);
+				req->set_status(IN_PARTITION_ICNT_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
+			} else {
+				rop_delay_t r;
+				r.req = req;
+				r.ready_cycle = cycle + m_config->rop_latency;
+				m_rop.push(r);
+				req->set_status(IN_PARTITION_ROP_DELAY,gpu_sim_cycle+gpu_tot_sim_cycle);
+			}
+    	}
     }
 }
 
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
index 3df54b1..2d13918 100644
--- a/src/gpgpu-sim/l2cache.h
+++ b/src/gpgpu-sim/l2cache.h
@@ -72,6 +72,7 @@ public:
    void print_stat( FILE *fp ) { m_dram->print_stat(fp); }
    void visualize() const { m_dram->visualize(); }
    void print( FILE *fp ) const;
+   void handle_memcpy_to_gpu( size_t dst_start_addr, unsigned subpart_id, mem_access_sector_mask_t mask );
 
    class memory_sub_partition * get_sub_partition(int sub_partition_id) 
    {
@@ -154,6 +155,7 @@ public:
    void cache_cycle( unsigned cycle );
 
    bool full() const;
+   bool full(unsigned size) const;
    void push( class mem_fetch* mf, unsigned long long clock_cycle );
    class mem_fetch* pop(); 
    class mem_fetch* top();
@@ -177,6 +179,12 @@ public:
    void accumulate_L2cache_stats(class cache_stats &l2_stats) const;
    void get_L2cache_sub_stats(struct cache_sub_stats &css) const;
 
+   void force_l2_tag_update(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask)
+   {
+        m_L2cache->force_tag_access( addr, m_memcpy_cycle_offset + time, mask );
+        m_memcpy_cycle_offset += 1;
+   }
+
 private:
 // data
    unsigned m_id;  //< the global sub partition ID
@@ -207,6 +215,15 @@ private:
    std::set<mem_fetch*> m_request_tracker;
 
    friend class L2interface;
+
+   std::vector<mem_fetch*> breakdown_request_to_sector_requests(mem_fetch* mf);
+
+   // This is a cycle offset that has to be applied to the l2 accesses to account for
+   // the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for kernel execution
+   // but we want cudamemcpy to go through the L2. Everytime an access is made from cudamemcpy
+   // this counter is incremented, and when the l2 is accessed (in both cudamemcpyies and otherwise)
+   // this value is added to the gpgpu-sim cycle counters.
+   unsigned m_memcpy_cycle_offset;
 };
 
 class L2interface : public mem_fetch_interface {
diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h
index 3dac87d..2235cdc 100644
--- a/src/gpgpu-sim/l2cache_trace.h
+++ b/src/gpgpu-sim/l2cache_trace.h
@@ -34,6 +34,9 @@
 #define MEMPART_PRINT_STR SIM_PRINT_STR " %d - "
 #define MEMPART_DTRACE(x)  ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) )
 
+#define MEM_SUBPART_PRINT_STR SIM_PRINT_STR " %d - "
+#define MEM_SUBPART_DTRACE(x)  ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)m_id) )
+
 // Intended to be called from inside components of a memory partition
 // Depends on a get_mpid() function
 #define MEMPART_DPRINTF(...) do {\
@@ -46,10 +49,23 @@
     }\
 } while (0)
 
+#define MEM_SUBPART_DPRINTF(...) do {\
+    if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) {\
+        printf( MEM_SUBPART_PRINT_STR,\
+                gpu_sim_cycle + gpu_tot_sim_cycle,\
+                Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT],\
+                m_id );\
+        printf(__VA_ARGS__);\
+    }\
+} while (0)
+
 #else
 
 #define MEMPART_DTRACE(x)  (false)
 #define MEMPART_DPRINTF(x, ...) do {} while (0)
 
+#define MEM_SUBPART_DTRACE(x)  (false)
+#define MEM_SUBPART_DPRINTF(x, ...) do {} while (0)
+
 #endif
 
diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc
index 580c051..c05a693 100644
--- a/src/gpgpu-sim/mem_fetch.cc
+++ b/src/gpgpu-sim/mem_fetch.cc
@@ -39,7 +39,9 @@ mem_fetch::mem_fetch( const mem_access_t &access,
                       unsigned wid,
                       unsigned sid, 
                       unsigned tpc, 
-                      const class memory_config *config )
+                      const class memory_config *config,
+					  mem_fetch *m_original_mf,
+					  mem_fetch *m_original_wr_mf)
 {
    m_request_uid = sm_next_mf_request_uid++;
    m_access = access;
@@ -61,6 +63,8 @@ mem_fetch::mem_fetch( const mem_access_t &access,
    m_status_change = gpu_sim_cycle + gpu_tot_sim_cycle;
    m_mem_config = config;
    icnt_flit_size = config->icnt_flit_size;
+   original_mf = m_original_mf;
+   original_wr_mf = m_original_wr_mf;
 }
 
 mem_fetch::~mem_fetch()
diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h
index db4a8e9..278cf32 100644
--- a/src/gpgpu-sim/mem_fetch.h
+++ b/src/gpgpu-sim/mem_fetch.h
@@ -55,7 +55,9 @@ public:
                unsigned wid,
                unsigned sid, 
                unsigned tpc, 
-               const class memory_config *config );
+               const class memory_config *config,
+			   mem_fetch *original_mf = NULL,
+			   mem_fetch *original_wr_mf = NULL);
    ~mem_fetch();
 
    void set_status( enum mem_fetch_status status, unsigned long long cycle );
@@ -113,6 +115,8 @@ public:
    const memory_config *get_mem_config(){return m_mem_config;}
 
    unsigned get_num_flits(bool simt_to_mem);
+   mem_fetch* original_mf;
+   mem_fetch* original_wr_mf;
 private:
    // request source information
    unsigned m_request_uid;
diff --git a/src/gpgpu-sim/mem_latency_stat.cc b/src/gpgpu-sim/mem_latency_stat.cc
index fde0eff..35d6d84 100644
--- a/src/gpgpu-sim/mem_latency_stat.cc
+++ b/src/gpgpu-sim/mem_latency_stat.cc
@@ -75,6 +75,10 @@ memory_stats_t::memory_stats_t( unsigned n_shader, const struct shader_core_conf
    max_mf_latency = 0;
    max_icnt2mem_latency = 0;
    max_icnt2sh_latency = 0;
+   tot_icnt2mem_latency = 0;
+   tot_icnt2sh_latency = 0;
+   tot_mrq_num = 0;
+   tot_mrq_latency = 0;
    memset(mrq_lat_table, 0, sizeof(unsigned)*32);
    memset(dq_lat_table, 0, sizeof(unsigned)*32);
    memset(mf_lat_table, 0, sizeof(unsigned)*32);
@@ -158,6 +162,7 @@ void memory_stats_t::memlatstat_read_done(mem_fetch *mf)
          mf_max_lat_table[mf->get_tlx_addr().chip][mf->get_tlx_addr().bk] = mf_latency;
       unsigned icnt2sh_latency;
       icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_return_timestamp();
+      tot_icnt2sh_latency += icnt2sh_latency;
       icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++;
       if (icnt2sh_latency > max_icnt2sh_latency)
          max_icnt2sh_latency = icnt2sh_latency;
@@ -191,6 +196,7 @@ void memory_stats_t::memlatstat_icnt2mem_pop(mem_fetch *mf)
    if (m_memory_config->gpgpu_memlatency_stat) {
       unsigned icnt2mem_latency;
       icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->get_timestamp();
+      tot_icnt2mem_latency += icnt2mem_latency;
       icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++;
       if (icnt2mem_latency > max_icnt2mem_latency)
          max_icnt2mem_latency = icnt2mem_latency;
@@ -216,14 +222,17 @@ void memory_stats_t::memlatstat_print( unsigned n_mem, unsigned gpu_mem_n_bk )
    unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses;
 
    if (m_memory_config->gpgpu_memlatency_stat) {
+	   printf("maxmflatency = %d \n", max_mf_latency);
+	   printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency);
       printf("maxmrqlatency = %d \n", max_mrq_latency);
-      printf("maxdqlatency = %d \n", max_dq_latency);
-      printf("maxmflatency = %d \n", max_mf_latency);
+      //printf("maxdqlatency = %d \n", max_dq_latency);
+      printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency);
       if (num_mfs) {
          printf("averagemflatency = %lld \n", mf_total_lat/num_mfs);
+         printf("avg_icnt2mem_latency = %lld \n", tot_icnt2mem_latency/num_mfs);
+         printf("avg_mrq_latency = %lld \n", tot_mrq_latency/tot_mrq_num);
+         printf("avg_icnt2sh_latency = %lld \n", tot_icnt2sh_latency/num_mfs);
       }
-      printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency);
-      printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency);
       printf("mrq_lat_table:");
       for (i=0; i< 32; i++) {
          printf("%d \t", mrq_lat_table[i]);
diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h
index 4968a3b..5b89202 100644
--- a/src/gpgpu-sim/mem_latency_stat.h
+++ b/src/gpgpu-sim/mem_latency_stat.h
@@ -56,6 +56,10 @@ public:
    unsigned max_dq_latency;
    unsigned max_mf_latency;
    unsigned max_icnt2mem_latency;
+   unsigned long long int tot_icnt2mem_latency;
+   unsigned long long int tot_icnt2sh_latency;
+   unsigned long long int tot_mrq_latency;
+   unsigned long long int tot_mrq_num;
    unsigned max_icnt2sh_latency;
    unsigned mrq_lat_table[32];
    unsigned dq_lat_table[32];
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index bb2cf0e..bf482fb 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -150,6 +150,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        m_simt_stack,
                                        &m_warp,
                                        &m_pipeline_reg[ID_OC_SP],
+									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i
@@ -164,6 +165,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                                     m_simt_stack,
                                                     &m_warp,
                                                     &m_pipeline_reg[ID_OC_SP],
+													&m_pipeline_reg[ID_OC_DP],
                                                     &m_pipeline_reg[ID_OC_SFU],
                                                     &m_pipeline_reg[ID_OC_MEM],
                                                     i,
@@ -179,6 +181,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        m_simt_stack,
                                        &m_warp,
                                        &m_pipeline_reg[ID_OC_SP],
+									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i
@@ -193,6 +196,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
 					  m_simt_stack,
 					  &m_warp,
 					  &m_pipeline_reg[ID_OC_SP],
+					  &m_pipeline_reg[ID_OC_DP],
 					  &m_pipeline_reg[ID_OC_SFU],
 					  &m_pipeline_reg[ID_OC_MEM],
 					  i
@@ -207,6 +211,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                        m_simt_stack,
                                        &m_warp,
                                        &m_pipeline_reg[ID_OC_SP],
+									   &m_pipeline_reg[ID_OC_DP],
                                        &m_pipeline_reg[ID_OC_SFU],
                                        &m_pipeline_reg[ID_OC_MEM],
                                        i,
@@ -228,8 +233,9 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
     }
     
     //op collector configuration
-    enum { SP_CUS, SFU_CUS, MEM_CUS, GEN_CUS };
+    enum { SP_CUS, DP_CUS, SFU_CUS, MEM_CUS, GEN_CUS };
     m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp);
+    m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp);
     m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu);
     m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem);
     m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen);
@@ -246,6 +252,15 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
         in_ports.clear(),out_ports.clear(),cu_sets.clear();
     }
     
+    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) {
+            in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
+            out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
+            cu_sets.push_back((unsigned)DP_CUS);
+            cu_sets.push_back((unsigned)GEN_CUS);
+            m_operand_collector.add_port(in_ports,out_ports,cu_sets);
+            in_ports.clear(),out_ports.clear(),cu_sets.clear();
+        }
+
     for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) {
         in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
         out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
@@ -280,7 +295,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
     m_operand_collector.init( m_config->gpgpu_num_reg_banks, this );
     
     // execute
-    m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_sfu_units + 1; // sp_unit, sfu, ldst_unit
+    m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + 1; // sp_unit, sfu, ldst_unit
     //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ];
     //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ];
     
@@ -292,12 +307,18 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
         m_issue_port.push_back(OC_EX_SP);
     }
     
+    for (int k = 0; k < m_config->gpgpu_num_dp_units; k++) {
+            m_fu.push_back(new dp_unit( &m_pipeline_reg[EX_WB], m_config, this ));
+            m_dispatch_port.push_back(ID_OC_DP);
+            m_issue_port.push_back(OC_EX_DP);
+        }
+
     for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) {
         m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this ));
         m_dispatch_port.push_back(ID_OC_SFU);
         m_issue_port.push_back(OC_EX_SFU);
     }
-    
+
     m_ldst_unit = new ldst_unit( m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, m_scoreboard, config, mem_config, stats, shader_id, tpc_id );
     m_fu.push_back(m_ldst_unit);
     m_dispatch_port.push_back(ID_OC_MEM);
@@ -677,7 +698,7 @@ void shader_core_ctx::fetch()
                 if( !m_warp[warp_id].functional_done() && !m_warp[warp_id].imiss_pending() && m_warp[warp_id].ibuffer_empty() ) {
                     address_type pc  = m_warp[warp_id].get_pc();
                     address_type ppc = pc + PROGRAM_MEM_START;
-                    unsigned nbytes=16; 
+                    unsigned nbytes=16;
                     unsigned offset_in_block = pc & (m_config->m_L1I_config.get_line_sz()-1);
                     if( (offset_in_block+nbytes) > m_config->m_L1I_config.get_line_sz() )
                         nbytes = (m_config->m_L1I_config.get_line_sz()-offset_in_block);
@@ -754,7 +775,7 @@ void shader_core_ctx::issue(){
      unsigned j;
      for (unsigned i = 0; i < schedulers.size(); i++) {
 	j = (Issue_Prio + i) % schedulers.size();
-	schedulers[j]->cycle();
+	  schedulers[j]->cycle();
      }
      Issue_Prio = (Issue_Prio+1)% schedulers.size();
 
@@ -879,6 +900,7 @@ void scheduler_unit::cycle()
         exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE;
 	unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp;
 	bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units;
+
         while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) {
             const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst();
             //Jin: handle cdp latency;
@@ -920,9 +942,11 @@ void scheduler_unit::cycle()
                                 previous_issued_inst_exec_type = exec_unit_type_t::MEM;
                             }
                         } else {
+
                             bool sp_pipe_avail = m_sp_out->has_free();
                             bool sfu_pipe_avail = m_sfu_out->has_free();
-                            if( sp_pipe_avail && (pI->op != SFU_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) {
+                            bool dp_pipe_avail = m_dp_out->has_free();
+                            if( sp_pipe_avail && (pI->op != SFU_OP && pI->op != DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) {
                                 
                                 //Jin: special for CDP api
                                 if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) {
@@ -948,7 +972,16 @@ void scheduler_unit::cycle()
                                 issued_inst=true;
                                 warp_inst_issued = true;
                                 previous_issued_inst_exec_type = exec_unit_type_t::SP;
-                            } else if ( (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) {
+                            } else if ( (m_shader->m_config->gpgpu_num_dp_units != 0) && (pI->op == DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::DP)) {
+                                if( dp_pipe_avail ) {
+                                    m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id);
+                                    issued++;
+                                    issued_inst=true;
+                                    warp_inst_issued = true;
+                                    previous_issued_inst_exec_type = exec_unit_type_t::DP;
+                                }
+                            }  //If the DP units = 0 (like in Fermi archi), then change DP inst to SFU inst
+                            else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) {
                                 if( sfu_pipe_avail ) {
                                     m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id);
                                     issued++;
@@ -1139,11 +1172,12 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader
                                Scoreboard* scoreboard, simt_stack** simt,
                                std::vector<shd_warp_t>* warp,
                                register_set* sp_out,
+							   register_set* dp_out,
                                register_set* sfu_out,
                                register_set* mem_out,
                                int id,
                                char* config_string )
-    : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id )
+    : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id )
 {
     unsigned m_prioritization_readin;
     int ret = sscanf( config_string,
@@ -1405,8 +1439,14 @@ ldst_unit::process_cache_access( cache_t* cache,
     mem_stage_stall_type result = NO_RC_FAIL;
     bool write_sent = was_write_sent(events);
     bool read_sent = was_read_sent(events);
-    if( write_sent ) 
-        m_core->inc_store_req( inst.warp_id() );
+    if( write_sent ) {
+    	unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)?
+    			(mf->get_data_size()/SECTOR_SIZE) : 1;
+
+		for(unsigned i=0; i< inc_ack; ++i)
+			m_core->inc_store_req( inst.warp_id() );
+
+    }
     if ( status == HIT ) {
         assert( !read_sent );
         inst.accessq_pop_back();
@@ -1418,7 +1458,7 @@ ldst_unit::process_cache_access( cache_t* cache,
         if( !write_sent ) 
             delete mf;
     } else if ( status == RESERVATION_FAIL ) {
-        result = COAL_STALL;
+        result = BK_CONF;
         assert( !read_sent );
         assert( !write_sent );
         delete mf;
@@ -1427,8 +1467,8 @@ ldst_unit::process_cache_access( cache_t* cache,
         //inst.clear_active( access.get_warp_mask() ); // threads in mf writeback when mf returns
         inst.accessq_pop_back();
     }
-    if( !inst.accessq_empty() )
-        result = BK_CONF;
+    if( !inst.accessq_empty() && result == NO_RC_FAIL)
+        result = COAL_STALL;
     return result;
 }
 
@@ -1523,7 +1563,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea
        assert( CACHE_UNDEFINED != inst.cache_op );
        stall_cond = process_memory_access_queue(m_L1D,inst);
    }
-   if( !inst.accessq_empty() ) 
+   if( !inst.accessq_empty() && stall_cond == NO_RC_FAIL)
        stall_cond = COAL_STALL;
    if (stall_cond != NO_RC_FAIL) {
       stall_reason = stall_cond;
@@ -1588,6 +1628,13 @@ void sp_unit::active_lanes_in_pipeline(){
 	m_core->incfuactivelanes_stat(active_count);
 	m_core->incfumemactivelanes_stat(active_count);
 }
+void dp_unit::active_lanes_in_pipeline(){
+	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
+	assert(active_count<=m_core->get_config()->warp_size);
+	m_core->incspactivelanes_stat(active_count);
+	m_core->incfuactivelanes_stat(active_count);
+	m_core->incfumemactivelanes_stat(active_count);
+}
 
 void sfu::active_lanes_in_pipeline(){
 	unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline();
@@ -1603,6 +1650,12 @@ sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,sh
     m_name = "SP "; 
 }
 
+dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core)
+    : pipelined_simd_unit(result_port,config,config->max_sfu_latency,core)
+{
+    m_name = "DP ";
+}
+
 void sp_unit :: issue(register_set& source_reg)
 {
     warp_inst_t** ready_reg = source_reg.get_ready();
@@ -1612,6 +1665,14 @@ void sp_unit :: issue(register_set& source_reg)
 	pipelined_simd_unit::issue(source_reg);
 }
 
+void dp_unit :: issue(register_set& source_reg)
+{
+    warp_inst_t** ready_reg = source_reg.get_ready();
+	//m_core->incexecstat((*ready_reg));
+	(*ready_reg)->op_pipe=DP__OP;
+	m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency);
+	pipelined_simd_unit::issue(source_reg);
+}
 
 pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core )
     : simd_function_unit(config) 
@@ -1909,12 +1970,12 @@ void ldst_unit::cycle()
 
    if( !m_response_fifo.empty() ) {
        mem_fetch *mf = m_response_fifo.front();
-       if (mf->istexture()) {
+       if (mf->get_access_type() == TEXTURE_ACC_R) {
            if (m_L1T->fill_port_free()) {
                m_L1T->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle);
                m_response_fifo.pop_front(); 
            }
-       } else if (mf->isconst())  {
+       } else if (mf->get_access_type() == CONST_ACC_R)  {
            if (m_L1C->fill_port_free()) {
                mf->set_status(IN_SHADER_FETCHED,gpu_sim_cycle+gpu_tot_sim_cycle);
                m_L1C->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle);
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 2fed420..5b41c06 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -326,12 +326,13 @@ public:
                    Scoreboard* scoreboard, simt_stack** simt, 
                    std::vector<shd_warp_t>* warp, 
                    register_set* sp_out,
+				   register_set* dp_out,
                    register_set* sfu_out,
                    register_set* mem_out,
                    int id) 
         : m_supervised_warps(), m_stats(stats), m_shader(shader),
         m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp),
-        m_sp_out(sp_out),m_sfu_out(sfu_out),m_mem_out(mem_out), m_id(id){}
+        m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_mem_out(mem_out), m_id(id){}
     virtual ~scheduler_unit(){}
     virtual void add_supervised_warp_id(int i) {
         m_supervised_warps.push_back(&warp(i));
@@ -403,6 +404,7 @@ protected:
     //warp_inst_t** m_pipeline_reg;
     std::vector<shd_warp_t>* m_warp;
     register_set* m_sp_out;
+    register_set* m_dp_out;
     register_set* m_sfu_out;
     register_set* m_mem_out;
 
@@ -415,10 +417,11 @@ public:
                     Scoreboard* scoreboard, simt_stack** simt,
                     std::vector<shd_warp_t>* warp,
                     register_set* sp_out,
+					register_set* dp_out,
                     register_set* sfu_out,
                     register_set* mem_out,
                     int id )
-	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id ){}
+	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){}
 	virtual ~lrr_scheduler () {}
 	virtual void order_warps ();
     virtual void done_adding_supervised_warps() {
@@ -432,10 +435,11 @@ public:
                     Scoreboard* scoreboard, simt_stack** simt,
                     std::vector<shd_warp_t>* warp,
                     register_set* sp_out,
+					register_set* dp_out,
                     register_set* sfu_out,
                     register_set* mem_out,
                     int id )
-	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id ){}
+	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){}
 	virtual ~gto_scheduler () {}
 	virtual void order_warps ();
     virtual void done_adding_supervised_warps() {
@@ -450,10 +454,11 @@ public:
                     Scoreboard* scoreboard, simt_stack** simt,
                     std::vector<shd_warp_t>* warp,
                     register_set* sp_out,
+					register_set* dp_out,
                     register_set* sfu_out,
                     register_set* mem_out,
                     int id )
-	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id ){}
+	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){}
 	virtual ~oldest_scheduler () {}
 	virtual void order_warps ();
         virtual void done_adding_supervised_warps() {
@@ -468,11 +473,12 @@ public:
                           Scoreboard* scoreboard, simt_stack** simt,
                           std::vector<shd_warp_t>* warp,
                           register_set* sp_out,
+						  register_set* dp_out,
                           register_set* sfu_out,
                           register_set* mem_out,
                           int id,
                           char* config_str )
-	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, sfu_out, mem_out, id ),
+	: scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ),
 	  m_pending_warps() 
     {
         unsigned inner_level_readin;
@@ -518,6 +524,7 @@ public:
                     Scoreboard* scoreboard, simt_stack** simt,
                     std::vector<shd_warp_t>* warp,
                     register_set* sp_out,
+					register_set* dp_out,
                     register_set* sfu_out,
                     register_set* mem_out,
                     int id,
@@ -1080,6 +1087,23 @@ public:
         switch(inst.op) {
         case SFU_OP: break;
         case ALU_SFU_OP: break;
+        case DP_OP: break;     //for compute <= 29 (i..e Fermi and GT200)
+        default: return false;
+        }
+        return pipelined_simd_unit::can_issue(inst);
+    }
+    virtual void active_lanes_in_pipeline();
+    virtual void issue(  register_set& source_reg );
+};
+
+class dp_unit : public pipelined_simd_unit
+{
+public:
+	dp_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core );
+    virtual bool can_issue( const warp_inst_t &inst ) const
+    {
+        switch(inst.op) {
+        case DP_OP: break;
         default: return false;
         }
         return pipelined_simd_unit::can_issue(inst);
@@ -1099,6 +1123,7 @@ public:
         case LOAD_OP: return false;
         case STORE_OP: return false;
         case MEMORY_BARRIER_OP: return false;
+        case DP_OP: return false;
         default: break;
         }
         return pipelined_simd_unit::can_issue(inst);
@@ -1226,9 +1251,11 @@ protected:
 
 enum pipeline_stage_name_t {
     ID_OC_SP=0,
+	ID_OC_DP,
     ID_OC_SFU,  
     ID_OC_MEM,  
     OC_EX_SP,
+	OC_EX_DP,
     OC_EX_SFU,
     OC_EX_MEM,
     EX_WB,
@@ -1237,9 +1264,11 @@ enum pipeline_stage_name_t {
 
 const char* const pipeline_stage_name_decode[] = {
     "ID_OC_SP",
+	"ID_OC_DP",
     "ID_OC_SFU",  
     "ID_OC_MEM",  
     "OC_EX_SP",
+	"OC_EX_DP",
     "OC_EX_SFU",
     "OC_EX_MEM",
     "EX_WB",
@@ -1328,21 +1357,25 @@ struct shader_core_config : public core_config
 
     //op collector
     int gpgpu_operand_collector_num_units_sp;
+    int gpgpu_operand_collector_num_units_dp;
     int gpgpu_operand_collector_num_units_sfu;
     int gpgpu_operand_collector_num_units_mem;
     int gpgpu_operand_collector_num_units_gen;
 
     unsigned int gpgpu_operand_collector_num_in_ports_sp;
+    unsigned int gpgpu_operand_collector_num_in_ports_dp;
     unsigned int gpgpu_operand_collector_num_in_ports_sfu;
     unsigned int gpgpu_operand_collector_num_in_ports_mem;
     unsigned int gpgpu_operand_collector_num_in_ports_gen;
 
     unsigned int gpgpu_operand_collector_num_out_ports_sp;
+    unsigned int gpgpu_operand_collector_num_out_ports_dp;
     unsigned int gpgpu_operand_collector_num_out_ports_sfu;
     unsigned int gpgpu_operand_collector_num_out_ports_mem;
     unsigned int gpgpu_operand_collector_num_out_ports_gen;
 
     int gpgpu_num_sp_units;
+    int gpgpu_num_dp_units;
     int gpgpu_num_sfu_units;
     int gpgpu_num_mem_units;
 
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 04845e7..ad4587a 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -91,6 +91,7 @@ void *gpgpu_sim_thread_sequential(void*)
 pthread_mutex_t g_sim_lock = PTHREAD_MUTEX_INITIALIZER;
 bool g_sim_active = false;
 bool g_sim_done = true;
+bool break_limit = false;
 
 void *gpgpu_sim_thread_concurrent(void*)
 {
@@ -144,11 +145,13 @@ void *gpgpu_sim_thread_concurrent(void*)
                 if(g_the_gpu->cycle_insn_cta_max_hit()){
                     g_stream_manager->stop_all_running_kernels();
                     g_sim_done = true;
+                    break_limit = true;
                 }
             }
 
             active=g_the_gpu->active() || !g_stream_manager->empty_protected();
-        } while( active );
+
+        } while( active && !g_sim_done);
         if(g_debug_execution >= 3) {
            printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n");
            fflush(stdout);
@@ -166,6 +169,11 @@ void *gpgpu_sim_thread_concurrent(void*)
        printf("GPGPU-Sim: *** simulation thread exiting ***\n");
        fflush(stdout);
     }
+    if(break_limit) {
+    	printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n");
+    	exit(1);
+    }
+
     sem_post(&g_sim_signal_exit);
     return NULL;
 }
@@ -179,7 +187,7 @@ void synchronize()
     bool done = false;
     do {
         pthread_mutex_lock(&g_sim_lock);
-        done = g_stream_manager->empty() && !g_sim_active;
+        done = ( g_stream_manager->empty() && !g_sim_active ) || g_sim_done;
         pthread_mutex_unlock(&g_sim_lock);
     } while (!done);
     printf("GPGPU-Sim: detected inactive GPU simulation thread\n");
diff --git a/src/trace_streams.tup b/src/trace_streams.tup
index bbc9bb3..3455a00 100644
--- a/src/trace_streams.tup
+++ b/src/trace_streams.tup
@@ -29,6 +29,7 @@ TS_TUP_BEGIN( trace_streams_type )
     TS_TUP( WARP_SCHEDULER ),
     TS_TUP( SCOREBOARD ),
     TS_TUP( MEMORY_PARTITION_UNIT ),
+    TS_TUP( MEMORY_SUBPARTITION_UNIT ),
     TS_TUP( INTERCONNECT ),
     TS_TUP( NUM_TRACE_STREAMS )
 TS_TUP_END( trace_streams_type )