From b2def455d573f66fbc38dabda4adbc3a56225910 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Sun, 9 Feb 2020 20:39:22 -0500 Subject: adding kepler sass, skip-first-kernel and update config file --- .../SM3_KEPLER_TITAN/config_kepler_islip.icnt | 73 ++++++++ .../tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config | 186 ++++++++++++++++++++ configs/tested-cfgs/SM6_TITANX/gpgpusim.config | 34 ++-- configs/tested-cfgs/SM6_TITANX/gpgpusim_old.config | 192 +++++++++++++++++++++ configs/tested-cfgs/SM7_QV100/gpgpusim.config | 2 +- configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config | 3 +- .../Turing_RTX2060/config_pascal_islip.icnt | 73 ++++++++ configs/tested-cfgs/Turing_RTX2060/gpgpusim.config | 186 ++++++++++++++++++++ src/gpgpu-sim/gpu-sim.cc | 3 + src/gpgpu-sim/gpu-sim.h | 4 +- src/trace-driven/gpgpusim_trace_driven_main.cc | 6 + src/trace-driven/kepler_opcode.h | 141 +++++++++++++++ src/trace-driven/pascal_opcode.h | 8 +- src/trace-driven/trace_driven.cc | 5 + src/trace-driven/trace_opcode.h | 7 +- 15 files changed, 899 insertions(+), 24 deletions(-) create mode 100644 configs/tested-cfgs/SM3_KEPLER_TITAN/config_kepler_islip.icnt create mode 100644 configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config create mode 100644 configs/tested-cfgs/SM6_TITANX/gpgpusim_old.config create mode 100644 configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt create mode 100644 configs/tested-cfgs/Turing_RTX2060/gpgpusim.config create mode 100644 src/trace-driven/kepler_opcode.h diff --git a/configs/tested-cfgs/SM3_KEPLER_TITAN/config_kepler_islip.icnt b/configs/tested-cfgs/SM3_KEPLER_TITAN/config_kepler_islip.icnt new file mode 100644 index 0000000..2fe3b53 --- /dev/null +++ b/configs/tested-cfgs/SM3_KEPLER_TITAN/config_kepler_islip.icnt @@ -0,0 +1,73 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 38; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 64; +input_buffer_size = 256; +ejection_buffer_size = 64; +boundary_buffer_size = 64; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config new file mode 100644 index 0000000..323d2d9 --- /dev/null +++ b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config @@ -0,0 +1,186 @@ +# This config models the KEPLER (TITAN) +# For more info about this card, see Nvidia White paper +# https://wr0.wr.inf.h-brs.de/wr/hardware/nodes3/nvidia/NVIDIA-Kepler-GK110-Architecture-Whitepaper.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 35 +-gpgpu_ignore_resources_limitation 1 + +# Device Limits +-gpgpu_stack_size_limit 1024 +-gpgpu_heap_size_limit 8388608 +-gpgpu_runtime_sync_depth_limit 2 +-gpgpu_runtime_pending_launch_count_limit 2048 + +# Compute Capability +-gpgpu_compute_capability_major 3 +-gpgpu_compute_capability_minor 5 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +# P102 has two semi-indp scheds per core, and two cores per cluster +-gpgpu_n_clusters 14 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Kepler clock domains +#-gpgpu_clock_domains ::: +# Kepler NVIDIA TITAN clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_700_series +-gpgpu_clock_domains 837.0:837.0:837.0:1502.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_occupancy_sm_number 62 + +# This implies a maximum of 32 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 16 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB +## Kepler has 6 SP SIMD units and 2 SFU units per SM. +# There is no int unit in kepler +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 6,4,0,2,1,6,4,0,2,1,12 +-gpgpu_num_sp_units 6 +-gpgpu_num_sfu_units 2 +-gpgpu_num_dp_units 4 + + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2 +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 2,8,8,8,130 +-ptx_opcode_initiation_sfu 2 +-ptx_opcode_latency_sfu 200 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# The defulat is to disable the L1 cache, unless cache modifieres are used +-gpgpu_cache:dl1 S:4:128:32,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefL1 S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefShared S:4:128:32,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 49152 +-gpgpu_shmem_sizeDefault 49152 +-gpgpu_shmem_size_PrefL1 16384 +-gpgpu_shmem_size_PrefShared 49152 +# By default, L1 cache is disabled in Kepler P102 and only enabled for local memory +# requests with .nc modifier or __ldg mehtod will be cached in L1 cache even with gmem_skip_L1D=1 +-gmem_skip_L1D 1 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 82 +-smem_latency 24 +-gpgpu_flush_l1_cache 1 + +# 32 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 1.5MB L2 cache +-gpgpu_cache:dl2 S:32:128:16,L:B:m:L:L,A:256:64,16:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 32:32:32:32 +-perf_sim_memcpy 1 +-memory_partition_indexing 0 + +# 4 KB Inst. +-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 12 +-gpgpu_operand_collector_num_units_sfu 6 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_units_dp 6 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_operand_collector_num_in_ports_sfu 2 +-gpgpu_operand_collector_num_out_ports_sfu 2 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_operand_collector_num_in_ports_dp 1 +-gpgpu_operand_collector_num_out_ports_dp 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +# Use kepler Coalsce arhitetecture +-gpgpu_coalesce_arch 35 + +## In Kepler, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 +-gpgpu_dual_issue_diff_exec_units 0 + +# interconnection +-network_mode 1 +-inter_config_file config_kepler_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 64 + +# for NVIDIA TITAN, bus width is 384bits (12 DRAM chips x 32 bits) +# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing, scaled to 2500MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=8:RCD=16:RAS=37:RP=16:RC=52: + CL=16:WL=6:CDLR=7:WR=16:nbkgrp=4:CCDL=4:RTPL=3" + +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Kepler TITAN has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Kepler +-power_simulation_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/tested-cfgs/SM6_TITANX/gpgpusim.config b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config index e6d8f1d..f8689c2 100644 --- a/configs/tested-cfgs/SM6_TITANX/gpgpusim.config +++ b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config @@ -25,7 +25,7 @@ # high level architecture configuration # P102 has two semi-indp scheds per core, and two cores per cluster -gpgpu_n_clusters 28 --gpgpu_n_cores_per_cluster 2 +-gpgpu_n_cores_per_cluster 1 -gpgpu_n_mem 12 -gpgpu_n_sub_partition_per_mchannel 2 @@ -36,12 +36,12 @@ -gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0 # shader core pipeline config --gpgpu_shader_registers 32768 +-gpgpu_shader_registers 65536 -gpgpu_occupancy_sm_number 62 # This implies a maximum of 32 warps/SM --gpgpu_shader_core_pipeline 1024:32 --gpgpu_shader_cta 16 +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 -gpgpu_simd_model 1 # Pipeline widths and number of FUs @@ -49,9 +49,9 @@ ## Pascal GP102 has 4 SP SIMD units and 4 SFU units per SM. In this config, we split SM into two shader cores, each has 2 SPs and 2 SFUs # There is no int unit in Pascal ## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 2,1,0,2,1,2,1,0,2,1,5 --gpgpu_num_sp_units 2 --gpgpu_num_sfu_units 2 +-gpgpu_pipeline_widths 4,1,0,4,1,4,1,0,4,1,9 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 4 -gpgpu_num_dp_units 1 @@ -74,16 +74,14 @@ # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo # Note: Hashing set index function (H) only applies to a set size of 32 or 64. -# Pascal GP102 has 96KB Shared memory divided over 2 cores, each has 48KB -# Pascal GP102 has 2 banks L1 cache, where each is 24KB L1 cache # The defulat is to disable the L1 cache, unless cache modifieres are used --gpgpu_cache:dl1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_cache:dl1PrefL1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_cache:dl1PrefShared S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_shmem_size 49152 --gpgpu_shmem_sizeDefault 49152 --gpgpu_shmem_size_PrefL1 49152 --gpgpu_shmem_size_PrefShared 49152 +-gpgpu_cache:dl1 S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefL1 S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefShared S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 98304 +-gpgpu_shmem_sizeDefault 98304 +-gpgpu_shmem_size_PrefL1 98304 +-gpgpu_shmem_size_PrefShared 98304 # By default, L1 cache is disabled in Pascal P102. # requests with .nc modifier or __ldg mehtod will be cached in L1 cache even with gmem_skip_L1D=1 -gmem_skip_L1D 1 @@ -136,7 +134,7 @@ # interconnection -network_mode 1 --inter_config_file config_fermi_islip.icnt +-inter_config_file config_pascal_islip.icnt # memory partition latency config -rop_latency 120 @@ -168,7 +166,7 @@ #-Write_Queue_Size 64:56:32 # Pascal 102 has four schedulers per core --gpgpu_num_sched_per_core 2 +-gpgpu_num_sched_per_core 4 # Two Level Scheduler with active and pending pools #-gpgpu_scheduler two_level_active:6:0:1 # Loose round robbin scheduler diff --git a/configs/tested-cfgs/SM6_TITANX/gpgpusim_old.config b/configs/tested-cfgs/SM6_TITANX/gpgpusim_old.config new file mode 100644 index 0000000..17ad779 --- /dev/null +++ b/configs/tested-cfgs/SM6_TITANX/gpgpusim_old.config @@ -0,0 +1,192 @@ +# This config models the Pascal GP102 (NVIDIA TITAN X) +# For more info about this card, see Nvidia White paper +# http://international.download.nvidia.com/geforce-com/international/pdfs/GeForce_GTX_1080_Whitepaper_FINAL.pdf + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 61 +-gpgpu_ignore_resources_limitation 1 + +# Device Limits +-gpgpu_stack_size_limit 1024 +-gpgpu_heap_size_limit 8388608 +-gpgpu_runtime_sync_depth_limit 2 +-gpgpu_runtime_pending_launch_count_limit 2048 + +# Compute Capability +-gpgpu_compute_capability_major 6 +-gpgpu_compute_capability_minor 1 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +# P102 has two semi-indp scheds per core, and two cores per cluster +-gpgpu_n_clusters 28 +-gpgpu_n_cores_per_cluster 2 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains ::: +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1417.0:1417.0:1417.0:2500.0 + +# shader core pipeline config +-gpgpu_shader_registers 32768 +-gpgpu_occupancy_sm_number 62 + +# This implies a maximum of 32 warps/SM +-gpgpu_shader_core_pipeline 1024:32 +-gpgpu_shader_cta 16 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 4 SFU units per SM. In this config, we split SM into two shader cores, each has 2 SPs and 2 SFUs +# There is no int unit in Pascal +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 2,1,0,2,1,2,1,0,2,1,5 +-gpgpu_num_sp_units 2 +-gpgpu_num_sfu_units 2 +-gpgpu_num_dp_units 1 + + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2 +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 1,1,1,1,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 8,8,8,8,130 +-ptx_opcode_initiation_sfu 4 +-ptx_opcode_latency_sfu 8 + + +# latencies and cache configs are adopted from: +# https://arxiv.org/pdf/1804.06826.pdf +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory divided over 2 cores, each has 48KB +# Pascal GP102 has 2 banks L1 cache, where each is 24KB L1 cache +# The defulat is to disable the L1 cache, unless cache modifieres are used +-gpgpu_cache:dl1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefL1 S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_cache:dl1PrefShared S:4:128:48,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 49152 +-gpgpu_shmem_sizeDefault 49152 +-gpgpu_shmem_size_PrefL1 49152 +-gpgpu_shmem_size_PrefShared 49152 +# By default, L1 cache is disabled in Pascal P102. +# requests with .nc modifier or __ldg mehtod will be cached in L1 cache even with gmem_skip_L1D=1 +-gmem_skip_L1D 1 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 82 +-smem_latency 24 +-gpgpu_flush_l1_cache 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 S:64:128:16,L:B:m:L:L,A:256:64,16:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 32:32:32:32 +-perf_sim_memcpy 1 +-memory_partition_indexing 0 + +# 4 KB Inst. +-gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 +# 48 KB Tex +# Note, TEX is deprected in Pascal, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod +-gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 + +# enable operand collector +-gpgpu_operand_collector_num_units_sp 12 +-gpgpu_operand_collector_num_units_sfu 6 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_units_dp 6 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_operand_collector_num_in_ports_sfu 2 +-gpgpu_operand_collector_num_out_ports_sfu 2 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_operand_collector_num_in_ports_dp 1 +-gpgpu_operand_collector_num_out_ports_dp 1 +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +# Use Pascal Coalsce arhitetecture +-gpgpu_coalesce_arch 61 + +## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units +-gpgpu_max_insn_issue_per_warp 2 +-gpgpu_dual_issue_diff_exec_units 1 + +# interconnection +-network_mode 1 +-inter_config_file config_pascal_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 64 + +# for NVIDIA TITAN X, bus width is 384bits (12 DRAM chips x 32 bits) +# 12 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing, scaled to 2500MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=8:RCD=16:RAS=37:RP=16:RC=52: + CL=16:WL=6:CDLR=7:WR=16:nbkgrp=4:CCDL=4:RTPL=3" + +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Pascal 102 has four schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Pascal 102 +-power_simulation_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/tested-cfgs/SM7_QV100/gpgpusim.config b/configs/tested-cfgs/SM7_QV100/gpgpusim.config index 584ef8d..41aea78 100644 --- a/configs/tested-cfgs/SM7_QV100/gpgpusim.config +++ b/configs/tested-cfgs/SM7_QV100/gpgpusim.config @@ -165,7 +165,7 @@ -gpgpu_dram_burst_length 2 -dram_data_command_freq_ratio 2 # HBM is DDR -gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@6;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.CBCSSSSS +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCB.CCCSSSSS # HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) # Timing for 1 GHZ diff --git a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config index 0d4a812..ba50287 100644 --- a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config +++ b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config @@ -20,7 +20,6 @@ -gpgpu_runtime_sync_depth_limit 2 -gpgpu_runtime_pending_launch_count_limit 2048 -gpgpu_kernel_launch_latency 5000 --gpgpu_TB_launch_latency 0 # Compute Capability -gpgpu_compute_capability_major 7 @@ -166,7 +165,7 @@ -gpgpu_dram_burst_length 2 -dram_data_command_freq_ratio 2 # HBM is DDR -gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@6;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.CBCSSSSS +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCB.CCCSSSSS # HBM timing are adopted from hynix JESD235 standered and nVidia HPCA 2017 paper (http://www.cs.utah.edu/~nil/pubs/hpca17.pdf) # Timing for 1 GHZ diff --git a/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt b/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt new file mode 100644 index 0000000..dec4789 --- /dev/null +++ b/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt @@ -0,0 +1,73 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 52; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 64; +input_buffer_size = 256; +ejection_buffer_size = 64; +boundary_buffer_size = 64; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config b/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config new file mode 100644 index 0000000..ab324d8 --- /dev/null +++ b/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config @@ -0,0 +1,186 @@ +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# Device Limits +-gpgpu_stack_size_limit 1024 +-gpgpu_heap_size_limit 8388608 +-gpgpu_runtime_sync_depth_limit 2 +-gpgpu_runtime_pending_launch_count_limit 2048 +-gpgpu_kernel_launch_latency 5000 +-gpgpu_TB_launch_latency 0 + +# Compute Capability +-gpgpu_compute_capability_major 7 +-gpgpu_compute_capability_minor 0 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 30 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# volta clock domains +#-gpgpu_clock_domains ::: +# Turing clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_20_series +-gpgpu_clock_domains 1365.0:1365.0:1365.0:3500.0 +# boost mode +# -gpgpu_clock_domains 1680.0:1680.0:1680.0:3500.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_registers_per_block 65536 +-gpgpu_occupancy_sm_number 70 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE +## Volta GV100 has 4 SP SIMD units, 4 SFU units, 4 DP units per core, 4 Tensor core units +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 4 +-gpgpu_num_dp_units 4 +-gpgpu_num_int_units 4 +-gpgpu_tensor_core_avail 1 +-gpgpu_num_tensor_core_units 4 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +# Throughput (initiation latency) are adopted from +# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 2,2,2,2,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 2,2,2,2,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 4,4,4,4,130 +-ptx_opcode_latency_sfu 100 +-ptx_opcode_initiation_sfu 8 +-ptx_opcode_latency_tesnor 64 +-ptx_opcode_initiation_tensor 64 + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +-adaptive_cache_config 0 +-l1_banks 4 +-gpgpu_cache:dl1 S:1:128:512,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 65536 +-gpgpu_shmem_sizeDefault 65536 +-gpgpu_shmem_per_block 65536 +-gmem_skip_L1D 0 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 20 +-smem_latency 20 +-gpgpu_flush_l1_cache 1 + +# 32 sets, each 128 bytes 32-way for each memory sub partition (96 KB per memory sub partition). This gives us 6MB L2 cache +-gpgpu_cache:dl2 S:32:128:32,L:B:m:L:L,A:192:4,32:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 +-perf_sim_memcpy 1 +-memory_partition_indexing 0 + +# 128 KB Inst. +-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 +# 128 KB Tex +# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod +-gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2 +# 64 KB Const +-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 +-perfect_inst_const_cache 1 + +# i.e. schedulers are isolated +-sub_core_model 1 +# disable specialized operand collectors and use generic operand collectors instead +-enable_specialized_operand_collector 0 +-gpgpu_operand_collector_num_units_gen 32 +-gpgpu_operand_collector_num_in_ports_gen 8 +-gpgpu_operand_collector_num_out_ports_gen 8 +# volta has 8 banks, 4 schedulers, two banks per scheduler +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 60 + +## In Volta, a warp scheduler can issue 1 inst per cycle +-gpgpu_max_insn_issue_per_warp 1 +-gpgpu_dual_issue_diff_exec_units 1 + +# interconnection +#-network_mode 1 +#-inter_config_file config_volta_islip.icnt +# for local xbar, use: +-network_mode 2 +-inct_in_buffer_limit 512 +-inct_out_buffer_limit 512 +-inct_subnets 2 +-arbiter_algo 1 + +# memory partition latency config +-rop_latency 160 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 192 + +# GDDR6 +# http://monitorinsider.com/GDDR6.html +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 2 +-gpgpu_dram_burst_length 16 +-dram_data_command_freq_ratio 4 # GDDR6 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing, scaled to 3500MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=4:RRD=10:RCD=20:RAS=50:RP=20:RC=62: + CL=20:WL=8:CDLR=9:WR=20:nbkgrp=4:CCDL=4:RTPL=4" + +# select lower bits for bnkgrp to increase bnkgrp parallelism +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# Volta has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Volta +-power_simulation_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 7f9985e..4e38f67 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -573,6 +573,9 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-trace_driven_mode", OPT_BOOL, &trace_driven_mode, "Turn on trace_driven_mode", "0"); + option_parser_register(opp, "-trace_skip_first_kernel", OPT_BOOL, + &trace_skip_first_kernel, "skip first intiliztion kernel in trace mode", + "0"); option_parser_register(opp, "-trace", OPT_CSTR, &g_traces_filename, "traces kernel file" "traces kernel file directory", diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 1ac4fdb..abc905e 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -348,7 +348,8 @@ public: size_t sync_depth_limit() const {return runtime_sync_depth_limit; } size_t pending_launch_count_limit() const {return runtime_pending_launch_count_limit;} - unsigned is_trace_driven_mode() const { return trace_driven_mode; } + bool is_trace_driven_mode() const { return trace_driven_mode; } + bool is_skip_first_kernel() const { return trace_skip_first_kernel; } char* get_traces_filename() const { return g_traces_filename; } bool flush_l1() const { return gpgpu_flush_l1_cache; } @@ -408,6 +409,7 @@ private: //trace driven mode options bool trace_driven_mode; + bool trace_skip_first_kernel; char *g_traces_filename; friend class gpgpu_sim; diff --git a/src/trace-driven/gpgpusim_trace_driven_main.cc b/src/trace-driven/gpgpusim_trace_driven_main.cc index 68b2ff7..0e3aced 100644 --- a/src/trace-driven/gpgpusim_trace_driven_main.cc +++ b/src/trace-driven/gpgpusim_trace_driven_main.cc @@ -50,6 +50,7 @@ int main ( int argc, const char **argv ) trace_parser tracer(m_gpgpu_sim->get_config().get_traces_filename(), m_gpgpu_sim, m_gpgpu_context); std::vector commandlist = tracer.parse_kernellist_file(); + bool first_kernel=true; for(unsigned i=0; iget_config().is_skip_first_kernel() && first_kernel) { + first_kernel = false; + continue; + } kernel_info = tracer.parse_kernel_info(commandlist[i]); m_gpgpu_sim->launch(kernel_info); } diff --git a/src/trace-driven/kepler_opcode.h b/src/trace-driven/kepler_opcode.h new file mode 100644 index 0000000..f85346f --- /dev/null +++ b/src/trace-driven/kepler_opcode.h @@ -0,0 +1,141 @@ +//developed by Mahmoud Khairy, Purdue Univ +//abdallm@purdue.edu + +#ifndef KEPLER_OPCODE_H +#define KEPLER_OPCODE_H + +#include "../abstract_hardware_model.h" +#include "trace_opcode.h" +#include +#include + +#define KEPLER_BINART_VERSION 35 +#define KEPLER_SHARED_MEMORY_VIRTIAL_ADDRESS_START 0x00007f2c60000000 + +//TO DO: moving this to a yml or def files + +///Kepler ISA +//see: https://docs.nvidia.com/cuda/cuda-binary-utilities/index.html +static const std::unordered_map Kepler_OpcodeMap = { + //Floating Point 32 Instructions + {"FFMA", OpcodeChar(OP_FFMA, SP_OP)}, + {"FADD", OpcodeChar(OP_FADD, SP_OP)}, + {"FCMP", OpcodeChar(OP_FCMP, SP_OP)}, + {"FMUL", OpcodeChar(OP_FMUL, SP_OP)}, + {"FMNMX", OpcodeChar(OP_FMNMX, SP_OP)}, + {"FSWZ", OpcodeChar(OP_FSWZ, SP_OP)}, + {"FSET", OpcodeChar(OP_FSET, SP_OP)}, + {"FSETP", OpcodeChar(OP_FSETP, SP_OP)}, + {"FCHK", OpcodeChar(OP_FCHK, SP_OP)}, + {"RRO", OpcodeChar(OP_RRO, SP_OP)}, + //SFU + {"MUFU", OpcodeChar(OP_MUFU, SFU_OP)}, + + + //Double Point Instructions + {"DFMA", OpcodeChar(OP_DFMA, DP_OP)}, + {"DADD", OpcodeChar(OP_DADD, DP_OP)}, + {"DMUL", OpcodeChar(OP_DMUL, DP_OP)}, + {"DMNMX", OpcodeChar(OP_DMNMX, DP_OP)}, + {"DSET", OpcodeChar(OP_DSET, DP_OP)}, + {"DSETP", OpcodeChar(OP_DSETP, DP_OP)}, + + //Integer Instructions + {"IMAD", OpcodeChar(OP_IMAD, INTP_OP)}, + {"IMADSP", OpcodeChar(OP_IMADSP, INTP_OP)}, + {"IMUL", OpcodeChar(OP_IMUL, INTP_OP)}, + {"IADD", OpcodeChar(OP_IADD, INTP_OP)}, + {"ISCADD", OpcodeChar(OP_ISCADD, INTP_OP)}, + {"ISAD", OpcodeChar(OP_ISAD, INTP_OP)}, + {"IMNMX", OpcodeChar(OP_IMNMX, INTP_OP)}, + {"BFE", OpcodeChar(OP_BFE, INTP_OP)}, + {"BFI", OpcodeChar(OP_BFI, INTP_OP)}, + {"SHR", OpcodeChar(OP_SHR, INTP_OP)}, + {"SHL", OpcodeChar(OP_SHL, INTP_OP)}, + {"SHF", OpcodeChar(OP_SHF, INTP_OP)}, + {"LOP", OpcodeChar(OP_LOP, INTP_OP)}, + {"FLO", OpcodeChar(OP_FLO, INTP_OP)}, + {"ISET", OpcodeChar(OP_ISET, INTP_OP)}, + {"ISETP", OpcodeChar(OP_ISETP, INTP_OP)}, + {"ICMP", OpcodeChar(OP_ICMP, INTP_OP)}, + {"POPC", OpcodeChar(OP_POPC, INTP_OP)}, + + //Conversion Instructions + {"F2F", OpcodeChar(OP_F2F, ALU_OP)}, + {"F2I", OpcodeChar(OP_F2I, ALU_OP)}, + {"I2F", OpcodeChar(OP_I2F, ALU_OP)}, + {"I2I", OpcodeChar(OP_I2I, ALU_OP)}, + + //Movement Instructions + {"MOV", OpcodeChar(OP_MOV, ALU_OP)}, + {"MOV32I", OpcodeChar(OP_MOV32I, ALU_OP)}, + {"SEL", OpcodeChar(OP_SEL, ALU_OP)}, + {"PRMT", OpcodeChar(OP_PRMT, ALU_OP)}, + {"SHFL", OpcodeChar(OP_SHFL, ALU_OP)}, + + //Predicate Instructions + {"P2R", OpcodeChar(OP_P2R, ALU_OP)}, + {"R2P", OpcodeChar(OP_R2P, ALU_OP)}, + {"CSET", OpcodeChar(OP_CSET, ALU_OP)}, + {"CSETP", OpcodeChar(OP_CSETP, ALU_OP)}, + {"PSET", OpcodeChar(OP_PSET, ALU_OP)}, + {"PSETP", OpcodeChar(OP_PSETP, ALU_OP)}, + + //Texture Instructions + //For now, we ignore texture loads, consider it as ALU_OP + {"TEX", OpcodeChar(OP_TEX, ALU_OP)}, + {"TLD", OpcodeChar(OP_TLD, ALU_OP)}, + {"TLD4", OpcodeChar(OP_TLD4, ALU_OP)}, + {"TXQ", OpcodeChar(OP_TXQ, ALU_OP)}, + + //Load/Store Instructions + //For now, we ignore constant loads, consider it as ALU_OP, TO DO + {"LDC", OpcodeChar(OP_LDC, ALU_OP)}, + {"LD", OpcodeChar(OP_LD, LOAD_OP)}, + {"LDG", OpcodeChar(OP_LDG, LOAD_OP)}, + {"LDL", OpcodeChar(OP_LDL, LOAD_OP)}, + {"LDS", OpcodeChar(OP_LDS, LOAD_OP)}, + {"LDSLK", OpcodeChar(OP_LDSLK, LOAD_OP)}, + {"ST", OpcodeChar(OP_ST, STORE_OP)}, + {"STL", OpcodeChar(OP_STL, STORE_OP)}, + {"STS", OpcodeChar(OP_STS, STORE_OP)}, + {"STSCUL", OpcodeChar(OP_STSCUL, STORE_OP)}, + {"ATOM", OpcodeChar(OP_ATOM, STORE_OP)}, + {"RED", OpcodeChar(OP_RED, STORE_OP)}, + {"CCTL", OpcodeChar(OP_CCTL, ALU_OP)}, + {"CCTLL", OpcodeChar(OP_CCTLL, ALU_OP)}, + {"MEMBAR", OpcodeChar(OP_MEMBAR, MEMORY_BARRIER_OP)}, + + //surface memory instructions + {"SUCLAMP", OpcodeChar(OP_SUCLAMP, LOAD_OP)}, + {"SUBFM", OpcodeChar(OP_SUBFM, LOAD_OP)}, + {"SUEAU", OpcodeChar(OP_SUEAU, LOAD_OP)}, + {"SULDGA", OpcodeChar(OP_SULDGA, LOAD_OP)}, + {"SUSTGA", OpcodeChar(OP_SUSTGA, STORE_OP)}, + + //Control Instructions + {"BRA", OpcodeChar(OP_BRA, BRANCH_OP)}, + {"BRX", OpcodeChar(OP_BRX, BRANCH_OP)}, + {"JMP", OpcodeChar(OP_JMP, BRANCH_OP)}, + {"JMX", OpcodeChar(OP_JMX, BRANCH_OP)}, + {"CAL", OpcodeChar(OP_CAL, CALL_OPS)}, + {"JCAL", OpcodeChar(OP_JCAL, CALL_OPS)}, + {"RET", OpcodeChar(OP_RET, RET_OPS)}, + {"BRK", OpcodeChar(OP_BRK, RET_OPS)}, + {"CONT", OpcodeChar(OP_CONT, RET_OPS)}, + {"SSY", OpcodeChar(OP_SSY, RET_OPS)}, + {"PBK", OpcodeChar(OP_PBK, RET_OPS)}, + {"PCNT", OpcodeChar(OP_PCNT, RET_OPS)}, + {"PRET", OpcodeChar(OP_PRET, RET_OPS)}, + {"BPT", OpcodeChar(OP_BPT, BRANCH_OP)}, + {"EXIT", OpcodeChar(OP_EXIT, EXIT_OPS)}, + + //Miscellaneous Instructions + {"NOP", OpcodeChar(OP_NOP, ALU_OP)}, + {"S2R", OpcodeChar(OP_S2R, ALU_OP)}, + {"B2R", OpcodeChar(OP_B2R, ALU_OP)}, + {"BAR", OpcodeChar(OP_BAR, BARRIER_OP)}, + {"VOTE", OpcodeChar(OP_VOTE, ALU_OP)}, +}; + +#endif diff --git a/src/trace-driven/pascal_opcode.h b/src/trace-driven/pascal_opcode.h index d4f787d..2cacb28 100644 --- a/src/trace-driven/pascal_opcode.h +++ b/src/trace-driven/pascal_opcode.h @@ -70,6 +70,7 @@ static const std::unordered_map Pascal_OpcodeMap = { {"ISCADD", OpcodeChar(OP_ISCADD, INTP_OP)}, {"ISCADD32I", OpcodeChar(OP_ISCADD32I, INTP_OP)}, {"ISETP", OpcodeChar(OP_ISETP, INTP_OP)}, + {"ISET", OpcodeChar(OP_ISET, INTP_OP)}, {"LEA", OpcodeChar(OP_LEA, INTP_OP)}, {"LOP", OpcodeChar(OP_LOP, INTP_OP)}, {"LOP3", OpcodeChar(OP_LOP3, INTP_OP)}, @@ -85,6 +86,8 @@ static const std::unordered_map Pascal_OpcodeMap = { {"IMADSP", OpcodeChar(OP_IMADSP, INTP_OP)}, {"SHL", OpcodeChar(OP_SHL, INTP_OP)}, {"XMAD", OpcodeChar(OP_XMAD, INTP_OP)}, + {"VMNMX", OpcodeChar(OP_VMNMX, INTP_OP)}, + //Conversion Instructions {"F2F", OpcodeChar(OP_F2F, ALU_OP)}, @@ -109,7 +112,8 @@ static const std::unordered_map Pascal_OpcodeMap = { {"R2P", OpcodeChar(OP_R2P, ALU_OP)}, {"CSET", OpcodeChar(OP_CSET, ALU_OP)}, {"CSETP", OpcodeChar(OP_CSETP, ALU_OP)}, - {"PSETP", OpcodeChar(OP_PSETP, ALU_OP)}, + {"PSET", OpcodeChar(OP_PSET, ALU_OP)}, + //Load/Store Instructions {"LD", OpcodeChar(OP_LD, LOAD_OP)}, @@ -157,6 +161,8 @@ static const std::unordered_map Pascal_OpcodeMap = { {"CALL", OpcodeChar(OP_CALL, CALL_OPS)}, {"EXIT", OpcodeChar(OP_EXIT, EXIT_OPS)}, {"JMP", OpcodeChar(OP_JMP, BRANCH_OP)}, + {"SSY", OpcodeChar(OP_SSY, BRANCH_OP)}, + {"SYNC", OpcodeChar(OP_SYNC, BRANCH_OP)}, {"JMX", OpcodeChar(OP_JMX, BRANCH_OP)}, {"KILL", OpcodeChar(OP_KILL, BRANCH_OP)}, {"NANOSLEEP", OpcodeChar(OP_NANOSLEEP, BRANCH_OP)}, diff --git a/src/trace-driven/trace_driven.cc b/src/trace-driven/trace_driven.cc index fb8afdd..7b5c523 100644 --- a/src/trace-driven/trace_driven.cc +++ b/src/trace-driven/trace_driven.cc @@ -23,6 +23,7 @@ #include "volta_opcode.h" #include "turing_opcode.h" #include "pascal_opcode.h" +#include "kepler_opcode.h" #include "../gpgpusim_entrypoint.h" @@ -221,6 +222,10 @@ trace_kernel_info_t::trace_kernel_info_t(dim3 gridDim, dim3 blockDim, unsigned m OpcodeMap = &Volta_OpcodeMap; else if(m_binary_verion == PASCAL_TITANX_BINART_VERSION || m_binary_verion == PASCAL_P100_BINART_VERSION) OpcodeMap = &Pascal_OpcodeMap; + else if(m_binary_verion == KEPLER_BINART_VERSION) + OpcodeMap = &Kepler_OpcodeMap; + else if(m_binary_verion == TURING_BINART_VERSION) + OpcodeMap = &Turing_OpcodeMap; else assert(0 && "unsupported binary version"); } diff --git a/src/trace-driven/trace_opcode.h b/src/trace-driven/trace_opcode.h index 2b40ace..d60d0ae 100644 --- a/src/trace-driven/trace_opcode.h +++ b/src/trace-driven/trace_opcode.h @@ -10,6 +10,7 @@ enum TraceInstrOpcode { + //volta (common insts for others cards as well) OP_FADD = 1, OP_FADD32I, OP_FCHK, OP_FFMA32I, OP_FFMA, OP_FMNMX, OP_FMUL, OP_FMUL32I, OP_FSEL, OP_FSET, OP_FSETP, OP_FSWZADD, OP_MUFU, OP_HADD2, OP_HADD2_32I, OP_HFMA2, OP_HFMA2_32I, OP_HMUL2, OP_HMUL2_32I, OP_HSET2, OP_HSETP2, OP_HMMA, OP_DADD, OP_DFMA, OP_DMUL, OP_DSETP, @@ -23,8 +24,12 @@ enum TraceInstrOpcode { OP_TMML, OP_TXD, OP_TXQ, OP_BMOV, OP_BPT, OP_BRA, OP_BREAK, OP_BRX, OP_BSSY, OP_BSYNC, OP_CALL, OP_EXIT, OP_JMP, OP_JMX, OP_KILL, OP_NANOSLEEP, OP_RET, OP_RPCMOV, OP_RTT, OP_WARPSYNC, OP_YIELD, OP_B2R, OP_BAR, OP_CS2R, OP_CSMTEST, OP_DEPBAR, OP_GETLMEMBASE, OP_LEPC, OP_NOP, OP_PMTRIG, OP_R2B, OP_S2R, OP_SETCTAID, OP_SETLMEMBASE, OP_VOTE, OP_VOTE_VTG, + //unique insts for pascal OP_RRO, OP_DMNMX, OP_DSET, OP_BFE, OP_BFI, OP_ICMP, OP_IMADSP, OP_SHL, OP_XMAD, OP_CSET, OP_CSETP, - OP_TEXS, OP_TLD4S, OP_TLDS, OP_CAL, OP_JCAL, OP_PRET, OP_BRK, OP_PBK, OP_CONT, OP_PCNT, OP_PEXIT, + OP_TEXS, OP_TLD4S, OP_TLDS, OP_CAL, OP_JCAL, OP_PRET, OP_BRK, OP_PBK, OP_CONT, OP_PCNT, OP_PEXIT, OP_SSY, OP_SYNC, OP_PSET + , OP_VMNMX, OP_ISET, + //unique insts for kepler + OP_FCMP, OP_FSWZ, OP_ISAD, OP_LDSLK, OP_STSCUL, OP_SUCLAMP, OP_SUBFM, OP_SUEAU, OP_SULDGA, OP_SUSTGA, SASS_NUM_OPCODES /* The total number of opcodes. */ }; typedef enum TraceInstrOpcode sass_op_type; -- cgit v1.3 From 7d56112add643f9178753750efb97e12e1e5524c Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Sun, 9 Feb 2020 20:59:18 -0500 Subject: adding missing kepler insts --- src/trace-driven/kepler_opcode.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/trace-driven/kepler_opcode.h b/src/trace-driven/kepler_opcode.h index f85346f..0805c12 100644 --- a/src/trace-driven/kepler_opcode.h +++ b/src/trace-driven/kepler_opcode.h @@ -19,9 +19,12 @@ static const std::unordered_map Kepler_OpcodeMap = { //Floating Point 32 Instructions {"FFMA", OpcodeChar(OP_FFMA, SP_OP)}, + {"FFMA32I", OpcodeChar(OP_FFMA32I, SP_OP)}, {"FADD", OpcodeChar(OP_FADD, SP_OP)}, + {"FADD32I", OpcodeChar(OP_FADD32I, SP_OP)}, {"FCMP", OpcodeChar(OP_FCMP, SP_OP)}, {"FMUL", OpcodeChar(OP_FMUL, SP_OP)}, + {"FMUL32I", OpcodeChar(OP_FMUL32I, SP_OP)}, {"FMNMX", OpcodeChar(OP_FMNMX, SP_OP)}, {"FSWZ", OpcodeChar(OP_FSWZ, SP_OP)}, {"FSET", OpcodeChar(OP_FSET, SP_OP)}, @@ -44,8 +47,11 @@ static const std::unordered_map Kepler_OpcodeMap = { {"IMAD", OpcodeChar(OP_IMAD, INTP_OP)}, {"IMADSP", OpcodeChar(OP_IMADSP, INTP_OP)}, {"IMUL", OpcodeChar(OP_IMUL, INTP_OP)}, + {"IMUL32I", OpcodeChar(OP_IMUL32I, INTP_OP)}, {"IADD", OpcodeChar(OP_IADD, INTP_OP)}, + {"IADD32I", OpcodeChar(OP_IADD32I, INTP_OP)}, {"ISCADD", OpcodeChar(OP_ISCADD, INTP_OP)}, + {"ISCADD32I", OpcodeChar(OP_ISCADD32I, INTP_OP)}, {"ISAD", OpcodeChar(OP_ISAD, INTP_OP)}, {"IMNMX", OpcodeChar(OP_IMNMX, INTP_OP)}, {"BFE", OpcodeChar(OP_BFE, INTP_OP)}, @@ -54,6 +60,7 @@ static const std::unordered_map Kepler_OpcodeMap = { {"SHL", OpcodeChar(OP_SHL, INTP_OP)}, {"SHF", OpcodeChar(OP_SHF, INTP_OP)}, {"LOP", OpcodeChar(OP_LOP, INTP_OP)}, + {"LOP32I", OpcodeChar(OP_LOP32I, INTP_OP)}, {"FLO", OpcodeChar(OP_FLO, INTP_OP)}, {"ISET", OpcodeChar(OP_ISET, INTP_OP)}, {"ISETP", OpcodeChar(OP_ISETP, INTP_OP)}, -- cgit v1.3 From 79bc1ca604153be11878b24b3a8d670860011762 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Mon, 10 Feb 2020 19:09:54 -0500 Subject: fixing kepler load in sass --- src/trace-driven/trace_driven.cc | 11 +++++++---- src/trace-driven/trace_driven.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/trace-driven/trace_driven.cc b/src/trace-driven/trace_driven.cc index 7b5c523..35e953e 100644 --- a/src/trace-driven/trace_driven.cc +++ b/src/trace-driven/trace_driven.cc @@ -286,7 +286,7 @@ bool trace_kernel_info_t::get_next_threadblock_traces(std::vectorgetShaderCoreConfig(), m_gpgpu_context); - inst.parse_from_string(line, OpcodeMap); + inst.parse_from_string(line, OpcodeMap, binary_verion); threadblock_traces[warp_id]->push_back(inst); } } @@ -323,7 +323,7 @@ unsigned trace_warp_inst_t::get_datawidth_from_opcode(const std::vector* OpcodeMap){ +bool trace_warp_inst_t::parse_from_string(std::string trace, const std::unordered_map* OpcodeMap, unsigned binary_verion){ std::stringstream ss; ss.str(trace); @@ -546,9 +546,12 @@ bool trace_warp_inst_t::parse_from_string(std::string trace, const std::unordere case OP_LD: //TO DO: set generic load based on the address //right now, we consider all loads are shared. - assert(mem_width>0); + assert(mem_width>0); data_size = get_datawidth_from_opcode(opcode_tokens); - space.set_type(shared_space); + if(binary_verion == KEPLER_BINART_VERSION) + space.set_type(global_space); + else + space.set_type(shared_space); if(m_opcode == OP_LD) memory_op = memory_load; else diff --git a/src/trace-driven/trace_driven.h b/src/trace-driven/trace_driven.h index 33f4baf..9539e6d 100644 --- a/src/trace-driven/trace_driven.h +++ b/src/trace-driven/trace_driven.h @@ -44,7 +44,7 @@ public: m_opcode=0; } - bool parse_from_string(std::string trace, const std::unordered_map* OpcodeMap); + bool parse_from_string(std::string trace, const std::unordered_map* OpcodeMap, unsigned binary_verion); private: void set_latency(unsigned cat); -- cgit v1.3 From 221cbdadf6b54cd75aaecd89d93a9c6f9f8c7dad Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Mon, 10 Feb 2020 19:18:14 -0500 Subject: fixing the kepler stats --- src/trace-driven/kepler_opcode.h | 1 + src/trace-driven/trace_opcode.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/trace-driven/kepler_opcode.h b/src/trace-driven/kepler_opcode.h index 0805c12..f2bbc90 100644 --- a/src/trace-driven/kepler_opcode.h +++ b/src/trace-driven/kepler_opcode.h @@ -50,6 +50,7 @@ static const std::unordered_map Kepler_OpcodeMap = { {"IMUL32I", OpcodeChar(OP_IMUL32I, INTP_OP)}, {"IADD", OpcodeChar(OP_IADD, INTP_OP)}, {"IADD32I", OpcodeChar(OP_IADD32I, INTP_OP)}, + {"ISUB", OpcodeChar(OP_ISUB, INTP_OP)}, {"ISCADD", OpcodeChar(OP_ISCADD, INTP_OP)}, {"ISCADD32I", OpcodeChar(OP_ISCADD32I, INTP_OP)}, {"ISAD", OpcodeChar(OP_ISAD, INTP_OP)}, diff --git a/src/trace-driven/trace_opcode.h b/src/trace-driven/trace_opcode.h index d60d0ae..3492fd3 100644 --- a/src/trace-driven/trace_opcode.h +++ b/src/trace-driven/trace_opcode.h @@ -29,7 +29,7 @@ enum TraceInstrOpcode { OP_TEXS, OP_TLD4S, OP_TLDS, OP_CAL, OP_JCAL, OP_PRET, OP_BRK, OP_PBK, OP_CONT, OP_PCNT, OP_PEXIT, OP_SSY, OP_SYNC, OP_PSET , OP_VMNMX, OP_ISET, //unique insts for kepler - OP_FCMP, OP_FSWZ, OP_ISAD, OP_LDSLK, OP_STSCUL, OP_SUCLAMP, OP_SUBFM, OP_SUEAU, OP_SULDGA, OP_SUSTGA, + OP_FCMP, OP_FSWZ, OP_ISAD, OP_LDSLK, OP_STSCUL, OP_SUCLAMP, OP_SUBFM, OP_SUEAU, OP_SULDGA, OP_SUSTGA, OP_ISUB, SASS_NUM_OPCODES /* The total number of opcodes. */ }; typedef enum TraceInstrOpcode sass_op_type; -- cgit v1.3 From 1d3ccd32e70c6a6c524abbe8ff0ac450481290d4 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Mon, 10 Feb 2020 21:28:09 -0500 Subject: removing assertion in random hashing --- src/gpgpu-sim/addrdec.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc index 670bd61..c34cb32 100644 --- a/src/gpgpu-sim/addrdec.cc +++ b/src/gpgpu-sim/addrdec.cc @@ -221,7 +221,7 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_ { //This is an unrealistic hashing using software hashtable //we generate a random set for each memory address and save the value in a big hashtable for future reuse - assert(!gap); + //assert(!gap); new_addr_type chip_address = (addr>>(ADDR_CHIP_S-log2sub_partition)); tr1_hash_map::const_iterator got = address_random_interleaving.find (chip_address); if ( got == address_random_interleaving.end() ) { -- cgit v1.3 From 5221ed98c5a7f0ceec3eb96216f4449635a74b22 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Thu, 13 Feb 2020 12:32:39 -0500 Subject: fixing pascal --- src/gpgpu-sim/shader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 23050d3..7fc31f3 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -106,7 +106,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } if(m_config->sub_core_model) { //in subcore model, each scheduler should has its own issue register, so num scheduler = reg width - assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); + //assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SFU].get_size() ); assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_MEM].get_size() ); if(m_config->gpgpu_tensor_core_avail) -- cgit v1.3 From 024cb02ecadc58c401b9cbeb558774c6ba7415aa Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Thu, 26 Mar 2020 14:11:10 -0400 Subject: fixing core fetch bw --- src/gpgpu-sim/shader.cc | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 7fc31f3..e206d9c 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -889,7 +889,7 @@ void shader_core_ctx::fetch() assert( status == RESERVATION_FAIL ); delete mf; } - break; + //break; } } } @@ -1067,7 +1067,18 @@ void scheduler_unit::cycle() exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; //In tis mode, we only allow dual issue to diff execution units (as in Maxwell and Pascal) - + + if(warp(warp_id).ibuffer_empty()) + SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails as ibuffer_empty\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); + + if(warp(warp_id).waiting()) + SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails as waiting for barrier\n", + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); + + if((*iter)->get_warp_id() ==2 ) + printf(" Hello! I am here! \n "); + while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); //Jin: handle cdp latency; @@ -1120,13 +1131,13 @@ void scheduler_unit::cycle() } } else { - bool sp_pipe_avail = m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool sfu_pipe_avail = m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool tensor_core_pipe_avail = m_tensor_core_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool dp_pipe_avail = m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id); - bool int_pipe_avail = m_int_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool sp_pipe_avail = (m_shader->m_config->gpgpu_num_sp_units > 0) && m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool sfu_pipe_avail = (m_shader->m_config->gpgpu_num_sfu_units > 0) && m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool tensor_core_pipe_avail = (m_shader->m_config->gpgpu_num_tensor_core_units > 0) && m_tensor_core_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool dp_pipe_avail = (m_shader->m_config->gpgpu_num_dp_units > 0) && m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool int_pipe_avail = (m_shader->m_config->gpgpu_num_int_units > 0) && m_int_out->has_free(m_shader->m_config->sub_core_model, m_id); - //This code need to be refactored + //This code needs to be refactored if(pI->op != TENSOR_CORE_OP && pI->op != SFU_OP && pI->op != DP_OP) { bool execute_on_SP = false; @@ -1196,7 +1207,7 @@ void scheduler_unit::cycle() previous_issued_inst_exec_type = exec_unit_type_t::SFU; } } - else if ( (pI->op == TENSOR_CORE_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) { + else if ( (pI->op == TENSOR_CORE_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::TENSOR) ) { if( tensor_core_pipe_avail ) { m_shader->issue_warp(*m_tensor_core_out,pI,active_mask,warp_id,m_id); issued++; -- cgit v1.3 From 38a5c4d70539cdacf1ce20394027d9b705ff9c76 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Thu, 26 Mar 2020 18:18:17 -0400 Subject: increase inst fetch thoughput --- src/gpgpu-sim/shader.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index e206d9c..a2850d0 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -106,7 +106,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } if(m_config->sub_core_model) { //in subcore model, each scheduler should has its own issue register, so num scheduler = reg width - //assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SFU].get_size() ); assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_MEM].get_size() ); if(m_config->gpgpu_tensor_core_avail) @@ -889,7 +889,7 @@ void shader_core_ctx::fetch() assert( status == RESERVATION_FAIL ); delete mf; } - //break; + break; } } } @@ -1074,10 +1074,7 @@ void scheduler_unit::cycle() if(warp(warp_id).waiting()) SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails as waiting for barrier\n", - (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - - if((*iter)->get_warp_id() ==2 ) - printf(" Hello! I am here! \n "); + (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); @@ -3178,8 +3175,11 @@ void shader_core_ctx::cycle() execute(); read_operands(); issue(); + for(int i=0; i<8; ++i) { decode(); fetch(); + printf("Hello! \n"); + } } // Flushes all content of the cache to memory -- cgit v1.3 From ed97d2a5b0096fb8afa4bc83fe2df515b2a2399c Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Tue, 31 Mar 2020 13:00:34 -0400 Subject: adding inst fetch throughput --- src/gpgpu-sim/gpu-sim.cc | 4 +++- src/gpgpu-sim/shader.cc | 9 +++++---- src/gpgpu-sim/shader.h | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 4e38f67..4ef9a7d 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -470,7 +470,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-perfect_inst_const_cache", OPT_BOOL, &perfect_inst_const_cache, "perfect inst and const cache mode, so all inst and const hits in the cache(default = disabled)", "0"); - + option_parser_register(opp, "-inst_fetch_throughput", OPT_INT32, &inst_fetch_throughput, + "the number of fetched intruction per warp each cycle", + "1"); } void gpgpu_sim_config::reg_options(option_parser_t opp) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index a2850d0..297f3bc 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3175,10 +3175,9 @@ void shader_core_ctx::cycle() execute(); read_operands(); issue(); - for(int i=0; i<8; ++i) { - decode(); - fetch(); - printf("Hello! \n"); + for(int i=0; i< m_config->inst_fetch_throughput; ++i) { + decode(); + fetch(); } } @@ -3235,6 +3234,7 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() ///// wavefront allocator from booksim... ---> // Loop through diagonals of request matrix + printf("####\n"); for ( int p = 0; p < _square; ++p ) { output = ( _pri + p ) % _square; @@ -3250,6 +3250,7 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() // Grant! _inmatch[input] = output; _outmatch[output] = input; + printf("Register File: granting banks %d to OC % \n", input, output); } output = ( output + 1 ) % _square; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index d41c220..43c448b 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1524,6 +1524,7 @@ class shader_core_config : public core_config bool gpgpu_concurrent_kernel_sm; bool perfect_inst_const_cache; + unsigned inst_fetch_throughput; }; -- cgit v1.3 From eb618e6bbe0bac135ba800a5880f18040f35acf8 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Tue, 31 Mar 2020 13:33:42 -0400 Subject: adding some RF stats --- src/gpgpu-sim/shader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 297f3bc..afd63e8 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3250,7 +3250,7 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() // Grant! _inmatch[input] = output; _outmatch[output] = input; - printf("Register File: granting banks %d to OC % \n", input, output); + printf("Register File: granting bank %d to OC %d, schedid %d, warpid %d, Regid %d\n", input, output, (m_queue[output].front()).get_sid(), (m_queue[output].front()).get_wid(), (m_queue[output].front()).get_reg()); } output = ( output + 1 ) % _square; -- cgit v1.3 From cc5d6636df7388e44c5203e5580a2aa50c0fd6b1 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Tue, 31 Mar 2020 16:38:14 -0400 Subject: fixing the RF assertion --- src/gpgpu-sim/shader.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index afd63e8..d9f7b94 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3237,7 +3237,7 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() printf("####\n"); for ( int p = 0; p < _square; ++p ) { - output = ( _pri + p ) % _square; + output = ( _pri + p ) % _outputs; // Step through the current diagonal for ( input = 0; input < _inputs; ++input ) { @@ -3250,15 +3250,15 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() // Grant! _inmatch[input] = output; _outmatch[output] = input; - printf("Register File: granting bank %d to OC %d, schedid %d, warpid %d, Regid %d\n", input, output, (m_queue[output].front()).get_sid(), (m_queue[output].front()).get_wid(), (m_queue[output].front()).get_reg()); + printf("Register File: granting bank %d to OC %d, schedid %d, warpid %d, Regid %d\n", input, output, (m_queue[input].front()).get_sid(), (m_queue[input].front()).get_wid(), (m_queue[input].front()).get_reg()); } - output = ( output + 1 ) % _square; + output = ( output + 1 ) % _outputs; } } // Round-robin the priority diagonal - _pri = ( _pri + 1 ) % _square; + _pri = ( _pri + 1 ) % _outputs; /// <--- end code from booksim -- cgit v1.3 From 3633eacca183608fe1a666212ba468b787c063a5 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Tue, 31 Mar 2020 20:43:41 -0400 Subject: adding new comments --- src/gpgpu-sim/gpu-sim.cc | 3 +++ src/gpgpu-sim/shader.cc | 13 +++++++------ src/gpgpu-sim/shader.h | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 4ef9a7d..94ea6d2 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -473,6 +473,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-inst_fetch_throughput", OPT_INT32, &inst_fetch_throughput, "the number of fetched intruction per warp each cycle", "1"); + option_parser_register(opp, "-gpgpu_reg_file_port_throughput", OPT_INT32, ®_file_port_throughput, + "the number ports of the register file", + "1"); } void gpgpu_sim_config::reg_options(option_parser_t opp) diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index d9f7b94..ed0c25e 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -384,12 +384,12 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_fu.push_back(new dp_unit( &m_pipeline_reg[EX_WB], m_config, this )); m_dispatch_port.push_back(ID_OC_DP); m_issue_port.push_back(OC_EX_DP); - } + } for (int k = 0; k < m_config->gpgpu_num_int_units; k++) { m_fu.push_back(new int_unit( &m_pipeline_reg[EX_WB], m_config, this )); m_dispatch_port.push_back(ID_OC_INT); m_issue_port.push_back(OC_EX_INT); - } + } for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this )); @@ -2413,7 +2413,8 @@ void ldst_unit::issue( register_set ®_set ) void ldst_unit::cycle() { writeback(); - m_operand_collector->step(); + for(int i=0; i< m_config->reg_file_port_throughput; ++i) + m_operand_collector->step(); for( unsigned stage=0; (stage+1)empty() && !m_pipeline_reg[stage+1]->empty() ) move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage+1]); @@ -3234,7 +3235,7 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() ///// wavefront allocator from booksim... ---> // Loop through diagonals of request matrix - printf("####\n"); + // printf("####\n"); for ( int p = 0; p < _square; ++p ) { output = ( _pri + p ) % _outputs; @@ -3245,12 +3246,12 @@ std::list opndcoll_rfu_t::arbiter_t::allocate_reads() assert( output < _outputs ); if ( ( output < _outputs ) && ( _inmatch[input] == -1 ) && - ( _outmatch[output] == -1 ) && + //( _outmatch[output] == -1 ) && //allow OC to read multiple reg banks at the same cycle ( _request[input][output]/*.label != -1*/ ) ) { // Grant! _inmatch[input] = output; _outmatch[output] = input; - printf("Register File: granting bank %d to OC %d, schedid %d, warpid %d, Regid %d\n", input, output, (m_queue[input].front()).get_sid(), (m_queue[input].front()).get_wid(), (m_queue[input].front()).get_reg()); + // printf("Register File: granting bank %d to OC %d, schedid %d, warpid %d, Regid %d\n", input, output, (m_queue[input].front()).get_sid(), (m_queue[input].front()).get_wid(), (m_queue[input].front()).get_reg()); } output = ( output + 1 ) % _outputs; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 43c448b..665e3a5 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1525,6 +1525,7 @@ class shader_core_config : public core_config bool perfect_inst_const_cache; unsigned inst_fetch_throughput; + unsigned reg_file_port_throughput; }; -- cgit v1.3 From bd647361e2b8eb2d72f0055df851fcdee53e94cc Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Tue, 31 Mar 2020 20:54:44 -0400 Subject: fixing the tensor unit failing --- src/gpgpu-sim/gpu-sim.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 94ea6d2..e5b9c9d 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -453,7 +453,7 @@ void shader_core_config::reg_options(class OptionParser * opp) "1"); option_parser_register(opp, "-gpgpu_num_tensor_core_units", OPT_INT32, &gpgpu_num_tensor_core_units, "Number of tensor_core units (default=1)", - "1"); + "0"); option_parser_register(opp, "-gpgpu_num_mem_units", OPT_INT32, &gpgpu_num_mem_units, "Number if ldst units (default=1) WARNING: not hooked up to anything", "1"); -- cgit v1.3 From f9e65c53a29b89c167e713afa6d5eb92709047af Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Thu, 2 Apr 2020 21:23:07 -0400 Subject: updating config files --- .../tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config | 79 +++++---- configs/tested-cfgs/SM6_TITANX/gpgpusim.config | 88 +++++----- .../SM75_RTX2060/config_turing_islip.icnt | 73 ++++++++ configs/tested-cfgs/SM75_RTX2060/gpgpusim.config | 187 +++++++++++++++++++++ configs/tested-cfgs/SM7_QV100/gpgpusim.config | 55 +++--- configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config | 71 ++++---- .../Turing_RTX2060/config_pascal_islip.icnt | 73 -------- configs/tested-cfgs/Turing_RTX2060/gpgpusim.config | 186 -------------------- 8 files changed, 397 insertions(+), 415 deletions(-) create mode 100644 configs/tested-cfgs/SM75_RTX2060/config_turing_islip.icnt create mode 100644 configs/tested-cfgs/SM75_RTX2060/gpgpusim.config delete mode 100644 configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt delete mode 100644 configs/tested-cfgs/Turing_RTX2060/gpgpusim.config diff --git a/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config index 323d2d9..1951a3f 100644 --- a/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config +++ b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config @@ -18,12 +18,13 @@ -gpgpu_compute_capability_major 3 -gpgpu_compute_capability_minor 5 -# SASS execution (only supported with CUDA >= 4.0) +# PTX execution-driven -gpgpu_ptx_convert_to_ptxplus 0 -gpgpu_ptx_save_converted_ptxplus 0 +# SASS trace-driven mode execution +#-trace_driven_mode 1 # high level architecture configuration -# P102 has two semi-indp scheds per core, and two cores per cluster -gpgpu_n_clusters 14 -gpgpu_n_cores_per_cluster 1 -gpgpu_n_mem 12 @@ -46,15 +47,13 @@ # Pipeline widths and number of FUs # ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB -## Kepler has 6 SP SIMD units and 2 SFU units per SM. -# There is no int unit in kepler -## we need to scale the number of pipeline registers to be equal to the number of SP units +## Kepler has 6 SP SIMD units, 4 DPs and 2 SFU units per SM. +# There is no INT unit in kepler -gpgpu_pipeline_widths 6,4,0,2,1,6,4,0,2,1,12 -gpgpu_num_sp_units 6 -gpgpu_num_sfu_units 2 -gpgpu_num_dp_units 4 - # Instruction latencies and initiation intervals # "ADD,MAX,MUL,MAD,DIV" # All Div operations are executed on SFU unit @@ -68,6 +67,38 @@ -ptx_opcode_initiation_sfu 2 -ptx_opcode_latency_sfu 200 +# enable operand collector +-gpgpu_operand_collector_num_units_sp 12 +-gpgpu_operand_collector_num_units_sfu 6 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_units_dp 6 +-gpgpu_operand_collector_num_in_ports_sp 2 +-gpgpu_operand_collector_num_out_ports_sp 2 +-gpgpu_operand_collector_num_in_ports_sfu 2 +-gpgpu_operand_collector_num_out_ports_sfu 2 +-gpgpu_operand_collector_num_in_ports_mem 1 +-gpgpu_operand_collector_num_out_ports_mem 1 +-gpgpu_operand_collector_num_in_ports_dp 1 +-gpgpu_operand_collector_num_out_ports_dp 1 +-gpgpu_num_reg_banks 16 +-gpgpu_reg_file_port_throughput 2 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +# Use kepler Coalsce arhitetecture +-gpgpu_coalesce_arch 35 + +## In Kepler, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 +-gpgpu_dual_issue_diff_exec_units 0 + +# Kepler TITAN has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Greedy then oldest scheduler +-gpgpu_scheduler gto + # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo # Note: Hashing set index function (H) only applies to a set size of 32 or 64. @@ -97,37 +128,12 @@ # 4 KB Inst. -gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 +-inst_fetch_throughput 8 # 48 KB Tex -gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 # 12 KB Const -gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 -# enable operand collector --gpgpu_operand_collector_num_units_sp 12 --gpgpu_operand_collector_num_units_sfu 6 --gpgpu_operand_collector_num_units_mem 8 --gpgpu_operand_collector_num_units_dp 6 --gpgpu_operand_collector_num_in_ports_sp 2 --gpgpu_operand_collector_num_out_ports_sp 2 --gpgpu_operand_collector_num_in_ports_sfu 2 --gpgpu_operand_collector_num_out_ports_sfu 2 --gpgpu_operand_collector_num_in_ports_mem 1 --gpgpu_operand_collector_num_out_ports_mem 1 --gpgpu_operand_collector_num_in_ports_dp 1 --gpgpu_operand_collector_num_out_ports_dp 1 --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 -# Use kepler Coalsce arhitetecture --gpgpu_coalesce_arch 35 - -## In Kepler, a warp scheduler can issue 2 insts per cycle --gpgpu_max_insn_issue_per_warp 2 --gpgpu_dual_issue_diff_exec_units 0 - # interconnection -network_mode 1 -inter_config_file config_kepler_islip.icnt @@ -161,15 +167,6 @@ #-Seperate_Write_Queue_Enable 1 #-Write_Queue_Size 64:56:32 -# Kepler TITAN has four schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - # stat collection -gpgpu_memlatency_stat 14 -gpgpu_runtime_stat 500 diff --git a/configs/tested-cfgs/SM6_TITANX/gpgpusim.config b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config index f8689c2..d54b7d4 100644 --- a/configs/tested-cfgs/SM6_TITANX/gpgpusim.config +++ b/configs/tested-cfgs/SM6_TITANX/gpgpusim.config @@ -13,6 +13,7 @@ -gpgpu_heap_size_limit 8388608 -gpgpu_runtime_sync_depth_limit 2 -gpgpu_runtime_pending_launch_count_limit 2048 +-gpgpu_kernel_launch_latency 5000 # Compute Capability -gpgpu_compute_capability_major 6 @@ -23,7 +24,6 @@ -gpgpu_ptx_save_converted_ptxplus 0 # high level architecture configuration -# P102 has two semi-indp scheds per core, and two cores per cluster -gpgpu_n_clusters 28 -gpgpu_n_cores_per_cluster 1 -gpgpu_n_mem 12 @@ -46,35 +46,58 @@ # Pipeline widths and number of FUs # ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB -## Pascal GP102 has 4 SP SIMD units and 4 SFU units per SM. In this config, we split SM into two shader cores, each has 2 SPs and 2 SFUs -# There is no int unit in Pascal -## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 4,1,0,4,1,4,1,0,4,1,9 +## Pascal GP102 has 4 SP SIMD units and 4 SFU units per SM. +# There is no INT unit in Pascal +-gpgpu_pipeline_widths 4,0,0,4,4,4,0,0,4,4,8 -gpgpu_num_sp_units 4 -gpgpu_num_sfu_units 4 --gpgpu_num_dp_units 1 - # Instruction latencies and initiation intervals # "ADD,MAX,MUL,MAD,DIV" # All Div operations are executed on SFU unit -# Throughput (initiation latency) are adopted from CUDA SDK document V8, section 5.4.1, Table 2 --ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_latency_int 4,13,4,4,145 -ptx_opcode_initiation_int 1,1,1,1,4 --ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_latency_fp 4,13,4,4,39 -ptx_opcode_initiation_fp 1,2,1,1,4 -ptx_opcode_latency_dp 8,19,8,8,330 -ptx_opcode_initiation_dp 8,8,8,8,130 -ptx_opcode_initiation_sfu 4 -ptx_opcode_latency_sfu 8 +# in sub_core_model, schedulers are isolated, each scheduler has its own register file and EUs +-sub_core_model 1 +# enable operand collector +# disable specialized operand collectors and use generic operand collectors instead +-enable_specialized_operand_collector 0 +-gpgpu_operand_collector_num_units_gen 8 +-gpgpu_operand_collector_num_in_ports_gen 8 +-gpgpu_operand_collector_num_out_ports_gen 8 +# 16 register banks, 4 banks per subcore +-gpgpu_num_reg_banks 16 +-gpgpu_reg_file_port_throughput 2 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +# Use Pascal Coalsce arhitetecture +-gpgpu_coalesce_arch 61 + +# Pascal 102 has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units +-gpgpu_max_insn_issue_per_warp 2 +-gpgpu_dual_issue_diff_exec_units 1 -# latencies and cache configs are adopted from: -# https://arxiv.org/pdf/1804.06826.pdf +## L1/shared memory configuration # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo # Note: Hashing set index function (H) only applies to a set size of 32 or 64. # The defulat is to disable the L1 cache, unless cache modifieres are used +-l1_banks 2 -gpgpu_cache:dl1 S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 -gpgpu_cache:dl1PrefL1 S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 -gpgpu_cache:dl1PrefShared S:4:128:96,L:L:s:N:L,A:256:8,16:0,32 @@ -92,45 +115,21 @@ -gpgpu_flush_l1_cache 1 # 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache --gpgpu_cache:dl2 S:64:128:16,L:B:m:L:L,A:256:64,16:0,32 +-gpgpu_cache:dl2 S:1:128:1024,L:B:m:L:L,A:256:64,16:0,32 -gpgpu_cache:dl2_texture_only 0 -gpgpu_dram_partition_queues 32:32:32:32 -perf_sim_memcpy 1 --memory_partition_indexing 0 +-memory_partition_indexing 4 # 4 KB Inst. -gpgpu_cache:il1 N:8:128:4,L:R:f:N:L,S:2:48,4 +-inst_fetch_throughput 8 # 48 KB Tex # Note, TEX is deprected in Pascal, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod -gpgpu_tex_cache:l1 N:16:128:24,L:R:m:N:L,T:128:4,128:2 # 12 KB Const -gpgpu_const_cache:l1 N:128:64:2,L:R:f:N:L,S:2:64,4 - -# enable operand collector --gpgpu_operand_collector_num_units_sp 12 --gpgpu_operand_collector_num_units_sfu 6 --gpgpu_operand_collector_num_units_mem 8 --gpgpu_operand_collector_num_units_dp 6 --gpgpu_operand_collector_num_in_ports_sp 2 --gpgpu_operand_collector_num_out_ports_sp 2 --gpgpu_operand_collector_num_in_ports_sfu 2 --gpgpu_operand_collector_num_out_ports_sfu 2 --gpgpu_operand_collector_num_in_ports_mem 1 --gpgpu_operand_collector_num_out_ports_mem 1 --gpgpu_operand_collector_num_in_ports_dp 1 --gpgpu_operand_collector_num_out_ports_dp 1 --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 -# Use Pascal Coalsce arhitetecture --gpgpu_coalesce_arch 61 - -## In Pascal, a warp scheduler can issue 2 insts per cycle using 2 diff execution units --gpgpu_max_insn_issue_per_warp 2 --gpgpu_dual_issue_diff_exec_units 1 +-perfect_inst_const_cache 1 # interconnection -network_mode 1 @@ -165,15 +164,6 @@ #-Seperate_Write_Queue_Enable 1 #-Write_Queue_Size 64:56:32 -# Pascal 102 has four schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - # stat collection -gpgpu_memlatency_stat 14 -gpgpu_runtime_stat 500 diff --git a/configs/tested-cfgs/SM75_RTX2060/config_turing_islip.icnt b/configs/tested-cfgs/SM75_RTX2060/config_turing_islip.icnt new file mode 100644 index 0000000..eed1c34 --- /dev/null +++ b/configs/tested-cfgs/SM75_RTX2060/config_turing_islip.icnt @@ -0,0 +1,73 @@ +//52*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 40; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 52; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 64; +input_buffer_size = 256; +ejection_buffer_size = 64; +boundary_buffer_size = 64; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 1; +output_speedup = 1; +internal_speedup = 2.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config new file mode 100644 index 0000000..9d3992a --- /dev/null +++ b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config @@ -0,0 +1,187 @@ +# This config models the Turing RTX 2060 +# For more info about turing architecture: +# https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/technologies/turing-architecture/NVIDIA-Turing-Architecture-Whitepaper.pdf +# "RTX on—The NVIDIA Turing GPU", IEEE MICRO 2020 + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# Device Limits +-gpgpu_stack_size_limit 1024 +-gpgpu_heap_size_limit 8388608 +-gpgpu_runtime_sync_depth_limit 2 +-gpgpu_runtime_pending_launch_count_limit 2048 +-gpgpu_kernel_launch_latency 5000 + +# Compute Capability +-gpgpu_compute_capability_major 7 +-gpgpu_compute_capability_minor 5 + +# PTX execution-driven +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 +# SASS trace-driven mode execution +#-trace_driven_mode 1 + +# high level architecture configuration +-gpgpu_n_clusters 30 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 12 +-gpgpu_n_sub_partition_per_mchannel 2 + +# volta clock domains +#-gpgpu_clock_domains ::: +-gpgpu_clock_domains 1365.0:1365.0:1365.0:3500.0 +# boost mode +# -gpgpu_clock_domains 1680.0:1680.0:1680.0:3500.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 +-gpgpu_registers_per_block 65536 +-gpgpu_occupancy_sm_number 75 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE +## Turing has 4 SP SIMD units, 4 INT units, 4 SFU units, 8 Tensor core units +## We need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,0,4,4,4,4,0,4,4,4,8,4,4 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 4 +-gpgpu_num_int_units 4 +-gpgpu_tensor_core_avail 1 +-gpgpu_num_tensor_core_units 4 + +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# All Div operations are executed on SFU unit +-ptx_opcode_latency_int 4,13,4,5,145 +-ptx_opcode_initiation_int 2,2,2,2,8 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 2,2,2,2,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 4,4,4,4,130 +-ptx_opcode_latency_sfu 100 +-ptx_opcode_initiation_sfu 8 +-ptx_opcode_latency_tesnor 64 +-ptx_opcode_initiation_tensor 64 + +# Trung has sub core model, in which each scheduler has its own register file and EUs +# i.e. schedulers are isolated +-sub_core_model 1 +# disable specialized operand collectors and use generic operand collectors instead +-enable_specialized_operand_collector 0 +-gpgpu_operand_collector_num_units_gen 8 +-gpgpu_operand_collector_num_in_ports_gen 8 +-gpgpu_operand_collector_num_out_ports_gen 8 +# turing has 8 banks dual-port, 4 schedulers, two banks per scheduler +# we increase #banks to 16 to mitigate the effect of Regisrer File Cache (RFC) which we do not implement in the current version +-gpgpu_num_reg_banks 16 +-gpgpu_reg_file_port_throughput 2 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 60 + +## In Turing, a warp scheduler can issue 1 inst per cycle +-gpgpu_max_insn_issue_per_warp 1 +-gpgpu_dual_issue_diff_exec_units 1 + +# Turing has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# ::,::::,::,:** +# ** Optional parameter - Required when mshr_type==Texture Fifo +-adaptive_cache_config 0 +-l1_banks 4 +-gpgpu_cache:dl1 S:1:128:512,L:L:s:N:L,A:256:8,16:0,32 +-gpgpu_shmem_size 65536 +-gpgpu_shmem_sizeDefault 65536 +-gpgpu_shmem_per_block 65536 +-gmem_skip_L1D 0 +-icnt_flit_size 40 +-gpgpu_n_cluster_ejection_buffer_size 32 +-l1_latency 20 +-smem_latency 20 +-gpgpu_flush_l1_cache 1 + +# 32 sets, each 128 bytes 32-way for each memory sub partition (96 KB per memory sub partition). This gives us 6MB L2 cache +-gpgpu_cache:dl2 S:32:128:32,L:B:m:L:L,A:192:4,32:0,32 +-gpgpu_cache:dl2_texture_only 0 +-gpgpu_dram_partition_queues 64:64:64:64 +-perf_sim_memcpy 1 +-memory_partition_indexing 0 + +# 128 KB Inst. +-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 +-inst_fetch_throughput 4 +# 128 KB Tex +# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod +-gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2 +# 64 KB Const +-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 +-perfect_inst_const_cache 1 + +# interconnection +#-network_mode 1 +#-inter_config_file config_turing_islip.icnt +# use built-in local xbar +-network_mode 2 +-inct_in_buffer_limit 512 +-inct_out_buffer_limit 512 +-inct_subnets 2 +-arbiter_algo 1 + +# memory partition latency config +-rop_latency 160 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 192 + +# Turing has GDDR6 +# http://monitorinsider.com/GDDR6.html +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 2 +-gpgpu_dram_burst_length 16 +-dram_data_command_freq_ratio 4 # GDDR6 is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing, scaled to 3500MHZ +-gpgpu_dram_timing_opt "nbk=16:CCD=4:RRD=10:RCD=20:RAS=50:RP=20:RC=62: + CL=20:WL=8:CDLR=9:WR=20:nbkgrp=4:CCDL=4:RTPL=4" + +# select lower bits for bnkgrp to increase bnkgrp parallelism +-dram_bnk_indexing_policy 0 +-dram_bnkgrp_indexing_policy 1 + +#-Seperate_Write_Queue_Enable 1 +#-Write_Queue_Size 64:56:32 + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs, disable it untill we create a real energy model for Volta +-power_simulation_enabled 0 + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/configs/tested-cfgs/SM7_QV100/gpgpusim.config b/configs/tested-cfgs/SM7_QV100/gpgpusim.config index 41aea78..1ed4fb2 100644 --- a/configs/tested-cfgs/SM7_QV100/gpgpusim.config +++ b/configs/tested-cfgs/SM7_QV100/gpgpusim.config @@ -25,7 +25,7 @@ -gpgpu_compute_capability_major 7 -gpgpu_compute_capability_minor 0 -# SASS execution (only supported with CUDA >= 4.0) +# PTX execution-driven -gpgpu_ptx_convert_to_ptxplus 0 -gpgpu_ptx_save_converted_ptxplus 0 @@ -37,8 +37,6 @@ # volta clock domains #-gpgpu_clock_domains ::: -# Volta NVIDIA TITANV clock domains are adopted from -# https://en.wikipedia.org/wiki/Volta_(microarchitecture) -gpgpu_clock_domains 1132.0:1132.0:1132.0:850.0 # boost mode # -gpgpu_clock_domains 1628.0:1628.0:1628.0:850.0 @@ -68,8 +66,6 @@ # Instruction latencies and initiation intervals # "ADD,MAX,MUL,MAD,DIV" # All Div operations are executed on SFU unit -# Throughput (initiation latency) are adopted from -# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf -ptx_opcode_latency_int 4,13,4,5,145 -ptx_opcode_initiation_int 2,2,2,2,8 -ptx_opcode_latency_fp 4,13,4,5,39 @@ -81,6 +77,30 @@ -ptx_opcode_latency_tesnor 64 -ptx_opcode_initiation_tensor 64 +# Volta has sub core model, in which each scheduler has its own register file and EUs +# i.e. schedulers are isolated +-sub_core_model 1 +# disable specialized operand collectors and use generic operand collectors instead +-enable_specialized_operand_collector 0 +-gpgpu_operand_collector_num_units_gen 8 +-gpgpu_operand_collector_num_in_ports_gen 8 +-gpgpu_operand_collector_num_out_ports_gen 8 +# volta has 8 banks, 4 schedulers, two banks per scheduler +# we increase #banks to 16 to mitigate the effect of Regisrer File Cache (RFC) which we do not implement in the current version +-gpgpu_num_reg_banks 16 +-gpgpu_reg_file_port_throughput 2 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 60 + +## In Volta, a warp scheduler can issue 1 inst per cycle +-gpgpu_max_insn_issue_per_warp 1 +-gpgpu_dual_issue_diff_exec_units 1 + +## L1/shared memory configuration # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo # Defualt config is 32KB DL1 and 96KB shared memory @@ -91,7 +111,6 @@ -adaptive_cache_config 1 # Volta unified cache has four banks -l1_banks 4 -#-mem_unit_ports 4 -gpgpu_cache:dl1 S:1:128:256,L:L:s:N:L,A:256:8,16:0,32 -gpgpu_shmem_size 98304 -gpgpu_shmem_sizeDefault 98304 @@ -112,6 +131,7 @@ # 128 KB Inst. -gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 +-inst_fetch_throughput 4 # 128 KB Tex # Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod -gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2 @@ -119,31 +139,10 @@ -gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 -perfect_inst_const_cache 1 -# Volta has sub core model, in which each scheduler has its own register file and EUs -# i.e. schedulers are isolated --sub_core_model 1 -# disable specialized operand collectors and use generic operand collectors instead --enable_specialized_operand_collector 0 --gpgpu_operand_collector_num_units_gen 32 --gpgpu_operand_collector_num_in_ports_gen 8 --gpgpu_operand_collector_num_out_ports_gen 8 -# volta has 8 banks, 4 schedulers, two banks per scheduler --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 --gpgpu_coalesce_arch 60 - -## In Volta, a warp scheduler can issue 1 inst per cycle --gpgpu_max_insn_issue_per_warp 1 --gpgpu_dual_issue_diff_exec_units 1 - # interconnection #-network_mode 1 #-inter_config_file config_volta_islip.icnt -# for local xbar, use: +# use built-in local xbar -network_mode 2 -inct_in_buffer_limit 512 -inct_out_buffer_limit 512 diff --git a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config index ba50287..0255f76 100644 --- a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config +++ b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config @@ -12,7 +12,6 @@ -gpgpu_ptx_instruction_classification 0 -gpgpu_ptx_sim_mode 0 -gpgpu_ptx_force_max_capability 70 --trace_driven_mode 1 # Device Limits -gpgpu_stack_size_limit 1024 @@ -25,7 +24,8 @@ -gpgpu_compute_capability_major 7 -gpgpu_compute_capability_minor 0 -# SASS execution (only supported with CUDA >= 4.0) +# SASS trace-driven mode support +-trace_driven_mode 1 -gpgpu_ptx_convert_to_ptxplus 0 -gpgpu_ptx_save_converted_ptxplus 0 @@ -37,8 +37,6 @@ # volta clock domains #-gpgpu_clock_domains ::: -# Volta NVIDIA TITANV clock domains are adopted from -# https://en.wikipedia.org/wiki/Volta_(microarchitecture) -gpgpu_clock_domains 1132.0:1132.0:1132.0:850.0 # boost mode # -gpgpu_clock_domains 1628.0:1628.0:1628.0:850.0 @@ -68,8 +66,6 @@ # Instruction latencies and initiation intervals # "ADD,MAX,MUL,MAD,DIV" # All Div operations are executed on SFU unit -# Throughput (initiation latency) are adopted from -# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf -ptx_opcode_latency_int 4,13,4,5,145 -ptx_opcode_initiation_int 2,2,2,2,8 -ptx_opcode_latency_fp 4,13,4,5,39 @@ -81,6 +77,35 @@ -ptx_opcode_latency_tesnor 6 -ptx_opcode_initiation_tensor 2 +# Volta has sub core model, in which each scheduler has its own register file and EUs +# i.e. schedulers are isolated +-sub_core_model 1 +# disable specialized operand collectors and use generic operand collectors instead +-enable_specialized_operand_collector 0 +-gpgpu_operand_collector_num_units_gen 8 +-gpgpu_operand_collector_num_in_ports_gen 8 +-gpgpu_operand_collector_num_out_ports_gen 8 +# volta has 8 banks, 4 schedulers, two banks per scheduler +# we increase #banks to 16 to mitigate the effect of Regisrer File Cache (RFC) which we do not implement in the current version +-gpgpu_num_reg_banks 16 +-gpgpu_reg_file_port_throughput 2 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 +-gpgpu_coalesce_arch 60 + +## In Volta, a warp scheduler can issue 1 inst per cycle +-gpgpu_max_insn_issue_per_warp 1 +-gpgpu_dual_issue_diff_exec_units 1 + +# Volta has four schedulers per core +-gpgpu_num_sched_per_core 4 +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +## L1/shared memory configuration # ::,::::,::,:** # ** Optional parameter - Required when mshr_type==Texture Fifo # Defualt config is 32KB DL1 and 96KB shared memory @@ -91,7 +116,6 @@ -adaptive_cache_config 1 # Volta unified cache has four banks -l1_banks 4 -#-mem_unit_ports 4 -gpgpu_cache:dl1 S:1:128:256,L:L:s:N:L,A:256:8,16:0,32 -gpgpu_shmem_size 98304 -gpgpu_shmem_sizeDefault 98304 @@ -112,6 +136,7 @@ # 128 KB Inst. -gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 +-inst_fetch_throughput 4 # 128 KB Tex # Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod -gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2 @@ -119,31 +144,10 @@ -gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 -perfect_inst_const_cache 1 -# Volta has sub core model, in which each scheduler has its own register file and EUs -# i.e. schedulers are isolated --sub_core_model 1 -# disable specialized operand collectors and use generic operand collectors instead --enable_specialized_operand_collector 0 --gpgpu_operand_collector_num_units_gen 32 --gpgpu_operand_collector_num_in_ports_gen 8 --gpgpu_operand_collector_num_out_ports_gen 8 -# volta has 8 banks, 4 schedulers, two banks per scheduler --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 --gpgpu_coalesce_arch 60 - -## In Volta, a warp scheduler can issue 1 inst per cycle --gpgpu_max_insn_issue_per_warp 1 --gpgpu_dual_issue_diff_exec_units 1 - # interconnection #-network_mode 1 #-inter_config_file config_volta_islip.icnt -# for local xbar, use: +# use built-in local xbar -network_mode 2 -inct_in_buffer_limit 512 -inct_out_buffer_limit 512 @@ -186,15 +190,6 @@ #-Seperate_Write_Queue_Enable 1 #-Write_Queue_Size 64:56:32 -# Volta has four schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - # stat collection -gpgpu_memlatency_stat 14 -gpgpu_runtime_stat 500 diff --git a/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt b/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt deleted file mode 100644 index dec4789..0000000 --- a/configs/tested-cfgs/Turing_RTX2060/config_pascal_islip.icnt +++ /dev/null @@ -1,73 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 40; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 52; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 64; -input_buffer_size = 256; -ejection_buffer_size = 64; -boundary_buffer_size = 64; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 1; -output_speedup = 1; -internal_speedup = 2.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config b/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config deleted file mode 100644 index ab324d8..0000000 --- a/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config +++ /dev/null @@ -1,186 +0,0 @@ -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 70 - -# Device Limits --gpgpu_stack_size_limit 1024 --gpgpu_heap_size_limit 8388608 --gpgpu_runtime_sync_depth_limit 2 --gpgpu_runtime_pending_launch_count_limit 2048 --gpgpu_kernel_launch_latency 5000 --gpgpu_TB_launch_latency 0 - -# Compute Capability --gpgpu_compute_capability_major 7 --gpgpu_compute_capability_minor 0 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 30 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 12 --gpgpu_n_sub_partition_per_mchannel 2 - -# volta clock domains -#-gpgpu_clock_domains ::: -# Turing clock domains are adopted from -# https://en.wikipedia.org/wiki/GeForce_20_series --gpgpu_clock_domains 1365.0:1365.0:1365.0:3500.0 -# boost mode -# -gpgpu_clock_domains 1680.0:1680.0:1680.0:3500.0 - -# shader core pipeline config --gpgpu_shader_registers 65536 --gpgpu_registers_per_block 65536 --gpgpu_occupancy_sm_number 70 - -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 --gpgpu_shader_cta 32 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE -## Volta GV100 has 4 SP SIMD units, 4 SFU units, 4 DP units per core, 4 Tensor core units -## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4 --gpgpu_num_sp_units 4 --gpgpu_num_sfu_units 4 --gpgpu_num_dp_units 4 --gpgpu_num_int_units 4 --gpgpu_tensor_core_avail 1 --gpgpu_num_tensor_core_units 4 - -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" -# All Div operations are executed on SFU unit -# Throughput (initiation latency) are adopted from -# http://on-demand.gputechconf.com/gtc/2018/presentation/s8122-dissecting-the-volta-gpu-architecture-through-microbenchmarking.pdf --ptx_opcode_latency_int 4,13,4,5,145 --ptx_opcode_initiation_int 2,2,2,2,8 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 2,2,2,2,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 4,4,4,4,130 --ptx_opcode_latency_sfu 100 --ptx_opcode_initiation_sfu 8 --ptx_opcode_latency_tesnor 64 --ptx_opcode_initiation_tensor 64 - -# ::,::::,::,:** -# ** Optional parameter - Required when mshr_type==Texture Fifo --adaptive_cache_config 0 --l1_banks 4 --gpgpu_cache:dl1 S:1:128:512,L:L:s:N:L,A:256:8,16:0,32 --gpgpu_shmem_size 65536 --gpgpu_shmem_sizeDefault 65536 --gpgpu_shmem_per_block 65536 --gmem_skip_L1D 0 --icnt_flit_size 40 --gpgpu_n_cluster_ejection_buffer_size 32 --l1_latency 20 --smem_latency 20 --gpgpu_flush_l1_cache 1 - -# 32 sets, each 128 bytes 32-way for each memory sub partition (96 KB per memory sub partition). This gives us 6MB L2 cache --gpgpu_cache:dl2 S:32:128:32,L:B:m:L:L,A:192:4,32:0,32 --gpgpu_cache:dl2_texture_only 0 --gpgpu_dram_partition_queues 64:64:64:64 --perf_sim_memcpy 1 --memory_partition_indexing 0 - -# 128 KB Inst. --gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4 -# 128 KB Tex -# Note, TEX is deprected in Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod --gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2 -# 64 KB Const --gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 --perfect_inst_const_cache 1 - -# i.e. schedulers are isolated --sub_core_model 1 -# disable specialized operand collectors and use generic operand collectors instead --enable_specialized_operand_collector 0 --gpgpu_operand_collector_num_units_gen 32 --gpgpu_operand_collector_num_in_ports_gen 8 --gpgpu_operand_collector_num_out_ports_gen 8 -# volta has 8 banks, 4 schedulers, two banks per scheduler --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 --gpgpu_coalesce_arch 60 - -## In Volta, a warp scheduler can issue 1 inst per cycle --gpgpu_max_insn_issue_per_warp 1 --gpgpu_dual_issue_diff_exec_units 1 - -# interconnection -#-network_mode 1 -#-inter_config_file config_volta_islip.icnt -# for local xbar, use: --network_mode 2 --inct_in_buffer_limit 512 --inct_out_buffer_limit 512 --inct_subnets 2 --arbiter_algo 1 - -# memory partition latency config --rop_latency 160 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 --gpgpu_frfcfs_dram_sched_queue_size 64 --gpgpu_dram_return_queue_size 192 - -# GDDR6 -# http://monitorinsider.com/GDDR6.html --gpgpu_n_mem_per_ctrlr 1 --gpgpu_dram_buswidth 2 --gpgpu_dram_burst_length 16 --dram_data_command_freq_ratio 4 # GDDR6 is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS - -# Use the same GDDR5 timing, scaled to 3500MHZ --gpgpu_dram_timing_opt "nbk=16:CCD=4:RRD=10:RCD=20:RAS=50:RP=20:RC=62: - CL=20:WL=8:CDLR=9:WR=20:nbkgrp=4:CCDL=4:RTPL=4" - -# select lower bits for bnkgrp to increase bnkgrp parallelism --dram_bnk_indexing_policy 0 --dram_bnkgrp_indexing_policy 1 - -#-Seperate_Write_Queue_Enable 1 -#-Write_Queue_Size 64:56:32 - -# Volta has four schedulers per core --gpgpu_num_sched_per_core 4 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs, disable it untill we create a real energy model for Volta --power_simulation_enabled 0 - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 - -- cgit v1.3 From fe8dd7eadbb67b9e917c92c9c540812db35fce37 Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Thu, 2 Apr 2020 21:41:01 -0400 Subject: updating the turing config --- configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config | 2 +- configs/tested-cfgs/SM75_RTX2060/gpgpusim.config | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config index 1951a3f..77617d6 100644 --- a/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config +++ b/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config @@ -40,7 +40,7 @@ -gpgpu_shader_registers 65536 -gpgpu_occupancy_sm_number 62 -# This implies a maximum of 32 warps/SM +# This implies a maximum of 64 warps/SM -gpgpu_shader_core_pipeline 2048:32 -gpgpu_shader_cta 16 -gpgpu_simd_model 1 diff --git a/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config index 9d3992a..e8329dd 100644 --- a/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config +++ b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config @@ -1,12 +1,12 @@ # This config models the Turing RTX 2060 # For more info about turing architecture: -# https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/technologies/turing-architecture/NVIDIA-Turing-Architecture-Whitepaper.pdf -# "RTX on—The NVIDIA Turing GPU", IEEE MICRO 2020 +# 1- https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/technologies/turing-architecture/NVIDIA-Turing-Architecture-Whitepaper.pdf +# 2- "RTX on—The NVIDIA Turing GPU", IEEE MICRO 2020 # functional simulator specification -gpgpu_ptx_instruction_classification 0 -gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 70 +-gpgpu_ptx_force_max_capability 75 # Device Limits -gpgpu_stack_size_limit 1024 @@ -33,17 +33,17 @@ # volta clock domains #-gpgpu_clock_domains ::: --gpgpu_clock_domains 1365.0:1365.0:1365.0:3500.0 +-gpgpu_clock_domains 1365.0:1365.0:1365.0:7000.0 # boost mode -# -gpgpu_clock_domains 1680.0:1680.0:1680.0:3500.0 +# -gpgpu_clock_domains 1680.0:1680.0:1680.0:7000.0 # shader core pipeline config -gpgpu_shader_registers 65536 -gpgpu_registers_per_block 65536 -gpgpu_occupancy_sm_number 75 -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 +# This implies a maximum of 32 warps/SM +-gpgpu_shader_core_pipeline 1024:32 -gpgpu_shader_cta 32 -gpgpu_simd_model 1 @@ -89,7 +89,7 @@ -gpgpu_shmem_num_banks 32 -gpgpu_shmem_limited_broadcast 0 -gpgpu_shmem_warp_parts 1 --gpgpu_coalesce_arch 60 +-gpgpu_coalesce_arch 75 ## In Turing, a warp scheduler can issue 1 inst per cycle -gpgpu_max_insn_issue_per_warp 1 @@ -156,7 +156,7 @@ -gpgpu_n_mem_per_ctrlr 1 -gpgpu_dram_buswidth 2 -gpgpu_dram_burst_length 16 --dram_data_command_freq_ratio 4 # GDDR6 is QDR +-dram_data_command_freq_ratio 2 # GDDR6 is configured as DDR in Turing -gpgpu_mem_address_mask 1 -gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS -- cgit v1.3 From 52204ff08a9c9a21a99fee3f976d2a419c014fec Mon Sep 17 00:00:00 2001 From: "Mahmoud Khairy A. Abdallah" Date: Mon, 18 May 2020 21:49:34 -0400 Subject: fixing some failing apps --- configs/tested-cfgs/SM75_RTX2060/gpgpusim.config | 6 +++--- configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config | 4 ++-- src/gpgpu-sim/gpu-sim.cc | 3 ++- src/gpgpu-sim/shader.cc | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config index e8329dd..b89971e 100644 --- a/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config +++ b/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config @@ -33,9 +33,9 @@ # volta clock domains #-gpgpu_clock_domains ::: --gpgpu_clock_domains 1365.0:1365.0:1365.0:7000.0 +-gpgpu_clock_domains 1365.0:1365.0:1365.0:3500.0 # boost mode -# -gpgpu_clock_domains 1680.0:1680.0:1680.0:7000.0 +# -gpgpu_clock_domains 1680.0:1680.0:1680.0:3500.0 # shader core pipeline config -gpgpu_shader_registers 65536 @@ -156,7 +156,7 @@ -gpgpu_n_mem_per_ctrlr 1 -gpgpu_dram_buswidth 2 -gpgpu_dram_burst_length 16 --dram_data_command_freq_ratio 2 # GDDR6 is configured as DDR in Turing +-dram_data_command_freq_ratio 4 -gpgpu_mem_address_mask 1 -gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS diff --git a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config index 0255f76..0df3eec 100644 --- a/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config +++ b/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config @@ -74,8 +74,8 @@ -ptx_opcode_initiation_dp 4,4,4,4,130 -ptx_opcode_latency_sfu 100 -ptx_opcode_initiation_sfu 8 --ptx_opcode_latency_tesnor 6 --ptx_opcode_initiation_tensor 2 +-ptx_opcode_latency_tesnor 8 +-ptx_opcode_initiation_tensor 4 # Volta has sub core model, in which each scheduler has its own register file and EUs # i.e. schedulers are isolated diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index e5b9c9d..cd5fa56 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1828,7 +1828,8 @@ void shader_core_ctx::dump_warp_state( FILE *fout ) const void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count ) { if (m_memory_config->m_perf_sim_memcpy) { - assert (dst_start_addr % 32 == 0); + //if(!m_config.trace_driven_mode) //in trace-driven mode, CUDA runtime can start nre data structure at any position + // assert (dst_start_addr % 32 == 0); for ( unsigned counter = 0; counter < count; counter += 32 ) { const unsigned wr_addr = dst_start_addr + counter; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index ed0c25e..65ec113 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3091,7 +3091,7 @@ unsigned int shader_core_config::max_cta( const kernel_info_t &k ) const case VOLTA: { //For Volta, we assign the remaining shared memory to L1 cache //For more info about adaptive cache, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x - assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared + //assert(gpgpu_shmem_size == 98304); //Volta has 96 KB shared //To Do: make it flexible and not tuned to 9KB share memory unsigned max_assoc = m_L1D_config.get_max_assoc(); -- cgit v1.3