Merge pull request #65 from tgrogers/dev

Bunch of changes made to 3.x that help with data collection

7 files changed, 93 insertions, 26 deletions

diff --git a/setup_environment b/setup_environment
index 96cc362..3f2476e 100644
--- a/setup_environment
+++ b/setup_environment
@@ -8,7 +8,8 @@ export GPGPUSIM_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )"
 GPGPUSIM_VERSION_STRING=`cat $GPGPUSIM_ROOT/version | awk '/Version/ {print $8}'`
 #Detect Git branch and commit #
 GIT_COMMIT=`git log -n 1 | head -1 | sed -re 's/commit (.*)/\1/'`
-GIT_FILES_CHANGED=`git diff --numstat --cached && git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1/'`
+GIT_FILES_CHANGED=`git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1./'`
+GIT_FILES_CHANGED+=`git diff --numstat --cached | wc | sed -re 's/^\s+([0-9]+).*/\1/'`
 GPGPUSIM_BUILD_STRING="gpgpu-sim_git-commit-$GIT_COMMIT-modified_$GIT_FILES_CHANGED"
 
 echo -n "GPGPU-Sim version $GPGPUSIM_VERSION_STRING (build $GPGPUSIM_BUILD_STRING) ";
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index d4ace76..a34b99b 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -1408,7 +1408,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
    if ( (g_ptx_sim_num_insn % 100000) == 0 ) {
       dim3 ctaid = get_ctaid();
       dim3 tid = get_tid();
-      printf("GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n",
+      DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n",
              g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z );
       fflush(stdout);
    }
diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index 5ea4190..8886398 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -517,18 +517,28 @@ void cache_stats::print_stats(FILE *fout, const char *cache_name) const{
     /// the provided name is used.
     /// The printed format is "<cache_name>[<request_type>][<request_status>] = <stat_value>"
     ///
+    std::vector< unsigned > total_access;
+    total_access.resize(NUM_MEM_ACCESS_TYPE, 0);
     std::string m_cache_name = cache_name;
     for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
         for (unsigned status = 0; status < NUM_CACHE_REQUEST_STATUS; ++status) {
-            if(m_stats[type][status] > 0){
-                fprintf(fout, "\t%s[%s][%s] = %u\n",
-                    m_cache_name.c_str(),
-                    mem_access_type_str((enum mem_access_type)type),
-                    cache_request_status_str((enum cache_request_status)status),
-                    m_stats[type][status]);
-            }
+            fprintf(fout, "\t%s[%s][%s] = %u\n",
+                m_cache_name.c_str(),
+                mem_access_type_str((enum mem_access_type)type),
+                cache_request_status_str((enum cache_request_status)status),
+                m_stats[type][status]);
+            if(status != RESERVATION_FAIL)
+                 total_access[type]+= m_stats[type][status];
         }
     }
+    for (unsigned type = 0; type < NUM_MEM_ACCESS_TYPE; ++type) {
+         if(total_access[type] > 0)
+          fprintf(fout, "\t%s[%s][%s] = %u\n",
+                m_cache_name.c_str(),
+                mem_access_type_str((enum mem_access_type)type),
+                "TOTAL_ACCESS",
+                total_access[type]);
+    }
 }
 
 void cache_sub_stats::print_port_stats(FILE *fout, const char *cache_name) const
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 3829861..bd4c00c 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -37,6 +37,7 @@
 
 
 #include "shader.h"
+#include "shader_trace.h"
 #include "dram.h"
 #include "mem_fetch.h"
 
@@ -88,6 +89,14 @@ unsigned long long  gpu_tot_sim_cycle = 0;
 // performance counter for stalls due to congestion.
 unsigned int gpu_stall_dramfull = 0; 
 unsigned int gpu_stall_icnt2sh = 0;
+unsigned long long partiton_reqs_in_parallel = 0;
+unsigned long long partiton_reqs_in_parallel_total = 0;
+unsigned long long partiton_reqs_in_parallel_util = 0;
+unsigned long long partiton_reqs_in_parallel_util_total = 0;
+unsigned long long  gpu_sim_cycle_parition_util = 0;
+unsigned long long  gpu_tot_sim_cycle_parition_util = 0;
+unsigned long long partiton_replys_in_parallel = 0;
+unsigned long long partiton_replys_in_parallel_total = 0;
 
 /* Clock Domains */
 
@@ -224,7 +233,7 @@ void shader_core_config::reg_options(class OptionParser * opp)
                    "per-shader L1 data cache config "
                    " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                    "none" );
-    option_parser_register(opp, "-gpgpu_cache:dl1PreShared", OPT_CSTR, &m_L1D_config.m_config_stringPrefShared,
+    option_parser_register(opp, "-gpgpu_cache:dl1PrefShared", OPT_CSTR, &m_L1D_config.m_config_stringPrefShared,
                    "per-shader L1 data cache config "
                    " {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}",
                    "none" );
@@ -745,6 +754,10 @@ void gpgpu_sim::init()
     gpu_sim_insn = 0;
     last_gpu_sim_insn = 0;
     m_total_cta_launched=0;
+    partiton_reqs_in_parallel = 0;
+    partiton_replys_in_parallel = 0;
+    partiton_reqs_in_parallel_util = 0;
+    gpu_sim_cycle_parition_util = 0;
 
     reinit_clock_domains();
     set_param_gpgpu_num_shaders(m_config.num_shader());
@@ -781,8 +794,16 @@ void gpgpu_sim::update_stats() {
     gpu_tot_sim_cycle += gpu_sim_cycle;
     gpu_tot_sim_insn += gpu_sim_insn;
     gpu_tot_issued_cta += m_total_cta_launched;
+    partiton_reqs_in_parallel_total += partiton_reqs_in_parallel;
+    partiton_replys_in_parallel_total += partiton_replys_in_parallel;
+    partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util;
+    gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ;
 
     gpu_sim_cycle = 0;
+    partiton_reqs_in_parallel = 0;
+    partiton_replys_in_parallel = 0;
+    partiton_reqs_in_parallel_util = 0;
+    gpu_sim_cycle_parition_util = 0;
     gpu_sim_insn = 0;
     m_total_cta_launched = 0;
 }
@@ -966,6 +987,21 @@ void gpgpu_sim::gpu_print_stat()
    printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull);
    printf("gpu_stall_icnt2sh    = %d\n", gpu_stall_icnt2sh );
 
+   printf("partiton_reqs_in_parallel = %lld\n", partiton_reqs_in_parallel);
+   printf("partiton_reqs_in_parallel_total    = %lld\n", partiton_reqs_in_parallel_total );
+   printf("partiton_level_parallism = %12.4f\n", (float)partiton_reqs_in_parallel / gpu_sim_cycle);
+   printf("partiton_level_parallism_total  = %12.4f\n", (float)(partiton_reqs_in_parallel+partiton_reqs_in_parallel_total) / (gpu_tot_sim_cycle+gpu_sim_cycle) );
+   printf("partiton_reqs_in_parallel_util = %lld\n", partiton_reqs_in_parallel_util);
+   printf("partiton_reqs_in_parallel_util_total    = %lld\n", partiton_reqs_in_parallel_util_total );
+   printf("gpu_sim_cycle_parition_util = %lld\n", gpu_sim_cycle_parition_util);
+   printf("gpu_tot_sim_cycle_parition_util    = %lld\n", gpu_tot_sim_cycle_parition_util );
+   printf("partiton_level_parallism_util = %12.4f\n", (float)partiton_reqs_in_parallel_util / gpu_sim_cycle_parition_util);
+   printf("partiton_level_parallism_util_total  = %12.4f\n", (float)(partiton_reqs_in_parallel_util+partiton_reqs_in_parallel_util_total) / (gpu_sim_cycle_parition_util+gpu_tot_sim_cycle_parition_util) );
+   printf("partiton_replys_in_parallel = %lld\n", partiton_replys_in_parallel);
+   printf("partiton_replys_in_parallel_total    = %lld\n", partiton_replys_in_parallel_total );
+   printf("L2_BW  = %12.4f GB/Sec\n", ((float)(partiton_replys_in_parallel * 32) / (gpu_sim_cycle * m_config.icnt_period)) / 1000000000);
+   printf("L2_BW_total  = %12.4f GB/Sec\n", ((float)((partiton_replys_in_parallel+partiton_replys_in_parallel_total) * 32) / ((gpu_tot_sim_cycle+gpu_sim_cycle) * m_config.icnt_period)) / 1000000000 );
+
    time_t curr_time;
    time(&curr_time);
    unsigned long long elapsed_time = MAX( curr_time - g_simulation_starttime, 1 );
@@ -1176,8 +1212,8 @@ bool shader_core_ctx::occupy_shader_resource_1block(kernel_info_t & k, bool occu
        m_occupied_regs += (padded_cta_size * ((kernel_info->regs+3)&~3));
        m_occupied_ctas++;
 
-       printf("GPGPU-Sim uArch: Shader %d occupied %d threads, %d shared mem, %d registers, %d ctas\n",
-            m_sid, m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas);  
+      SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Occupied %d threads, %d shared mem, %d registers, %d ctas\n",
+            m_occupied_n_threads, m_occupied_shmem, m_occupied_regs, m_occupied_ctas);  
    }
 
    return true;
@@ -1302,8 +1338,8 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     m_n_active_cta++;
 
     shader_CTA_count_log(m_sid, 1);
-    printf("GPGPU-Sim uArch: core:%3d, cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", 
-        m_sid, free_cta_hw_id, start_thread, end_thread, gpu_sim_cycle, gpu_tot_sim_cycle );
+    SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: cta:%2u, start_tid:%4u, end_tid:%4u, initialized @(%lld,%lld)\n", 
+        free_cta_hw_id, start_thread, end_thread, gpu_sim_cycle, gpu_tot_sim_cycle );
 
 }
 
@@ -1367,6 +1403,7 @@ void gpgpu_sim::cycle()
       for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) 
          m_cluster[i]->icnt_cycle(); 
    }
+    unsigned partiton_replys_in_parallel_per_cycle = 0;
     if (clock_mask & ICNT) {
         // pop from memory controller to interconnect
         for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) {
@@ -1379,6 +1416,7 @@ void gpgpu_sim::cycle()
                     mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle);
                     ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size );
                     m_memory_sub_partition[i]->pop();
+                    partiton_replys_in_parallel_per_cycle++;
                 } else {
                     gpu_stall_icnt2sh++;
                 }
@@ -1387,6 +1425,7 @@ void gpgpu_sim::cycle()
             }
         }
     }
+    partiton_replys_in_parallel += partiton_replys_in_parallel_per_cycle;
 
    if (clock_mask & DRAM) {
       for (unsigned i=0;i<m_memory_config->m_n_mem;i++){
@@ -1399,6 +1438,7 @@ void gpgpu_sim::cycle()
    }
 
    // L2 operations follow L2 clock domain
+   unsigned partiton_reqs_in_parallel_per_cycle = 0;
    if (clock_mask & L2) {
        m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].clear();
       for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) {
@@ -1409,11 +1449,17 @@ void gpgpu_sim::cycle()
           } else {
               mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) );
               m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle );
+              partiton_reqs_in_parallel_per_cycle++;
           }
           m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle);
           m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]);
        }
    }
+   partiton_reqs_in_parallel += partiton_reqs_in_parallel_per_cycle;
+   if(partiton_reqs_in_parallel_per_cycle > 0){
+	   partiton_reqs_in_parallel_util += partiton_reqs_in_parallel_per_cycle;
+	   gpu_sim_cycle_parition_util++;
+   }
 
    if (clock_mask & ICNT) {
       icnt_transfer();
@@ -1500,7 +1546,8 @@ void gpgpu_sim::cycle()
             hrs     = elapsed_time/3600 - 24*days;
             minutes = elapsed_time/60 - 60*(hrs + 24*days);
             sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days));
-            printf("GPGPU-Sim uArch: cycles simulated: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
+
+            DPRINTF(LIVENESS, "GPGPU-Sim uArch: cycles simulated: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
                    gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, 
                    (double)gpu_sim_insn/(double)gpu_sim_cycle,
                    (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time),
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index d17e51d..4640d65 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1969,12 +1969,12 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t
       m_barriers.deallocate_barrier(cta_num);
       shader_CTA_count_unlog(m_sid, 1);
 
-      printf("GPGPU-Sim uArch: Shader %d finished CTA #%d (%lld,%lld), %u CTAs running\n", m_sid, cta_num, gpu_sim_cycle, gpu_tot_sim_cycle,
-             m_n_active_cta );
+     SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Finished CTA #%d (%lld,%lld), %u CTAs running\n",
+        cta_num, gpu_sim_cycle, gpu_tot_sim_cycle, m_n_active_cta);
 
       if( m_n_active_cta == 0 ) {
-          printf("GPGPU-Sim uArch: Shader %u empty (last released kernel %u \'%s\').\n", m_sid, kernel->get_uid(),
-                 kernel->name().c_str() );
+        SHADER_DPRINTF(LIVENESS, "GPGPU-Sim uArch: Empty (last released kernel %u \'%s\').\n",
+            kernel->get_uid(), kernel->name().c_str());
           fflush(stdout);
 
           //Shader can only be empty when no more cta are dispatched
@@ -1989,8 +1989,10 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num, kernel_info_t
       kernel->dec_running();
       if( !m_gpu->kernel_more_cta_left(kernel) ) {
           if( !kernel->running() ) {
-              printf("GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(),
-                kernel->name().c_str(), m_sid );
+              SHADER_DPRINTF(LIVENESS,
+                "GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", kernel->get_uid(),
+                kernel->name().c_str(), m_sid);
+
               if(m_kernel == kernel)
                 m_kernel = NULL;
               m_gpu->set_kernel_done( kernel );
diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index ad4587a..ede9f20 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -93,8 +93,15 @@ bool g_sim_active = false;
 bool g_sim_done = true;
 bool break_limit = false;
 
+static void termination_callback()
+{
+    printf("GPGPU-Sim: *** exit detected ***\n");
+    fflush(stdout);
+}
+
 void *gpgpu_sim_thread_concurrent(void*)
 {
+    atexit(termination_callback);
     // concurrent kernel execution simulation thread
     do {
         if(g_debug_execution >= 3) {
@@ -156,8 +163,8 @@ void *gpgpu_sim_thread_concurrent(void*)
            printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n");
            fflush(stdout);
         }
-        g_the_gpu->print_stats();
         if(sim_cycles) {
+            g_the_gpu->print_stats();
             g_the_gpu->update_stats();
             print_simulation_time();
         }
@@ -165,10 +172,9 @@ void *gpgpu_sim_thread_concurrent(void*)
         g_sim_active = false;
         pthread_mutex_unlock(&g_sim_lock);
     } while( !g_sim_done );
-    if(g_debug_execution >= 3) {
-       printf("GPGPU-Sim: *** simulation thread exiting ***\n");
-       fflush(stdout);
-    }
+    printf("GPGPU-Sim: *** simulation thread exiting ***\n");
+    fflush(stdout);
+
     if(break_limit) {
     	printf("GPGPU-Sim: ** break due to reaching the maximum cycles (or instructions) **\n");
     	exit(1);
diff --git a/src/trace_streams.tup b/src/trace_streams.tup
index c41690e..fd68a16 100644
--- a/src/trace_streams.tup
+++ b/src/trace_streams.tup
@@ -29,5 +29,6 @@ TS_TUP_BEGIN( trace_streams_type )
     TS_TUP( WARP_SCHEDULER ),
     TS_TUP( SCOREBOARD ),
     TS_TUP( MEMORY_PARTITION_UNIT ),
+    TS_TUP( LIVENESS ),
     TS_TUP( NUM_TRACE_STREAMS )
 TS_TUP_END( trace_streams_type )