Adding in an occupancy metric to match the nvprof metric

5 files changed, 76 insertions, 6 deletions

diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index 0b77f47..70d3790 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -414,7 +414,7 @@ struct sector_cache_block : public cache_block_t {
 	}
 
     virtual void print_status() {
-    	 printf("m_block_addr is %u, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]);
+    	 printf("m_block_addr is %llu, status = %u %u %u %u\n", m_block_addr, m_status[0], m_status[1], m_status[2], m_status[3]);
     }
 
 
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 08d4525..9e0801c 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -845,6 +845,7 @@ void gpgpu_sim::update_stats() {
     partiton_replys_in_parallel_total += partiton_replys_in_parallel;
     partiton_reqs_in_parallel_util_total += partiton_reqs_in_parallel_util;
     gpu_tot_sim_cycle_parition_util += gpu_sim_cycle_parition_util ;
+    gpu_tot_occupancy += gpu_occupancy;
 
     gpu_sim_cycle = 0;
     partiton_reqs_in_parallel = 0;
@@ -853,6 +854,7 @@ void gpgpu_sim::update_stats() {
     gpu_sim_cycle_parition_util = 0;
     gpu_sim_insn = 0;
     m_total_cta_launched = 0;
+    gpu_occupancy = occupancy_stats();
 }
 
 void gpgpu_sim::print_stats()
@@ -1026,6 +1028,9 @@ void gpgpu_sim::gpu_print_stat()
    printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn+gpu_sim_insn);
    printf("gpu_tot_ipc = %12.4f\n", (float)(gpu_tot_sim_insn+gpu_sim_insn) / (gpu_tot_sim_cycle+gpu_sim_cycle));
    printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta + m_total_cta_launched);
+   printf("gpu_occupancy = %.4f\% \n", gpu_occupancy.get_occ_fraction() * 100);
+   printf("gpu_tot_occupancy = %.4f\% \n", (gpu_occupancy + gpu_tot_occupancy).get_occ_fraction() * 100);
+
 
    extern unsigned long long g_max_total_param_size;
    fprintf(statfout, "max_total_param_size = %llu\n", g_max_total_param_size);
@@ -1529,6 +1534,8 @@ void gpgpu_sim::cycle()
          // Update core icnt/cache stats for GPUWattch
          m_cluster[i]->get_icnt_stats(m_power_stats->pwr_mem_stat->n_simt_to_mem[CURRENT_STAT_IDX][i], m_power_stats->pwr_mem_stat->n_mem_to_simt[CURRENT_STAT_IDX][i]);
          m_cluster[i]->get_cache_stats(m_power_stats->pwr_mem_stat->core_cache_stats[CURRENT_STAT_IDX]);
+         m_cluster[i]->get_current_occupancy(gpu_occupancy.aggregate_warp_slot_filled, gpu_occupancy.aggregate_theoretical_warp_slots);
+
       }
       float temp=0;
       for (unsigned i=0;i<m_shader_config->num_shader();i++){
@@ -1594,15 +1601,20 @@ void gpgpu_sim::cycle()
          time_t curr_time;
          time(&curr_time);
          unsigned long long  elapsed_time = MAX(curr_time - g_simulation_starttime, 1);
-         if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq ) {
+         if ( (elapsed_time - last_liveness_message_time) >= m_config.liveness_message_freq && DTRACE(LIVENESS) ) {
             days    = elapsed_time/(3600*24);
             hrs     = elapsed_time/3600 - 24*days;
             minutes = elapsed_time/60 - 60*(hrs + 24*days);
             sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days));
-
-            DPRINTF(LIVENESS, "GPGPU-Sim uArch: cycles simulated: %lld  inst.: %lld (ipc=%4.1f) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
-                   gpu_tot_sim_cycle + gpu_sim_cycle, gpu_tot_sim_insn + gpu_sim_insn, 
+            
+            unsigned long long active = 0, total = 0;
+            for (unsigned i=0;i<m_shader_config->n_simt_clusters;i++) {
+                m_cluster[i]->get_current_occupancy(active, total);
+            }
+            DPRINTF(LIVENESS, "uArch: inst.: %lld (ipc=%4.1f, occ=%0.4f\% [%llu / %llu]) sim_rate=%u (inst/sec) elapsed = %u:%u:%02u:%02u / %s", 
+                   gpu_tot_sim_insn + gpu_sim_insn, 
                    (double)gpu_sim_insn/(double)gpu_sim_cycle,
+                   float(active)/float(total) * 100, active, total,
                    (unsigned)((gpu_tot_sim_insn+gpu_sim_insn) / elapsed_time),
                    (unsigned)days,(unsigned)hrs,(unsigned)minutes,(unsigned)sec,
                    ctime(&curr_time));
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 1778008..1bae1fa 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -383,6 +383,32 @@ private:
     friend class gpgpu_sim;
 };
 
+struct occupancy_stats {
+    occupancy_stats() : aggregate_warp_slot_filled(0), aggregate_theoretical_warp_slots(0){}
+    occupancy_stats( unsigned long long wsf, unsigned long long tws )
+        : aggregate_warp_slot_filled(wsf), aggregate_theoretical_warp_slots(tws){}
+
+    unsigned long long aggregate_warp_slot_filled;
+    unsigned long long aggregate_theoretical_warp_slots;
+
+    float get_occ_fraction() const {
+        return float(aggregate_warp_slot_filled) / float(aggregate_theoretical_warp_slots);
+    }
+
+    occupancy_stats& operator+=(const occupancy_stats& rhs) {
+        aggregate_warp_slot_filled += rhs.aggregate_warp_slot_filled;
+        aggregate_theoretical_warp_slots += rhs.aggregate_theoretical_warp_slots;
+        return *this;
+    }
+
+    occupancy_stats operator+(const occupancy_stats& rhs) const{
+        return occupancy_stats( aggregate_warp_slot_filled + rhs.aggregate_warp_slot_filled,
+                                aggregate_theoretical_warp_slots + rhs.aggregate_theoretical_warp_slots
+                               );
+    }
+};
+
+
 class gpgpu_sim : public gpgpu_t {
 public:
    gpgpu_sim( const gpgpu_sim_config &config );
@@ -521,6 +547,9 @@ public:
    unsigned long long  gpu_tot_sim_insn;
    unsigned long long  gpu_sim_insn_last_update;
    unsigned gpu_sim_insn_last_update_sid;
+   occupancy_stats gpu_occupancy;
+   occupancy_stats gpu_tot_occupancy;
+
 
    FuncCache get_cache_config(std::string kernel_name);
    void set_cache_config(std::string kernel_name, FuncCache cacheConfig );
@@ -549,4 +578,5 @@ public:
    }
 };
 
+
 #endif
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 0e2e1c2..bb8e4b6 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -74,7 +74,7 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu,
                                   shader_core_stats *stats )
    : core_t( gpu, NULL, config->warp_size, config->n_thread_per_shader ),
      m_barriers( this, config->max_warps_per_shader, config->max_cta_per_core, config->max_barriers_per_cta, config->warp_size ),
-     m_dynamic_warp_id(0)
+     m_dynamic_warp_id(0), m_active_warps(0)
 {
     m_cluster = cluster;
     m_config = config;
@@ -357,6 +357,7 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
        m_occupied_ctas = 0;
        m_occupied_hwtid.reset();
        m_occupied_cta_to_hwtid.clear();
+       m_active_warps = 0;
 
    }
    for (unsigned i = start_thread; i<end_thread; i++) {
@@ -391,6 +392,7 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign
             m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id);
             ++m_dynamic_warp_id;
             m_not_completed += n_active;
+            ++m_active_warps;
       }
    }
 }
@@ -418,6 +420,18 @@ void shader_core_ctx::get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsig
     m_simt_stack[warp_id]->get_pdom_stack_top_info(pc,rpc);
 }
 
+float shader_core_ctx::get_current_occupancy( unsigned long long & active, unsigned long long & total ) const
+{
+    // To match the achieved_occupancy in nvprof, only SMs that are active are counted toward the occupancy.
+    if ( m_active_warps > 0 ) {
+        total += m_warp.size();
+        active += m_active_warps;
+        return float(active) / float(total);
+    } else {
+        return 0;
+    }
+}
+
 void shader_core_stats::print( FILE* fout ) const
 {
 	unsigned long long  thread_icount_uarch=0;
@@ -692,6 +706,8 @@ void shader_core_ctx::fetch()
                     }
                     if( did_exit ) 
                         m_warp[warp_id].set_done_exit();
+                        --m_active_warps;
+                        assert(m_active_warps >= 0);
                 }
 
                 // this code fetches instructions from the i-cache or generates memory requests
@@ -3524,6 +3540,15 @@ void simt_core_cluster::print_not_completed( FILE *fp ) const
     }
 }
 
+
+float simt_core_cluster::get_current_occupancy( unsigned long long& active, unsigned long long& total ) const {
+    float aggregate = 0.f;
+    for( unsigned i=0; i < m_config->n_simt_cores_per_cluster; i++ ) {
+        aggregate+=m_core[i]->get_current_occupancy( active, total );
+    }
+    return aggregate / m_config->n_simt_cores_per_cluster;
+}
+
 unsigned simt_core_cluster::get_n_active_cta() const
 {
     unsigned n=0;
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index e07096e..f7b13e2 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1691,6 +1691,7 @@ public:
     // accessors
     virtual bool warp_waiting_at_barrier( unsigned warp_id ) const;
     void get_pdom_stack_top_info( unsigned tid, unsigned *pc, unsigned *rpc ) const;
+    float get_current_occupancy( unsigned long long & active, unsigned long long & total ) const;
 
 // used by pipeline timing model components:
     // modifiers
@@ -1897,6 +1898,7 @@ public:
     std::vector<register_set> m_pipeline_reg;
     Scoreboard               *m_scoreboard;
     opndcoll_rfu_t            m_operand_collector;
+    int m_active_warps;
 
     //schedule
     std::vector<scheduler_unit*>  schedulers;
@@ -1986,6 +1988,7 @@ public:
     void get_L1T_sub_stats(struct cache_sub_stats &css) const;
 
     void get_icnt_stats(long &n_simt_to_mem, long &n_mem_to_simt) const;
+    float get_current_occupancy( unsigned long long& active, unsigned long long & total ) const;
 
 private:
     unsigned m_cluster_id;