Merge branch 'dev' of https://github.com/gpgpu-sim/gpgpu-sim_distribution into dev

4 files changed, 75 insertions, 15 deletions

diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 6cb4779..ec570bf 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -58,6 +58,7 @@
 #include "../debug.h"
 #include "../gpgpusim_entrypoint.h"
 #include "../cuda-sim/cuda-sim.h"
+#include "../cuda-sim/ptx_ir.h"
 #include "../trace.h"
 #include "mem_latency_stat.h"
 #include "power_stat.h"
@@ -284,6 +285,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, 
                  "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)",
                  "8192");
+    option_parser_register(opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block,
+                 "Maximum number of registers per CTA. (default 8192)",
+                 "8192");
     option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation,
                  "gpgpu_ignore_resources_limitation (default 0)",
                  "0");
@@ -305,6 +309,9 @@ void shader_core_config::reg_options(class OptionParser * opp)
     option_parser_register(opp, "-gpgpu_n_ldst_response_buffer_size", OPT_UINT32, &ldst_unit_response_queue_size, 
                  "number of response packets in ld/st unit ejection buffer",
                  "2");
+    option_parser_register(opp, "-gpgpu_shmem_per_block", OPT_UINT32, &gpgpu_shmem_per_block,
+                 "Size of shared memory per thread block or CTA (default 48kB)",
+                 "49152");
     option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size,
                  "Size of shared memory per shader core (default 16kB)",
                  "16384");
@@ -757,11 +764,21 @@ int gpgpu_sim::shared_mem_size() const
    return m_shader_config->gpgpu_shmem_size;
 }
 
+int gpgpu_sim::shared_mem_per_block() const
+{
+   return m_shader_config->gpgpu_shmem_per_block;
+}
+
 int gpgpu_sim::num_registers_per_core() const
 {
    return m_shader_config->gpgpu_shader_registers;
 }
 
+int gpgpu_sim::num_registers_per_block() const
+{
+   return m_shader_config->gpgpu_registers_per_block;
+}
+
 int gpgpu_sim::wrp_size() const
 {
    return m_shader_config->warp_size;
@@ -1418,22 +1435,44 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel )
     // bind functional simulation state of threads to hardware resources (simulation) 
     warp_set_t warps;
     unsigned nthreads_in_block= 0;
+    function_info *kernel_func_info = kernel.entry();
+    symbol_table * symtab= kernel_func_info->get_symtab();
+    unsigned ctaid= kernel.get_next_cta_id_single();
+    checkpoint *g_checkpoint=  new checkpoint();
     for (unsigned i = start_thread; i<end_thread; i++) {
         m_threadState[i].m_cta_id = free_cta_hw_id;
         unsigned warp_id = i/m_config->warp_size;
         nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu());
         m_threadState[i].m_active = true; 
+        // load thread local memory and register file
+        if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+        {
+            char fname[2048];
+            snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid );
+            m_thread[i]->resume_reg_thread(fname,symtab);
+            char f1name[2048];
+            snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid);
+            g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); 
+        }
+        //
         warps.set( warp_id );
     }
     assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max
     m_cta_status[free_cta_hw_id]=nthreads_in_block;
 
+    if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+    {
+        char f1name[2048];
+        snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid);
+        
+        g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name);  
+    }
     // now that we know which warps are used in this CTA, we can allocate
     // resources for use in CTA-wide barrier operations
     m_barriers.allocate_barrier(free_cta_hw_id,warps);
 
     // initialize the SIMT stacks and fetch hardware
-    init_warps( free_cta_hw_id, start_thread, end_thread);
+    init_warps( free_cta_hw_id, start_thread, end_thread, ctaid, cta_size, kernel.get_uid());
     m_n_active_cta++;
 
     shader_CTA_count_log(m_sid, 1);
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index 1bae1fa..6ce5524 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -335,6 +335,7 @@ public:
     unsigned num_shader() const { return m_shader_config.num_shader(); }
     unsigned num_cluster() const { return m_shader_config.n_simt_clusters; }
     unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; }
+    unsigned checkpoint_option;
 
 private:
     void init_clock_domains(void ); 
@@ -436,7 +437,9 @@ public:
    void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc );
 
    int shared_mem_size() const;
+   int shared_mem_per_block() const;
    int num_registers_per_core() const;
+   int num_registers_per_block() const;
    int wrp_size() const;
    int shader_clock() const;
    const struct cudaDeviceProp *get_prop() const;
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 12ea9d8..3db988b 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -440,11 +440,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
    }
 }
 
-void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread )
+void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id )
 {
     address_type start_pc = next_pc(start_thread);
     if (m_config->model == POST_DOMINATOR) {
         unsigned start_warp = start_thread / m_config->warp_size;
+        unsigned warp_per_cta =  cta_size / m_config->warp_size;
         unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0);
         for (unsigned i = start_warp; i < end_warp; ++i) {
             unsigned n_active=0;
@@ -459,6 +460,21 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign
                 }
             }
             m_simt_stack[i]->launch(start_pc,active_threads);
+
+              if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+               { 
+                char fname[2048];
+                snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid );
+                unsigned pc,rpc;
+                m_simt_stack[i]->resume(fname);
+                m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc);
+                for (unsigned t = 0; t < m_config->warp_size; t++) {
+                  m_thread[i * m_config->warp_size + t]->set_npc(pc);
+                  m_thread[i * m_config->warp_size + t]->update_pc();
+                }   
+                start_pc=pc;
+              }
+               
             m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id);
             ++m_dynamic_warp_id;
             m_not_completed += n_active;
@@ -3508,19 +3524,20 @@ int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_sh
 	   return bank % num_banks;
 }
 
-bool opndcoll_rfu_t::writeback( const warp_inst_t &inst )
+bool opndcoll_rfu_t::writeback( warp_inst_t &inst )
 {
    assert( !inst.empty() );
    std::list<unsigned> regs = m_shader->get_regs_written(inst);
-   std::list<unsigned>::iterator r;
-   unsigned n=0;
-   for( r=regs.begin(); r!=regs.end();r++,n++ ) {
-      unsigned reg = *r;
-      unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id());
-      if( m_arbiter.bank_idle(bank) ) {
-          m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg,m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id()));
-      } else {
-          return false;
+   for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) {
+      int reg_num = inst.arch_reg.dst[op]; // this math needs to match that used in function_info::ptx_decode_inst
+      if( reg_num >= 0 ){ // valid register
+         unsigned bank = register_bank(reg_num,inst.warp_id(),m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id());
+         if( m_arbiter.bank_idle(bank) ) {
+             m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg_num,m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id()));
+             inst.arch_reg.dst[op] = -1;
+         } else {
+             return false;
+         }
       }
    }
    for(unsigned i=0;i<(unsigned)regs.size();i++){
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 45cf8e4..9abd223 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -576,7 +576,7 @@ public:
    void init( unsigned num_banks, shader_core_ctx *shader );
 
    // modifiers
-   bool writeback( const warp_inst_t &warp ); // might cause stall 
+   bool writeback( warp_inst_t &warp ); 
 
    void step()
    {
@@ -1445,7 +1445,8 @@ struct shader_core_config : public core_config
     unsigned max_cta_per_core; //Limit on number of concurrent CTAs in shader core
     unsigned max_barriers_per_cta;
     char * gpgpu_scheduler_string;
-
+    unsigned gpgpu_shmem_per_block;
+    unsigned gpgpu_registers_per_block;
     char* pipeline_widths_string;
     int pipe_widths[N_PIPELINE_STAGES];
 
@@ -1955,7 +1956,7 @@ public:
 	 }
 
     int test_res_bus(int latency);
-    void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread);
+    void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id);
     virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid);
     address_type next_pc( int tid ) const;
     void fetch();