changes for checkpoint support

8 files changed, 191 insertions, 7 deletions

diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index acb376a..dc1dc5c 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -35,10 +35,70 @@
 #include "gpgpu-sim/gpu-sim.h"
 #include "option_parser.h"
 #include <algorithm>
+#include <sys/stat.h>
+#include <sstream>
+#include <iostream>
 
 unsigned mem_access_t::sm_next_access_uid = 0;   
 unsigned warp_inst_t::sm_next_uid = 0;
 
+checkpoint::checkpoint()
+{
+
+    struct stat st = {0};
+
+    if (stat("checkpoint_files", &st) == -1) {
+        mkdir("checkpoint_files", 0777);
+    }
+
+}
+void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name)
+{
+
+    FILE * fp2 = fopen(f1name, "r");
+    assert(fp2!=NULL);
+      char line [ 128 ]; /* or other suitable maximum line size */
+      unsigned int offset ;
+      while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */
+      {
+         unsigned int index;
+         char * pch;
+         pch = strtok (line," ");
+         if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l')
+         {
+
+           pch = strtok (NULL, " ");
+           
+           std::stringstream ss;
+            ss << std::hex << pch;
+            ss >> index;
+
+           offset=0;
+         }
+         else {
+            unsigned int  data;
+            std::stringstream ss;
+            ss << std::hex << pch;
+            ss >> data;
+            temp_mem->write_only(offset,index, 4,&data);
+            offset= offset+4;
+         }
+         //fputs ( line, stdout ); /* write the line */
+      }
+      fclose ( fp2 );
+}
+
+
+
+void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format)
+{
+
+      FILE * fp3 = fopen(fname, "w");
+      assert(fp3!=NULL);
+      mem->print(format,fp3);
+      fclose(fp3);
+}
+
 void move_warp( warp_inst_t *&dst, warp_inst_t *&src )
 {
    assert( dst->empty() );
@@ -64,6 +124,31 @@ void gpgpu_functional_sim_config::reg_options(class OptionParser * opp)
 	                 &m_experimental_lib_support,
 	                 "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]",
 	                 "0");
+  option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, 
+               " checkpointing flag (0 = no checkpoint)",
+               "0");
+  option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, 
+               " checkpointing during execution of which kernel (1- 1st kernel)",
+               "1");
+  option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, 
+               " checkpointing after # of CTA (< less than total CTA)",
+               "0");
+  option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, 
+               " resume flag (0 = no resume)",
+               "0");
+  option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, 
+               " Resume from which kernel (1= 1st kernel)",
+               "0");
+   option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, 
+               " resume from which CTA ",
+               "0");
+      option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, 
+               " resume from which CTA ",
+               "0");
+         option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, 
+               " resume from which CTA ",
+               "0");
+
     option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL,
                  &m_ptx_convert_to_ptxplus,
                  "Convert SASS (native ISA) to ptxplus and run ptxplus",
@@ -93,10 +178,20 @@ gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config )
     : m_function_model_config(config)
 {
    m_global_mem = new memory_space_impl<8192>("global",64*1024);
+   
    m_tex_mem = new memory_space_impl<8192>("tex",64*1024);
    m_surf_mem = new memory_space_impl<8192>("surf",64*1024);
 
    m_dev_malloc=GLOBAL_HEAP_START; 
+   checkpoint_option = m_function_model_config.get_checkpoint_option();
+   checkpoint_kernel = m_function_model_config.get_checkpoint_kernel();
+   checkpoint_CTA = m_function_model_config.get_checkpoint_CTA();
+   resume_option = m_function_model_config.get_resume_option();
+   resume_kernel = m_function_model_config.get_resume_kernel();
+   resume_CTA = m_function_model_config.get_resume_CTA();
+   checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t();
+   checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y();
+
 
    if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) 
       ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w");
@@ -733,6 +828,51 @@ void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask )
     m_stack.push_back(new_stack_entry);
 }
 
+void simt_stack::resume( char * fname )
+{
+    reset();    
+
+
+
+      FILE * fp2 = fopen(fname, "r");
+      assert(fp2!=NULL);
+
+      char line [ 200 ]; /* or other suitable maximum line size */
+
+      while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */
+      {
+          simt_stack_entry new_stack_entry;
+          char * pch;
+          pch = strtok (line," ");
+          for (unsigned j=0; j<m_warp_size; j++)
+          {
+                if (pch[0]=='1')
+                    new_stack_entry.m_active_mask.set(j);
+                else
+                    new_stack_entry.m_active_mask.reset(j);
+                pch = strtok (NULL," ");
+                
+          }  
+          
+         new_stack_entry.m_pc=atoi(pch);
+         pch = strtok (NULL," "); 
+         new_stack_entry.m_calldepth=atoi(pch);
+         pch = strtok (NULL," "); 
+         new_stack_entry.m_recvg_pc=atoi(pch);
+         pch = strtok (NULL," "); 
+         new_stack_entry.m_branch_div_cycle=atoi(pch);
+         pch = strtok (NULL," "); 
+         if(pch[0]=='0')
+            new_stack_entry.m_type= STACK_ENTRY_TYPE_NORMAL;
+         else
+            new_stack_entry.m_type= STACK_ENTRY_TYPE_CALL;
+         m_stack.push_back(new_stack_entry);
+      }
+      fclose ( fp2 );
+
+    
+}
+
 const simt_mask_t &simt_stack::get_active_mask() const
 {
     assert(m_stack.size() > 0);
@@ -777,6 +917,20 @@ void simt_stack::print (FILE *fout) const
         ptx_print_insn( stack_entry.m_pc, fout );
         fprintf(fout,"\n");
     }
+
+}
+
+void simt_stack::print_checkpoint (FILE *fout) const
+{
+    for ( unsigned k=0; k < m_stack.size(); k++ ) {
+        simt_stack_entry stack_entry = m_stack[k];
+       
+        for (unsigned j=0; j<m_warp_size; j++)
+            fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j)?'1':'0') );
+        fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc,stack_entry.m_calldepth,stack_entry.m_recvg_pc,stack_entry.m_branch_div_cycle,stack_entry.m_type );
+        fprintf(fout, "%d %d\n",m_warp_id, m_warp_size );
+        
+    }
 }
 
 void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc )
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index 9049a84..abd32f9 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -98,7 +98,7 @@ public:
         delete[] m_warpAtBarrier;
     }
     //! executes all warps till completion 
-    void execute();
+    void execute(int inst_count, unsigned ctaid_cp);
     virtual void warp_exit( unsigned warp_id );
     virtual bool warp_waiting_at_barrier( unsigned warp_id ) const  
     {
@@ -108,7 +108,7 @@ public:
 private:
     void executeWarp(unsigned, bool &, bool &);
     //initializes threads in the CTA block which we are executing
-    void initializeCTA();
+    void initializeCTA(unsigned ctaid_cp);
     virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid)
     {
     if(m_thread[tid]==NULL || m_thread[tid]->is_done()){
diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc
index 7bdf4d9..9554f55 100644
--- a/src/cuda-sim/memory.cc
+++ b/src/cuda-sim/memory.cc
@@ -44,9 +44,16 @@ template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::strin
    assert( m_log2_block_size != (unsigned)-1 );
 }
 
+template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data)
+{
+   m_data[index].write(offset,length,(const unsigned char*)data);
+}
+
 template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI)
 {
+
    mem_addr_t index = addr >> m_log2_block_size;
+
    if ( (addr+length) <= (index+1)*BSIZE ) {
       // fast route for intra-block access 
       unsigned offset = addr & (BSIZE-1);
@@ -142,8 +149,9 @@ template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, s
 template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const
 {
    typename map_t::const_iterator i_page;
-   for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
-      fprintf(fout, "%s - %#x:", m_name.c_str(), i_page->first);
+
+   for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
+      fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first);
       i_page->second.print(format, fout);
    }
 }
diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h
index f785b8b..ab588bc 100644
--- a/src/cuda-sim/memory.h
+++ b/src/cuda-sim/memory.h
@@ -81,7 +81,7 @@ public:
    {
       unsigned int *i_data = (unsigned int*)m_data;
       for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) {
-         if (d % 8 == 0) {
+         if (d % 1 == 0) {
             fprintf(fout, "\n");
          }
          fprintf(fout, format, i_data[d]);
@@ -104,6 +104,7 @@ class memory_space
 public:
    virtual ~memory_space() {}
    virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0;
+   virtual void write_only( mem_addr_t index, mem_addr_t offset,  size_t length, const void *data ) = 0;
    virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0;
    virtual void print( const char *format, FILE *fout ) const = 0;
    virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0;
@@ -114,8 +115,10 @@ public:
    memory_space_impl( std::string name, unsigned hash_size );
 
    virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI );
+   virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data);
    virtual void read( mem_addr_t addr, size_t length, void *data ) const;
    virtual void print( const char *format, FILE *fout ) const;
+   
    virtual void set_watch( addr_t addr, unsigned watchpoint ); 
 
 private:
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index 0a6e8eb..d226fbe 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -306,6 +306,8 @@ public:
 
    const ptx_version &get_ptx_version() const;
    void set_reg( const symbol *reg, const ptx_reg_t &value );
+   void print_reg_thread (char * fname);
+   void resume_reg_thread(char * fname,  symbol_table * symtab);
    ptx_reg_t get_reg( const symbol *reg );
    ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag );
    void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI );
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index eddc2f4..6ce5524 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -335,6 +335,7 @@ public:
     unsigned num_shader() const { return m_shader_config.num_shader(); }
     unsigned num_cluster() const { return m_shader_config.n_simt_clusters; }
     unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; }
+    unsigned checkpoint_option;
 
 private:
     void init_clock_domains(void ); 
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index cbc8388..faccf18 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -391,11 +391,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
    }
 }
 
-void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread )
+void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id )
 {
     address_type start_pc = next_pc(start_thread);
     if (m_config->model == POST_DOMINATOR) {
         unsigned start_warp = start_thread / m_config->warp_size;
+        unsigned warp_per_cta =  cta_size / m_config->warp_size;
         unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0);
         for (unsigned i = start_warp; i < end_warp; ++i) {
             unsigned n_active=0;
@@ -410,6 +411,21 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign
                 }
             }
             m_simt_stack[i]->launch(start_pc,active_threads);
+
+              if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+               { 
+                char fname[2048];
+                snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid );
+                unsigned pc,rpc;
+                m_simt_stack[i]->resume(fname);
+                m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc);
+                for (unsigned t = 0; t < m_config->warp_size; t++) {
+                  m_thread[i * m_config->warp_size + t]->set_npc(pc);
+                  m_thread[i * m_config->warp_size + t]->update_pc();
+                }   
+                start_pc=pc;
+              }
+               
             m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id);
             ++m_dynamic_warp_id;
             m_not_completed += n_active;
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 6e06322..46106f8 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1897,7 +1897,7 @@ public:
 	 }
 
     int test_res_bus(int latency);
-    void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread);
+    void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id);
     virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid);
     address_type next_pc( int tid ) const;
     void fetch();