summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CHANGES4
-rw-r--r--aerialvision/organizedata.py2
-rw-r--r--checkpoint.md101
-rw-r--r--doc/checkpoint.pngbin0 -> 23434 bytes
-rw-r--r--src/abstract_hardware_model.cc154
-rw-r--r--src/cuda-sim/cuda-sim.h4
-rw-r--r--src/cuda-sim/memory.cc12
-rw-r--r--src/cuda-sim/memory.h5
-rw-r--r--src/cuda-sim/ptx_sim.h2
-rw-r--r--src/gpgpu-sim/gpu-sim.h1
-rw-r--r--src/gpgpu-sim/shader.cc18
-rw-r--r--src/gpgpu-sim/shader.h2
12 files changed, 296 insertions, 9 deletions
diff --git a/CHANGES b/CHANGES
index a208986..c5b97fc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -5,8 +5,10 @@ Version 4.0.0 (development branch) versus 3.2.3
2- Partially-support some SASS_60 in the PTXP_PLUS (not completed yet)
3- Added parsing support for wmma.load,wmma.mma and wmma.store ptx instructions
4- Implmented cudaLaunchKernel for CUTLASS library
+5- Added support for cuDNN and Pytorch library
+6- Added checkpoint support ([Fore more details](checkpoint.md))
-GPU Core Functional Simulation
-1- Implemented bfi and prmt instruction
+1- Implemented bfe, d4pa, bfi and prmt instruction
2- Implemented wmma.load and wmma.store supporting all the layout configuration for TITANV GPU
3- Implemented wmma.mma instructions supporting all of its 32 configuration for TITANV GPU
4- debug support for wmma instruction using debug_tensorcore flag
diff --git a/aerialvision/organizedata.py b/aerialvision/organizedata.py
index ea947cd..090b90f 100644
--- a/aerialvision/organizedata.py
+++ b/aerialvision/organizedata.py
@@ -97,7 +97,7 @@ def organizedata(fileVars):
'sparse':OrganizeSparse, # Vector data with 2D index (used by DRAM access stats)
'custom':0
}
- data_type_char = {int:'L', float:'d'}
+ data_type_char = {int:'I', float:'f'}
print "Organizing data into internal format..."
diff --git a/checkpoint.md b/checkpoint.md
new file mode 100644
index 0000000..3420fe8
--- /dev/null
+++ b/checkpoint.md
@@ -0,0 +1,101 @@
+# checkpoint documentation #
+
+The following diagram shows the how checkpoiting works and different variables which can be user defined
+
+![Checkpoint](doc/checkpoint.png)
+
+A program can be run in functional simulation mode upto some point and then GPU states are stored in files so that program can be resumed from same point in performance simulation mode
+
+**Following details are stored in "checkpoint\_files" folder in you run area**
+
+1. Global memory per kernel
+
+2. Local memory per thread
+
+3. Shared memory per CTA
+
+4. Register file per thread
+
+5. SIMT stack per warp
+
+The varibales shown in the diagram can be set in gpgpusim.config file.
+
+**Whether checkpoint should be executed or not**
+
+-checkpoint\_option 0
+
+
+**At which kernel checkpoint should be executed . (x from the figure)**
+
+-checkpoint\_kernel 1
+
+
+**How many CTA are executed 100% before checkpoint. (M from the figure)**
+
+-checkpoint\_CTA 50
+
+
+**Whether resume should be executed or not**
+
+-resume\_option 0
+
+
+**From which CTA to resume (M from the figure)**
+
+-resume\_CTA 50
+
+
+**From which kernel to resume (x from the figure)**
+
+-resume\_kernel 1
+
+**From M to t, CTA are executed partially (t from the figure)**
+
+-checkpoint\_CTA\_t 100
+
+**How many instruction are executed before checkpoint in partial CTA**
+
+-checkpoint\_insn\_Y 104
+
+**Suppose, each thread is executing 26 instructions and there are 256 threads in a block. You want to checkpoint after 13 instructions in each thread, then Y should be set to = 13\*256/warp\_size = 104, if 32 is the warp size**
+
+For an example, in check samples/0\_Simple/vectorAdd folder
+
+vectorAdd.cu launches 2 kernels with 256 CTA each and 256 threads per CTA and 26 instructions per thread.
+
+Checkpoint in 1st kernel after 50 full CTA and and 50 partial CTA (13\*256 instructions per CTA)
+
+Then following options should be added to gpgpusim.config
+
+-gpgpu\_ptx\_sim\_mode 1
+
+-checkpoint\_option 1
+
+-checkpoint\_kernel 1
+
+-checkpoint\_CTA 50
+
+-resume\_option 0
+
+-checkpoint\_CTA\_t 100
+
+-checkpoint\_insn\_Y 104
+
+**This will simulate 4,99,200 (50\*256\*26 + 50\*256\*13) instruction and only block 0 to 49 will pass in kernel 1**
+
+
+And, after that for resuming from samw point in performance simulation
+
+-gpgpu\_ptx\_sim\_mode 0
+
+-checkpoint\_option 0
+
+-resume\_option 1
+
+-resume\_CTA 50
+
+-resume\_kernel 1
+
+-checkpoint\_CTA\_t 100
+
+**This will simulate 12,04,736 instructions in kernel 1 (50\*256\*0 + 50\*256\*13 + 156\*256\*26 ) and 17,03,936 (256\*256\*26) instructions in kernel 2 and block 0 to 255 will pass in both the kernels**
diff --git a/doc/checkpoint.png b/doc/checkpoint.png
new file mode 100644
index 0000000..db327f6
--- /dev/null
+++ b/doc/checkpoint.png
Binary files differ
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc
index acb376a..dc1dc5c 100644
--- a/src/abstract_hardware_model.cc
+++ b/src/abstract_hardware_model.cc
@@ -35,10 +35,70 @@
#include "gpgpu-sim/gpu-sim.h"
#include "option_parser.h"
#include <algorithm>
+#include <sys/stat.h>
+#include <sstream>
+#include <iostream>
unsigned mem_access_t::sm_next_access_uid = 0;
unsigned warp_inst_t::sm_next_uid = 0;
+checkpoint::checkpoint()
+{
+
+ struct stat st = {0};
+
+ if (stat("checkpoint_files", &st) == -1) {
+ mkdir("checkpoint_files", 0777);
+ }
+
+}
+void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name)
+{
+
+ FILE * fp2 = fopen(f1name, "r");
+ assert(fp2!=NULL);
+ char line [ 128 ]; /* or other suitable maximum line size */
+ unsigned int offset ;
+ while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */
+ {
+ unsigned int index;
+ char * pch;
+ pch = strtok (line," ");
+ if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l')
+ {
+
+ pch = strtok (NULL, " ");
+
+ std::stringstream ss;
+ ss << std::hex << pch;
+ ss >> index;
+
+ offset=0;
+ }
+ else {
+ unsigned int data;
+ std::stringstream ss;
+ ss << std::hex << pch;
+ ss >> data;
+ temp_mem->write_only(offset,index, 4,&data);
+ offset= offset+4;
+ }
+ //fputs ( line, stdout ); /* write the line */
+ }
+ fclose ( fp2 );
+}
+
+
+
+void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format)
+{
+
+ FILE * fp3 = fopen(fname, "w");
+ assert(fp3!=NULL);
+ mem->print(format,fp3);
+ fclose(fp3);
+}
+
void move_warp( warp_inst_t *&dst, warp_inst_t *&src )
{
assert( dst->empty() );
@@ -64,6 +124,31 @@ void gpgpu_functional_sim_config::reg_options(class OptionParser * opp)
&m_experimental_lib_support,
"Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]",
"0");
+ option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option,
+ " checkpointing flag (0 = no checkpoint)",
+ "0");
+ option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel,
+ " checkpointing during execution of which kernel (1- 1st kernel)",
+ "1");
+ option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA,
+ " checkpointing after # of CTA (< less than total CTA)",
+ "0");
+ option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option,
+ " resume flag (0 = no resume)",
+ "0");
+ option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel,
+ " Resume from which kernel (1= 1st kernel)",
+ "0");
+ option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA,
+ " resume from which CTA ",
+ "0");
+ option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t,
+ " resume from which CTA ",
+ "0");
+ option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y,
+ " resume from which CTA ",
+ "0");
+
option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL,
&m_ptx_convert_to_ptxplus,
"Convert SASS (native ISA) to ptxplus and run ptxplus",
@@ -93,10 +178,20 @@ gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config )
: m_function_model_config(config)
{
m_global_mem = new memory_space_impl<8192>("global",64*1024);
+
m_tex_mem = new memory_space_impl<8192>("tex",64*1024);
m_surf_mem = new memory_space_impl<8192>("surf",64*1024);
m_dev_malloc=GLOBAL_HEAP_START;
+ checkpoint_option = m_function_model_config.get_checkpoint_option();
+ checkpoint_kernel = m_function_model_config.get_checkpoint_kernel();
+ checkpoint_CTA = m_function_model_config.get_checkpoint_CTA();
+ resume_option = m_function_model_config.get_resume_option();
+ resume_kernel = m_function_model_config.get_resume_kernel();
+ resume_CTA = m_function_model_config.get_resume_CTA();
+ checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t();
+ checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y();
+
if(m_function_model_config.get_ptx_inst_debug_to_file() != 0)
ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w");
@@ -733,6 +828,51 @@ void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask )
m_stack.push_back(new_stack_entry);
}
+void simt_stack::resume( char * fname )
+{
+ reset();
+
+
+
+ FILE * fp2 = fopen(fname, "r");
+ assert(fp2!=NULL);
+
+ char line [ 200 ]; /* or other suitable maximum line size */
+
+ while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */
+ {
+ simt_stack_entry new_stack_entry;
+ char * pch;
+ pch = strtok (line," ");
+ for (unsigned j=0; j<m_warp_size; j++)
+ {
+ if (pch[0]=='1')
+ new_stack_entry.m_active_mask.set(j);
+ else
+ new_stack_entry.m_active_mask.reset(j);
+ pch = strtok (NULL," ");
+
+ }
+
+ new_stack_entry.m_pc=atoi(pch);
+ pch = strtok (NULL," ");
+ new_stack_entry.m_calldepth=atoi(pch);
+ pch = strtok (NULL," ");
+ new_stack_entry.m_recvg_pc=atoi(pch);
+ pch = strtok (NULL," ");
+ new_stack_entry.m_branch_div_cycle=atoi(pch);
+ pch = strtok (NULL," ");
+ if(pch[0]=='0')
+ new_stack_entry.m_type= STACK_ENTRY_TYPE_NORMAL;
+ else
+ new_stack_entry.m_type= STACK_ENTRY_TYPE_CALL;
+ m_stack.push_back(new_stack_entry);
+ }
+ fclose ( fp2 );
+
+
+}
+
const simt_mask_t &simt_stack::get_active_mask() const
{
assert(m_stack.size() > 0);
@@ -777,6 +917,20 @@ void simt_stack::print (FILE *fout) const
ptx_print_insn( stack_entry.m_pc, fout );
fprintf(fout,"\n");
}
+
+}
+
+void simt_stack::print_checkpoint (FILE *fout) const
+{
+ for ( unsigned k=0; k < m_stack.size(); k++ ) {
+ simt_stack_entry stack_entry = m_stack[k];
+
+ for (unsigned j=0; j<m_warp_size; j++)
+ fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j)?'1':'0') );
+ fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc,stack_entry.m_calldepth,stack_entry.m_recvg_pc,stack_entry.m_branch_div_cycle,stack_entry.m_type );
+ fprintf(fout, "%d %d\n",m_warp_id, m_warp_size );
+
+ }
}
void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc )
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index 9049a84..abd32f9 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -98,7 +98,7 @@ public:
delete[] m_warpAtBarrier;
}
//! executes all warps till completion
- void execute();
+ void execute(int inst_count, unsigned ctaid_cp);
virtual void warp_exit( unsigned warp_id );
virtual bool warp_waiting_at_barrier( unsigned warp_id ) const
{
@@ -108,7 +108,7 @@ public:
private:
void executeWarp(unsigned, bool &, bool &);
//initializes threads in the CTA block which we are executing
- void initializeCTA();
+ void initializeCTA(unsigned ctaid_cp);
virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid)
{
if(m_thread[tid]==NULL || m_thread[tid]->is_done()){
diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc
index 7bdf4d9..9554f55 100644
--- a/src/cuda-sim/memory.cc
+++ b/src/cuda-sim/memory.cc
@@ -44,9 +44,16 @@ template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::strin
assert( m_log2_block_size != (unsigned)-1 );
}
+template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data)
+{
+ m_data[index].write(offset,length,(const unsigned char*)data);
+}
+
template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI)
{
+
mem_addr_t index = addr >> m_log2_block_size;
+
if ( (addr+length) <= (index+1)*BSIZE ) {
// fast route for intra-block access
unsigned offset = addr & (BSIZE-1);
@@ -142,8 +149,9 @@ template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, s
template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const
{
typename map_t::const_iterator i_page;
- for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
- fprintf(fout, "%s - %#x:", m_name.c_str(), i_page->first);
+
+ for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) {
+ fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first);
i_page->second.print(format, fout);
}
}
diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h
index f785b8b..ab588bc 100644
--- a/src/cuda-sim/memory.h
+++ b/src/cuda-sim/memory.h
@@ -81,7 +81,7 @@ public:
{
unsigned int *i_data = (unsigned int*)m_data;
for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) {
- if (d % 8 == 0) {
+ if (d % 1 == 0) {
fprintf(fout, "\n");
}
fprintf(fout, format, i_data[d]);
@@ -104,6 +104,7 @@ class memory_space
public:
virtual ~memory_space() {}
virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0;
+ virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0;
virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0;
virtual void print( const char *format, FILE *fout ) const = 0;
virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0;
@@ -114,8 +115,10 @@ public:
memory_space_impl( std::string name, unsigned hash_size );
virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI );
+ virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data);
virtual void read( mem_addr_t addr, size_t length, void *data ) const;
virtual void print( const char *format, FILE *fout ) const;
+
virtual void set_watch( addr_t addr, unsigned watchpoint );
private:
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index 0a6e8eb..d226fbe 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -306,6 +306,8 @@ public:
const ptx_version &get_ptx_version() const;
void set_reg( const symbol *reg, const ptx_reg_t &value );
+ void print_reg_thread (char * fname);
+ void resume_reg_thread(char * fname, symbol_table * symtab);
ptx_reg_t get_reg( const symbol *reg );
ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag );
void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI );
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index eddc2f4..6ce5524 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -335,6 +335,7 @@ public:
unsigned num_shader() const { return m_shader_config.num_shader(); }
unsigned num_cluster() const { return m_shader_config.n_simt_clusters; }
unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; }
+ unsigned checkpoint_option;
private:
void init_clock_domains(void );
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index cbc8388..faccf18 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -391,11 +391,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re
}
}
-void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread )
+void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id )
{
address_type start_pc = next_pc(start_thread);
if (m_config->model == POST_DOMINATOR) {
unsigned start_warp = start_thread / m_config->warp_size;
+ unsigned warp_per_cta = cta_size / m_config->warp_size;
unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0);
for (unsigned i = start_warp; i < end_warp; ++i) {
unsigned n_active=0;
@@ -410,6 +411,21 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign
}
}
m_simt_stack[i]->launch(start_pc,active_threads);
+
+ if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t )
+ {
+ char fname[2048];
+ snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid );
+ unsigned pc,rpc;
+ m_simt_stack[i]->resume(fname);
+ m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc);
+ for (unsigned t = 0; t < m_config->warp_size; t++) {
+ m_thread[i * m_config->warp_size + t]->set_npc(pc);
+ m_thread[i * m_config->warp_size + t]->update_pc();
+ }
+ start_pc=pc;
+ }
+
m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id);
++m_dynamic_warp_id;
m_not_completed += n_active;
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 6e06322..46106f8 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -1897,7 +1897,7 @@ public:
}
int test_res_bus(int latency);
- void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread);
+ void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id);
virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid);
address_type next_pc( int tid ) const;
void fetch();
generated by cgit v1.3 (git 2.53.0) at 2026-07-10 21:25:07 +0000