From 960a1492fbbf2a4d098c8d060bc22782211ca60b Mon Sep 17 00:00:00 2001 From: Suchita Pati Date: Tue, 10 Apr 2018 14:52:49 -0500 Subject: added config -gpgpu_shmem_per_block and fixed cudaGetDeviceAttributes function --- src/gpgpu-sim/gpu-sim.cc | 8 ++++++++ src/gpgpu-sim/gpu-sim.h | 1 + src/gpgpu-sim/shader.h | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) (limited to 'src/gpgpu-sim') diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 3829861..5d63ab7 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -265,6 +265,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_n_ldst_response_buffer_size", OPT_UINT32, &ldst_unit_response_queue_size, "number of response packets in ld/st unit ejection buffer", "2"); + option_parser_register(opp, "-gpgpu_shmem_per_block", OPT_UINT32, &gpgpu_shmem_per_block, + "Size of shared memory per thread block or CTA (default 48kB)", + "49152"); option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, "Size of shared memory per shader core (default 16kB)", "16384"); @@ -661,6 +664,11 @@ int gpgpu_sim::shared_mem_size() const return m_shader_config->gpgpu_shmem_size; } +int gpgpu_sim::shared_mem_per_block() const +{ + return m_shader_config->gpgpu_shmem_per_block; +} + int gpgpu_sim::num_registers_per_core() const { return m_shader_config->gpgpu_shader_registers; diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 7d92c66..8d1c4fc 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -389,6 +389,7 @@ public: void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ); int shared_mem_size() const; + int shared_mem_per_block() const; int num_registers_per_core() const; int wrp_size() const; int shader_clock() const; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index bdd8dbe..fbddd18 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1285,7 +1285,7 @@ struct shader_core_config : public core_config unsigned max_cta_per_core; //Limit on number of concurrent CTAs in shader core unsigned max_barriers_per_cta; char * gpgpu_scheduler_string; - + unsigned gpgpu_shmem_per_block; char* pipeline_widths_string; int pipe_widths[N_PIPELINE_STAGES]; -- cgit v1.3 From 70e036c6007135c43d213139e1a6963090721f59 Mon Sep 17 00:00:00 2001 From: Suchita Pati Date: Tue, 1 May 2018 14:58:35 -0500 Subject: Added support for -gpgpu_registers_per_block config --- libcuda/cuda_runtime_api.cc | 27 ++++++++------------------- src/gpgpu-sim/gpu-sim.cc | 8 ++++++++ src/gpgpu-sim/gpu-sim.h | 1 + src/gpgpu-sim/shader.h | 1 + 4 files changed, 18 insertions(+), 19 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index ef46f00..e3c2542 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -349,24 +349,13 @@ class _cuda_device_id *GPGPUSim_Init() prop->maxGridSize[2] = 0x40000000; prop->totalConstMem = 0x40000000; prop->textureAlignment = 0; - /* - - * TODO: Update the .config and xml files of all GPU config files with new value of sharedMemPerBlock. - - * Previously, this was thought as sharedMemPerMultiprocessor and is being used in many places. - - * Check whether all the instances of shared_mem_size(), gpgpu_shmem_size or sharedMemPerBlock are meant to use sharedMemPerBlock or sharedMemPerMultiprocessor. - - */ - - prop->sharedMemPerBlock = the_gpu->shared_mem_per_block(); - - #if (CUDART_VERSION > 5000) - - prop->sharedMemPerMultiprocessor = the_gpu->shared_mem_size(); - - #endif - +// * TODO: Update the .config and xml files of all GPU config files with new value of sharedMemPerBlock and regsPerBlock + prop->sharedMemPerBlock = the_gpu->shared_mem_per_block(); +#if (CUDART_VERSION > 5000) + prop->regsPerMultiprocessor = the_gpu->num_registers_per_core(); + prop->sharedMemPerMultiprocessor = the_gpu->shared_mem_size(); +#endif + prop->sharedMemPerBlock = the_gpu->shared_mem_per_block(); prop->regsPerBlock = the_gpu->num_registers_per_core(); prop->warpSize = the_gpu->wrp_size(); prop->clockRate = the_gpu->shader_clock(); @@ -912,7 +901,7 @@ __host__ cudaError_t CUDARTAPI cudaDeviceGetAttribute(int *value, enum cudaDevic *value= prop->sharedMemPerMultiprocessor; break; case 82: - *value= prop->regsPerBlock; + *value= prop->regsPerMultiprocessor; break; default: printf("ERROR: Attribute number %d unimplemented \n",attr); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 5d63ab7..b283964 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -247,6 +247,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", "8192"); + option_parser_register(opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block, + "Maximum number of registers per thread block. (default 8192)", + "8192"); option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, "Maximum number of concurrent CTAs in shader (default 8)", "8"); @@ -674,6 +677,11 @@ int gpgpu_sim::num_registers_per_core() const return m_shader_config->gpgpu_shader_registers; } +int gpgpu_sim::num_registers_per_block() const +{ + return m_shader_config->gpgpu_registers_per_block; +} + int gpgpu_sim::wrp_size() const { return m_shader_config->warp_size; diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 8d1c4fc..5908fd5 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -391,6 +391,7 @@ public: int shared_mem_size() const; int shared_mem_per_block() const; int num_registers_per_core() const; + int num_registers_per_block() const; int wrp_size() const; int shader_clock() const; const struct cudaDeviceProp *get_prop() const; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index fbddd18..6201840 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1286,6 +1286,7 @@ struct shader_core_config : public core_config unsigned max_barriers_per_cta; char * gpgpu_scheduler_string; unsigned gpgpu_shmem_per_block; + unsigned gpgpu_registers_per_block; char* pipeline_widths_string; int pipe_widths[N_PIPELINE_STAGES]; -- cgit v1.3 From f405cb9484a8b0f961bd7c143bebf1fcb17546da Mon Sep 17 00:00:00 2001 From: Suchita Pati Date: Wed, 2 May 2018 12:23:59 -0500 Subject: Minor change in config description --- src/gpgpu-sim/gpu-sim.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/gpgpu-sim') diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index b283964..07a0c57 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -248,7 +248,7 @@ void shader_core_config::reg_options(class OptionParser * opp) "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", "8192"); option_parser_register(opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block, - "Maximum number of registers per thread block. (default 8192)", + "Maximum number of registers per CTA. (default 8192)", "8192"); option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &max_cta_per_core, "Maximum number of concurrent CTAs in shader (default 8)", -- cgit v1.3 From a9af79ac84b69fa18dd395349b88f0d984f0a505 Mon Sep 17 00:00:00 2001 From: J Date: Tue, 7 Aug 2018 15:12:52 -0700 Subject: working fix for deadlock due to operand collector, parser changes to support culaunchkernel --- libcuda/cuda_runtime_api.cc | 74 ++++++--------------------------------------- src/cuda-sim/ptx_ir.h | 12 ++++++++ src/cuda-sim/ptx_parser.cc | 34 +++++++++++++++++++++ src/gpgpu-sim/shader.cc | 6 ++++ 4 files changed, 62 insertions(+), 64 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index c87c6c3..c12aaeb 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -3033,7 +3033,6 @@ CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const cha function_info* f = symtab->lookup_function( std::string(name) ); //just need to add given pointer to map for cudaLaunch context->register_hostFun_function( (const char*) hfunc, f); - g_cuda_launch_stack.push_back( kernel_config() ); *hfunc = (CUfunction)hfunc; return CUDA_SUCCESS; } @@ -3047,60 +3046,6 @@ CUresult CUDAAPI cuModuleUnload(CUmodule hmod) return CUDA_SUCCESS; } -CUresult CUDAAPI cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d = new dim3(x,y,z); - config.set_block_dim(d); - - return CUDA_SUCCESS; -} - -CUresult CUDAAPI cuParamSetSize(CUfunction hfunc, unsigned int numbytes) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - //check if size matches given args - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - gpgpu_ptx_sim_arg_list_t args = config.get_args(); - size_t total_size = 0; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - total_size += a->m_nbytes; - } - return (numbytes==total_size) ? CUDA_SUCCESS : CUDA_ERROR_INVALID_VALUE; -} - -CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned int numbytes) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - cudaSetupArgument((const void *) ptr, (size_t) numbytes, (size_t) offset); - return CUDA_SUCCESS; -} - -CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height) -{ - if(g_debug_execution >= 3){ - announce_call(__my_func__); - } - const char *hostFun = (const char*) f; - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d = new dim3(grid_width,grid_height,1); - config.set_grid_dim(d); - - cudaLaunch(hostFun); - return CUDA_SUCCESS; -} - - CUresult CUDAAPI cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void **kernelParams, void **extra) @@ -3108,17 +3053,18 @@ CUresult CUDAAPI cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned in if(g_debug_execution >= 3){ announce_call(__my_func__); } - if (sharedMemBytes!=0||hStream!=NULL||kernelParams!=NULL||extra!=NULL){ - printf("GPGPU-Sim CUDA DRIVER API: Warning: Currently do not support \nsharedMemBytes, hStream, kernelParams, and extra parameters.\n"); + if (extra!=NULL){ + printf("GPGPU-Sim CUDA DRIVER API: ERROR: Currently do not support void** extra.\n"); + abort(); } const char *hostFun = (const char*) f; - gpgpusim_ptx_assert( !g_cuda_launch_stack.empty(), "empty launch stack" ); - kernel_config &config = g_cuda_launch_stack.back(); - dim3 *d_b = new dim3(blockDimX, blockDimY, blockDimZ); - config.set_block_dim(d_b); - dim3 *d_g = new dim3(gridDimX, gridDimY, gridDimZ); - config.set_grid_dim(d_g); - + CUctx_st *context = GPGPUSim_Context(); + function_info *entry = context->get_kernel(hostFun); + cudaConfigureCall(dim3(gridDimX, gridDimY, gridDimZ), dim3(blockDimX, blockDimY, blockDimZ), sharedMemBytes, (cudaStream_t) hStream); + for(unsigned i = 0; i < entry->num_args(); i++){ + std::pair p = entry->get_param_config(i); + cudaSetupArgument(kernelParams[i], p.first, p.second); + } cudaLaunch(hostFun); return CUDA_SUCCESS; } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index ef4cf48..e726ab9 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -1293,6 +1293,17 @@ public: bool is_pdom_set() const { return pdom_done; } //return pdom flag void set_pdom() { pdom_done = true; } //set pdom flag + void add_config_param( size_t size, unsigned alignment ){ + unsigned offset = 0; + if (m_param_configs.size()>0){ + unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } + private: unsigned m_uid; unsigned m_local_mem_framesize; @@ -1306,6 +1317,7 @@ private: unsigned m_instr_mem_size; std::map m_kernel_params; std::map m_ptx_kernel_param_info; + std::vector< std::pair > m_param_configs; const symbol *m_return_var_sym; std::vector m_args; std::list m_instructions; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index c418fac..e6d6325 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -61,6 +61,7 @@ memory_space_t g_ptr_spec = undefined_space; int g_scalar_type_spec = -1; int g_vector_spec = -1; int g_alignment_spec = -1; +int g_size = -1; int g_extern_spec = 0; // variable declaration stuff: @@ -116,6 +117,7 @@ void init_directive_state() g_vector_spec=-1; g_opcode=-1; g_alignment_spec = -1; + g_size = -1; g_extern_spec = 0; g_scalar_type.clear(); g_operands.clear(); @@ -373,6 +375,9 @@ int pad_address (new_addr_type address, unsigned size, unsigned maxalign) { void add_identifier( const char *identifier, int array_dim, unsigned array_ident ) { + if(array_ident==ARRAY_IDENTIFIER){ + g_size *= array_dim; + } if( g_func_decl && (g_func_info == NULL) ) { // return variable decl... assert( g_add_identifier_cached__identifier == NULL ); @@ -562,10 +567,13 @@ void add_constptr(const char* identifier1, const char* identifier2, int offset) void add_function_arg() { + assert(g_size>0); if( g_func_info ) { PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); g_func_info->add_arg(g_last_symbol); + g_func_info->add_config_param( g_size, g_alignment_spec ); } + } void add_extern_spec() @@ -617,6 +625,32 @@ void add_vector_spec(int spec ) void add_scalar_type_spec( int type_spec ) { + //save size of parameter + switch ( type_spec ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + g_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + g_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + g_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + g_size = 8; break; + case BB128_TYPE: + g_size = 16; break; + } PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str()); g_scalar_type.push_back( type_spec ); if ( g_scalar_type.size() > 1 ) { diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 4640d65..da85bae 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3034,6 +3034,12 @@ bool opndcoll_rfu_t::writeback( const warp_inst_t &inst ) for( r=regs.begin(); r!=regs.end();r++,n++ ) { unsigned reg = *r; unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift); + unsigned count = 0; + while( !m_arbiter.bank_idle(bank) ) { + assert((++count) Date: Tue, 7 Aug 2018 17:02:39 -0700 Subject: fix for deadlock that should serialize bank accesses --- src/gpgpu-sim/shader.cc | 27 +++++++++++---------------- src/gpgpu-sim/shader.h | 2 +- 2 files changed, 12 insertions(+), 17 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index da85bae..0a5e11f 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3025,25 +3025,20 @@ int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_sh return bank % num_banks; } -bool opndcoll_rfu_t::writeback( const warp_inst_t &inst ) +bool opndcoll_rfu_t::writeback( warp_inst_t &inst ) { assert( !inst.empty() ); std::list regs = m_shader->get_regs_written(inst); - std::list::iterator r; - unsigned n=0; - for( r=regs.begin(); r!=regs.end();r++,n++ ) { - unsigned reg = *r; - unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift); - unsigned count = 0; - while( !m_arbiter.bank_idle(bank) ) { - assert((++count)= 0 ){ // valid register + unsigned bank = register_bank(reg_num,inst.warp_id(),m_num_banks,m_bank_warp_shift); + if( m_arbiter.bank_idle(bank) ) { + m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg_num,m_num_banks,m_bank_warp_shift)); + inst.arch_reg.dst[op] = -1; + } else { + return false; + } } } for(unsigned i=0;i<(unsigned)regs.size();i++){ diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 6201840..958abc4 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -525,7 +525,7 @@ public: void init( unsigned num_banks, shader_core_ctx *shader ); // modifiers - bool writeback( const warp_inst_t &warp ); // might cause stall + bool writeback( warp_inst_t &warp ); void step() { -- cgit v1.3 From 801438706272f5518f34bc3d6def462c6805a2fb Mon Sep 17 00:00:00 2001 From: Deval Shah Date: Fri, 9 Nov 2018 21:19:34 -0800 Subject: changes for checkpoint support --- CHANGES | 4 +- aerialvision/organizedata.py | 2 +- checkpoint.md | 101 +++++++++++++++++++++++++++ doc/checkpoint.png | Bin 0 -> 23434 bytes src/abstract_hardware_model.cc | 154 +++++++++++++++++++++++++++++++++++++++++ src/cuda-sim/cuda-sim.h | 4 +- src/cuda-sim/memory.cc | 12 +++- src/cuda-sim/memory.h | 5 +- src/cuda-sim/ptx_sim.h | 2 + src/gpgpu-sim/gpu-sim.h | 1 + src/gpgpu-sim/shader.cc | 18 ++++- src/gpgpu-sim/shader.h | 2 +- 12 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 checkpoint.md create mode 100644 doc/checkpoint.png (limited to 'src/gpgpu-sim') diff --git a/CHANGES b/CHANGES index a208986..c5b97fc 100644 --- a/CHANGES +++ b/CHANGES @@ -5,8 +5,10 @@ Version 4.0.0 (development branch) versus 3.2.3 2- Partially-support some SASS_60 in the PTXP_PLUS (not completed yet) 3- Added parsing support for wmma.load,wmma.mma and wmma.store ptx instructions 4- Implmented cudaLaunchKernel for CUTLASS library +5- Added support for cuDNN and Pytorch library +6- Added checkpoint support ([Fore more details](checkpoint.md)) -GPU Core Functional Simulation -1- Implemented bfi and prmt instruction +1- Implemented bfe, d4pa, bfi and prmt instruction 2- Implemented wmma.load and wmma.store supporting all the layout configuration for TITANV GPU 3- Implemented wmma.mma instructions supporting all of its 32 configuration for TITANV GPU 4- debug support for wmma instruction using debug_tensorcore flag diff --git a/aerialvision/organizedata.py b/aerialvision/organizedata.py index ea947cd..090b90f 100644 --- a/aerialvision/organizedata.py +++ b/aerialvision/organizedata.py @@ -97,7 +97,7 @@ def organizedata(fileVars): 'sparse':OrganizeSparse, # Vector data with 2D index (used by DRAM access stats) 'custom':0 } - data_type_char = {int:'L', float:'d'} + data_type_char = {int:'I', float:'f'} print "Organizing data into internal format..." diff --git a/checkpoint.md b/checkpoint.md new file mode 100644 index 0000000..3420fe8 --- /dev/null +++ b/checkpoint.md @@ -0,0 +1,101 @@ +# checkpoint documentation # + +The following diagram shows the how checkpoiting works and different variables which can be user defined + +![Checkpoint](doc/checkpoint.png) + +A program can be run in functional simulation mode upto some point and then GPU states are stored in files so that program can be resumed from same point in performance simulation mode + +**Following details are stored in "checkpoint\_files" folder in you run area** + +1. Global memory per kernel + +2. Local memory per thread + +3. Shared memory per CTA + +4. Register file per thread + +5. SIMT stack per warp + +The varibales shown in the diagram can be set in gpgpusim.config file. + +**Whether checkpoint should be executed or not** + +-checkpoint\_option 0 + + +**At which kernel checkpoint should be executed . (x from the figure)** + +-checkpoint\_kernel 1 + + +**How many CTA are executed 100% before checkpoint. (M from the figure)** + +-checkpoint\_CTA 50 + + +**Whether resume should be executed or not** + +-resume\_option 0 + + +**From which CTA to resume (M from the figure)** + +-resume\_CTA 50 + + +**From which kernel to resume (x from the figure)** + +-resume\_kernel 1 + +**From M to t, CTA are executed partially (t from the figure)** + +-checkpoint\_CTA\_t 100 + +**How many instruction are executed before checkpoint in partial CTA** + +-checkpoint\_insn\_Y 104 + +**Suppose, each thread is executing 26 instructions and there are 256 threads in a block. You want to checkpoint after 13 instructions in each thread, then Y should be set to = 13\*256/warp\_size = 104, if 32 is the warp size** + +For an example, in check samples/0\_Simple/vectorAdd folder + +vectorAdd.cu launches 2 kernels with 256 CTA each and 256 threads per CTA and 26 instructions per thread. + +Checkpoint in 1st kernel after 50 full CTA and and 50 partial CTA (13\*256 instructions per CTA) + +Then following options should be added to gpgpusim.config + +-gpgpu\_ptx\_sim\_mode 1 + +-checkpoint\_option 1 + +-checkpoint\_kernel 1 + +-checkpoint\_CTA 50 + +-resume\_option 0 + +-checkpoint\_CTA\_t 100 + +-checkpoint\_insn\_Y 104 + +**This will simulate 4,99,200 (50\*256\*26 + 50\*256\*13) instruction and only block 0 to 49 will pass in kernel 1** + + +And, after that for resuming from samw point in performance simulation + +-gpgpu\_ptx\_sim\_mode 0 + +-checkpoint\_option 0 + +-resume\_option 1 + +-resume\_CTA 50 + +-resume\_kernel 1 + +-checkpoint\_CTA\_t 100 + +**This will simulate 12,04,736 instructions in kernel 1 (50\*256\*0 + 50\*256\*13 + 156\*256\*26 ) and 17,03,936 (256\*256\*26) instructions in kernel 2 and block 0 to 255 will pass in both the kernels** diff --git a/doc/checkpoint.png b/doc/checkpoint.png new file mode 100644 index 0000000..db327f6 Binary files /dev/null and b/doc/checkpoint.png differ diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index acb376a..dc1dc5c 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -35,10 +35,70 @@ #include "gpgpu-sim/gpu-sim.h" #include "option_parser.h" #include +#include +#include +#include unsigned mem_access_t::sm_next_access_uid = 0; unsigned warp_inst_t::sm_next_uid = 0; +checkpoint::checkpoint() +{ + + struct stat st = {0}; + + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); + } + +} +void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) +{ + + FILE * fp2 = fopen(f1name, "r"); + assert(fp2!=NULL); + char line [ 128 ]; /* or other suitable maximum line size */ + unsigned int offset ; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + unsigned int index; + char * pch; + pch = strtok (line," "); + if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') + { + + pch = strtok (NULL, " "); + + std::stringstream ss; + ss << std::hex << pch; + ss >> index; + + offset=0; + } + else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset,index, 4,&data); + offset= offset+4; + } + //fputs ( line, stdout ); /* write the line */ + } + fclose ( fp2 ); +} + + + +void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) +{ + + FILE * fp3 = fopen(fname, "w"); + assert(fp3!=NULL); + mem->print(format,fp3); + fclose(fp3); +} + void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) { assert( dst->empty() ); @@ -64,6 +124,31 @@ void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) &m_experimental_lib_support, "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", + "0"); + option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", + "1"); + option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", + "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", + "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", + "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, "Convert SASS (native ISA) to ptxplus and run ptxplus", @@ -93,10 +178,20 @@ gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config ) : m_function_model_config(config) { m_global_mem = new memory_space_impl<8192>("global",64*1024); + m_tex_mem = new memory_space_impl<8192>("tex",64*1024); m_surf_mem = new memory_space_impl<8192>("surf",64*1024); m_dev_malloc=GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); @@ -733,6 +828,51 @@ void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) m_stack.push_back(new_stack_entry); } +void simt_stack::resume( char * fname ) +{ + reset(); + + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + + char line [ 200 ]; /* or other suitable maximum line size */ + + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + simt_stack_entry new_stack_entry; + char * pch; + pch = strtok (line," "); + for (unsigned j=0; j 0); @@ -777,6 +917,20 @@ void simt_stack::print (FILE *fout) const ptx_print_insn( stack_entry.m_pc, fout ); fprintf(fout,"\n"); } + +} + +void simt_stack::print_checkpoint (FILE *fout) const +{ + for ( unsigned k=0; k < m_stack.size(); k++ ) { + simt_stack_entry stack_entry = m_stack[k]; + + for (unsigned j=0; jis_done()){ diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 7bdf4d9..9554f55 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -44,9 +44,16 @@ template memory_space_impl::memory_space_impl( std::strin assert( m_log2_block_size != (unsigned)-1 ); } +template void memory_space_impl::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) +{ + m_data[index].write(offset,length,(const unsigned char*)data); +} + template void memory_space_impl::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; + if ( (addr+length) <= (index+1)*BSIZE ) { // fast route for intra-block access unsigned offset = addr & (BSIZE-1); @@ -142,8 +149,9 @@ template void memory_space_impl::read( mem_addr_t addr, s template void memory_space_impl::print( const char *format, FILE *fout ) const { typename map_t::const_iterator i_page; - for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s - %#x:", m_name.c_str(), i_page->first); + + for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); i_page->second.print(format, fout); } } diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index f785b8b..ab588bc 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -81,7 +81,7 @@ public: { unsigned int *i_data = (unsigned int*)m_data; for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 8 == 0) { + if (d % 1 == 0) { fprintf(fout, "\n"); } fprintf(fout, format, i_data[d]); @@ -104,6 +104,7 @@ class memory_space public: virtual ~memory_space() {} virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; virtual void print( const char *format, FILE *fout ) const = 0; virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; @@ -114,8 +115,10 @@ public: memory_space_impl( std::string name, unsigned hash_size ); virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); virtual void read( mem_addr_t addr, size_t length, void *data ) const; virtual void print( const char *format, FILE *fout ) const; + virtual void set_watch( addr_t addr, unsigned watchpoint ); private: diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index 0a6e8eb..d226fbe 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -306,6 +306,8 @@ public: const ptx_version &get_ptx_version() const; void set_reg( const symbol *reg, const ptx_reg_t &value ); + void print_reg_thread (char * fname); + void resume_reg_thread(char * fname, symbol_table * symtab); ptx_reg_t get_reg( const symbol *reg ); ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ); void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ); diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index eddc2f4..6ce5524 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -335,6 +335,7 @@ public: unsigned num_shader() const { return m_shader_config.num_shader(); } unsigned num_cluster() const { return m_shader_config.n_simt_clusters; } unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; } + unsigned checkpoint_option; private: void init_clock_domains(void ); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index cbc8388..faccf18 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -391,11 +391,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re } } -void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread ) +void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id ) { address_type start_pc = next_pc(start_thread); if (m_config->model == POST_DOMINATOR) { unsigned start_warp = start_thread / m_config->warp_size; + unsigned warp_per_cta = cta_size / m_config->warp_size; unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0); for (unsigned i = start_warp; i < end_warp; ++i) { unsigned n_active=0; @@ -410,6 +411,21 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign } } m_simt_stack[i]->launch(start_pc,active_threads); + + if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid ); + unsigned pc,rpc; + m_simt_stack[i]->resume(fname); + m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc); + for (unsigned t = 0; t < m_config->warp_size; t++) { + m_thread[i * m_config->warp_size + t]->set_npc(pc); + m_thread[i * m_config->warp_size + t]->update_pc(); + } + start_pc=pc; + } + m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id); ++m_dynamic_warp_id; m_not_completed += n_active; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 6e06322..46106f8 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1897,7 +1897,7 @@ public: } int test_res_bus(int latency); - void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread); + void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id); virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid); address_type next_pc( int tid ) const; void fetch(); -- cgit v1.3 From 7a9c450e6b905af9ca6cdd3c7b79ad5aec535a5a Mon Sep 17 00:00:00 2001 From: Deval Shah Date: Fri, 9 Nov 2018 21:27:34 -0800 Subject: resolving merge conflicts --- libcuda/cuda_runtime_api.cc | 35 +--------------- src/abstract_hardware_model.h | 30 ++------------ src/cuda-sim/cuda-sim.cc | 94 ++++--------------------------------------- src/cuda-sim/instructions.cc | 56 -------------------------- src/gpgpu-sim/gpu-sim.cc | 25 ++---------- 5 files changed, 16 insertions(+), 224 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 4b50e34..f00fe52 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1425,40 +1425,7 @@ __host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) dim3 gridDim = config.grid_dim(); dim3 blockDim = config.block_dim(); - gpgpu_t *gpu = context->get_device()->get_gpgpu(); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - class memory_space *global_mem; - global_mem = gpu->get_global_memory(); - - if(gpu->resume_option ==1 && (grid->get_uid()==gpu->resume_kernel)) - { - - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); - - g_checkpoint->load_global_mem(global_mem, f1name); - for (int i=0;iresume_CTA;i++) - grid->increment_cta_id(); - } - if(gpu->resume_option==1 && (grid->get_uid()resume_kernel)) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); - - g_checkpoint->load_global_mem(global_mem, f1name); - printf("Skipping kernel %d as resuming from kernel %d\n",grid->get_uid(),gpu->resume_kernel ); - g_cuda_launch_stack.pop_back(); - return g_last_cudaError = cudaSuccess; - - } - if(gpu->checkpoint_option==1 && (grid->get_uid()>gpu->checkpoint_kernel)) - { - printf("Skipping kernel %d as checkpoint from kernel %d\n",grid->get_uid(),gpu->checkpoint_kernel ); - g_cuda_launch_stack.pop_back(); - return g_last_cudaError = cudaSuccess; - - } + printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", kname.c_str(), stream?stream->get_uid():0, gridDim.x,gridDim.y,gridDim.z,blockDim.x,blockDim.y,blockDim.z ); stream_operation op(grid,g_ptx_sim_mode,stream); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 2350db4..45fba76 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -381,8 +381,7 @@ public: void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; unsigned get_rp() const; void print(FILE *fp) const; - void resume(char * fname) ; - void print_checkpoint (FILE *fout) const; + protected: unsigned m_warp_id; @@ -502,28 +501,14 @@ public: const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } unsigned get_texcache_linesize() const { return m_texcache_linesize; } - int get_checkpoint_option() const {return checkpoint_option; } - int get_checkpoint_kernel() const {return checkpoint_kernel; } - int get_checkpoint_CTA() const {return checkpoint_CTA; } - int get_resume_option() const {return resume_option; } - int get_resume_kernel() const {return resume_kernel; } - int get_resume_CTA() const {return resume_CTA; } - int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } - int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } + private: // PTX options int m_ptx_convert_to_ptxplus; int m_ptx_use_cuobjdump; int m_experimental_lib_support; unsigned m_ptx_force_max_capability; - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - int resume_option; - int resume_kernel; - int resume_CTA; - int checkpoint_CTA_t; - int checkpoint_insn_Y; + int g_ptx_inst_debug_to_file; char* g_ptx_inst_debug_file; int g_ptx_inst_debug_thread_uid; @@ -535,14 +520,7 @@ private: class gpgpu_t { public: gpgpu_t( const gpgpu_functional_sim_config &config ); - int checkpoint_option; - int checkpoint_kernel; - int checkpoint_CTA; - int resume_option; - int resume_kernel; - int resume_CTA; - int checkpoint_CTA_t; - int checkpoint_insn_Y; + void* gpu_malloc( size_t size ); void* gpu_mallocarray( size_t count ); void gpu_memset( size_t dst_start_addr, int c, size_t count ); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 642e301..6a6b307 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -2125,8 +2125,7 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //before we execute, we should do PDOM analysis for functional simulation scenario. function_info *kernel_func_info = kernel.entry(); const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); + if (kernel_func_info->is_pdom_set()) { printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); @@ -2143,21 +2142,12 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) - int inst_count=50; - int cp_op= g_the_gpu->checkpoint_option; - int cp_CTA = g_the_gpu->checkpoint_CTA; - int cp_kernel= g_the_gpu->checkpoint_kernel; - cp_count= g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ unsigned temp=kernel.get_next_cta_id_single(); - if(cp_op==0 || (cp_op==1 && cta_launched= 5000) launch_all_device_kernels(); #endif - } - else - { - kernel.increment_cta_id(); - } - cta_launched++; + } - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x"); - } + @@ -2227,7 +2207,6 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); for(int i=0; i< m_warp_count; i++){ m_warpAtBarrier[i]=false; @@ -2240,10 +2219,7 @@ void functionalCoreSim::initializeCTA(unsigned ctaid_cp) for(unsigned i=0; ithreads_per_cta();i++) { ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); + ctaLiveThreads++; } @@ -2266,40 +2242,25 @@ void functionalCoreSim::createWarp(unsigned warpId) char fname[2048]; snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - if(cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; iset_npc(pc); - m_thread[i]->update_pc(); - } - } m_liveThreadCount[warpId]= liveThreadsCount; } void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - cp_count= m_gpu->checkpoint_insn_Y; - cp_cta_resume= m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); - int count=0; + while(true){ bool someOneLive= false; bool allAtBarrier = true; for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } + if(!someOneLive) break; if(allAtBarrier){ for(unsigned i=0;ientry()->get_symtab(); - - - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;iprint_checkpoint(fp); - fclose(fp); - } - } } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 31a33c6..f57a3f7 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -183,63 +183,7 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) m_last_set_operand_value = value; } -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); - - int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - typename reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size ); - - } - //m_regs.pop_back(); - } - fclose(fp); - - } - -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - unsigned size; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - char * decl= pch; - pch = strtok (NULL," "); - size = atoi(pch); - - - m_regs.back()[reg] = data; - } - fclose ( fp2 ); -} - ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) { diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 79a6fcd..c706f23 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1414,38 +1414,19 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) // bind functional simulation state of threads to hardware resources (simulation) warp_set_t warps; unsigned nthreads_in_block= 0; - function_info *kernel_func_info = kernel.entry(); - symbol_table * symtab= kernel_func_info->get_symtab(); - unsigned ctaid= kernel.get_next_cta_id_single(); - checkpoint *g_checkpoint= new checkpoint(); + for (unsigned i = start_thread; iwarp_size; nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); m_threadState[i].m_active = true; - // load thread local memory and register file - if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); - m_thread[i]->resume_reg_thread(fname,symtab); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); - g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); - } - // + warps.set( warp_id ); } assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max m_cta_status[free_cta_hw_id]=nthreads_in_block; - if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); - - g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); - } + // now that we know which warps are used in this CTA, we can allocate // resources for use in CTA-wide barrier operations m_barriers.allocate_barrier(free_cta_hw_id,warps); -- cgit v1.3 From 642818ae5ff61c1544bcce9e7ba2dd0aea47ea6a Mon Sep 17 00:00:00 2001 From: Deval Shah Date: Fri, 9 Nov 2018 21:29:18 -0800 Subject: Adding checkpoint support --- libcuda/cuda_runtime_api.cc | 35 +++++++++++++++- src/abstract_hardware_model.h | 30 ++++++++++++-- src/cuda-sim/cuda-sim.cc | 94 +++++++++++++++++++++++++++++++++++++++---- src/cuda-sim/instructions.cc | 56 ++++++++++++++++++++++++++ src/gpgpu-sim/gpu-sim.cc | 25 ++++++++++-- 5 files changed, 224 insertions(+), 16 deletions(-) (limited to 'src/gpgpu-sim') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index f00fe52..4b50e34 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1425,7 +1425,40 @@ __host__ cudaError_t CUDARTAPI cudaLaunch( const char *hostFun ) dim3 gridDim = config.grid_dim(); dim3 blockDim = config.block_dim(); - + gpgpu_t *gpu = context->get_device()->get_gpgpu(); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + class memory_space *global_mem; + global_mem = gpu->get_global_memory(); + + if(gpu->resume_option ==1 && (grid->get_uid()==gpu->resume_kernel)) + { + + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); + + g_checkpoint->load_global_mem(global_mem, f1name); + for (int i=0;iresume_CTA;i++) + grid->increment_cta_id(); + } + if(gpu->resume_option==1 && (grid->get_uid()resume_kernel)) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", grid->get_uid()); + + g_checkpoint->load_global_mem(global_mem, f1name); + printf("Skipping kernel %d as resuming from kernel %d\n",grid->get_uid(),gpu->resume_kernel ); + g_cuda_launch_stack.pop_back(); + return g_last_cudaError = cudaSuccess; + + } + if(gpu->checkpoint_option==1 && (grid->get_uid()>gpu->checkpoint_kernel)) + { + printf("Skipping kernel %d as checkpoint from kernel %d\n",grid->get_uid(),gpu->checkpoint_kernel ); + g_cuda_launch_stack.pop_back(); + return g_last_cudaError = cudaSuccess; + + } printf("GPGPU-Sim PTX: pushing kernel \'%s\' to stream %u, gridDim= (%u,%u,%u) blockDim = (%u,%u,%u) \n", kname.c_str(), stream?stream->get_uid():0, gridDim.x,gridDim.y,gridDim.z,blockDim.x,blockDim.y,blockDim.z ); stream_operation op(grid,g_ptx_sim_mode,stream); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 45fba76..2350db4 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -381,7 +381,8 @@ public: void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; unsigned get_rp() const; void print(FILE *fp) const; - + void resume(char * fname) ; + void print_checkpoint (FILE *fout) const; protected: unsigned m_warp_id; @@ -501,14 +502,28 @@ public: const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } unsigned get_texcache_linesize() const { return m_texcache_linesize; } - + int get_checkpoint_option() const {return checkpoint_option; } + int get_checkpoint_kernel() const {return checkpoint_kernel; } + int get_checkpoint_CTA() const {return checkpoint_CTA; } + int get_resume_option() const {return resume_option; } + int get_resume_kernel() const {return resume_kernel; } + int get_resume_CTA() const {return resume_CTA; } + int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } private: // PTX options int m_ptx_convert_to_ptxplus; int m_ptx_use_cuobjdump; int m_experimental_lib_support; unsigned m_ptx_force_max_capability; - + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; int g_ptx_inst_debug_to_file; char* g_ptx_inst_debug_file; int g_ptx_inst_debug_thread_uid; @@ -520,7 +535,14 @@ private: class gpgpu_t { public: gpgpu_t( const gpgpu_functional_sim_config &config ); - + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; void* gpu_malloc( size_t size ); void* gpu_mallocarray( size_t count ); void gpu_memset( size_t dst_start_addr, int c, size_t count ); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 6a6b307..642e301 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -2125,7 +2125,8 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //before we execute, we should do PDOM analysis for functional simulation scenario. function_info *kernel_func_info = kernel.entry(); const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); if (kernel_func_info->is_pdom_set()) { printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); @@ -2142,12 +2143,21 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) + int inst_count=50; + int cp_op= g_the_gpu->checkpoint_option; + int cp_CTA = g_the_gpu->checkpoint_CTA; + int cp_kernel= g_the_gpu->checkpoint_kernel; + cp_count= g_the_gpu->checkpoint_insn_Y; + cp_cta_resume= g_the_gpu->checkpoint_CTA_t; + int cta_launched =0; //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ unsigned temp=kernel.get_next_cta_id_single(); + if(cp_op==0 || (cp_op==1 && cta_launched= 5000) launch_all_device_kernels(); #endif - + } + else + { + kernel.increment_cta_id(); + } + cta_launched++; } - + if(cp_op==1) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); + g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x"); + } @@ -2207,6 +2227,7 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { int ctaLiveThreads=0; + symbol_table * symtab= m_kernel->entry()->get_symtab(); for(int i=0; i< m_warp_count; i++){ m_warpAtBarrier[i]=false; @@ -2219,7 +2240,10 @@ void functionalCoreSim::initializeCTA(unsigned ctaid_cp) for(unsigned i=0; ithreads_per_cta();i++) { ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); + if(cp_cta_resume==1) + m_thread[i]->resume_reg_thread(fname,symtab); ctaLiveThreads++; } @@ -2242,25 +2266,40 @@ void functionalCoreSim::createWarp(unsigned warpId) char fname[2048]; snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); + if(cp_cta_resume==1) + { + unsigned pc,rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + for(int i=warpId*m_warp_size; iset_npc(pc); + m_thread[i]->update_pc(); + } + } m_liveThreadCount[warpId]= liveThreadsCount; } void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - + cp_count= m_gpu->checkpoint_insn_Y; + cp_cta_resume= m_gpu->checkpoint_CTA_t; initializeCTA(ctaid_cp); - + int count=0; while(true){ bool someOneLive= false; bool allAtBarrier = true; for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) + { + someOneLive=false; + break; + } if(!someOneLive) break; if(allAtBarrier){ for(unsigned i=0;ientry()->get_symtab(); + + + unsigned ctaid =m_kernel->get_next_cta_id_single(); + if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x"); + for(int i=0; i<32*m_warp_count;i++) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } + + for(int i=0;iprint_checkpoint(fp); + fclose(fp); + } + } } diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index f57a3f7..31a33c6 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -183,7 +183,63 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) m_last_set_operand_value = value; } +void ptx_thread_info::print_reg_thread(char * fname) +{ + + FILE *fp= fopen(fname,"w"); + assert(fp!=NULL); + + int size = m_regs.size(); + + if(size>0) + { + reg_map_t reg = m_regs.back(); + + typename reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) + { + const std::string &name = it->first->name(); + const std::string &dec= it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size ); + + } + //m_regs.pop_back(); + } + fclose(fp); + + } + +void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) +{ + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + //m_regs.push_back( reg_map_t() ); + char line [ 200 ]; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) + { + symbol *reg; + char * pch; + unsigned size; + pch = strtok (line," "); + char * name =pch; + reg= symtab->lookup(name); + ptx_reg_t data; + pch = strtok (NULL," "); + data = atoi(pch); + pch = strtok (NULL," "); + char * decl= pch; + pch = strtok (NULL," "); + size = atoi(pch); + + + m_regs.back()[reg] = data; + } + fclose ( fp2 ); +} + ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) { diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index c706f23..79a6fcd 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1414,19 +1414,38 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) // bind functional simulation state of threads to hardware resources (simulation) warp_set_t warps; unsigned nthreads_in_block= 0; - + function_info *kernel_func_info = kernel.entry(); + symbol_table * symtab= kernel_func_info->get_symtab(); + unsigned ctaid= kernel.get_next_cta_id_single(); + checkpoint *g_checkpoint= new checkpoint(); for (unsigned i = start_thread; iwarp_size; nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); m_threadState[i].m_active = true; - + // load thread local memory and register file + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); + m_thread[i]->resume_reg_thread(fname,symtab); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); + g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); + } + // warps.set( warp_id ); } assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max m_cta_status[free_cta_hw_id]=nthreads_in_block; - + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaidcheckpoint_CTA_t ) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); + + g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); + } // now that we know which warps are used in this CTA, we can allocate // resources for use in CTA-wide barrier operations m_barriers.allocate_barrier(free_cta_hw_id,warps); -- cgit v1.3