From 112a3a2e3c9d17af420d2389f68a35407c692744 Mon Sep 17 00:00:00 2001 From: tgrogers Date: Fri, 29 Jun 2018 17:34:22 -0400 Subject: Allowing modern GPU configrations to properly run PTXPLUS. There are several problems with this: 1) To get proper occupancy, based on register file usage, you must querry ptxas with the sm version that corresponds to the register usage in the config file. To enable this, a new config option has been added that determines what sm version you pass to ptxas to compute reg-usage. This configuration option is always required in the gpgpusim.config file 2) If you are running PTXPLUS with a modern card (i.e. volta/pascal), you need ptxas from CUDA 9.1. However since PTXPLUS only supports sm_13 - you need a version of CUDA where cudaobjectdump supports sm_1x. This ended at CUDA 5 - and PTXPLUS requires CUDA 4.2. Therefore, to run PTXPLUS on a modern card, you need CUDA 4.2 + modern CUDA installed. To fascilitate this, a new envronment varaible is added and the setup envrionment script prints an appropraite warning if you are using a newer CUDA. We have tried to make this as fail-proof as possible - and die appropraitely when something is wrong. --- setup_environment | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'setup_environment') diff --git a/setup_environment b/setup_environment index 3f2476e..939fb85 100644 --- a/setup_environment +++ b/setup_environment @@ -53,6 +53,8 @@ if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030 ]; then return fi + + if [ $# = '1' ] ; then export GPGPUSIM_CONFIG=gcc-$CC_VERSION/cuda-$CUDA_VERSION_NUMBER/$1 @@ -134,4 +136,15 @@ fi echo "setup_environment succeeded"; +if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then + echo "" + echo "----------------------------------------------------------------------------" + echo "WARNING - If you only care about PTX execution, ignore this warning."; + echo "If you are using PTXPLUS, only sm_13 is supported and it requires that the app binaries are compiled in CUDA 4.2 or less."; + echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must"; + echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH"; + echo "has been set properly. You must set this manually."; + echo "----------------------------------------------------------------------------" +fi + export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=1 -- cgit v1.3 From d42cb5648f77cd2caed9c656f4d6ad75c23b6f5a Mon Sep 17 00:00:00 2001 From: tgrogers Date: Fri, 29 Jun 2018 23:43:31 -0400 Subject: Fixing the PTXPLUS + new CUDA execution case. --- libcuda/cuda_runtime_api.cc | 61 ++++++++++++++++++++++++++++++++------------- setup_environment | 16 ++++++------ src/cuda-sim/ptx_loader.cc | 6 ++--- 3 files changed, 55 insertions(+), 28 deletions(-) (limited to 'setup_environment') diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 94705dc..5964a4d 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -1367,17 +1367,17 @@ void extract_code_using_cuobjdump(){ // Needs the option '-all' to extract PTX from CDP-enabled binary extern bool g_cdp_enabled; const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER")); - if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 42 ) { + if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) { const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH"); if ( NULL == cuda_42_path ) { printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly " - "point to it via the environment variable $CUDA_42_INSTALL_PATH."); + "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n"); exit(1); } else { command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; } } else { - command = "$CUDA_INSTALL_PATH/bin/cuobjdump"; + command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; } if(!g_cdp_enabled) command += " -ptx -elf -sass " + app_binary + " > " + fname; @@ -1778,21 +1778,46 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) if (sizeof(void*) == 4) printf("GPGPU-Sim PTX: FatBin file name extraction has not been tested on 32-bit system.\n"); - #if (CUDART_VERSION <= 6000) - // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC) - typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); - __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin; - - // Extract the source code file name that generate the given FatBin. - // - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin). - // - An integer inside the fatbin structure contains the relative offset to the source code file name. - // - This offset differs among different CUDA and GCC versions. - char * pfatbin = (char*) fatDeviceText->d; - int offset = *((int*)(pfatbin+48)); - char * filename = (pfatbin+16+offset); - #else - const char * filename = "default"; - #endif + // This code will get the CUDA version the app was compiled with. + // We need this to determine how to handle the parsing of the binary. + // Making this a runtime variable based on the app, enables GPGPU-Sim compiled + // with a newer version of CUDA to run apps compiled with older versions of + // CUDA. This is especially useful for PTXPLUS execution. + char fname[1024]; + snprintf(fname,1024,"_app_cuda_version_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + int app_cuda_version = 0; + std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed 's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname; + system(app_cuda_version_command.c_str()); + FILE * cmd = fopen(fname, "r"); + char buf[256]; + while (fgets(buf, sizeof(buf), cmd) != 0) { + std::cout << buf; + app_cuda_version = atoi(buf); + } + fclose(cmd); + if ( app_cuda_version == 0 ) { + printf( "Error - Cannot detect the app's CUDA version.\n" ); + exit(1); + } + const char* filename; + if ( app_cuda_version < 6 ) { + // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC) + typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); + __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin; + + // Extract the source code file name that generate the given FatBin. + // - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin). + // - An integer inside the fatbin structure contains the relative offset to the source code file name. + // - This offset differs among different CUDA and GCC versions. + char * pfatbin = (char*) fatDeviceText->d; + int offset = *((int*)(pfatbin+48)); + filename = (pfatbin+16+offset); + } else { + filename = "default"; + } + // The extracted file name is associated with a fat_cubin_handle passed // into cudaLaunch(). Inside cudaLaunch(), the associated file name is // used to find the PTX/SASS section from cuobjdump, which contains the diff --git a/setup_environment b/setup_environment index 939fb85..8bcb9aa 100644 --- a/setup_environment +++ b/setup_environment @@ -47,7 +47,7 @@ fi CC_VERSION=`gcc --version | head -1 | awk '{for(i=1;i<=NF;i++){ if(match($i,/^[0-9]\.[0-9]\.[0-9]$/)) {print $i; exit 0}}}'` CUDA_VERSION_STRING=`$CUDA_INSTALL_PATH/bin/nvcc --version | awk '/release/ {print $5;}' | sed 's/,//'`; -CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'` +export CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'` if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030 ]; then echo "ERROR ** GPGPU-Sim version $GPGPUSIM_VERSION_STRING not tested with CUDA version $CUDA_VERSION_STRING (please see README)"; return @@ -134,17 +134,19 @@ else echo "configured without a power model."; fi -echo "setup_environment succeeded"; - if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then - echo "" - echo "----------------------------------------------------------------------------" - echo "WARNING - If you only care about PTX execution, ignore this warning."; - echo "If you are using PTXPLUS, only sm_13 is supported and it requires that the app binaries are compiled in CUDA 4.2 or less."; + echo ""; + echo "----------------------------------------------------------------------------"; + echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." + echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH." + echo "The following text describes why:"; + echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less."; echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must"; echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH"; echo "has been set properly. You must set this manually."; echo "----------------------------------------------------------------------------" fi +echo "setup_environment succeeded"; + export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=1 diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 99174cf..22f03b9 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -73,7 +73,7 @@ void ptx_reg_options(option_parser_t opp) "Saved converted ptxplus to a file", "0"); option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register usage for computing GPU occupancy." + "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " "This parameter is required in the config.", "0"); } @@ -327,8 +327,8 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num #if CUDART_VERSION >= 3000 if ( g_occupancy_sm_number == 0 ) { - printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy.\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage." + printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" "A register size/SM mismatch may result in occupancy differences." ); exit(1); } -- cgit v1.3 From 691c57ea8596da35da3ce90a7b58796f3791036c Mon Sep 17 00:00:00 2001 From: tgrogers Date: Sat, 30 Jun 2018 10:12:20 -0400 Subject: changing the flow s.t. you cannot compile the simulator with a different version of CUDA from the app. --- diff.diff | 173 ++++++++++++++++++++++++++++++++++++++++++++ libcuda/cuda_runtime_api.cc | 69 +++++++++--------- setup_environment | 28 ++++--- src/cuda-sim/ptx_loader.cc | 2 +- 4 files changed, 225 insertions(+), 47 deletions(-) create mode 100644 diff.diff (limited to 'setup_environment') diff --git a/diff.diff b/diff.diff new file mode 100644 index 0000000..8ab27fe --- /dev/null +++ b/diff.diff @@ -0,0 +1,173 @@ +diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc +index 5964a4d..53436d3 100644 +--- a/libcuda/cuda_runtime_api.cc ++++ b/libcuda/cuda_runtime_api.cc +@@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess; + + extern stream_manager *g_stream_manager; + ++ ++ + void register_ptx_function( const char *name, function_info *impl ) + { + // no longer need this +@@ -1341,6 +1343,29 @@ std::string get_app_binary(){ + return self_exe_path; + } + ++static int get_app_cuda_version() { ++ int app_cuda_version = 0; ++ char fname[1024]; ++ snprintf(fname,1024,"_app_cuda_version_XXXXXX"); ++ int fd=mkstemp(fname); ++ close(fd); ++ std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed 's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname; ++ system(app_cuda_version_command.c_str()); ++ FILE * cmd = fopen(fname, "r"); ++ char buf[256]; ++ while (fgets(buf, sizeof(buf), cmd) != 0) { ++ std::cout << buf; ++ app_cuda_version = atoi(buf); ++ } ++ fclose(cmd); ++ if ( app_cuda_version == 0 ) { ++ printf( "Error - Cannot detect the app's CUDA version.\n" ); ++ exit(1); ++ } ++ return app_cuda_version; ++} ++ ++ + //! Call cuobjdump to extract everything (-elf -sass -ptx) + /*! + * This Function extract the whole PTX (for all the files) using cuobjdump +@@ -1366,23 +1391,10 @@ void extract_code_using_cuobjdump(){ + // Running cuobjdump using dynamic link to current process + // Needs the option '-all' to extract PTX from CDP-enabled binary + extern bool g_cdp_enabled; +- const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER")); +- if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) { +- const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH"); +- if ( NULL == cuda_42_path ) { +- printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly " +- "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n"); +- exit(1); +- } else { +- command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; +- } +- } else { +- command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; +- } + if(!g_cdp_enabled) +- command += " -ptx -elf -sass " + app_binary + " > " + fname; ++ command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname; + else +- command += " -ptx -elf -sass -all " + app_binary + " > " + fname; ++ command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname; + bool parse_output = true; + int result = system(command.c_str()); + if(result) { +@@ -1765,6 +1777,7 @@ void cuobjdumpParseBinary(unsigned int handle){ + //TODO: Remove temporarily files as per configurations + } + ++ + void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) + { + #if (CUDART_VERSION < 2010) +@@ -1783,26 +1796,10 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) + // Making this a runtime variable based on the app, enables GPGPU-Sim compiled + // with a newer version of CUDA to run apps compiled with older versions of + // CUDA. This is especially useful for PTXPLUS execution. +- char fname[1024]; +- snprintf(fname,1024,"_app_cuda_version_XXXXXX"); +- int fd=mkstemp(fname); +- close(fd); +- int app_cuda_version = 0; +- std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed 's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname; +- system(app_cuda_version_command.c_str()); +- FILE * cmd = fopen(fname, "r"); +- char buf[256]; +- while (fgets(buf, sizeof(buf), cmd) != 0) { +- std::cout << buf; +- app_cuda_version = atoi(buf); +- } +- fclose(cmd); +- if ( app_cuda_version == 0 ) { +- printf( "Error - Cannot detect the app's CUDA version.\n" ); +- exit(1); +- } ++ int app_cuda_version = get_app_cuda_version(); ++ assert( app_cuda_version == CUDART_VERSION / 1000 && "The app must be compiled with same major version as the simulator." ); + const char* filename; +- if ( app_cuda_version < 6 ) { ++#if CUDART_VERSION < 6000 + // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC) + typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); + __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin; +@@ -1814,9 +1811,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) + char * pfatbin = (char*) fatDeviceText->d; + int offset = *((int*)(pfatbin+48)); + filename = (pfatbin+16+offset); +- } else { ++#else + filename = "default"; +- } ++#endif + + // The extracted file name is associated with a fat_cubin_handle passed + // into cudaLaunch(). Inside cudaLaunch(), the associated file name is +diff --git a/setup_environment b/setup_environment +index 8bcb9aa..e5cd4fa 100644 +--- a/setup_environment ++++ b/setup_environment +@@ -134,18 +134,26 @@ else + echo "configured without a power model."; + fi + ++export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH ++echo ""; ++echo "----------------------------------------------------------------------------"; ++echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." + if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then +- echo ""; +- echo "----------------------------------------------------------------------------"; +- echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." +- echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH." +- echo "The following text describes why:"; +- echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less."; +- echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must"; +- echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH"; +- echo "has been set properly. You must set this manually."; +- echo "----------------------------------------------------------------------------" ++ echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2." ++ echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable." ++else ++ echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable" ++ echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)" ++ echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\"" + fi ++echo "The following text describes why:"; ++echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less."; ++echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen."; ++echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x"; ++echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage" ++echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card." ++echo ""; ++echo "----------------------------------------------------------------------------"; + + echo "setup_environment succeeded"; + +diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc +index 22f03b9..d8e04d4 100644 +--- a/src/cuda-sim/ptx_loader.cc ++++ b/src/cuda-sim/ptx_loader.cc +@@ -339,7 +339,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); + #endif + +- snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", ++ snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + result = system(commandline); diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc index 5964a4d..53436d3 100644 --- a/libcuda/cuda_runtime_api.cc +++ b/libcuda/cuda_runtime_api.cc @@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess; extern stream_manager *g_stream_manager; + + void register_ptx_function( const char *name, function_info *impl ) { // no longer need this @@ -1341,6 +1343,29 @@ std::string get_app_binary(){ return self_exe_path; } +static int get_app_cuda_version() { + int app_cuda_version = 0; + char fname[1024]; + snprintf(fname,1024,"_app_cuda_version_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed 's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname; + system(app_cuda_version_command.c_str()); + FILE * cmd = fopen(fname, "r"); + char buf[256]; + while (fgets(buf, sizeof(buf), cmd) != 0) { + std::cout << buf; + app_cuda_version = atoi(buf); + } + fclose(cmd); + if ( app_cuda_version == 0 ) { + printf( "Error - Cannot detect the app's CUDA version.\n" ); + exit(1); + } + return app_cuda_version; +} + + //! Call cuobjdump to extract everything (-elf -sass -ptx) /*! * This Function extract the whole PTX (for all the files) using cuobjdump @@ -1366,23 +1391,10 @@ void extract_code_using_cuobjdump(){ // Running cuobjdump using dynamic link to current process // Needs the option '-all' to extract PTX from CDP-enabled binary extern bool g_cdp_enabled; - const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER")); - if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) { - const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH"); - if ( NULL == cuda_42_path ) { - printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly " - "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n"); - exit(1); - } else { - command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; - } - } else { - command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump"; - } if(!g_cdp_enabled) - command += " -ptx -elf -sass " + app_binary + " > " + fname; + command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname; else - command += " -ptx -elf -sass -all " + app_binary + " > " + fname; + command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname; bool parse_output = true; int result = system(command.c_str()); if(result) { @@ -1765,6 +1777,7 @@ void cuobjdumpParseBinary(unsigned int handle){ //TODO: Remove temporarily files as per configurations } + void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) { #if (CUDART_VERSION < 2010) @@ -1783,26 +1796,10 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) // Making this a runtime variable based on the app, enables GPGPU-Sim compiled // with a newer version of CUDA to run apps compiled with older versions of // CUDA. This is especially useful for PTXPLUS execution. - char fname[1024]; - snprintf(fname,1024,"_app_cuda_version_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - int app_cuda_version = 0; - std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed 's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname; - system(app_cuda_version_command.c_str()); - FILE * cmd = fopen(fname, "r"); - char buf[256]; - while (fgets(buf, sizeof(buf), cmd) != 0) { - std::cout << buf; - app_cuda_version = atoi(buf); - } - fclose(cmd); - if ( app_cuda_version == 0 ) { - printf( "Error - Cannot detect the app's CUDA version.\n" ); - exit(1); - } + int app_cuda_version = get_app_cuda_version(); + assert( app_cuda_version == CUDART_VERSION / 1000 && "The app must be compiled with same major version as the simulator." ); const char* filename; - if ( app_cuda_version < 6 ) { +#if CUDART_VERSION < 6000 // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC) typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin; @@ -1814,9 +1811,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin ) char * pfatbin = (char*) fatDeviceText->d; int offset = *((int*)(pfatbin+48)); filename = (pfatbin+16+offset); - } else { +#else filename = "default"; - } +#endif // The extracted file name is associated with a fat_cubin_handle passed // into cudaLaunch(). Inside cudaLaunch(), the associated file name is diff --git a/setup_environment b/setup_environment index 8bcb9aa..e5cd4fa 100644 --- a/setup_environment +++ b/setup_environment @@ -134,18 +134,26 @@ else echo "configured without a power model."; fi +export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH +echo ""; +echo "----------------------------------------------------------------------------"; +echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then - echo ""; - echo "----------------------------------------------------------------------------"; - echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." - echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH." - echo "The following text describes why:"; - echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less."; - echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must"; - echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH"; - echo "has been set properly. You must set this manually."; - echo "----------------------------------------------------------------------------" + echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2." + echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable." +else + echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable" + echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)" + echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\"" fi +echo "The following text describes why:"; +echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less."; +echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen."; +echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x"; +echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage" +echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card." +echo ""; +echo "----------------------------------------------------------------------------"; echo "setup_environment succeeded"; diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 22f03b9..d8e04d4 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -339,7 +339,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); #endif - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); -- cgit v1.3 From 263fd9540b90be452640b54c19c7403c2281f3d9 Mon Sep 17 00:00:00 2001 From: tgrogers Date: Sat, 30 Jun 2018 10:17:18 -0400 Subject: Preventing bad git commit numbers from printing when you run setup_env from a dir that is not the gpgpusim_distr dir --- setup_environment | 3 +++ 1 file changed, 3 insertions(+) (limited to 'setup_environment') diff --git a/setup_environment b/setup_environment index e5cd4fa..65f8340 100644 --- a/setup_environment +++ b/setup_environment @@ -7,10 +7,13 @@ export GPGPUSIM_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )" GPGPUSIM_VERSION_STRING=`cat $GPGPUSIM_ROOT/version | awk '/Version/ {print $8}'` #Detect Git branch and commit # +CURRENT_PWD=`pwd` +cd $GPGPUSIM_ROOT GIT_COMMIT=`git log -n 1 | head -1 | sed -re 's/commit (.*)/\1/'` GIT_FILES_CHANGED=`git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1./'` GIT_FILES_CHANGED+=`git diff --numstat --cached | wc | sed -re 's/^\s+([0-9]+).*/\1/'` GPGPUSIM_BUILD_STRING="gpgpu-sim_git-commit-$GIT_COMMIT-modified_$GIT_FILES_CHANGED" +cd $CURRENT_PWD echo -n "GPGPU-Sim version $GPGPUSIM_VERSION_STRING (build $GPGPUSIM_BUILD_STRING) "; -- cgit v1.3 From a72f1d829205b9bfc4070cbb2ff46e7d7aba1fc6 Mon Sep 17 00:00:00 2001 From: tgrogers Date: Sat, 30 Jun 2018 12:54:21 -0400 Subject: Cleaning up the setup env message a bit and having the simulator properly dump to stderr on error --- setup_environment | 2 +- src/cuda-sim/ptx_loader.cc | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'setup_environment') diff --git a/setup_environment b/setup_environment index 65f8340..55f3ea6 100644 --- a/setup_environment +++ b/setup_environment @@ -140,7 +140,7 @@ fi export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH echo ""; echo "----------------------------------------------------------------------------"; -echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA." +echo "INFO - If you only care about PTX execution, ignore this message. GPGPU-Sim supports PTX execution in modern CUDA." if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2." echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable." diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index d8e04d4..11d9013 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -125,7 +125,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam fflush(stdout); printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); result = system(commandline); - if(result){printf("GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} + if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} // Get ptxplus from file @@ -147,7 +147,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); int rm_result = system(rm_commandline); if( rm_result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); exit(1); } } @@ -198,7 +198,7 @@ void fix_duplicate_errors(char fname2[1024]) { printf("Running: %s\n", commandline); int result = system(commandline); if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); exit(1); } @@ -287,7 +287,7 @@ void fix_duplicate_errors(char fname2[1024]) { printf("Running: %s\n", commandline); result = system(commandline); if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); exit(1); } } @@ -313,9 +313,9 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num printf("Running: %s\n", commandline2); int result = system(commandline2); if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + fprintf(stderr, " Ensure you have write access to simulation directory\n"); + fprintf(stderr, " and have \'cat\' and \'sed\' in your path.\n"); exit(1); } @@ -327,9 +327,9 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num #if CUDART_VERSION >= 3000 if ( g_occupancy_sm_number == 0 ) { - printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences." ); + fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences." ); exit(1); } extern bool g_cdp_enabled; @@ -357,8 +357,8 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num result = system(commandline); } if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + fprintf(stderr, " Ensure ptxas is in your path.\n"); exit(1); } } @@ -372,7 +372,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); result = system(commandline); if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result); + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result); exit(1); } } -- cgit v1.3 From c37cd1e3fa9a3e4392ffc5453e9fd1696fd38396 Mon Sep 17 00:00:00 2001 From: tgrogers Date: Sat, 30 Jun 2018 13:53:16 -0400 Subject: Adding in a check to not blow away the PTXAS variable if you already had it set --- setup_environment | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setup_environment') diff --git a/setup_environment b/setup_environment index 55f3ea6..2dafba5 100644 --- a/setup_environment +++ b/setup_environment @@ -137,7 +137,9 @@ else echo "configured without a power model."; fi -export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH +if [ ! -z "$PTXAS_CUDA_INSTALL_PATH" ]; then + export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH; +fi echo ""; echo "----------------------------------------------------------------------------"; echo "INFO - If you only care about PTX execution, ignore this message. GPGPU-Sim supports PTX execution in modern CUDA." -- cgit v1.3 From 5a6957fa2a45cd1856550e9426cb6bd743f19e98 Mon Sep 17 00:00:00 2001 From: tgrogers Date: Sat, 30 Jun 2018 23:25:20 -0400 Subject: oops - you should set this if it does not exist, not the other way around... --- setup_environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setup_environment') diff --git a/setup_environment b/setup_environment index 2dafba5..f1af978 100644 --- a/setup_environment +++ b/setup_environment @@ -137,7 +137,7 @@ else echo "configured without a power model."; fi -if [ ! -z "$PTXAS_CUDA_INSTALL_PATH" ]; then +if [ -z "$PTXAS_CUDA_INSTALL_PATH" ]; then export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH; fi echo ""; -- cgit v1.3