Fixing the PTXPLUS + new CUDA execution case.

3 files changed, 54 insertions, 27 deletions

diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index 94705dc..5964a4d 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -1367,17 +1367,17 @@ void extract_code_using_cuobjdump(){
 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
 	extern bool g_cdp_enabled;
     const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER"));
-    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 42 ) {
+    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) {
         const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH");
         if ( NULL == cuda_42_path ) {
             printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly "
-                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.");
+                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n");
             exit(1);
         } else {
             command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
         }
     } else {
-        command = "$CUDA_INSTALL_PATH/bin/cuobjdump";
+        command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
     }
 	if(!g_cdp_enabled)
         command += " -ptx -elf -sass " + app_binary + " > " + fname;
@@ -1778,21 +1778,46 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		if (sizeof(void*) == 4) 
 			printf("GPGPU-Sim PTX: FatBin file name extraction has not been tested on 32-bit system.\n"); 
 
-		#if (CUDART_VERSION <= 6000)
-		// FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
-		typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
-		__fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+        // This code will get the CUDA version the app was compiled with.
+        // We need this to determine how to handle the parsing of the binary.
+        // Making this a runtime variable based on the app, enables GPGPU-Sim compiled
+        // with a newer version of CUDA to run apps compiled with older versions of
+        // CUDA. This is especially useful for PTXPLUS execution.
+        char fname[1024];
+        snprintf(fname,1024,"_app_cuda_version_XXXXXX");
+        int fd=mkstemp(fname);
+        close(fd);
+        int app_cuda_version = 0;
+        std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
+        system(app_cuda_version_command.c_str());
+        FILE * cmd = fopen(fname, "r");
+        char buf[256];
+        while (fgets(buf, sizeof(buf), cmd) != 0) {
+            std::cout << buf;
+            app_cuda_version = atoi(buf);
+        }
+        fclose(cmd);
+        if ( app_cuda_version == 0 ) {
+            printf( "Error - Cannot detect the app's CUDA version.\n" );
+            exit(1);
+        }
+        const char* filename;
+        if ( app_cuda_version < 6 ) {
+            // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
+            typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
+            __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+
+            // Extract the source code file name that generate the given FatBin. 
+            // - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin).
+            // - An integer inside the fatbin structure contains the relative offset to the source code file name.
+            // - This offset differs among different CUDA and GCC versions. 
+            char * pfatbin = (char*) fatDeviceText->d; 
+            int offset = *((int*)(pfatbin+48)); 
+            filename = (pfatbin+16+offset);
+        } else {
+            filename = "default";
+        }
 
-		// Extract the source code file name that generate the given FatBin. 
-		// - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin).
-		// - An integer inside the fatbin structure contains the relative offset to the source code file name.
-		// - This offset differs among different CUDA and GCC versions. 
-		char * pfatbin = (char*) fatDeviceText->d; 
-		int offset = *((int*)(pfatbin+48)); 
-		char * filename = (pfatbin+16+offset); 
-		#else
-		const char * filename = "default";
-		#endif
 		// The extracted file name is associated with a fat_cubin_handle passed
 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
 		// used to find the PTX/SASS section from cuobjdump, which contains the
diff --git a/setup_environment b/setup_environment
index 939fb85..8bcb9aa 100644
--- a/setup_environment
+++ b/setup_environment
@@ -47,7 +47,7 @@ fi
 CC_VERSION=`gcc --version | head -1 | awk '{for(i=1;i<=NF;i++){ if(match($i,/^[0-9]\.[0-9]\.[0-9]$/))  {print $i; exit 0}}}'`
 
 CUDA_VERSION_STRING=`$CUDA_INSTALL_PATH/bin/nvcc --version | awk '/release/ {print $5;}' | sed 's/,//'`;
-CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'`
+export CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'`
 if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030  ]; then
 	echo "ERROR ** GPGPU-Sim version $GPGPUSIM_VERSION_STRING not tested with CUDA version $CUDA_VERSION_STRING (please see README)";
 	return
@@ -134,17 +134,19 @@ else
 	echo "configured without a power model.";
 fi
 
-echo "setup_environment succeeded";
-
 if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
-    echo ""
-    echo "----------------------------------------------------------------------------"
-    echo "WARNING - If you only care about PTX execution, ignore this warning.";
-    echo "If you are using PTXPLUS, only sm_13 is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
+    echo "";
+    echo "----------------------------------------------------------------------------";
+    echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
+    echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH."
+    echo "The following text describes why:";
+    echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
     echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must";
     echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH";
     echo "has been set properly. You must set this manually.";
     echo "----------------------------------------------------------------------------"
 fi
 
+echo "setup_environment succeeded";
+
 export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=1
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 99174cf..22f03b9 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -73,7 +73,7 @@ void ptx_reg_options(option_parser_t opp)
                 "Saved converted ptxplus to a file",
                 "0");
    option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number,
-                "The SM number to pass to ptxas when getting register usage for computing GPU occupancy."
+                "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. "
                 "This parameter is required in the config.",
                 "0");
 }
@@ -327,8 +327,8 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
 
 #if CUDART_VERSION >= 3000
     if ( g_occupancy_sm_number == 0 ) {
-        printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy.\n"
-                "The register file size is specifically tied to the sm version used to querry ptxas for register usage."
+        printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n"
+                "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n"
                 "A register size/SM mismatch may result in occupancy differences." );
         exit(1);
     }