changing the flow s.t. you cannot compile the simulator with a different version of CUDA from the app.

4 files changed, 225 insertions, 47 deletions

diff --git a/diff.diff b/diff.diff
new file mode 100644
index 0000000..8ab27fe
--- /dev/null
+++ b/diff.diff
@@ -0,0 +1,173 @@
+diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
+index 5964a4d..53436d3 100644
+--- a/libcuda/cuda_runtime_api.cc
++++ b/libcuda/cuda_runtime_api.cc
+@@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess;
+ 
+ extern stream_manager *g_stream_manager;
+ 
++
++
+ void register_ptx_function( const char *name, function_info *impl )
+ {
+ 	// no longer need this
+@@ -1341,6 +1343,29 @@ std::string get_app_binary(){
+    return self_exe_path; 
+ }
+ 
++static int get_app_cuda_version() {
++    int app_cuda_version = 0;
++    char fname[1024];
++    snprintf(fname,1024,"_app_cuda_version_XXXXXX");
++    int fd=mkstemp(fname);
++    close(fd);
++    std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
++    system(app_cuda_version_command.c_str());
++    FILE * cmd = fopen(fname, "r");
++    char buf[256];
++    while (fgets(buf, sizeof(buf), cmd) != 0) {
++        std::cout << buf;
++        app_cuda_version = atoi(buf);
++    }
++    fclose(cmd);
++    if ( app_cuda_version == 0 ) {
++        printf( "Error - Cannot detect the app's CUDA version.\n" );
++        exit(1);
++    }
++    return app_cuda_version;
++}
++
++
+ //! Call cuobjdump to extract everything (-elf -sass -ptx)
+ /*!
+  *	This Function extract the whole PTX (for all the files) using cuobjdump
+@@ -1366,23 +1391,10 @@ void extract_code_using_cuobjdump(){
+ 	// Running cuobjdump using dynamic link to current process
+ 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
+ 	extern bool g_cdp_enabled;
+-    const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER"));
+-    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) {
+-        const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH");
+-        if ( NULL == cuda_42_path ) {
+-            printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly "
+-                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n");
+-            exit(1);
+-        } else {
+-            command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
+-        }
+-    } else {
+-        command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
+-    }
+ 	if(!g_cdp_enabled)
+-        command += " -ptx -elf -sass " + app_binary + " > " + fname;
++        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname;
+ 	else
+-        command += " -ptx -elf -sass -all " + app_binary + " > " + fname;
++        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname;
+ 	bool parse_output = true; 
+ 	int result = system(command.c_str());
+ 	if(result) {
+@@ -1765,6 +1777,7 @@ void cuobjdumpParseBinary(unsigned int handle){
+     //TODO: Remove temporarily files as per configurations
+ }
+ 
++
+ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+ {
+ #if (CUDART_VERSION < 2010)
+@@ -1783,26 +1796,10 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+         // Making this a runtime variable based on the app, enables GPGPU-Sim compiled
+         // with a newer version of CUDA to run apps compiled with older versions of
+         // CUDA. This is especially useful for PTXPLUS execution.
+-        char fname[1024];
+-        snprintf(fname,1024,"_app_cuda_version_XXXXXX");
+-        int fd=mkstemp(fname);
+-        close(fd);
+-        int app_cuda_version = 0;
+-        std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
+-        system(app_cuda_version_command.c_str());
+-        FILE * cmd = fopen(fname, "r");
+-        char buf[256];
+-        while (fgets(buf, sizeof(buf), cmd) != 0) {
+-            std::cout << buf;
+-            app_cuda_version = atoi(buf);
+-        }
+-        fclose(cmd);
+-        if ( app_cuda_version == 0 ) {
+-            printf( "Error - Cannot detect the app's CUDA version.\n" );
+-            exit(1);
+-        }
++        int app_cuda_version = get_app_cuda_version();
++        assert( app_cuda_version == CUDART_VERSION / 1000  && "The app must be compiled with same major version as the simulator." );
+         const char* filename;
+-        if ( app_cuda_version < 6 ) {
++#if CUDART_VERSION < 6000
+             // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
+             typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
+             __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+@@ -1814,9 +1811,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+             char * pfatbin = (char*) fatDeviceText->d; 
+             int offset = *((int*)(pfatbin+48)); 
+             filename = (pfatbin+16+offset);
+-        } else {
++#else
+             filename = "default";
+-        }
++#endif
+ 
+ 		// The extracted file name is associated with a fat_cubin_handle passed
+ 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
+diff --git a/setup_environment b/setup_environment
+index 8bcb9aa..e5cd4fa 100644
+--- a/setup_environment
++++ b/setup_environment
+@@ -134,18 +134,26 @@ else
+ 	echo "configured without a power model.";
+ fi
+ 
++export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH
++echo "";
++echo "----------------------------------------------------------------------------";
++echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
+ if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
+-    echo "";
+-    echo "----------------------------------------------------------------------------";
+-    echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
+-    echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH."
+-    echo "The following text describes why:";
+-    echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
+-    echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must";
+-    echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH";
+-    echo "has been set properly. You must set this manually.";
+-    echo "----------------------------------------------------------------------------"
++    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2."
++    echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable."
++else
++    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable"
++    echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)"
++    echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\""
+ fi
++echo "The following text describes why:";
++echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less.";
++echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen.";
++echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x";
++echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage"
++echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card."
++echo "";
++echo "----------------------------------------------------------------------------";
+ 
+ echo "setup_environment succeeded";
+ 
+diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
+index 22f03b9..d8e04d4 100644
+--- a/src/cuda-sim/ptx_loader.cc
++++ b/src/cuda-sim/ptx_loader.cc
+@@ -339,7 +339,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
+         snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
+ #endif
+ 
+-    snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
++    snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
+              extra_flags, fname2, tempfile_ptxinfo);
+     printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
+     result = system(commandline);
diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index 5964a4d..53436d3 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess;
 
 extern stream_manager *g_stream_manager;
 
+
+
 void register_ptx_function( const char *name, function_info *impl )
 {
 	// no longer need this
@@ -1341,6 +1343,29 @@ std::string get_app_binary(){
    return self_exe_path; 
 }
 
+static int get_app_cuda_version() {
+    int app_cuda_version = 0;
+    char fname[1024];
+    snprintf(fname,1024,"_app_cuda_version_XXXXXX");
+    int fd=mkstemp(fname);
+    close(fd);
+    std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
+    system(app_cuda_version_command.c_str());
+    FILE * cmd = fopen(fname, "r");
+    char buf[256];
+    while (fgets(buf, sizeof(buf), cmd) != 0) {
+        std::cout << buf;
+        app_cuda_version = atoi(buf);
+    }
+    fclose(cmd);
+    if ( app_cuda_version == 0 ) {
+        printf( "Error - Cannot detect the app's CUDA version.\n" );
+        exit(1);
+    }
+    return app_cuda_version;
+}
+
+
 //! Call cuobjdump to extract everything (-elf -sass -ptx)
 /*!
  *	This Function extract the whole PTX (for all the files) using cuobjdump
@@ -1366,23 +1391,10 @@ void extract_code_using_cuobjdump(){
 	// Running cuobjdump using dynamic link to current process
 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
 	extern bool g_cdp_enabled;
-    const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER"));
-    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) {
-        const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH");
-        if ( NULL == cuda_42_path ) {
-            printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly "
-                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n");
-            exit(1);
-        } else {
-            command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
-        }
-    } else {
-        command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
-    }
 	if(!g_cdp_enabled)
-        command += " -ptx -elf -sass " + app_binary + " > " + fname;
+        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname;
 	else
-        command += " -ptx -elf -sass -all " + app_binary + " > " + fname;
+        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname;
 	bool parse_output = true; 
 	int result = system(command.c_str());
 	if(result) {
@@ -1765,6 +1777,7 @@ void cuobjdumpParseBinary(unsigned int handle){
     //TODO: Remove temporarily files as per configurations
 }
 
+
 void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 {
 #if (CUDART_VERSION < 2010)
@@ -1783,26 +1796,10 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
         // Making this a runtime variable based on the app, enables GPGPU-Sim compiled
         // with a newer version of CUDA to run apps compiled with older versions of
         // CUDA. This is especially useful for PTXPLUS execution.
-        char fname[1024];
-        snprintf(fname,1024,"_app_cuda_version_XXXXXX");
-        int fd=mkstemp(fname);
-        close(fd);
-        int app_cuda_version = 0;
-        std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
-        system(app_cuda_version_command.c_str());
-        FILE * cmd = fopen(fname, "r");
-        char buf[256];
-        while (fgets(buf, sizeof(buf), cmd) != 0) {
-            std::cout << buf;
-            app_cuda_version = atoi(buf);
-        }
-        fclose(cmd);
-        if ( app_cuda_version == 0 ) {
-            printf( "Error - Cannot detect the app's CUDA version.\n" );
-            exit(1);
-        }
+        int app_cuda_version = get_app_cuda_version();
+        assert( app_cuda_version == CUDART_VERSION / 1000  && "The app must be compiled with same major version as the simulator." );
         const char* filename;
-        if ( app_cuda_version < 6 ) {
+#if CUDART_VERSION < 6000
             // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
             typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
             __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
@@ -1814,9 +1811,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
             char * pfatbin = (char*) fatDeviceText->d; 
             int offset = *((int*)(pfatbin+48)); 
             filename = (pfatbin+16+offset);
-        } else {
+#else
             filename = "default";
-        }
+#endif
 
 		// The extracted file name is associated with a fat_cubin_handle passed
 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
diff --git a/setup_environment b/setup_environment
index 8bcb9aa..e5cd4fa 100644
--- a/setup_environment
+++ b/setup_environment
@@ -134,18 +134,26 @@ else
 	echo "configured without a power model.";
 fi
 
+export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH
+echo "";
+echo "----------------------------------------------------------------------------";
+echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
 if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
-    echo "";
-    echo "----------------------------------------------------------------------------";
-    echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
-    echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH."
-    echo "The following text describes why:";
-    echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
-    echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must";
-    echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH";
-    echo "has been set properly. You must set this manually.";
-    echo "----------------------------------------------------------------------------"
+    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2."
+    echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable."
+else
+    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable"
+    echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)"
+    echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\""
 fi
+echo "The following text describes why:";
+echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less.";
+echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen.";
+echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x";
+echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage"
+echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card."
+echo "";
+echo "----------------------------------------------------------------------------";
 
 echo "setup_environment succeeded";
 
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 22f03b9..d8e04d4 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -339,7 +339,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
         snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
 #endif
 
-    snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
+    snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
              extra_flags, fname2, tempfile_ptxinfo);
     printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
     result = system(commandline);