Allowing modern GPU configrations to properly run PTXPLUS.

There are several problems with this:

1) To get proper occupancy, based on register file usage, you must querry ptxas with
   the sm version that corresponds to the register usage in the config file. To enable this,
   a new config option has been added that determines what sm version you pass to ptxas to compute reg-usage.
   This configuration option is always required in the gpgpusim.config file
2) If you are running PTXPLUS with a modern card (i.e. volta/pascal), you need ptxas from CUDA 9.1.
   However since PTXPLUS only supports sm_13 - you need a version of CUDA where cudaobjectdump
   supports sm_1x. This ended at CUDA 5 - and PTXPLUS requires CUDA 4.2. Therefore, to run PTXPLUS
   on a modern card, you need CUDA 4.2 + modern CUDA installed. To fascilitate this, a new envronment
   varaible is added and the setup envrionment script prints an appropraite warning if you are using a newer CUDA.

We have tried to make this as fail-proof as possible - and die appropraitely when something is wrong.

11 files changed, 67 insertions, 22 deletions

diff --git a/configs/GTX480/gpgpusim.config b/configs/GTX480/gpgpusim.config
index 436cb41..ee90c12 100644
--- a/configs/GTX480/gpgpusim.config
+++ b/configs/GTX480/gpgpusim.config
@@ -22,6 +22,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 32768
+-gpgpu_occupancy_sm_number 20
 
 # This implies a maximum of 48 warps/SM
 -gpgpu_shader_core_pipeline 1536:32 
diff --git a/configs/GeForceGTX1080Ti/gpgpusim.config b/configs/GeForceGTX1080Ti/gpgpusim.config
index 47c2b6a..fb044c6 100644
--- a/configs/GeForceGTX1080Ti/gpgpusim.config
+++ b/configs/GeForceGTX1080Ti/gpgpusim.config
@@ -3,7 +3,7 @@
 # functional simulator specification
 -gpgpu_ptx_instruction_classification 0
 -gpgpu_ptx_sim_mode 0
--gpgpu_ptx_force_max_capability 20 
+-gpgpu_ptx_force_max_capability 60
 
 # SASS execution (only supported with CUDA >= 4.0)
 -gpgpu_ptx_convert_to_ptxplus 0
@@ -23,6 +23,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 65536
+-gpgpu_occupancy_sm_number 60
 
 # This implies a maximum of 64 warps/SM
 -gpgpu_shader_core_pipeline 2048:32 
diff --git a/configs/GeForceGTX750Ti/gpgpusim.config b/configs/GeForceGTX750Ti/gpgpusim.config
index 8b030b6..c675aab 100644
--- a/configs/GeForceGTX750Ti/gpgpusim.config
+++ b/configs/GeForceGTX750Ti/gpgpusim.config
@@ -21,6 +21,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 65536
+-gpgpu_occupancy_sm_number 52
 
 # This implies a maximum of 64 warps/SM
 -gpgpu_shader_core_pipeline 2048:32 
diff --git a/configs/QuadroFX5600/gpgpusim.config b/configs/QuadroFX5600/gpgpusim.config
index cb87b65..6f836ee 100644
--- a/configs/QuadroFX5600/gpgpusim.config
+++ b/configs/QuadroFX5600/gpgpusim.config
@@ -11,6 +11,9 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 16384
+-gpgpu_occupancy_sm_number 12
+
+-gpgpu_occupancy_sm_number 12
 #8192 (registers per block as written by device Query and which used in this option in our other configurations but this break some benchmarks execution! it does not affect performance modeling though)
 -gpgpu_shader_core_pipeline 768:32 
 -gpgpu_shader_cta 8
diff --git a/configs/QuadroFX5800/gpgpusim.config b/configs/QuadroFX5800/gpgpusim.config
index 82243c2..fef1110 100644
--- a/configs/QuadroFX5800/gpgpusim.config
+++ b/configs/QuadroFX5800/gpgpusim.config
@@ -11,6 +11,8 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 16384
+-gpgpu_occupancy_sm_number 13
+
 -gpgpu_shader_core_pipeline 1024:32 
 -gpgpu_shader_cta 8
 -gpgpu_simd_model 1
diff --git a/configs/TeslaC2050/gpgpusim.config b/configs/TeslaC2050/gpgpusim.config
index 442ab8b..6ac2c12 100644
--- a/configs/TeslaC2050/gpgpusim.config
+++ b/configs/TeslaC2050/gpgpusim.config
@@ -25,6 +25,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 32768
+-gpgpu_occupancy_sm_number 20
 
 # This implies a maximum of 48 warps/SM
 -gpgpu_shader_core_pipeline 1536:32 
diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index 2b0f944..94705dc 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -1350,39 +1350,52 @@ std::string get_app_binary(){
  *	enabled
  * */
 void extract_code_using_cuobjdump(){
-	CUctx_st *context = GPGPUSim_Context();
-	char command[1000];
+    CUctx_st *context = GPGPUSim_Context();
+    std::string command;
 
-   std::string app_binary = get_app_binary(); 
+    std::string app_binary = get_app_binary(); 
 
 	char fname[1024];
 	snprintf(fname,1024,"_cuobjdump_complete_output_XXXXXX");
 	int fd=mkstemp(fname);
 	close(fd);
 	// Running cuobjdump using dynamic link to current process
-	snprintf(command,1000,"md5sum %s ", app_binary.c_str());
-	printf("Running md5sum using \"%s\"\n", command);
-	system(command);
+    command = "md5sum " + app_binary;
+	printf("Running md5sum using \"%s\"\n", command.c_str());
+	system(command.c_str());
 	// Running cuobjdump using dynamic link to current process
 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
 	extern bool g_cdp_enabled;
+    const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER"));
+    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 42 ) {
+        const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH");
+        if ( NULL == cuda_42_path ) {
+            printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly "
+                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.");
+            exit(1);
+        } else {
+            command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
+        }
+    } else {
+        command = "$CUDA_INSTALL_PATH/bin/cuobjdump";
+    }
 	if(!g_cdp_enabled)
-	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname);
+        command += " -ptx -elf -sass " + app_binary + " > " + fname;
 	else
-	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all %s > %s", app_binary.c_str(), fname);
+        command += " -ptx -elf -sass -all " + app_binary + " > " + fname;
 	bool parse_output = true; 
-	int result = system(command);
+	int result = system(command.c_str());
 	if(result) {
 		if (context->get_device()->get_gpgpu()->get_config().experimental_lib_support() && (result == 65280)) {  
 			// Some CUDA application may exclusively use kernels provided by CUDA
 			// libraries (e.g. CUBLAS).  Skipping cuobjdump extraction from the
 			// executable for this case. 
 			// 65280 is the return code from cuobjdump denoting the specific error (tested on CUDA 4.0/4.1/4.2)
-			printf("WARNING: Failed to execute: %s\n", command); 
+			printf("WARNING: Failed to execute: %s\n", command.c_str()); 
 			printf("         Executable binary does not contain any GPU kernel.\n"); 
 			parse_output = false; 
 		} else {
-			printf("ERROR: Failed to execute: %s\n", command); 
+			printf("ERROR: Failed to execute: %s\n", command.c_str()); 
 			exit(1);
 		}
 	}
@@ -1435,7 +1448,7 @@ void extract_code_using_cuobjdump(){
 			std::cout << "Running cuobjdump on " << line << std::endl;
 			std::cout << "Using command: " << cmd.str() << std::endl;
 			result = system(cmd.str().c_str());
-			if(result) {printf("ERROR: Failed to execute: %s\n", command); exit(1);}
+			if(result) {printf("ERROR: Failed to execute: %s\n", command.c_str()); exit(1);}
 			std::cout << "Done" << std::endl;
 
 			std::cout << "Trying to parse " << libcodfn.str() << std::endl;
@@ -1737,13 +1750,13 @@ void cuobjdumpParseBinary(unsigned int handle){
         symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxplus_str, handle);
         printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
         context->add_binary(symtab, handle);
-        gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability );
+        gpgpu_ptxinfo_load_from_string( ptxcode, handle );
         delete[] ptxplus_str;
     } else {
         symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxcode, handle);
         printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
         context->add_binary(symtab, handle);
-        gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability );
+        gpgpu_ptxinfo_load_from_string( ptxcode, handle );
     }
     load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
     load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
@@ -1847,7 +1860,7 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 			} else {
 				symtab=gpgpu_ptx_sim_load_ptx_from_string(ptx,source_num);
 				context->add_binary(symtab,fat_cubin_handle);
-				gpgpu_ptxinfo_load_from_string( ptx, source_num, max_capability );
+				gpgpu_ptxinfo_load_from_string( ptx, source_num );
 			}
 			source_num++;
 			load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
diff --git a/libopencl/opencl_runtime_api.cc b/libopencl/opencl_runtime_api.cc
index 63f7c81..97a54d8 100644
--- a/libopencl/opencl_runtime_api.cc
+++ b/libopencl/opencl_runtime_api.cc
@@ -577,7 +577,7 @@ void _cl_program::Build(const char *options)
       }
       info.m_asm = tmp;
       info.m_symtab = gpgpu_ptx_sim_load_ptx_from_string( tmp, source_num );
-      gpgpu_ptxinfo_load_from_string( tmp, source_num);
+      gpgpu_ptxinfo_load_from_string( tmp, source_num );
       free(tmp);
    }
    printf("GPGPU-Sim OpenCL API: finished compiling OpenCL kernels.\n"); 
diff --git a/setup_environment b/setup_environment
index 3f2476e..939fb85 100644
--- a/setup_environment
+++ b/setup_environment
@@ -53,6 +53,8 @@ if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030  ]; then
 	return
 fi
 
+
+
 if [ $# = '1' ] ;
 then
     export GPGPUSIM_CONFIG=gcc-$CC_VERSION/cuda-$CUDA_VERSION_NUMBER/$1
@@ -134,4 +136,15 @@ fi
 
 echo "setup_environment succeeded";
 
+if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
+    echo ""
+    echo "----------------------------------------------------------------------------"
+    echo "WARNING - If you only care about PTX execution, ignore this warning.";
+    echo "If you are using PTXPLUS, only sm_13 is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
+    echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must";
+    echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH";
+    echo "has been set properly. You must set this manually.";
+    echo "----------------------------------------------------------------------------"
+fi
+
 export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=1
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 6c1b595..99174cf 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -54,6 +54,7 @@ extern int ptxinfo_debug;
 extern FILE *ptxinfo_in;
 
 static bool g_save_embedded_ptx;
+static int g_occupancy_sm_number;
 bool g_keep_intermediate_files;
 bool m_ptx_save_converted_ptxplus;
 
@@ -71,6 +72,10 @@ void ptx_reg_options(option_parser_t opp)
                 &m_ptx_save_converted_ptxplus,
                 "Saved converted ptxplus to a file",
                 "0");
+   option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number,
+                "The SM number to pass to ptxas when getting register usage for computing GPU occupancy."
+                "This parameter is required in the config.",
+                "0");
 }
 
 void print_ptx_file( const char *p, unsigned source_num, const char *filename )
@@ -287,7 +292,7 @@ void fix_duplicate_errors(char fname2[1024]) {
 	}
 }
 
-void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version )
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num )
 {
     char fname[1024];
     snprintf(fname,1024,"_ptx_XXXXXX");
@@ -321,12 +326,17 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
     extra_flags[0]=0;
 
 #if CUDART_VERSION >= 3000
-    if (sm_version == 0) sm_version = 20;
+    if ( g_occupancy_sm_number == 0 ) {
+        printf( "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy.\n"
+                "The register file size is specifically tied to the sm version used to querry ptxas for register usage."
+                "A register size/SM mismatch may result in occupancy differences." );
+        exit(1);
+    }
     extern bool g_cdp_enabled;
     if(!g_cdp_enabled)
-        snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+        snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number);
     else
-        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
+        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
 #endif
 
     snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h
index d3d0c92..cb02eda 100644
--- a/src/cuda-sim/ptx_loader.h
+++ b/src/cuda-sim/ptx_loader.h
@@ -32,7 +32,7 @@
 extern bool g_override_embedded_ptx;
  
 class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num );
-void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version=20 );
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num );
 char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str);
 bool keep_intermediate_files();