Merge pull request #76 from tgrogers/dev

Changes to allow correct occupancy with newer cards, while still running PTXPlus

13 files changed, 376 insertions, 102 deletions

diff --git a/.travis.yml b/.travis.yml
index 3f2fb6c..7a12a1c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,17 +9,35 @@ language: cpp
 matrix:
   include:
     - services: docker
-      env: CONFIG=configs.gtx480.yml
+      env:
+        - CONFIG=configs.gtx480.yml
+        - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
 # This config is just taking far too long...
 #    - services: docker
-#      env: CONFIG=configs.gtx750ti.yml
+#      env:
+#       - CONFIG=configs.gtx750ti.yml
+#       - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+#        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
     - services: docker
-      env: CONFIG=configs.quadro5600.yml
+      env:
+        - CONFIG=configs.quadro5600.yml
+        - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
     - services: docker
-      env: CONFIG=configs.quadro5800.yml
+      env:
+        - CONFIG=configs.quadro5800.yml
+        - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
     - services: docker
-      env: CONFIG=configs.teslac2050.yml
+      env:
+        - CONFIG=configs.teslac2050.yml
+        - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
     - services: docker
-      env: CONFIG=configs.gtx1080ti.yml
+      env:
+        - CONFIG=configs.gtx1080ti.yml
+        - CUDA_INSTALL_PATH=/usr/local/cuda-4.2/
+        - PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1/
 
-script: docker run -v `pwd`:/home/runner/gpgpu-sim_distribution:rw tgrogers/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'export CUDA_INSTALL_PATH=/usr/local/cuda-4.2/ && source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/$CONFIG -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress'"
+script: docker run -v `pwd`:/home/runner/gpgpu-sim_distribution:rw tgrogers/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'export CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH && export PTXAS_CUDA_INSTALL_PATH=$PTXAS_CUDA_INSTALL_PATH && source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/$CONFIG -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress'"
diff --git a/configs/GTX480/gpgpusim.config b/configs/GTX480/gpgpusim.config
index 436cb41..ee90c12 100644
--- a/configs/GTX480/gpgpusim.config
+++ b/configs/GTX480/gpgpusim.config
@@ -22,6 +22,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 32768
+-gpgpu_occupancy_sm_number 20
 
 # This implies a maximum of 48 warps/SM
 -gpgpu_shader_core_pipeline 1536:32 
diff --git a/configs/GeForceGTX1080Ti/gpgpusim.config b/configs/GeForceGTX1080Ti/gpgpusim.config
index 47c2b6a..fb044c6 100644
--- a/configs/GeForceGTX1080Ti/gpgpusim.config
+++ b/configs/GeForceGTX1080Ti/gpgpusim.config
@@ -3,7 +3,7 @@
 # functional simulator specification
 -gpgpu_ptx_instruction_classification 0
 -gpgpu_ptx_sim_mode 0
--gpgpu_ptx_force_max_capability 20 
+-gpgpu_ptx_force_max_capability 60
 
 # SASS execution (only supported with CUDA >= 4.0)
 -gpgpu_ptx_convert_to_ptxplus 0
@@ -23,6 +23,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 65536
+-gpgpu_occupancy_sm_number 60
 
 # This implies a maximum of 64 warps/SM
 -gpgpu_shader_core_pipeline 2048:32 
diff --git a/configs/GeForceGTX750Ti/gpgpusim.config b/configs/GeForceGTX750Ti/gpgpusim.config
index 8b030b6..c675aab 100644
--- a/configs/GeForceGTX750Ti/gpgpusim.config
+++ b/configs/GeForceGTX750Ti/gpgpusim.config
@@ -21,6 +21,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 65536
+-gpgpu_occupancy_sm_number 52
 
 # This implies a maximum of 64 warps/SM
 -gpgpu_shader_core_pipeline 2048:32 
diff --git a/configs/QuadroFX5600/gpgpusim.config b/configs/QuadroFX5600/gpgpusim.config
index cb87b65..6f836ee 100644
--- a/configs/QuadroFX5600/gpgpusim.config
+++ b/configs/QuadroFX5600/gpgpusim.config
@@ -11,6 +11,9 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 16384
+-gpgpu_occupancy_sm_number 12
+
+-gpgpu_occupancy_sm_number 12
 #8192 (registers per block as written by device Query and which used in this option in our other configurations but this break some benchmarks execution! it does not affect performance modeling though)
 -gpgpu_shader_core_pipeline 768:32 
 -gpgpu_shader_cta 8
diff --git a/configs/QuadroFX5800/gpgpusim.config b/configs/QuadroFX5800/gpgpusim.config
index 82243c2..fef1110 100644
--- a/configs/QuadroFX5800/gpgpusim.config
+++ b/configs/QuadroFX5800/gpgpusim.config
@@ -11,6 +11,8 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 16384
+-gpgpu_occupancy_sm_number 13
+
 -gpgpu_shader_core_pipeline 1024:32 
 -gpgpu_shader_cta 8
 -gpgpu_simd_model 1
diff --git a/configs/TeslaC2050/gpgpusim.config b/configs/TeslaC2050/gpgpusim.config
index 442ab8b..6ac2c12 100644
--- a/configs/TeslaC2050/gpgpusim.config
+++ b/configs/TeslaC2050/gpgpusim.config
@@ -25,6 +25,7 @@
 
 # shader core pipeline config
 -gpgpu_shader_registers 32768
+-gpgpu_occupancy_sm_number 20
 
 # This implies a maximum of 48 warps/SM
 -gpgpu_shader_core_pipeline 1536:32 
diff --git a/diff.diff b/diff.diff
new file mode 100644
index 0000000..8ab27fe
--- /dev/null
+++ b/diff.diff
@@ -0,0 +1,173 @@
+diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
+index 5964a4d..53436d3 100644
+--- a/libcuda/cuda_runtime_api.cc
++++ b/libcuda/cuda_runtime_api.cc
+@@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess;
+ 
+ extern stream_manager *g_stream_manager;
+ 
++
++
+ void register_ptx_function( const char *name, function_info *impl )
+ {
+ 	// no longer need this
+@@ -1341,6 +1343,29 @@ std::string get_app_binary(){
+    return self_exe_path; 
+ }
+ 
++static int get_app_cuda_version() {
++    int app_cuda_version = 0;
++    char fname[1024];
++    snprintf(fname,1024,"_app_cuda_version_XXXXXX");
++    int fd=mkstemp(fname);
++    close(fd);
++    std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
++    system(app_cuda_version_command.c_str());
++    FILE * cmd = fopen(fname, "r");
++    char buf[256];
++    while (fgets(buf, sizeof(buf), cmd) != 0) {
++        std::cout << buf;
++        app_cuda_version = atoi(buf);
++    }
++    fclose(cmd);
++    if ( app_cuda_version == 0 ) {
++        printf( "Error - Cannot detect the app's CUDA version.\n" );
++        exit(1);
++    }
++    return app_cuda_version;
++}
++
++
+ //! Call cuobjdump to extract everything (-elf -sass -ptx)
+ /*!
+  *	This Function extract the whole PTX (for all the files) using cuobjdump
+@@ -1366,23 +1391,10 @@ void extract_code_using_cuobjdump(){
+ 	// Running cuobjdump using dynamic link to current process
+ 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
+ 	extern bool g_cdp_enabled;
+-    const int current_cuda = atoi(getenv("CUDA_VERSION_NUMBER"));
+-    if ( context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() && current_cuda > 4020 ) {
+-        const char* cuda_42_path = getenv("CUDA_42_INSTALL_PATH");
+-        if ( NULL == cuda_42_path ) {
+-            printf("You are using a new version of CUDA and PTXPLUS. You are required to have CUDA SDK 4.2 and explicitly "
+-                   "point to it via the environment variable $CUDA_42_INSTALL_PATH.\n\n");
+-            exit(1);
+-        } else {
+-            command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
+-        }
+-    } else {
+-        command = "$CUDA_42_INSTALL_PATH/bin/cuobjdump";
+-    }
+ 	if(!g_cdp_enabled)
+-        command += " -ptx -elf -sass " + app_binary + " > " + fname;
++        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname;
+ 	else
+-        command += " -ptx -elf -sass -all " + app_binary + " > " + fname;
++        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname;
+ 	bool parse_output = true; 
+ 	int result = system(command.c_str());
+ 	if(result) {
+@@ -1765,6 +1777,7 @@ void cuobjdumpParseBinary(unsigned int handle){
+     //TODO: Remove temporarily files as per configurations
+ }
+ 
++
+ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+ {
+ #if (CUDART_VERSION < 2010)
+@@ -1783,26 +1796,10 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+         // Making this a runtime variable based on the app, enables GPGPU-Sim compiled
+         // with a newer version of CUDA to run apps compiled with older versions of
+         // CUDA. This is especially useful for PTXPLUS execution.
+-        char fname[1024];
+-        snprintf(fname,1024,"_app_cuda_version_XXXXXX");
+-        int fd=mkstemp(fname);
+-        close(fd);
+-        int app_cuda_version = 0;
+-        std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
+-        system(app_cuda_version_command.c_str());
+-        FILE * cmd = fopen(fname, "r");
+-        char buf[256];
+-        while (fgets(buf, sizeof(buf), cmd) != 0) {
+-            std::cout << buf;
+-            app_cuda_version = atoi(buf);
+-        }
+-        fclose(cmd);
+-        if ( app_cuda_version == 0 ) {
+-            printf( "Error - Cannot detect the app's CUDA version.\n" );
+-            exit(1);
+-        }
++        int app_cuda_version = get_app_cuda_version();
++        assert( app_cuda_version == CUDART_VERSION / 1000  && "The app must be compiled with same major version as the simulator." );
+         const char* filename;
+-        if ( app_cuda_version < 6 ) {
++#if CUDART_VERSION < 6000
+             // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
+             typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
+             __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+@@ -1814,9 +1811,9 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
+             char * pfatbin = (char*) fatDeviceText->d; 
+             int offset = *((int*)(pfatbin+48)); 
+             filename = (pfatbin+16+offset);
+-        } else {
++#else
+             filename = "default";
+-        }
++#endif
+ 
+ 		// The extracted file name is associated with a fat_cubin_handle passed
+ 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
+diff --git a/setup_environment b/setup_environment
+index 8bcb9aa..e5cd4fa 100644
+--- a/setup_environment
++++ b/setup_environment
+@@ -134,18 +134,26 @@ else
+ 	echo "configured without a power model.";
+ fi
+ 
++export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH
++echo "";
++echo "----------------------------------------------------------------------------";
++echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
+ if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
+-    echo "";
+-    echo "----------------------------------------------------------------------------";
+-    echo "WARNING - If you only care about PTX execution, ignore this warning. GPGPU-Sim supports PTX execution in modern CUDA."
+-    echo "If you want to run PTXPLUS, download SDK 4.2 (in addition to your modern CUDA) and set \$CUDA_42_INSTALL_PATH."
+-    echo "The following text describes why:";
+-    echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app binaries are compiled in CUDA 4.2 or less.";
+-    echo "New versions of CUDA tools have dropped parsing support for sm_1x - therefore to use PTXPLUS with a modern card configuration, you must";
+-    echo "have both CUDA 4.2 and a modern CUDA installed. When running PTXPLUS under a CUDA version >= 4.2, the simulator will fail unless \$CUDA_42_INSTALL_PATH";
+-    echo "has been set properly. You must set this manually.";
+-    echo "----------------------------------------------------------------------------"
++    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2."
++    echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable."
++else
++    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable"
++    echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)"
++    echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\""
+ fi
++echo "The following text describes why:";
++echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less.";
++echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen.";
++echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x";
++echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage"
++echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card."
++echo "";
++echo "----------------------------------------------------------------------------";
+ 
+ echo "setup_environment succeeded";
+ 
+diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
+index 22f03b9..d8e04d4 100644
+--- a/src/cuda-sim/ptx_loader.cc
++++ b/src/cuda-sim/ptx_loader.cc
+@@ -339,7 +339,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
+         snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
+ #endif
+ 
+-    snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
++    snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
+              extra_flags, fname2, tempfile_ptxinfo);
+     printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
+     result = system(commandline);
diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index 5ef6115..edd4fb9 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -182,6 +182,8 @@ cudaError_t g_last_cudaError = cudaSuccess;
 
 extern stream_manager *g_stream_manager;
 
+
+
 void register_ptx_function( const char *name, function_info *impl )
 {
 	// no longer need this
@@ -1341,6 +1343,29 @@ std::string get_app_binary(){
    return self_exe_path; 
 }
 
+static int get_app_cuda_version() {
+    int app_cuda_version = 0;
+    char fname[1024];
+    snprintf(fname,1024,"_app_cuda_version_XXXXXX");
+    int fd=mkstemp(fname);
+    close(fd);
+    std::string app_cuda_version_command = "ldd " + get_app_binary() + " | grep libcudart.so | sed  's/.*libcudart.so.\\(.*\\) =>.*/\\1/' > " + fname;
+    system(app_cuda_version_command.c_str());
+    FILE * cmd = fopen(fname, "r");
+    char buf[256];
+    while (fgets(buf, sizeof(buf), cmd) != 0) {
+        std::cout << buf;
+        app_cuda_version = atoi(buf);
+    }
+    fclose(cmd);
+    if ( app_cuda_version == 0 ) {
+        printf( "Error - Cannot detect the app's CUDA version.\n" );
+        exit(1);
+    }
+    return app_cuda_version;
+}
+
+
 //! Call cuobjdump to extract everything (-elf -sass -ptx)
 /*!
  *	This Function extract the whole PTX (for all the files) using cuobjdump
@@ -1350,39 +1375,39 @@ std::string get_app_binary(){
  *	enabled
  * */
 void extract_code_using_cuobjdump(){
-	CUctx_st *context = GPGPUSim_Context();
-	char command[1000];
+    CUctx_st *context = GPGPUSim_Context();
+    std::string command;
 
-   std::string app_binary = get_app_binary(); 
+    std::string app_binary = get_app_binary(); 
 
 	char fname[1024];
 	snprintf(fname,1024,"_cuobjdump_complete_output_XXXXXX");
 	int fd=mkstemp(fname);
 	close(fd);
 	// Running cuobjdump using dynamic link to current process
-	snprintf(command,1000,"md5sum %s ", app_binary.c_str());
-	printf("Running md5sum using \"%s\"\n", command);
-	system(command);
+    command = "md5sum " + app_binary;
+	printf("Running md5sum using \"%s\"\n", command.c_str());
+	system(command.c_str());
 	// Running cuobjdump using dynamic link to current process
 	// Needs the option '-all' to extract PTX from CDP-enabled binary 
 	extern bool g_cdp_enabled;
 	if(!g_cdp_enabled)
-	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass %s > %s", app_binary.c_str(), fname);
+        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass " + app_binary + " > " + fname;
 	else
-	    snprintf(command,1000,"$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all %s > %s", app_binary.c_str(), fname);
+        command = "$CUDA_INSTALL_PATH/bin/cuobjdump -ptx -elf -sass -all " + app_binary + " > " + fname;
 	bool parse_output = true; 
-	int result = system(command);
+	int result = system(command.c_str());
 	if(result) {
 		if (context->get_device()->get_gpgpu()->get_config().experimental_lib_support() && (result == 65280)) {  
 			// Some CUDA application may exclusively use kernels provided by CUDA
 			// libraries (e.g. CUBLAS).  Skipping cuobjdump extraction from the
 			// executable for this case. 
 			// 65280 is the return code from cuobjdump denoting the specific error (tested on CUDA 4.0/4.1/4.2)
-			printf("WARNING: Failed to execute: %s\n", command); 
+			printf("WARNING: Failed to execute: %s\n", command.c_str()); 
 			printf("         Executable binary does not contain any GPU kernel.\n"); 
 			parse_output = false; 
 		} else {
-			printf("ERROR: Failed to execute: %s\n", command); 
+			printf("ERROR: Failed to execute: %s\n", command.c_str()); 
 			exit(1);
 		}
 	}
@@ -1435,7 +1460,7 @@ void extract_code_using_cuobjdump(){
 			std::cout << "Running cuobjdump on " << line << std::endl;
 			std::cout << "Using command: " << cmd.str() << std::endl;
 			result = system(cmd.str().c_str());
-			if(result) {printf("ERROR: Failed to execute: %s\n", command); exit(1);}
+			if(result) {printf("ERROR: Failed to execute: %s\n", command.c_str()); exit(1);}
 			std::cout << "Done" << std::endl;
 
 			std::cout << "Trying to parse " << libcodfn.str() << std::endl;
@@ -1696,61 +1721,63 @@ void cuobjdumpRegisterFatBinary(unsigned int handle, const char* filename){
 //! Either submit PTX for simulation or convert SASS to PTXPlus and submit it
 void cuobjdumpParseBinary(unsigned int handle){
 
-	if(fatbin_registered[handle]) return;
-	fatbin_registered[handle] = true;
-	CUctx_st *context = GPGPUSim_Context();
-	std::string fname = fatbinmap[handle];
+    if(fatbin_registered[handle]) return;
+    fatbin_registered[handle] = true;
+    CUctx_st *context = GPGPUSim_Context();
+    std::string fname = fatbinmap[handle];
 
-	if (name_symtab.find(fname) != name_symtab.end()) {
-		symbol_table *symtab = name_symtab[fname];
-		context->add_binary(symtab, handle);
-		return;
-	}
+    if (name_symtab.find(fname) != name_symtab.end()) {
+        symbol_table *symtab = name_symtab[fname];
+        context->add_binary(symtab, handle);
+        return;
+    }
 
-	unsigned max_capability = 0;
-	for (	std::list<cuobjdumpSection*>::iterator iter = cuobjdumpSectionList.begin();
-			iter != cuobjdumpSectionList.end();
-			iter++){
-		unsigned capability = (*iter)->getArch();
-		if (capability > max_capability) max_capability = capability;
-	}
-	if (max_capability > 20) printf("WARNING: No guarantee that PTX will be parsed for SM version %u\n", max_capability);
+    unsigned max_capability = 0;
+    for ( std::list<cuobjdumpSection*>::iterator iter = cuobjdumpSectionList.begin();
+          iter != cuobjdumpSectionList.end();
+          iter++ ){
+        unsigned capability = (*iter)->getArch();
+        if (capability > max_capability) max_capability = capability;
+    }
+    printf("Using PTX version = %u\n", max_capability);
+    if (max_capability > 20) printf("WARNING: No guarantee that PTX will be parsed for SM version %u\n", max_capability);
 
-	cuobjdumpPTXSection* ptx = findPTXSection(fname);
-	symbol_table *symtab;
-	char *ptxcode;
-	const char *override_ptx_name = getenv("PTX_SIM_KERNELFILE"); 
-	if (override_ptx_name == NULL or getenv("PTX_SIM_USE_PTX_FILE") == NULL) {
-		ptxcode = readfile(ptx->getPTXfilename());
-	} else {
-		printf("GPGPU-Sim PTX: overriding embedded ptx with '%s' (PTX_SIM_USE_PTX_FILE is set)\n", override_ptx_name);
-		ptxcode = readfile(override_ptx_name);
-	}
-	if(context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() ) {
-		cuobjdumpELFSection* elfsection = findELFSection(ptx->getIdentifier());
-		assert (elfsection!= NULL);
-		char *ptxplus_str = gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(
-				ptx->getPTXfilename(),
-				elfsection->getELFfilename(),
-				elfsection->getSASSfilename());
-		symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxplus_str, handle);
-		printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
-		context->add_binary(symtab, handle);
-		gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability );
-		delete[] ptxplus_str;
-	} else {
-		symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxcode, handle);
-		printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
-		context->add_binary(symtab, handle);
-		gpgpu_ptxinfo_load_from_string( ptxcode, handle, max_capability );
-	}
-	load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
-	load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
-	name_symtab[fname] = symtab;
+    cuobjdumpPTXSection* ptx = findPTXSection(fname);
+    symbol_table *symtab;
+    char *ptxcode;
+    const char *override_ptx_name = getenv("PTX_SIM_KERNELFILE"); 
+    if (override_ptx_name == NULL or getenv("PTX_SIM_USE_PTX_FILE") == NULL) {
+        ptxcode = readfile(ptx->getPTXfilename());
+    } else {
+        printf("GPGPU-Sim PTX: overriding embedded ptx with '%s' (PTX_SIM_USE_PTX_FILE is set)\n", override_ptx_name);
+        ptxcode = readfile(override_ptx_name);
+    }
+    if(context->get_device()->get_gpgpu()->get_config().convert_to_ptxplus() ) {
+        cuobjdumpELFSection* elfsection = findELFSection(ptx->getIdentifier());
+        assert (elfsection!= NULL);
+        char *ptxplus_str = gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(
+                ptx->getPTXfilename(),
+                elfsection->getELFfilename(),
+                elfsection->getSASSfilename());
+        symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxplus_str, handle);
+        printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
+        context->add_binary(symtab, handle);
+        gpgpu_ptxinfo_load_from_string( ptxcode, handle );
+        delete[] ptxplus_str;
+    } else {
+        symtab=gpgpu_ptx_sim_load_ptx_from_string(ptxcode, handle);
+        printf("Adding %s with cubin handle %u\n", ptx->getPTXfilename().c_str(), handle);
+        context->add_binary(symtab, handle);
+        gpgpu_ptxinfo_load_from_string( ptxcode, handle );
+    }
+    load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
+    load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
+    name_symtab[fname] = symtab;
 
-	//TODO: Remove temporarily files as per configurations
+    //TODO: Remove temporarily files as per configurations
 }
 
+
 void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 {
 #if (CUDART_VERSION < 2010)
@@ -1764,21 +1791,30 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 		if (sizeof(void*) == 4) 
 			printf("GPGPU-Sim PTX: FatBin file name extraction has not been tested on 32-bit system.\n"); 
 
-		#if (CUDART_VERSION <= 6000)
-		// FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
-		typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
-		__fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+        // This code will get the CUDA version the app was compiled with.
+        // We need this to determine how to handle the parsing of the binary.
+        // Making this a runtime variable based on the app, enables GPGPU-Sim compiled
+        // with a newer version of CUDA to run apps compiled with older versions of
+        // CUDA. This is especially useful for PTXPLUS execution.
+        int app_cuda_version = get_app_cuda_version();
+        assert( app_cuda_version == CUDART_VERSION / 1000  && "The app must be compiled with same major version as the simulator." );
+        const char* filename;
+#if CUDART_VERSION < 6000
+            // FatBin handle from the .fatbin.c file (one of the intermediate files generated by NVCC)
+            typedef struct {int m; int v; const unsigned long long* d; char* f;} __fatDeviceText __attribute__ ((aligned (8))); 
+            __fatDeviceText * fatDeviceText = (__fatDeviceText *) fatCubin;
+
+            // Extract the source code file name that generate the given FatBin. 
+            // - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin).
+            // - An integer inside the fatbin structure contains the relative offset to the source code file name.
+            // - This offset differs among different CUDA and GCC versions. 
+            char * pfatbin = (char*) fatDeviceText->d; 
+            int offset = *((int*)(pfatbin+48)); 
+            filename = (pfatbin+16+offset);
+#else
+            filename = "default";
+#endif
 
-		// Extract the source code file name that generate the given FatBin. 
-		// - Obtains the pointer to the actual fatbin structure from the FatBin handle (fatCubin).
-		// - An integer inside the fatbin structure contains the relative offset to the source code file name.
-		// - This offset differs among different CUDA and GCC versions. 
-		char * pfatbin = (char*) fatDeviceText->d; 
-		int offset = *((int*)(pfatbin+48)); 
-		char * filename = (pfatbin+16+offset); 
-		#else
-		const char * filename = "default";
-		#endif
 		// The extracted file name is associated with a fat_cubin_handle passed
 		// into cudaLaunch().  Inside cudaLaunch(), the associated file name is
 		// used to find the PTX/SASS section from cuobjdump, which contains the
@@ -1846,7 +1882,7 @@ void** CUDARTAPI __cudaRegisterFatBinary( void *fatCubin )
 			} else {
 				symtab=gpgpu_ptx_sim_load_ptx_from_string(ptx,source_num);
 				context->add_binary(symtab,fat_cubin_handle);
-				gpgpu_ptxinfo_load_from_string( ptx, source_num, max_capability );
+				gpgpu_ptxinfo_load_from_string( ptx, source_num );
 			}
 			source_num++;
 			load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
diff --git a/libopencl/opencl_runtime_api.cc b/libopencl/opencl_runtime_api.cc
index 63f7c81..97a54d8 100644
--- a/libopencl/opencl_runtime_api.cc
+++ b/libopencl/opencl_runtime_api.cc
@@ -577,7 +577,7 @@ void _cl_program::Build(const char *options)
       }
       info.m_asm = tmp;
       info.m_symtab = gpgpu_ptx_sim_load_ptx_from_string( tmp, source_num );
-      gpgpu_ptxinfo_load_from_string( tmp, source_num);
+      gpgpu_ptxinfo_load_from_string( tmp, source_num );
       free(tmp);
    }
    printf("GPGPU-Sim OpenCL API: finished compiling OpenCL kernels.\n"); 
diff --git a/setup_environment b/setup_environment
index 3f2476e..f1af978 100644
--- a/setup_environment
+++ b/setup_environment
@@ -7,10 +7,13 @@ export GPGPUSIM_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )"
 
 GPGPUSIM_VERSION_STRING=`cat $GPGPUSIM_ROOT/version | awk '/Version/ {print $8}'`
 #Detect Git branch and commit #
+CURRENT_PWD=`pwd`
+cd $GPGPUSIM_ROOT
 GIT_COMMIT=`git log -n 1 | head -1 | sed -re 's/commit (.*)/\1/'`
 GIT_FILES_CHANGED=`git diff --numstat | wc | sed -re 's/^\s+([0-9]+).*/\1./'`
 GIT_FILES_CHANGED+=`git diff --numstat --cached | wc | sed -re 's/^\s+([0-9]+).*/\1/'`
 GPGPUSIM_BUILD_STRING="gpgpu-sim_git-commit-$GIT_COMMIT-modified_$GIT_FILES_CHANGED"
+cd $CURRENT_PWD
 
 echo -n "GPGPU-Sim version $GPGPUSIM_VERSION_STRING (build $GPGPUSIM_BUILD_STRING) ";
 
@@ -47,12 +50,14 @@ fi
 CC_VERSION=`gcc --version | head -1 | awk '{for(i=1;i<=NF;i++){ if(match($i,/^[0-9]\.[0-9]\.[0-9]$/))  {print $i; exit 0}}}'`
 
 CUDA_VERSION_STRING=`$CUDA_INSTALL_PATH/bin/nvcc --version | awk '/release/ {print $5;}' | sed 's/,//'`;
-CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'`
+export CUDA_VERSION_NUMBER=`echo $CUDA_VERSION_STRING | sed 's/\./ /' | awk '{printf("%02u%02u", 10*int($1), 10*$2);}'`
 if [ $CUDA_VERSION_NUMBER -gt 9100 -o $CUDA_VERSION_NUMBER -lt 2030  ]; then
 	echo "ERROR ** GPGPU-Sim version $GPGPUSIM_VERSION_STRING not tested with CUDA version $CUDA_VERSION_STRING (please see README)";
 	return
 fi
 
+
+
 if [ $# = '1' ] ;
 then
     export GPGPUSIM_CONFIG=gcc-$CC_VERSION/cuda-$CUDA_VERSION_NUMBER/$1
@@ -132,6 +137,29 @@ else
 	echo "configured without a power model.";
 fi
 
+if [ -z "$PTXAS_CUDA_INSTALL_PATH" ]; then
+    export PTXAS_CUDA_INSTALL_PATH=$CUDA_INSTALL_PATH;
+fi
+echo "";
+echo "----------------------------------------------------------------------------";
+echo "INFO - If you only care about PTX execution, ignore this message. GPGPU-Sim supports PTX execution in modern CUDA."
+if [ $CUDA_VERSION_NUMBER -gt 4200 ]; then
+    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration, the apps and simulator must be compiled with CUDA 4.2."
+    echo "You can still run a PASCAL configuration when compiling with 4.2 by setting the \$PTXAS_CUDA_INSTALL_PATH directory environment variable."
+else
+    echo "If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable"
+    echo "\$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)"
+    echo "For example: \"export \$PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1\""
+fi
+echo "The following text describes why:";
+echo "If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less.";
+echo "The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen.";
+echo "The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x";
+echo "When running using modern config (i.e. pascal) and PTXPLUS with CUDA 4.2, the \$PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage"
+echo "(and hence occupancy) using a version of CUDA that knows the register usage on the real card."
+echo "";
+echo "----------------------------------------------------------------------------";
+
 echo "setup_environment succeeded";
 
 export GPGPUSIM_SETUP_ENVIRONMENT_WAS_RUN=1
diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc
index 6c1b595..11d9013 100644
--- a/src/cuda-sim/ptx_loader.cc
+++ b/src/cuda-sim/ptx_loader.cc
@@ -54,6 +54,7 @@ extern int ptxinfo_debug;
 extern FILE *ptxinfo_in;
 
 static bool g_save_embedded_ptx;
+static int g_occupancy_sm_number;
 bool g_keep_intermediate_files;
 bool m_ptx_save_converted_ptxplus;
 
@@ -71,6 +72,10 @@ void ptx_reg_options(option_parser_t opp)
                 &m_ptx_save_converted_ptxplus,
                 "Saved converted ptxplus to a file",
                 "0");
+   option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number,
+                "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. "
+                "This parameter is required in the config.",
+                "0");
 }
 
 void print_ptx_file( const char *p, unsigned source_num, const char *filename )
@@ -120,7 +125,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam
 	fflush(stdout);
 	printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline);
 	result = system(commandline);
-	if(result){printf("GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);}
+	if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);}
 
 
 	// Get ptxplus from file
@@ -142,7 +147,7 @@ char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilenam
 		printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline);
 		int rm_result = system(rm_commandline);
 		if( rm_result != 0 ) {
-			printf("GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result);
+			fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result);
 			exit(1);
 		}
 	}
@@ -193,7 +198,7 @@ void fix_duplicate_errors(char fname2[1024]) {
 	printf("Running: %s\n", commandline);
 	int result = system(commandline);
 	if (result != 0) {
-		printf("GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile);
+		fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile);
 		exit(1);
 	}
 
@@ -282,12 +287,12 @@ void fix_duplicate_errors(char fname2[1024]) {
 	printf("Running: %s\n", commandline);
 	result = system(commandline);
 	if (result != 0) {
-		printf("GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile);
+		fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile);
 		exit(1);
 	}
 }
 
-void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version )
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num )
 {
     char fname[1024];
     snprintf(fname,1024,"_ptx_XXXXXX");
@@ -308,9 +313,9 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
     printf("Running: %s\n", commandline2);
     int result = system(commandline2);
     if( result != 0 ) {
-       printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result);
-       printf("               Ensure you have write access to simulation directory\n");
-       printf("               and have \'cat\' and \'sed\' in your path.\n");
+       fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result);
+       fprintf(stderr, "               Ensure you have write access to simulation directory\n");
+       fprintf(stderr, "               and have \'cat\' and \'sed\' in your path.\n");
        exit(1);
     }
 
@@ -321,15 +326,20 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
     extra_flags[0]=0;
 
 #if CUDART_VERSION >= 3000
-    if (sm_version == 0) sm_version = 20;
+    if ( g_occupancy_sm_number == 0 ) {
+        fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n"
+                         "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n"
+                         "A register size/SM mismatch may result in occupancy differences." );
+        exit(1);
+    }
     extern bool g_cdp_enabled;
     if(!g_cdp_enabled)
-        snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version);
+        snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number);
     else
-        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version);
+        snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number);
 #endif
 
-    snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
+    snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file  /dev/null 2> %s",
              extra_flags, fname2, tempfile_ptxinfo);
     printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline);
     result = system(commandline);
@@ -347,8 +357,8 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
     		result = system(commandline);
 	}
 	if (result != 0) {
-		printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result);
-		printf("               Ensure ptxas is in your path.\n");
+		fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result);
+		fprintf(stderr, "               Ensure ptxas is in your path.\n");
 		exit(1);
 	}
     }
@@ -362,7 +372,7 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num
         printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline);
         result = system(commandline);
         if( result != 0 ) {
-    	    printf("GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result);
+    	    fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result);
     	    exit(1);
         }
     }
diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h
index d3d0c92..cb02eda 100644
--- a/src/cuda-sim/ptx_loader.h
+++ b/src/cuda-sim/ptx_loader.h
@@ -32,7 +32,7 @@
 extern bool g_override_embedded_ptx;
  
 class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num );
-void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version=20 );
+void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num );
 char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str);
 bool keep_intermediate_files();