From e4e51983f21cec01f0568eb4f0bf3dbb9e615289 Mon Sep 17 00:00:00 2001 From: Andrew Boktor Date: Mon, 15 Jun 2020 13:52:59 -0700 Subject: Making README into README.md and adding markdown formatting --- README | 482 ----------------------------------------------------------- README.md | 499 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 499 insertions(+), 482 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index ff6664f..0000000 --- a/README +++ /dev/null @@ -1,482 +0,0 @@ -Welcome to GPGPU-Sim, a cycle-level simulator modeling contemporary graphics -processing units (GPUs) running GPU computing workloads written in CUDA or -OpenCL. Also included in GPGPU-Sim is a performance visualization tool called -AerialVision and a configurable and extensible energy model called GPUWattch. -GPGPU-Sim and GPUWattch have been rigorously validated with performance and -power measurements of real hardware GPUs. - -This version of GPGPU-Sim has been tested with CUDA version 4.2, -5.0, 5.5, 6.0 and 7.5, 8.0, 9.0, 9.1 - -Please see the copyright notice in the file COPYRIGHT distributed with this -release in the same directory as this file. - -If you use GPGPU-Sim in your research, please cite: - -Ali Bakhoda, George Yuan, Wilson W. L. Fung, Henry Wong, Tor M. Aamodt, -Analyzing CUDA Workloads Using a Detailed GPU Simulator, in IEEE International -Symposium on Performance Analysis of Systems and Software (ISPASS), Boston, MA, -April 19-21, 2009. - -If you use cuDNN and Pytorch support, the Checkpoint function or the Debigging tool for functional simulation error in GPGPU-Sim for your research, -please cite: -Jonathan Lew, Deval Shah, Suchita Pati, Shaylin Cattell, Mengchi Zhang, Amruth Sandhupatla, Christopher Ng, Negar Goli, Matthew D. Sinclair, Timothy G. Rogers, Tor M. Aamodt -Analyzing Machine Learning Workloads Using a Detailed GPU Simulator, arXiv:1811.08933, -https://arxiv.org/abs/1811.08933 - -If you use the memory system in GPGPU-Sim, or the Volta/Pascal models, -please cite: -Mahmoud Khairy, Jain Akshay, Tor Aamodt, Timothy G Rogers, -Exploring Modern GPU Memory System Design Challenges through Accurate Modeling, arXiv:1810.07269, -https://arxiv.org/abs/1810.07269 - -If you use the Tensor Core in GPGPU-Sim or CUTLASS Library for your research -please cite: -Md Aamir Raihan, Negar Goli, Tor Aamodt, -Modeling Deep Learning Accelerator Enabled GPUs, arXiv:1811.08309, -https://arxiv.org/abs/1811.08309 - -If you use the GPUWattch energy model in your research, please cite: - -Jingwen Leng, Tayler Hetherington, Ahmed ElTantawy, Syed Gilani, Nam Sung Kim, -Tor M. Aamodt, Vijay Janapa Reddi, GPUWattch: Enabling Energy Optimizations in -GPGPUs, In proceedings of the ACM/IEEE International Symposium on Computer -Architecture (ISCA 2013), Tel-Aviv, Israel, June 23-27, 2013. - -If you use the support for CUDA dynamic parallelism in your research, please cite: - -Jin Wang and Sudhakar Yalamanchili, Characterization and Analysis of Dynamic -Parallelism in Unstructured GPU Applications, 2014 IEEE International Symposium -on Workload Characterization (IISWC), November 2014. - -If you use figures plotted using AerialVision in your publications, please cite: - -Aaron Ariel, Wilson W. L. Fung, Andrew Turner, Tor M. Aamodt, Visualizing -Complex Dynamics in Many-Core Accelerator Architectures, In Proceedings of the -IEEE International Symposium on Performance Analysis of Systems and Software -(ISPASS), pp. 164-174, White Plains, NY, March 28-30, 2010. - -This file contains instructions on installing, building and running GPGPU-Sim. -Detailed documentation on what GPGPU-Sim models, how to configure it, and a -guide to the source code can be found here: . -Instructions for building doxygen source code documentation are included below. -Detailed documentation on GPUWattch including how to configure it and a guide -to the source code can be found here: . - -If you have questions, please sign up for the google groups page (see -gpgpu-sim.org), but note that use of this simulator does not imply any level of -support. Questions answered on a best effort basis. - -To submit a bug report, go here: http://www.gpgpu-sim.org/bugs/ - -See Section 2 "INSTALLING, BUILDING and RUNNING GPGPU-Sim" below to get started. - -See file CHANGES for updates in this and earlier versions. - - -1. CONTRIBUTIONS and HISTORY - -== GPGPU-Sim == - -GPGPU-Sim was created by Tor Aamodt's research group at the University of -British Columbia. Many have directly contributed to development of GPGPU-Sim -including: Tor Aamodt, Wilson W.L. Fung, Ali Bakhoda, George Yuan, Ivan Sham, -Henry Wong, Henry Tran, Andrew Turner, Aaron Ariel, Inderpret Singh, Tim -Rogers, Jimmy Kwa, Andrew Boktor, Ayub Gubran Tayler Hetherington and others. - -GPGPU-Sim models the features of a modern graphics processor that are relevant -to non-graphics applications. The first version of GPGPU-Sim was used in a -MICRO'07 paper and follow-on ACM TACO paper on dynamic warp formation. That -version of GPGPU-Sim used the SimpleScalar PISA instruction set for functional -simulation, and various configuration files indicating which loops should be -spawned as kernels on the GPU, along with reconvergence points required for -SIMT execution to provide a programming model simlar to CUDA/OpenCL. Creating -benchmarks for the original GPGPU-Sim simulator was a very time consuming -process and the validity of code generation for CPU run on a GPU was questioned -by some. These issues motivated the development an interface for directly -running CUDA applications to leverage the growing number of applications being -developed to use CUDA. We subsequently added support for OpenCL and removed -all SimpleScalar code. - -The interconnection network is simulated using the booksim simulator developed -by Bill Dally's research group at Stanford. - -To produce output that matches the output from running the same CUDA program on -the GPU, we have implemented several PTX instructions using the CUDA Math -library (part of the CUDA toolkit). Code to interface with the CUDA Math -library is contained in cuda-math.h, which also includes several structures -derived from vector_types.h (one of the CUDA header files). - -== GPUWattch Energy Model == - -GPUWattch (introduced in GPGPU-Sim 3.2.0) was developed by researchers at the -University of British Columbia, the University of Texas at Austin, and the -University of Wisconsin-Madison. Contributors to GPUWattch include Tor -Aamodt's research group at the University of British Columbia: Tayler -Hetherington and Ahmed ElTantawy; Vijay Reddi's research group at the -University of Texas at Austin: Jingwen Leng; and Nam Sung Kim's research group -at the University of Wisconsin-Madison: Syed Gilani. - -GPUWattch leverages McPAT, which was developed by Sheng Li et al. at the -University of Notre Dame, Hewlett-Packard Labs, Seoul National University, and -the University of California, San Diego. The paper can be found at -http://www.hpl.hp.com/research/mcpat/micro09.pdf. - - - -2. INSTALLING, BUILDING and RUNNING GPGPU-Sim - -Assuming all dependencies required by GPGPU-Sim are installed on your system, -to build GPGPU-Sim all you need to do is add the following line to your -~/.bashrc file (assuming the CUDA Toolkit was installed in /usr/local/cuda): - - export CUDA_INSTALL_PATH=/usr/local/cuda - -then type - - bash - source setup_environment - make - -If the above fails, see "Step 1" and "Step 2" below. - -If the above worked, see "Step 3" below, which explains how to run a CUDA -benchmark on GPGPU-Sim. - -Step 1: Dependencies -==================== - -GPGPU-Sim was developed on SUSE Linux (this release was tested with SUSE -version 11.3) and has been used on several other Linux platforms (both 32-bit -and 64-bit systems). In principle, GPGPU-Sim should work with any linux -distribution as long as the following software dependencies are satisfied. - -Download and install the CUDA Toolkit. It is recommended to use version 3.1 for -normal PTX simulation and version 4.0 for cuobjdump support and/or to use -PTXPlus (Harware instruction set support). Note that it is possible to have -multiple versions of the CUDA toolkit installed on a single system -- just -install them in different directories and set your CUDA_INSTALL_PATH -environment variable to point to the version you want to use. - -[Optional] If you want to run OpenCL on the simulator, download and install -NVIDIA's OpenCL driver from . Update your -PATH and LD_LIBRARY_PATH as indicated by the NVIDIA install scripts. Note that -you will need to use the lib64 directory if you are using a 64-bit machine. We -have tested OpenCL on GPGPU-Sim using NVIDIA driver version 256.40 - -This version of GPGPU-Sim has been updated to support more recent versions of -the NVIDIA drivers (tested on version 295.20). - -GPGPU-Sim dependencies: -* gcc -* g++ -* make -* makedepend -* xutils -* bison -* flex -* zlib -* CUDA Toolkit - -GPGPU-Sim documentation dependencies: -* doxygen -* graphvi - -AerialVision dependencies: -* python-pmw -* python-ply -* python-numpy -* libpng12-dev -* python-matplotlib - -We used gcc/g++ version 4.5.1, bison version 2.4.1, and flex version 2.5.35. - -If you are using Ubuntu, the following commands will install all required -dependencies besides the CUDA Toolkit. - -GPGPU-Sim dependencies: -"sudo apt-get install build-essential xutils-dev bison zlib1g-dev flex -libglu1-mesa-dev" - -GPGPU-Sim documentation dependencies: -"sudo apt-get install doxygen graphviz" - -AerialVision dependencies: -"sudo apt-get install python-pmw python-ply python-numpy libpng12-dev -python-matplotlib" - -CUDA SDK dependencies: -"sudo apt-get install libxi-dev libxmu-dev libglut3-dev" - -If you are running applications which use NVIDIA libraries such as cuDNN and -cuBLAS, install them too. - -Finally, ensure CUDA_INSTALL_PATH is set to the location where you installed -the CUDA Toolkit (e.g., /usr/local/cuda) and that $CUDA_INSTALL_PATH/bin is in -your PATH. You probably want to modify your .bashrc file to incude the -following (this assumes the CUDA Toolkit was installed in /usr/local/cuda): - - export CUDA_INSTALL_PATH=/usr/local/cuda - export PATH=$CUDA_INSTALL_PATH/bin - -If running applications which use cuDNN or cuBLAS: - export CUDNN_PATH= - export LD_LIBRARY_PATH=$CUDA_INSTALL_PATH/lib64:$CUDA_INSTALL_PATH/lib:$CUDNN_PATH/lib64 - - -Step 2: Build -============= - -To build the simulator, you first need to configure how you want it to be -built. From the root directory of the simulator, type the following commands in -a bash shell (you can check you are using a bash shell by running the command -"echo $SHELL", which should print "/bin/bash"): - -source setup_environment - -replace with debug or release. Use release if you need faster -simulation and debug if you need to run the simulator in gdb. If nothing is -specified, release will be used by default. - -Now you are ready to build the simulator, just run - -make - -After make is done, the simulator would be ready to use. To clean the build, -run - -make clean - -To build the doxygen generated documentations, run - -make docs - -to clean the docs run - -make cleandocs - -The documentation resides at doc/doxygen/html. - -Step 3: Run -============ - -Before we run, we need to make sure the application's executable file is dynamically linked to CUDA runtime library. This can be done during compilation of your program by introducing the nvcc flag "--cudart shared" in makefile (quotes should be excluded). - -To confirm the same, type the follwoing command: - -ldd - -You should see that your application is using libcudart.so file in GPGPUSim directory. - -If running applications which use cuDNN or cuBLAS: - - * Modify the Makefile or the compilation command of the application to change - all the dynamic links to static ones, for example: - * -L$(CUDA_PATH)/lib64 -lcublas to - -L$(CUDA_PATH)/lib64 -lcublas_static - - * -L$(CUDNN_PATH)/lib64 -lcudnn to - -L$(CUDNN_PATH)/lib64 -lcudnn_static - - * Modify the Makefile or the compilation command such that the following - flags are used by the nvcc compiler: - -gencode arch=compute_61,code=compute_61 - - (the number 61 refers to the SM version. You would need to set it based - on the GPGPU-Sim config "-gpgpu ptx force max capability" you use) - -Copy the contents of configs/QuadroFX5800/ or configs/GTX480/ to your -application's working directory. These files configure the microarchitecture -models to resemble the respective GPGPU architectures. - -To use ptxplus (native ISA) change the following options in the configuration -file to "1" (Note: you need CUDA version 4.0) as follows: - --gpgpu_ptx_use_cuobjdump 1 --gpgpu_ptx_convert_to_ptxplus 1 - -Now To run a CUDA application on the simulator, simply execute - -source setup_environment - -Use the same you used while building the simulator. Then just -launch the executable as you would if it was to run on the hardware. By -running "source setup_environment " you change your LD_LIBRARY_PATH -to point to GPGPU-Sim's instead of CUDA or OpenCL runtime so that you do NOT -need to re-compile your application simply to run it on GPGPU-Sim. - -To revert back to running on the hardware, remove GPGPU-Sim from your -LD_LIBRARY_PATH environment variable. - -The following GPGPU-Sim configuration options are used to enable GPUWattch - - - power_simulation_enabled 1 (1=Enabled, 0=Not enabled) - - gpuwattch_xml_file .xml - -The GPUWattch XML configuration file name is set to gpuwattch.xml by default and -currently only supplied for GTX480 (default=gpuwattch_gtx480.xml). Please refer to - for more information. - -Running OpenCL applications is identical to running CUDA applications. However, -OpenCL applications need to communicate with the NVIDIA driver in order to -build OpenCL at runtime. GPGPU-Sim supports offloading this compilation to a -remote machine. The hostname of this machine can be specified using the -environment variable OPENCL_REMOTE_GPU_HOST. This variable should also be set -through the setup_environment script. If you are offloading to a remote machine, -you might want to setup passwordless ssh login to that machine in order to -avoid having too retype your password for every execution of an OpenCL -application. - -If you need to run the set of applications in the NVIDIA CUDA SDK code -samples then you will need to download, install and build the SDK. - -The CUDA applications from the ISPASS 2009 paper mentioned above are -distributed separately on github under the repo ispass2009-benchmarks. -The README.ISPASS-2009 file distributed with the benchmarks now contains -updated instructions for running the benchmarks on GPGPU-Sim v3.x. - - -3. (OPTIONAL) Updating GPGPU-Sim (ADVANCED USERS ONLY) - -If you have made modifications to the simulator and wish to incorporate new -features/bugfixes from subsequent releases the following instructions may help. -They are meant only as a starting point and only recommended for users -comfortable with using source control who have experience modifying and -debugging GPGPU-Sim. - -WARNING: Before following the procedure below, back up your modifications to -GPGPU-Sim. The following procedure may cause you to lose all your changes. In -general, merging code changes can require manual intervention and even in the -case where a merge proceeds automatically it may introduce errors. If many -edits have been made the merge process can be a painful manual process. Hence, -you will almost certainly want to have a copy of your code as it existed before -you followed the procedure below in case you need to start over again. You -will need to consult the documentation for git in addition to these -instructions in the case of any complications. - -STOP. BACK UP YOUR CHANGES BEFORE PROCEEDING. YOU HAVE BEEN WARNED. TWICE. - -To update GPGPU-Sim you need git to be installed on your system. Below we -assume that you ran the following command to get the source code of GPGPU-Sim: - -git clone git://dev.ece.ubc.ca/gpgpu-sim - -Since running the above command you have made local changes and we have -published changes to GPGPU-Sim on the above git server. You have looked at the -changes we made, looking at both the new CHANGES file and probably even the -source code differences. You decide you want to incorporate our changes into -your modified version of GPGPU-Sim. - -Before updating your source code, we recommend you remove any object files: - -make clean - -Then, run the following command in the root directory of GPGPU-Sim: - -git pull - -While git is pulling the latest changes, conflicts might arise due to changes -that you made that conflict with the latest updates. In this case, you need to -resolved those conflicts manually. You can either edit the conflicting files -directly using your favorite text editor, or you can use the following command -to open a graphical merge tool to do the merge: - -git mergetool - -3.1 Testing updated version of GPGPU-Sim - -Now you should test that the merged version "works". This means following the -steps for building GPGPU-Sim in the *new* README file (not this version) since -they may have changed. Assuming the code compiles without errors/warnings the -next step is to do some regression testing. At UBC we have an extensive set of -regression tests we run against our internal development branch when we make -changes. In the future we may make this set of regression tests publically -available. For now, you will want to compile the merged code and re-run all of -the applications you care about (implying these applications worked for you -before you did the merge). You want to do this before making further changes to -identify any compile time or runtime errors that occur due to the code merging -process. - - -4. MISCELLANEOUS - -4.1 Speeding up the execution - -Some applications take several hours to execute on GPGPUSim. This is because the simulator has to dump the PTX, analyze them and get resource usage statistics. This can be avoided everytime we execute the program in the following way: - -Step 1: Execute the program by enabling “-save_embedded_ptx 1” in config file, execute the code and let cuobjdump command dump all necessary files. After this process, you will get 2 new files namely: _cuobjdump_complete_output_ and _1.ptx - -Step 2: Create new environment variables or include the below in your .bashrc file: - a. export PTX_SIM_USE_PTX_FILE=_1.ptx - b. export PTX_SIM_KERNELFILE=_1.ptx - c. export CUOBJDUMP_SIM_FILE=_cuobjdump_complete_output_ - -Step 3: Disable -save_embedded_ptx flag, execute the code again. This will skip the dumping by cuobjdump and directly goes to executing the program thus saving time. - - -4.2 Debugging failing GPGPU-Sim Regressions - -Credits: Tor M Aamodt - -To debug failing GPGPU-Sim regression tests you need to run them locally. The fastest way to do this, assuming you are working with GPGPU-Sim versions more recent than the GPGPU-Sim dev branch circa March 28, 2018 (commit hash 2221d208a745a098a60b0d24c05007e92aaba092), is to install Docker. The instructions below were tested with Docker CE version 18.03 on Ubuntu and Mac OS. Docker will enable you to run the same set of regressions used by GPGPU-Sim when submitting a pull request to https://github.com/gpgpu-sim/gpgpu-sim_distribution and also allow you to log in and launch GPGPU-Sim in gdb so you can inspect failures. - -1. Install Docker. On Ubuntu 14.04 and 16.04 the following instructions work: https://docs.docker.com/install/linux/docker-ce/ubuntu/#uninstall-old-versions - -2. Clone GPGPU-Sim from your fork of GPGPU-Sim. For example: - - git clone https://github.com//gpgpu-sim_distribution.git - - -3. Run the following command (this is all one line) to run the regressions in docker: - - docker run --privileged -v `pwd`:/home/runner/gpgpu-sim_distribution:rw aamodt/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/configs.gtx1080ti.yml -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress’; tail -f /dev/null" - -Explanation: The last part of this command, "tail -f /dev/null" will keep the docker container running after the regressions finish. This enables you to log into the container to run the same tests inside gdb so you can debug. The "--privileged" part enables you to use breakpoints inside gdb in a container. The "-v" part maps the current directory (with the GPGPU-Sim source code you want to test) into the container. The string "aamodt/gpgpu-sim_regress:latest" is a tag for a container setup to run regressions which will be downloaded from docker hub. The portion starting with /bin/bash is a set of commands run inside a bash shell inside the container. E.g., the command start_torque.sh starts up a queue manager inside the container. - -If the above command stops with the message "fatal: unable to access 'https://github.com/tgrogers/gpgpu-sim_simulations.git/': Could not resolve host: github.com" this likely means your computer sits behind a firewall which is blocking access to Google's name servers (e.g., 8.8.8.8). To get around this you will need to modify th above command to point to your local DNS server. Lookup your DNS server IP address which we will call below. On Ubuntu run "ifconfig" to lookup the network interface connecting your computer to the network. Then run "nmcli device show " to find the IP address of your DNS server. Modify the above command to include "--dns " after "run", E.g., - - docker run --dns --privileged -v `pwd`:/home/runner/gpgpu-sim_distribution:rw aamodt/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/configs.gtx1080ti.yml -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress’; tail -f /dev/null" - -4. Find the CONTAINER ID associated with your docker container by running "docker ps". - -5. Log into the container by running the command: - - docker exec -it /bin/bash -c "su -l runner" - -The container is running Ubuntu 16.04 and has screen, cscope and vim installed (if you find a favorite Linux tool missing, it is fairly easy to create derived containers that have additional tools). - -6. Lookup the directory of the regression test you want to debug by going to the regression log file directory: - - cd /home/runner/gpgpu-sim_simulations/util/job_launching/logfiles - -7. The file "failed_job_log_sim_log.regress..txt" includes information about the failed test including its simulation directory. For the following example, I'll assume the first failing test was "hotspot-rodinia-2.0-ft-30_6_40___data_result_30_6_40_txt--GTX1080Ti" for which the simulation directory is /home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/hotspot-rodinia-2.0-ft/30_6_40___data_result_30_6_40_txt/GTX1080Ti/ - -8. Change to the simulation directory using: - - cd - -E.g., "cd /home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/hotspot-rodinia-2.0-ft/30_6_40___data_result_30_6_40_txt/GTX1080Ti/" - -This directory should contain a file called "torque.sim" that contains commands used to launch the simulation during regression tests. We will modify this file to enable us to re-run the regression test in gdb. This directory should also contain a file containing the standard output during the regression test. This file will end in .o where is the torque queue manager job number. For the running example for me this file is called "hotspot-rodinia-2.0-ft-30_6_40___data_result_30_6_40_txt.o2". Open this file to determine the LD_LIBRARY_PATH settings used when launching the simulation. Look for a line that starts "doing: export LD_LIBRARY_PATH" and copy the entire line starting with "export LD_LIBRARY_PATH ..." - -9. Paste the "export LD_LIBRARY_PATH ..." line into the bash shell to set LD_LIBRARY_PATH. E.g., - - export LD_LIBRARY_PATH=/home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/gpgpu-sim-builds/libcudart_gpgpu-sim_git-commit-177d02254ae38b6331b17dd6cd139b570a03c589_modified_0.so:/gpgpu-sim/usr/local/gcc-4.5.4/lib64:/gpgpu-sim/usr/local/gcc-4.5.4/lib:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/lib64/:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/4.5.4/:/usr/lib/x86_64-linux-gnu:/home/runner/gpgpu-sim_distribution/lib/gcc-4.5.4/cuda-4020/release:/gpgpu-sim/usr/local/gcc-4.5.4/lib64:/gpgpu-sim/usr/local/gcc-4.5.4/lib:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/lib64/:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/4.5.4/:/usr/lib/x86_64-linux-gnu: - -10. In the same shell, build the debug version of GPGPU-Sim then return to the directory above: - - pushd ~/gpgpu-sim_distribution/ - source setup_environment debug - make - popd - -11. Open and edit torque.sim and preface the very last line with "gdb --args ". After editing the last line in torque.sim should look something like: - - gdb --args /home/runner/gpgpu-sim_simulations/util/job_launching/../../benchmarks/bin/4.2/release/hotspot-rodinia-2.0-ft 30 6 40 ./data/result_30_6_40.txt - -12. Re-run the regression test in gdb by sourcing the torque.sim file: - - . torque.sim - -This will put you in at the (gdb) prompt. Setup any breakpoints needed and run. - diff --git a/README.md b/README.md new file mode 100644 index 0000000..048fb3c --- /dev/null +++ b/README.md @@ -0,0 +1,499 @@ +Welcome to GPGPU-Sim, a cycle-level simulator modeling contemporary graphics +processing units (GPUs) running GPU computing workloads written in CUDA or +OpenCL. Also included in GPGPU-Sim is a performance visualization tool called +AerialVision and a configurable and extensible energy model called GPUWattch. +GPGPU-Sim and GPUWattch have been rigorously validated with performance and +power measurements of real hardware GPUs. + +This version of GPGPU-Sim has been tested with CUDA version 4.2, +5.0, 5.5, 6.0 and 7.5, 8.0, 9.0, 9.1 + +Please see the copyright notice in the file COPYRIGHT distributed with this +release in the same directory as this file. + +If you use GPGPU-Sim in your research, please cite: + +Ali Bakhoda, George Yuan, Wilson W. L. Fung, Henry Wong, Tor M. Aamodt, +Analyzing CUDA Workloads Using a Detailed GPU Simulator, in IEEE International +Symposium on Performance Analysis of Systems and Software (ISPASS), Boston, MA, +April 19-21, 2009. + +If you use cuDNN and Pytorch support, the Checkpoint function or the Debigging tool for functional simulation error in GPGPU-Sim for your research, +please cite: +Jonathan Lew, Deval Shah, Suchita Pati, Shaylin Cattell, Mengchi Zhang, Amruth Sandhupatla, Christopher Ng, Negar Goli, Matthew D. Sinclair, Timothy G. Rogers, Tor M. Aamodt +Analyzing Machine Learning Workloads Using a Detailed GPU Simulator, arXiv:1811.08933, +https://arxiv.org/abs/1811.08933 + +If you use the memory system in GPGPU-Sim, or the Volta/Pascal models, +please cite: +Mahmoud Khairy, Jain Akshay, Tor Aamodt, Timothy G Rogers, +Exploring Modern GPU Memory System Design Challenges through Accurate Modeling, arXiv:1810.07269, +https://arxiv.org/abs/1810.07269 + +If you use the Tensor Core in GPGPU-Sim or CUTLASS Library for your research +please cite: +Md Aamir Raihan, Negar Goli, Tor Aamodt, +Modeling Deep Learning Accelerator Enabled GPUs, arXiv:1811.08309, +https://arxiv.org/abs/1811.08309 + +If you use the GPUWattch energy model in your research, please cite: + +Jingwen Leng, Tayler Hetherington, Ahmed ElTantawy, Syed Gilani, Nam Sung Kim, +Tor M. Aamodt, Vijay Janapa Reddi, GPUWattch: Enabling Energy Optimizations in +GPGPUs, In proceedings of the ACM/IEEE International Symposium on Computer +Architecture (ISCA 2013), Tel-Aviv, Israel, June 23-27, 2013. + +If you use the support for CUDA dynamic parallelism in your research, please cite: + +Jin Wang and Sudhakar Yalamanchili, Characterization and Analysis of Dynamic +Parallelism in Unstructured GPU Applications, 2014 IEEE International Symposium +on Workload Characterization (IISWC), November 2014. + +If you use figures plotted using AerialVision in your publications, please cite: + +Aaron Ariel, Wilson W. L. Fung, Andrew Turner, Tor M. Aamodt, Visualizing +Complex Dynamics in Many-Core Accelerator Architectures, In Proceedings of the +IEEE International Symposium on Performance Analysis of Systems and Software +(ISPASS), pp. 164-174, White Plains, NY, March 28-30, 2010. + +This file contains instructions on installing, building and running GPGPU-Sim. +Detailed documentation on what GPGPU-Sim models, how to configure it, and a +guide to the source code can be found here: . +Instructions for building doxygen source code documentation are included below. +Detailed documentation on GPUWattch including how to configure it and a guide +to the source code can be found here: . + +If you have questions, please sign up for the google groups page (see +gpgpu-sim.org), but note that use of this simulator does not imply any level of +support. Questions answered on a best effort basis. + +To submit a bug report, go here: http://www.gpgpu-sim.org/bugs/ + +See Section 2 "INSTALLING, BUILDING and RUNNING GPGPU-Sim" below to get started. + +See file CHANGES for updates in this and earlier versions. + +# CONTRIBUTIONS and HISTORY + +## GPGPU-Sim + +GPGPU-Sim was created by Tor Aamodt's research group at the University of +British Columbia. Many have directly contributed to development of GPGPU-Sim +including: Tor Aamodt, Wilson W.L. Fung, Ali Bakhoda, George Yuan, Ivan Sham, +Henry Wong, Henry Tran, Andrew Turner, Aaron Ariel, Inderpret Singh, Tim +Rogers, Jimmy Kwa, Andrew Boktor, Ayub Gubran Tayler Hetherington and others. + +GPGPU-Sim models the features of a modern graphics processor that are relevant +to non-graphics applications. The first version of GPGPU-Sim was used in a +MICRO'07 paper and follow-on ACM TACO paper on dynamic warp formation. That +version of GPGPU-Sim used the SimpleScalar PISA instruction set for functional +simulation, and various configuration files indicating which loops should be +spawned as kernels on the GPU, along with reconvergence points required for +SIMT execution to provide a programming model simlar to CUDA/OpenCL. Creating +benchmarks for the original GPGPU-Sim simulator was a very time consuming +process and the validity of code generation for CPU run on a GPU was questioned +by some. These issues motivated the development an interface for directly +running CUDA applications to leverage the growing number of applications being +developed to use CUDA. We subsequently added support for OpenCL and removed +all SimpleScalar code. + +The interconnection network is simulated using the booksim simulator developed +by Bill Dally's research group at Stanford. + +To produce output that matches the output from running the same CUDA program on +the GPU, we have implemented several PTX instructions using the CUDA Math +library (part of the CUDA toolkit). Code to interface with the CUDA Math +library is contained in cuda-math.h, which also includes several structures +derived from vector_types.h (one of the CUDA header files). + +## GPUWattch Energy Model + +GPUWattch (introduced in GPGPU-Sim 3.2.0) was developed by researchers at the +University of British Columbia, the University of Texas at Austin, and the +University of Wisconsin-Madison. Contributors to GPUWattch include Tor +Aamodt's research group at the University of British Columbia: Tayler +Hetherington and Ahmed ElTantawy; Vijay Reddi's research group at the +University of Texas at Austin: Jingwen Leng; and Nam Sung Kim's research group +at the University of Wisconsin-Madison: Syed Gilani. + +GPUWattch leverages McPAT, which was developed by Sheng Li et al. at the +University of Notre Dame, Hewlett-Packard Labs, Seoul National University, and +the University of California, San Diego. The paper can be found at +http://www.hpl.hp.com/research/mcpat/micro09.pdf. + +# INSTALLING, BUILDING and RUNNING GPGPU-Sim + +Assuming all dependencies required by GPGPU-Sim are installed on your system, +to build GPGPU-Sim all you need to do is add the following line to your +~/.bashrc file (assuming the CUDA Toolkit was installed in /usr/local/cuda): + +``` + export CUDA_INSTALL_PATH=/usr/local/cuda +``` + +then type + +``` + bash + source setup_environment + make +``` + +If the above fails, see "Step 1" and "Step 2" below. + +If the above worked, see "Step 3" below, which explains how to run a CUDA +benchmark on GPGPU-Sim. + +## Step 1: Dependencies + +GPGPU-Sim was developed on SUSE Linux (this release was tested with SUSE +version 11.3) and has been used on several other Linux platforms (both 32-bit +and 64-bit systems). In principle, GPGPU-Sim should work with any linux +distribution as long as the following software dependencies are satisfied. + +Download and install the CUDA Toolkit. It is recommended to use version 3.1 for +normal PTX simulation and version 4.0 for cuobjdump support and/or to use +PTXPlus (Harware instruction set support). Note that it is possible to have +multiple versions of the CUDA toolkit installed on a single system -- just +install them in different directories and set your CUDA_INSTALL_PATH +environment variable to point to the version you want to use. + +[Optional] If you want to run OpenCL on the simulator, download and install +NVIDIA's OpenCL driver from . Update your +PATH and LD_LIBRARY_PATH as indicated by the NVIDIA install scripts. Note that +you will need to use the lib64 directory if you are using a 64-bit machine. We +have tested OpenCL on GPGPU-Sim using NVIDIA driver version 256.40 + +This version of GPGPU-Sim has been updated to support more recent versions of +the NVIDIA drivers (tested on version 295.20). + +GPGPU-Sim dependencies: +- gcc +- g++ +- make +- makedepend +- xutils +- bison +- flex +- zlib +- CUDA Toolkit + +GPGPU-Sim documentation dependencies: +- doxygen +- graphvi + +AerialVision dependencies: +- python-pmw +- python-ply +- python-numpy +- libpng12-dev +- python-matplotlib + +We used gcc/g++ version 4.5.1, bison version 2.4.1, and flex version 2.5.35. + +If you are using Ubuntu, the following commands will install all required +dependencies besides the CUDA Toolkit. + +GPGPU-Sim dependencies: + + sudo apt-get install build-essential xutils-dev bison zlib1g-dev flex libglu1-mesa-dev + +GPGPU-Sim documentation dependencies: + + sudo apt-get install doxygen graphviz + +AerialVision dependencies: + + sudo apt-get install python-pmw python-ply python-numpy libpng12-dev python-matplotlib + +CUDA SDK dependencies: + + sudo apt-get install libxi-dev libxmu-dev libglut3-dev + +If you are running applications which use NVIDIA libraries such as cuDNN and +cuBLAS, install them too. + +Finally, ensure CUDA_INSTALL_PATH is set to the location where you installed +the CUDA Toolkit (e.g., /usr/local/cuda) and that \$CUDA_INSTALL_PATH/bin is in +your PATH. You probably want to modify your .bashrc file to incude the +following (this assumes the CUDA Toolkit was installed in /usr/local/cuda): + + export CUDA_INSTALL_PATH=/usr/local/cuda + export PATH=$CUDA_INSTALL_PATH/bin + +If running applications which use cuDNN or cuBLAS: + + export CUDNN_PATH= + export LD_LIBRARY_PATH=$CUDA_INSTALL_PATH/lib64:$CUDA_INSTALL_PATH/lib:$CUDNN_PATH/lib64 + + + +## Step 2: Build + +To build the simulator, you first need to configure how you want it to be +built. From the root directory of the simulator, type the following commands in +a bash shell (you can check you are using a bash shell by running the command +"echo \$SHELL", which should print "/bin/bash"): + +source setup_environment + +replace with debug or release. Use release if you need faster +simulation and debug if you need to run the simulator in gdb. If nothing is +specified, release will be used by default. + +Now you are ready to build the simulator, just run + + make + + +After make is done, the simulator would be ready to use. To clean the build, +run + + make clean + +To build the doxygen generated documentations, run + + make docs + +To clean the docs run + + make cleandocs + + +The documentation resides at doc/doxygen/html. + +## Step 3: Run + +Before we run, we need to make sure the application's executable file is dynamically linked to CUDA runtime library. This can be done during compilation of your program by introducing the nvcc flag "--cudart shared" in makefile (quotes should be excluded). + +To confirm the same, type the follwoing command: + +ldd + +You should see that your application is using libcudart.so file in GPGPUSim directory. + +If running applications which use cuDNN or cuBLAS: + +* Modify the Makefile or the compilation command of the application to change + all the dynamic links to static ones, for example: + * `-L$(CUDA_PATH)/lib64 -lcublas` to + `-L$(CUDA_PATH)/lib64 -lcublas_static` + + * `-L$(CUDNN_PATH)/lib64 -lcudnn` to + `-L$(CUDNN_PATH)/lib64 -lcudnn_static` + +* Modify the Makefile or the compilation command such that the following + flags are used by the nvcc compiler: + `-gencode arch=compute_61,code=compute_61` + + (the number 61 refers to the SM version. You would need to set it based + on the GPGPU-Sim config `-gpgpu-ptx-force-max-capability` you use) + +Copy the contents of configs/QuadroFX5800/ or configs/GTX480/ to your +application's working directory. These files configure the microarchitecture +models to resemble the respective GPGPU architectures. + +To use ptxplus (native ISA) change the following options in the configuration +file to "1" (Note: you need CUDA version 4.0) as follows: + + -gpgpu_ptx_use_cuobjdump 1 + -gpgpu_ptx_convert_to_ptxplus 1 + +Now To run a CUDA application on the simulator, simply execute + + source setup_environment + +Use the same you used while building the simulator. Then just +launch the executable as you would if it was to run on the hardware. By +running `source setup_environment ` you change your LD_LIBRARY_PATH +to point to GPGPU-Sim's instead of CUDA or OpenCL runtime so that you do NOT +need to re-compile your application simply to run it on GPGPU-Sim. + +To revert back to running on the hardware, remove GPGPU-Sim from your +LD_LIBRARY_PATH environment variable. + +The following GPGPU-Sim configuration options are used to enable GPUWattch + + -power_simulation_enabled 1 (1=Enabled, 0=Not enabled) + -gpuwattch_xml_file .xml + + +The GPUWattch XML configuration file name is set to gpuwattch.xml by default and +currently only supplied for GTX480 (default=gpuwattch_gtx480.xml). Please refer to + for more information. + +Running OpenCL applications is identical to running CUDA applications. However, +OpenCL applications need to communicate with the NVIDIA driver in order to +build OpenCL at runtime. GPGPU-Sim supports offloading this compilation to a +remote machine. The hostname of this machine can be specified using the +environment variable OPENCL_REMOTE_GPU_HOST. This variable should also be set +through the setup_environment script. If you are offloading to a remote machine, +you might want to setup passwordless ssh login to that machine in order to +avoid having too retype your password for every execution of an OpenCL +application. + +If you need to run the set of applications in the NVIDIA CUDA SDK code +samples then you will need to download, install and build the SDK. + +The CUDA applications from the ISPASS 2009 paper mentioned above are +distributed separately on github under the repo ispass2009-benchmarks. +The README.ISPASS-2009 file distributed with the benchmarks now contains +updated instructions for running the benchmarks on GPGPU-Sim v3.x. + +# (OPTIONAL) Contributing to GPGPU-Sim (ADVANCED USERS ONLY) + +If you have made modifications to the simulator and wish to incorporate new +features/bugfixes from subsequent releases the following instructions may help. +They are meant only as a starting point and only recommended for users +comfortable with using source control who have experience modifying and +debugging GPGPU-Sim. + +WARNING: Before following the procedure below, back up your modifications to +GPGPU-Sim. The following procedure may cause you to lose all your changes. In +general, merging code changes can require manual intervention and even in the +case where a merge proceeds automatically it may introduce errors. If many +edits have been made the merge process can be a painful manual process. Hence, +you will almost certainly want to have a copy of your code as it existed before +you followed the procedure below in case you need to start over again. You +will need to consult the documentation for git in addition to these +instructions in the case of any complications. + +STOP. BACK UP YOUR CHANGES BEFORE PROCEEDING. YOU HAVE BEEN WARNED. TWICE. + +To update GPGPU-Sim you need git to be installed on your system. Below we +assume that you ran the following command to get the source code of GPGPU-Sim: + +``` + git clone git://dev.ece.ubc.ca/gpgpu-sim +``` + +Since running the above command you have made local changes and we have +published changes to GPGPU-Sim on the above git server. You have looked at the +changes we made, looking at both the new CHANGES file and probably even the +source code differences. You decide you want to incorporate our changes into +your modified version of GPGPU-Sim. + +Before updating your source code, we recommend you remove any object files: + +``` + make clean +``` + +Then, run the following command in the root directory of GPGPU-Sim: + +``` + git pull +``` + +While git is pulling the latest changes, conflicts might arise due to changes +that you made that conflict with the latest updates. In this case, you need to +resolved those conflicts manually. You can either edit the conflicting files +directly using your favorite text editor, or you can use the following command +to open a graphical merge tool to do the merge: + +``` + git mergetool +``` + +## Testing updated version of GPGPU-Sim + +Now you should test that the merged version "works". This means following the +steps for building GPGPU-Sim in the _new_ README file (not this version) since +they may have changed. Assuming the code compiles without errors/warnings the +next step is to do some regression testing. At UBC we have an extensive set of +regression tests we run against our internal development branch when we make +changes. In the future we may make this set of regression tests publically +available. For now, you will want to compile the merged code and re-run all of +the applications you care about (implying these applications worked for you +before you did the merge). You want to do this before making further changes to +identify any compile time or runtime errors that occur due to the code merging +process. + + +# MISCELLANEOUS + +## Speeding up the execution + +Some applications take several hours to execute on GPGPUSim. This is because the simulator has to dump the PTX, analyze them and get resource usage statistics. This can be avoided everytime we execute the program in the following way: + +1. Execute the program by enabling “-save_embedded_ptx 1” in config file, execute the code and let cuobjdump command dump all necessary files. After this process, you will get 2 new files namely: _cuobjdump_complete_output_ and _1.ptx + +2. Create new environment variables or include the below in your .bashrc file: + 1. export PTX_SIM_USE_PTX_FILE=_1.ptx + 2. export PTX_SIM_KERNELFILE=_1.ptx + 3. export CUOBJDUMP_SIM_FILE=_cuobjdump_complete_output_ + +3. Disable -save_embedded_ptx flag, execute the code again. This will skip the dumping by cuobjdump and directly goes to executing the program thus saving time. + + +## Debugging failing GPGPU-Sim Regressions + +Credits: Tor M Aamodt + +To debug failing GPGPU-Sim regression tests you need to run them locally. The fastest way to do this, assuming you are working with GPGPU-Sim versions more recent than the GPGPU-Sim dev branch circa March 28, 2018 (commit hash 2221d208a745a098a60b0d24c05007e92aaba092), is to install Docker. The instructions below were tested with Docker CE version 18.03 on Ubuntu and Mac OS. Docker will enable you to run the same set of regressions used by GPGPU-Sim when submitting a pull request to https://github.com/gpgpu-sim/gpgpu-sim_distribution and also allow you to log in and launch GPGPU-Sim in gdb so you can inspect failures. + +1. Install Docker. On Ubuntu 14.04 and 16.04 the following instructions work: https://docs.docker.com/install/linux/docker-ce/ubuntu/#uninstall-old-versions + +2. Clone GPGPU-Sim from your fork of GPGPU-Sim. For example: + + git clone https://github.com//gpgpu-sim_distribution.git + + +3. Run the following command (this is all one line) to run the regressions in docker: + ``` + docker run --privileged -v `pwd`:/home/runner/gpgpu-sim_distribution:rw aamodt/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/configs.gtx1080ti.yml -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress’; tail -f /dev/null" + ``` + Explanation: The last part of this command, "tail -f /dev/null" will keep the docker container running after the regressions finish. This enables you to log into the container to run the same tests inside gdb so you can debug. The "--privileged" part enables you to use breakpoints inside gdb in a container. The "-v" part maps the current directory (with the GPGPU-Sim source code you want to test) into the container. The string "aamodt/gpgpu-sim_regress:latest" is a tag for a container setup to run regressions which will be downloaded from docker hub. The portion starting with /bin/bash is a set of commands run inside a bash shell inside the container. E.g., the command start_torque.sh starts up a queue manager inside the container. + + If the above command stops with the message "fatal: unable to access 'https://github.com/tgrogers/gpgpu-sim_simulations.git/': Could not resolve host: github.com" this likely means your computer sits behind a firewall which is blocking access to Google's name servers (e.g., 8.8.8.8). To get around this you will need to modify th above command to point to your local DNS server. Lookup your DNS server IP address which we will call below. On Ubuntu run "ifconfig" to lookup the network interface connecting your computer to the network. Then run "nmcli device show " to find the IP address of your DNS server. Modify the above command to include "--dns " after "run", E.g., + ``` + docker run --dns --privileged -v `pwd`:/home/runner/gpgpu-sim_distribution:rw aamodt/gpgpu-sim_regress:latest /bin/bash -c "./start_torque.sh; chown -R runner /home/runner/gpgpu-sim_distribution; su - runner -c 'source /home/runner/gpgpu-sim_distribution/setup_environment && make -j -C /home/runner/gpgpu-sim_distribution && cd /home/runner/gpgpu-sim_simulations/ && git pull && /home/runner/gpgpu-sim_simulations/util/job_launching/run_simulations.py -c /home/runner/gpgpu-sim_simulations/util/job_launching/regression_recipies/rodinia_2.0-ft/configs.gtx1080ti.yml -N regress && /home/runner/gpgpu-sim_simulations/util/job_launching/monitor_func_test.py -v -N regress’; tail -f /dev/null" + ``` + +4. Find the CONTAINER ID associated with your docker container by running "docker ps". + +5. Log into the container by running the command: + ``` + docker exec -it /bin/bash -c "su -l runner"` + ``` + The container is running Ubuntu 16.04 and has screen, cscope and vim installed (if you find a favorite Linux tool missing, it is fairly easy to create derived containers that have additional tools). + +6. Lookup the directory of the regression test you want to debug by going to the regression log file directory: + ``` + cd /home/runner/gpgpu-sim_simulations/util/job_launching/logfiles + ``` + +7. The file "failed_job_log_sim_log.regress..txt" includes information about the failed test including its simulation directory. For the following example, I'll assume the first failing test was "hotspot-rodinia-2.0-ft-30_6_40___data_result_30_6_40_txt--GTX1080Ti" for which the simulation directory is /home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/hotspot-rodinia-2.0-ft/30_6_40___data_result_30_6_40_txt/GTX1080Ti/ + +8. Change to the simulation directory using: + ``` + cd + ``` + E.g., `cd /home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/hotspot-rodinia-2.0-ft/30_6_40___data_result_30_6_40_txt/GTX1080Ti/` + + This directory should contain a file called "torque.sim" that contains commands used to launch the simulation during regression tests. We will modify this file to enable us to re-run the regression test in gdb. This directory should also contain a file containing the standard output during the regression test. This file will end in .o where is the torque queue manager job number. For the running example for me this file is called "hotspot-rodinia-2.0-ft-30_6_40___data_result_30_6_40_txt.o2". Open this file to determine the LD_LIBRARY_PATH settings used when launching the simulation. Look for a line that starts "doing: export LD_LIBRARY_PATH" and copy the entire line starting with "export LD_LIBRARY_PATH ..." + +9. Paste the "export LD_LIBRARY_PATH ..." line into the bash shell to set LD_LIBRARY_PATH. E.g., + ``` + export LD_LIBRARY_PATH=/home/runner/gpgpu-sim_simulations/util/job_launching/../../sim_run_4.2/gpgpu-sim-builds/libcudart_gpgpu-sim_git-commit-177d02254ae38b6331b17dd6cd139b570a03c589_modified_0.so:/gpgpu-sim/usr/local/gcc-4.5.4/lib64:/gpgpu-sim/usr/local/gcc-4.5.4/lib:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/lib64/:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/4.5.4/:/usr/lib/x86_64-linux-gnu:/home/runner/gpgpu-sim_distribution/lib/gcc-4.5.4/cuda-4020/release:/gpgpu-sim/usr/local/gcc-4.5.4/lib64:/gpgpu-sim/usr/local/gcc-4.5.4/lib:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/lib64/:/gpgpu-sim/usr/local/gcc-4.5.4/lib/gcc/x86_64-unknown-linux-gnu/4.5.4/:/usr/lib/x86_64-linux-gnu: + ``` + +10. In the same shell, build the debug version of GPGPU-Sim then return to the directory above: + ``` + pushd ~/gpgpu-sim_distribution/ + source setup_environment debug + make + popd + ``` + +11. Open and edit torque.sim and preface the very last line with "gdb --args ". After editing the last line in torque.sim should look something like: + ``` + gdb --args /home/runner/gpgpu-sim_simulations/util/job_launching/../../benchmarks/bin/4.2/release/hotspot-rodinia-2.0-ft 30 6 40 ./data/result_30_6_40.txt + ``` + +12. Re-run the regression test in gdb by sourcing the torque.sim file: + ``` + . torque.sim + ``` + This will put you in at the (gdb) prompt. Setup any breakpoints needed and run. + -- cgit v1.3 From 52bee41cfea17f13e93cc1fb812c125fd44bac7b Mon Sep 17 00:00:00 2001 From: Lucy Liu Date: Fri, 3 Jul 2020 13:09:18 -0700 Subject: add activemask, bfind, vmin, and vmax implementations from Francois --- cuobjdump_to_ptxplus/cuobjdumpInst.cc | 41 +++++----- src/abstract_hardware_model.h | 1 + src/cuda-sim/cuda-sim.cc | 6 +- src/cuda-sim/instructions.cc | 137 ++++++++++++++++++++++++++++++++-- src/cuda-sim/opcodes.def | 11 +-- src/cuda-sim/ptx.l | 9 ++- src/cuda-sim/ptx.y | 36 ++++----- src/cuda-sim/ptx_ir.cc | 3 +- src/cuda-sim/ptx_ir.h | 3 + src/cuda-sim/ptx_parser.cc | 5 +- 10 files changed, 195 insertions(+), 57 deletions(-) diff --git a/cuobjdump_to_ptxplus/cuobjdumpInst.cc b/cuobjdump_to_ptxplus/cuobjdumpInst.cc index 392f829..969313c 100644 --- a/cuobjdump_to_ptxplus/cuobjdumpInst.cc +++ b/cuobjdump_to_ptxplus/cuobjdumpInst.cc @@ -742,18 +742,18 @@ void cuobjdumpInst::printCuobjdumpOutputModifiers(const char* defaultMod) { std::list::iterator typemod = m_typeModifiers->begin(); if (*typemod == ".U16" or *typemod == ".S16") { - std::list::iterator dest_op = m_operands->begin(); - std::string& destination = *dest_op; + std::list::iterator dest_op = m_operands->begin(); + std::string& destination = *dest_op; if (destination[destination.length()-1] == 'l') { - output(".lo"); // write to the lower 16-bits + output(".lo"); // write to the lower 16-bits } else if (destination[destination.length()-1] == 'h') { - output(".hi"); // write to the upper 16-bits + output(".hi"); // write to the upper 16-bits } else { - output(".wide"); // write to the whole 32-bits + output(".wide"); // write to the whole 32-bits } - return; + return; } - output(defaultMod); // default output modifier for mul + output(defaultMod); // default output modifier for mul } std::string int_default_mod () { return ".u32" ;} @@ -1357,9 +1357,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1393,9 +1393,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: type = (type1Size > type2Size) ? type1 : type2; strcpy(tempString, type.c_str()); if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1407,7 +1407,7 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: { printCuobjdumpPredicate(); output("mul"); - printCuobjdumpOutputModifiers(".lo"); + printCuobjdumpOutputModifiers(".lo"); printCuobjdumpBaseModifiers(); if(m_typeModifiers->size() == 0) @@ -1434,9 +1434,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1468,10 +1468,10 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else printCuobjdumpTypeModifiers(); @@ -1506,10 +1506,10 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else printCuobjdumpTypeModifiers(); @@ -2071,6 +2071,11 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: printCuobjdumpTypeModifiers(); printCuobjdumpOperands(); output(";"); + } else if(m_base == "ACTIVEMASK") { + printCuobjdumpPredicate(); + output("activemask.b32"); + printCuobjdumpOperands(); + output(";"); } else if(m_base == "DFMA") { diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 46534ab..c6e3b43 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -1132,6 +1132,7 @@ class warp_inst_t : public inst_t { void print(FILE *fout) const; unsigned get_uid() const { return m_uid; } unsigned get_schd_id() const { return m_scheduler_id; } + active_mask_t get_warp_active_mask() const { return m_warp_active_mask; } protected: unsigned m_uid; diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 75dd3c8..451feb5 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -346,11 +346,11 @@ void function_info::ptx_assemble() { printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); connect_basic_blocks(); - bool modified = false; + bool modified = false; do { find_dominators(); find_idominators(); - modified = connect_break_targets(); + modified = connect_break_targets(); } while (modified == true); if ( g_debug_execution>=50 ) { @@ -1741,7 +1741,7 @@ void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if (pI->get_opcode() == VOTE_OP) { + if( pI->get_opcode() == VOTE_OP || pI->get_opcode() == ACTIVEMASK_OP ) { pJ = new ptx_instruction(*pI); *((warp_inst_t *)pJ) = inst; // copy active mask information pI = pJ; diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index bf9a040..886c6c0 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -166,6 +166,7 @@ void inst_not_implemented(const ptx_instruction *pI); ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); +void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, int op_code); void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); @@ -1709,8 +1710,40 @@ void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { } thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); +void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) +{ + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const unsigned i_type = pI->get_type(); + + const ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + const int msb = ( i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + + unsigned long a = 0; + switch (i_type) + { + case S32_TYPE: a = src1_data.s32; break; + case U32_TYPE: a = src1_data.u32; break; + case S64_TYPE: a = src1_data.s64; break; + case U64_TYPE: a = src1_data.u64; break; + default: assert(false); abort(); + } + + // negate negative signed inputs + if ( ( i_type == S32_TYPE || i_type == S64_TYPE ) && ( a & ( 1 << msb ) ) ) { + a = ~a; + } + uint32_t d_data = 0xffffffff; + for (uint32_t i = msb; i >= 0; i--) { + if (a & (1<set_operand_value(dst, d_data, U32_TYPE, thread, pI); + + } void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { @@ -6301,11 +6334,17 @@ void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { inst_not_implemented(pI); } -void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); + +#define VMAX 0 +#define VMIN 1 + +void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) +{ + video_mem_instruction(pI, thread, VMAX); } -void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { - inst_not_implemented(pI); +void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) +{ + video_mem_instruction(pI, thread, VMIN); } void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { inst_not_implemented(pI); @@ -6400,6 +6439,15 @@ void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { } } +void activemask_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + active_mask_t l_activemask_bitset = pI->get_warp_active_mask(); + uint32_t l_activemask_uint = static_cast(l_activemask_bitset.to_ulong()); + + const operand_info &dst = pI->dst(); + thread->set_operand_value(dst, l_activemask_uint, U32_TYPE, thread, pI); +} + void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { ptx_reg_t src1_data, src2_data, data; @@ -6477,3 +6525,80 @@ ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, return result; } + +void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, int op_code) +{ + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + const operand_info &src3 = pI->src3(); // c + + const unsigned i_type = pI->get_type(); + + std::list scalar_type; + std::list options; + + ptx_reg_t a, b, ta, tb, c, data; + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + // TODO: implement this + // ta = partSelectSignExtend( a, atype ); + // tb = partSelectSignExtend( b, btype ); + ta = a; + tb = b; + + options = pI->get_options(); + assert(options.size() == 1); + + auto option = options.begin(); + assert(*option == ATOMIC_MAX || *option == ATOMIC_MIN); + + switch ( i_type ) { + case S32_TYPE: { + // assert all operands are S32_TYPE: + scalar_type = pI->get_scalar_type(); + for (auto scalar : scalar_type) + { + assert(scalar == S32_TYPE); + } + assert(scalar_type.size() == 3); + scalar_type.clear(); + + switch (op_code) + { + case VMAX: + data.s32 = MY_MAX_I(ta.s32, tb.s32); + break; + case VMIN: + data.s32 = MY_MIN_I(ta.s32, tb.s32); + break; + default: + assert(0); + } + + switch (*option) + { + case ATOMIC_MAX: + data.s32 = MY_MAX_I(data.s32, c.s32); + break; + case ATOMIC_MIN: + data.s32 = MY_MIN_I(data.s32, c.s32); + break; + default: + assert(0); // not yet implemented + } + break; + + } + default: + assert(0); // not yet implemented + } + + thread->set_operand_value(dst, data, i_type, thread, pI); + + return; +} + diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index c4d6a83..aa85512 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -1,10 +1,10 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda +// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda // The University of British Columbia // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: -// +// // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. // Redistributions in binary form must reproduce the above copyright notice, this @@ -13,7 +13,7 @@ // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -27,7 +27,7 @@ /*6th operand of each OP_DEF reflects its classification */ -/*Type +/*Type ALU 1 MAD 2 Control 3 @@ -129,6 +129,7 @@ OP_DEF(VSHL_OP,vshl_impl,"vshl",0,11) OP_DEF(VSHR_OP,vshr_impl,"vshr",0,11) OP_DEF(VSUB_OP,vsub_impl,"vsub",0,11) OP_DEF(VOTE_OP,vote_impl,"vote",0,3) +OP_DEF(ACTIVEMASK_OP,activemask_impl,"activemask",1,3) OP_DEF(XOR_OP,xor_impl,"xor",1,1) OP_DEF(NOP_OP,nop_impl,"nop",0,7) OP_DEF(BREAK_OP,break_impl,"break",0,3) diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 2dadda4..3592501 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -158,6 +158,7 @@ vshl TC; yylval->int_value = VSHL_OP; return OPCODE; vshr TC; yylval->int_value = VSHR_OP; return OPCODE; vsub TC; yylval->int_value = VSUB_OP; return OPCODE; vote TC; yylval->int_value = VOTE_OP; return OPCODE; +activemask TC; yylval->int_value = ACTIVEMASK_OP; return OPCODE; xor TC; yylval->int_value = XOR_OP; return OPCODE; nop TC; yylval->int_value = NOP_OP; return OPCODE; break TC; yylval->int_value = BREAK_OP; return OPCODE; @@ -252,7 +253,7 @@ breakaddr TC; yylval->int_value = BREAKADDR_OP; return OPCODE; [$%][a-zA-Z0-9_$]+ TC; yylval->string_value = strdup(yytext); return IDENTIFIER; [0-9]+\.[0-9]+ TC; sscanf(yytext,"%lf", &yylval->double_value); return DOUBLE_OPERAND; - + 0[xX][0-9a-fA-F]+U? TC; CHECK_UNSIGNED; sscanf(yytext,"%x", &yylval->int_value); return INT_OPERAND; 0[0-7]+U? TC; printf("GPGPU-Sim: ERROR ** parsing octal not (yet) implemented\n"); abort(); return INT_OPERAND; 0[bB][01]+U? TC; printf("GPGPU-Sim: ERROR ** parsing binary not (yet) implemented\n"); abort(); return INT_OPERAND; @@ -438,9 +439,9 @@ breakaddr TC; yylval->int_value = BREAKADDR_OP; return OPCODE; "*/" BEGIN(INITIAL); "CPTX_BEGIN" printf("BEGINNING CUSTOM PTX.\n"); BEGIN(INITIAL); [^C*\n]+ // eat comment in chunks -"C" // eat the lone C +"C" // eat the lone C "*" // eat the lone star -\n TC; +\n TC; } { @@ -470,7 +471,7 @@ int ptx_error( yyscan_t yyscanner, ptx_recognizer* recognizer, const char *s ) if( recognizer->linebuf[i] == '\t' ) printf("\t"); else printf(" "); } - + printf("^\n\n"); fflush(stdout); //exit(1); diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index b38f783..90accc9 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -50,8 +50,8 @@ class ptx_recognizer; %token STRING %token OPCODE %token WMMA_DIRECTIVE -%token LAYOUT -%token CONFIGURATION +%token LAYOUT +%token CONFIGURATION %token ALIGN_DIRECTIVE %token BRANCHTARGETS_DIRECTIVE %token BYTE_DIRECTIVE @@ -295,7 +295,7 @@ ptr_space_spec: GLOBAL_DIRECTIVE { recognizer->add_ptr_spec(global_space); } ptr_align_spec: ALIGN_DIRECTIVE INT_OPERAND -statement_block: LEFT_BRACE statement_list RIGHT_BRACE +statement_block: LEFT_BRACE statement_list RIGHT_BRACE statement_list: directive_statement { recognizer->add_directive(); } | statement_list prototype_block {printf("Prototype statement detected. WARNING: this is not supported yet on GPGPU-SIM\n"); } @@ -315,7 +315,7 @@ directive_statement: variable_declaration SEMI_COLON | TARGET_DIRECTIVE IDENTIFIER { recognizer->target_header($2); } | FILE_DIRECTIVE INT_OPERAND STRING { recognizer->add_file($2,$3); } | FILE_DIRECTIVE INT_OPERAND STRING COMMA INT_OPERAND COMMA INT_OPERAND { recognizer->add_file($2,$3); } - | LOC_DIRECTIVE INT_OPERAND INT_OPERAND INT_OPERAND + | LOC_DIRECTIVE INT_OPERAND INT_OPERAND INT_OPERAND | PRAGMA_DIRECTIVE STRING SEMI_COLON { recognizer->add_pragma($2); } | function_decl SEMI_COLON {/*Do nothing*/} ; @@ -336,7 +336,7 @@ identifier_spec: IDENTIFIER { recognizer->add_identifier($1,0,NON_ARRAY_IDENTIFI int i,lbase,l; char *id = NULL; lbase = strlen($1); - for( i=0; i < $3; i++ ) { + for( i=0; i < $3; i++ ) { l = lbase + (int)log10(i+1)+10; id = (char*) malloc(l); snprintf(id,l,"%s%u",$1,i); @@ -348,10 +348,10 @@ identifier_spec: IDENTIFIER { recognizer->add_identifier($1,0,NON_ARRAY_IDENTIFI | IDENTIFIER LEFT_SQUARE_BRACKET INT_OPERAND RIGHT_SQUARE_BRACKET { recognizer->add_identifier($1,$3,ARRAY_IDENTIFIER); recognizer->func_header_info($1); recognizer->func_header_info_int("[",$3); recognizer->func_header_info("]");} ; -var_spec_list: var_spec +var_spec_list: var_spec | var_spec_list var_spec; -var_spec: space_spec +var_spec: space_spec | type_spec | align_spec | VISIBLE_DIRECTIVE @@ -376,8 +376,8 @@ addressable_spec: CONST_DIRECTIVE { recognizer->add_space_spec(const_space,$1); | TEX_DIRECTIVE { recognizer->add_space_spec(tex_space,0); } ; -type_spec: scalar_type - | vector_spec scalar_type +type_spec: scalar_type + | vector_spec scalar_type ; vector_spec: V2_TYPE { recognizer->add_option(V2_TYPE); recognizer->func_header_info(".v2");} @@ -417,14 +417,14 @@ literal_list: literal_operand // TODO: This is currently hardcoded to handle and ignore one specific case // that all prototype statements follow in the PTX from Pytorch. As a -// workaround, this parses and ignores both the prototype declaration -// and calling of the prototype (which conveniently comes right after the -// declaration for all cases.) This should be changed to handle both +// workaround, this parses and ignores both the prototype declaration +// and calling of the prototype (which conveniently comes right after the +// declaration for all cases.) This should be changed to handle both // declaring the prototype, and actually calling it. prototype_block: prototype_decl prototype_call -prototype_decl: IDENTIFIER COLON CALLPROTOTYPE_DIRECTIVE LEFT_PAREN prototype_param RIGHT_PAREN IDENTIFIER LEFT_PAREN prototype_param RIGHT_PAREN SEMI_COLON - +prototype_decl: IDENTIFIER COLON CALLPROTOTYPE_DIRECTIVE LEFT_PAREN prototype_param RIGHT_PAREN IDENTIFIER LEFT_PAREN prototype_param RIGHT_PAREN SEMI_COLON + prototype_call: OPCODE LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA operand COMMA LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA IDENTIFIER SEMI_COLON | OPCODE IDENTIFIER COMMA LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA IDENTIFIER SEMI_COLON @@ -439,7 +439,7 @@ instruction_statement: instruction SEMI_COLON instruction: opcode_spec LEFT_PAREN operand RIGHT_PAREN { recognizer->set_return(); } COMMA operand COMMA LEFT_PAREN operand_list RIGHT_PAREN | opcode_spec operand COMMA LEFT_PAREN operand_list RIGHT_PAREN | opcode_spec operand COMMA LEFT_PAREN RIGHT_PAREN - | opcode_spec operand_list + | opcode_spec operand_list | opcode_spec ; @@ -468,8 +468,8 @@ option: type_spec | compare_spec | addressable_spec | rounding_mode - | wmma_spec - | prmt_spec + | wmma_spec + | prmt_spec | SYNC_OPTION { recognizer->add_option(SYNC_OPTION); } | ARRIVE_OPTION { recognizer->add_option(ARRIVE_OPTION); } | RED_OPTION { recognizer->add_option(RED_OPTION); } @@ -609,7 +609,7 @@ vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { recognizer- ; tex_operand: LEFT_SQUARE_BRACKET IDENTIFIER COMMA { recognizer->add_scalar_operand($2); } - vector_operand + vector_operand RIGHT_SQUARE_BRACKET ; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index aa1c25a..e5b5fb7 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1147,7 +1147,8 @@ static std::list check_operands( const std::list &operands, gpgpu_context *ctx) { static int g_warn_literal_operands_two_type_inst; if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || - (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { + (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP) || + (opcode == VMIN_OP) || (opcode == VMAX_OP) ) { // just make sure these do not have have const operands... if (!g_warn_literal_operands_two_type_inst) { std::list::const_iterator o; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 6627847..26283a6 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -965,6 +965,9 @@ class ptx_instruction : public warp_inst_t { bool get_pred_neg() const { return m_neg_pred; } int get_pred_mod() const { return m_pred_mod; } const char *get_source() const { return m_source.c_str(); } + + const std::list get_scalar_type() const {return m_scalar_type;} + const std::list get_options() const {return m_options;} typedef std::vector::const_iterator const_iterator; diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 3ae8de3..549c08c 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -624,8 +624,9 @@ void ptx_recognizer::add_scalar_type_spec(int type_spec) { parse_assert((g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) || (g_opcode == TEX_OP) || (g_opcode == MMA_OP) || - (g_opcode == DP4A_OP), - "only cvt, set, slct, tex and dp4a can have more than one " + (g_opcode == DP4A_OP) || (g_opcode == VMIN_OP) || + (g_opcode == VMAX_OP), + "only cvt, set, slct, tex, vmin, vmax and dp4a can have more than one " "type specifier."); } g_scalar_type_spec = type_spec; -- cgit v1.3 From a8f98c3d111c9238ad79908e690b22c5e43f1522 Mon Sep 17 00:00:00 2001 From: Lucy Liu Date: Fri, 3 Jul 2020 13:53:02 -0700 Subject: removed whitespace changes --- cuobjdump_to_ptxplus/cuobjdumpInst.cc | 36 +++++++++++++++++------------------ src/cuda-sim/cuda-sim.cc | 6 +++--- src/cuda-sim/instructions.cc | 1 + src/cuda-sim/opcodes.def | 10 +++++----- src/cuda-sim/ptx.l | 8 ++++---- src/cuda-sim/ptx.y | 36 +++++++++++++++++------------------ src/cuda-sim/ptx_ir.h | 2 +- 7 files changed, 50 insertions(+), 49 deletions(-) diff --git a/cuobjdump_to_ptxplus/cuobjdumpInst.cc b/cuobjdump_to_ptxplus/cuobjdumpInst.cc index 969313c..eb70199 100644 --- a/cuobjdump_to_ptxplus/cuobjdumpInst.cc +++ b/cuobjdump_to_ptxplus/cuobjdumpInst.cc @@ -742,18 +742,18 @@ void cuobjdumpInst::printCuobjdumpOutputModifiers(const char* defaultMod) { std::list::iterator typemod = m_typeModifiers->begin(); if (*typemod == ".U16" or *typemod == ".S16") { - std::list::iterator dest_op = m_operands->begin(); - std::string& destination = *dest_op; + std::list::iterator dest_op = m_operands->begin(); + std::string& destination = *dest_op; if (destination[destination.length()-1] == 'l') { - output(".lo"); // write to the lower 16-bits + output(".lo"); // write to the lower 16-bits } else if (destination[destination.length()-1] == 'h') { - output(".hi"); // write to the upper 16-bits + output(".hi"); // write to the upper 16-bits } else { - output(".wide"); // write to the whole 32-bits + output(".wide"); // write to the whole 32-bits } - return; + return; } - output(defaultMod); // default output modifier for mul + output(defaultMod); // default output modifier for mul } std::string int_default_mod () { return ".u32" ;} @@ -1357,9 +1357,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1393,9 +1393,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: type = (type1Size > type2Size) ? type1 : type2; strcpy(tempString, type.c_str()); if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1407,7 +1407,7 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: { printCuobjdumpPredicate(); output("mul"); - printCuobjdumpOutputModifiers(".lo"); + printCuobjdumpOutputModifiers(".lo"); printCuobjdumpBaseModifiers(); if(m_typeModifiers->size() == 0) @@ -1434,9 +1434,9 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else { printCuobjdumpTypeModifiers(); @@ -1468,10 +1468,10 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else printCuobjdumpTypeModifiers(); @@ -1506,10 +1506,10 @@ void cuobjdumpInst::printCuobjdumpPtxPlus(std::list labelList, std: /*if(type1Size==16 && type2Size==16) output(".lo");*/ if(tempString[1] >= 'A' && tempString[1] <= 'Z') - tempString[1] += 32; + tempString[1] += 32; output(tempString); - } + } else printCuobjdumpTypeModifiers(); diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 451feb5..71f0703 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -346,11 +346,11 @@ void function_info::ptx_assemble() { printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); connect_basic_blocks(); - bool modified = false; + bool modified = false; do { find_dominators(); find_idominators(); - modified = connect_break_targets(); + modified = connect_break_targets(); } while (modified == true); if ( g_debug_execution>=50 ) { @@ -1741,7 +1741,7 @@ void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if( pI->get_opcode() == VOTE_OP || pI->get_opcode() == ACTIVEMASK_OP ) { + if (pI->get_opcode() == VOTE_OP || pI->get_opcode() == ACTIVEMASK_OP) { pJ = new ptx_instruction(*pI); *((warp_inst_t *)pJ) = inst; // copy active mask information pI = pJ; diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 886c6c0..1090ba4 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -166,6 +166,7 @@ void inst_not_implemented(const ptx_instruction *pI); ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); + void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, int op_code); void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index aa85512..f5bf156 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -1,10 +1,10 @@ -// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda +// Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda // The University of British Columbia // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: -// +// // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. // Redistributions in binary form must reproduce the above copyright notice, this @@ -13,7 +13,7 @@ // Neither the name of The University of British Columbia nor the names of its // contributors may be used to endorse or promote products derived from this // software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -27,7 +27,7 @@ /*6th operand of each OP_DEF reflects its classification */ -/*Type +/*Type ALU 1 MAD 2 Control 3 diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index 3592501..6754045 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -253,7 +253,7 @@ breakaddr TC; yylval->int_value = BREAKADDR_OP; return OPCODE; [$%][a-zA-Z0-9_$]+ TC; yylval->string_value = strdup(yytext); return IDENTIFIER; [0-9]+\.[0-9]+ TC; sscanf(yytext,"%lf", &yylval->double_value); return DOUBLE_OPERAND; - + 0[xX][0-9a-fA-F]+U? TC; CHECK_UNSIGNED; sscanf(yytext,"%x", &yylval->int_value); return INT_OPERAND; 0[0-7]+U? TC; printf("GPGPU-Sim: ERROR ** parsing octal not (yet) implemented\n"); abort(); return INT_OPERAND; 0[bB][01]+U? TC; printf("GPGPU-Sim: ERROR ** parsing binary not (yet) implemented\n"); abort(); return INT_OPERAND; @@ -439,9 +439,9 @@ breakaddr TC; yylval->int_value = BREAKADDR_OP; return OPCODE; "*/" BEGIN(INITIAL); "CPTX_BEGIN" printf("BEGINNING CUSTOM PTX.\n"); BEGIN(INITIAL); [^C*\n]+ // eat comment in chunks -"C" // eat the lone C +"C" // eat the lone C "*" // eat the lone star -\n TC; +\n TC; } { @@ -471,7 +471,7 @@ int ptx_error( yyscan_t yyscanner, ptx_recognizer* recognizer, const char *s ) if( recognizer->linebuf[i] == '\t' ) printf("\t"); else printf(" "); } - + printf("^\n\n"); fflush(stdout); //exit(1); diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 90accc9..b38f783 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -50,8 +50,8 @@ class ptx_recognizer; %token STRING %token OPCODE %token WMMA_DIRECTIVE -%token LAYOUT -%token CONFIGURATION +%token LAYOUT +%token CONFIGURATION %token ALIGN_DIRECTIVE %token BRANCHTARGETS_DIRECTIVE %token BYTE_DIRECTIVE @@ -295,7 +295,7 @@ ptr_space_spec: GLOBAL_DIRECTIVE { recognizer->add_ptr_spec(global_space); } ptr_align_spec: ALIGN_DIRECTIVE INT_OPERAND -statement_block: LEFT_BRACE statement_list RIGHT_BRACE +statement_block: LEFT_BRACE statement_list RIGHT_BRACE statement_list: directive_statement { recognizer->add_directive(); } | statement_list prototype_block {printf("Prototype statement detected. WARNING: this is not supported yet on GPGPU-SIM\n"); } @@ -315,7 +315,7 @@ directive_statement: variable_declaration SEMI_COLON | TARGET_DIRECTIVE IDENTIFIER { recognizer->target_header($2); } | FILE_DIRECTIVE INT_OPERAND STRING { recognizer->add_file($2,$3); } | FILE_DIRECTIVE INT_OPERAND STRING COMMA INT_OPERAND COMMA INT_OPERAND { recognizer->add_file($2,$3); } - | LOC_DIRECTIVE INT_OPERAND INT_OPERAND INT_OPERAND + | LOC_DIRECTIVE INT_OPERAND INT_OPERAND INT_OPERAND | PRAGMA_DIRECTIVE STRING SEMI_COLON { recognizer->add_pragma($2); } | function_decl SEMI_COLON {/*Do nothing*/} ; @@ -336,7 +336,7 @@ identifier_spec: IDENTIFIER { recognizer->add_identifier($1,0,NON_ARRAY_IDENTIFI int i,lbase,l; char *id = NULL; lbase = strlen($1); - for( i=0; i < $3; i++ ) { + for( i=0; i < $3; i++ ) { l = lbase + (int)log10(i+1)+10; id = (char*) malloc(l); snprintf(id,l,"%s%u",$1,i); @@ -348,10 +348,10 @@ identifier_spec: IDENTIFIER { recognizer->add_identifier($1,0,NON_ARRAY_IDENTIFI | IDENTIFIER LEFT_SQUARE_BRACKET INT_OPERAND RIGHT_SQUARE_BRACKET { recognizer->add_identifier($1,$3,ARRAY_IDENTIFIER); recognizer->func_header_info($1); recognizer->func_header_info_int("[",$3); recognizer->func_header_info("]");} ; -var_spec_list: var_spec +var_spec_list: var_spec | var_spec_list var_spec; -var_spec: space_spec +var_spec: space_spec | type_spec | align_spec | VISIBLE_DIRECTIVE @@ -376,8 +376,8 @@ addressable_spec: CONST_DIRECTIVE { recognizer->add_space_spec(const_space,$1); | TEX_DIRECTIVE { recognizer->add_space_spec(tex_space,0); } ; -type_spec: scalar_type - | vector_spec scalar_type +type_spec: scalar_type + | vector_spec scalar_type ; vector_spec: V2_TYPE { recognizer->add_option(V2_TYPE); recognizer->func_header_info(".v2");} @@ -417,14 +417,14 @@ literal_list: literal_operand // TODO: This is currently hardcoded to handle and ignore one specific case // that all prototype statements follow in the PTX from Pytorch. As a -// workaround, this parses and ignores both the prototype declaration -// and calling of the prototype (which conveniently comes right after the -// declaration for all cases.) This should be changed to handle both +// workaround, this parses and ignores both the prototype declaration +// and calling of the prototype (which conveniently comes right after the +// declaration for all cases.) This should be changed to handle both // declaring the prototype, and actually calling it. prototype_block: prototype_decl prototype_call -prototype_decl: IDENTIFIER COLON CALLPROTOTYPE_DIRECTIVE LEFT_PAREN prototype_param RIGHT_PAREN IDENTIFIER LEFT_PAREN prototype_param RIGHT_PAREN SEMI_COLON - +prototype_decl: IDENTIFIER COLON CALLPROTOTYPE_DIRECTIVE LEFT_PAREN prototype_param RIGHT_PAREN IDENTIFIER LEFT_PAREN prototype_param RIGHT_PAREN SEMI_COLON + prototype_call: OPCODE LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA operand COMMA LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA IDENTIFIER SEMI_COLON | OPCODE IDENTIFIER COMMA LEFT_PAREN IDENTIFIER RIGHT_PAREN COMMA IDENTIFIER SEMI_COLON @@ -439,7 +439,7 @@ instruction_statement: instruction SEMI_COLON instruction: opcode_spec LEFT_PAREN operand RIGHT_PAREN { recognizer->set_return(); } COMMA operand COMMA LEFT_PAREN operand_list RIGHT_PAREN | opcode_spec operand COMMA LEFT_PAREN operand_list RIGHT_PAREN | opcode_spec operand COMMA LEFT_PAREN RIGHT_PAREN - | opcode_spec operand_list + | opcode_spec operand_list | opcode_spec ; @@ -468,8 +468,8 @@ option: type_spec | compare_spec | addressable_spec | rounding_mode - | wmma_spec - | prmt_spec + | wmma_spec + | prmt_spec | SYNC_OPTION { recognizer->add_option(SYNC_OPTION); } | ARRIVE_OPTION { recognizer->add_option(ARRIVE_OPTION); } | RED_OPTION { recognizer->add_option(RED_OPTION); } @@ -609,7 +609,7 @@ vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { recognizer- ; tex_operand: LEFT_SQUARE_BRACKET IDENTIFIER COMMA { recognizer->add_scalar_operand($2); } - vector_operand + vector_operand RIGHT_SQUARE_BRACKET ; diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index 26283a6..8c4ad4d 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -965,7 +965,7 @@ class ptx_instruction : public warp_inst_t { bool get_pred_neg() const { return m_neg_pred; } int get_pred_mod() const { return m_pred_mod; } const char *get_source() const { return m_source.c_str(); } - + const std::list get_scalar_type() const {return m_scalar_type;} const std::list get_options() const {return m_options;} -- cgit v1.3 From f48a2d7ab623d2046306ec310c490f935dc48dae Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sat, 4 Jul 2020 16:29:05 -0700 Subject: edit for c++11 stuff added to instructions.cc --- src/cuda-sim/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile index 85d1c8c..2305ef0 100644 --- a/src/cuda-sim/Makefile +++ b/src/cuda-sim/Makefile @@ -48,6 +48,7 @@ ifeq ($(DEBUG),1) endif OPT += -I$(CUDA_INSTALL_PATH)/include -I$(OUTPUT_DIR)/ -I. -I$(SIM_OBJ_FILES_DIR) OPT += -fPIC +OPT += std=c++11 ifeq ($(TRACE),1) OPT += -DTRACING_ON=1 -- cgit v1.3 From f273d54336fc57d0c22741d96bc25b030c5f7f99 Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sat, 4 Jul 2020 16:49:33 -0700 Subject: trying again --- src/cuda-sim/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile index 2305ef0..1ce6df0 100644 --- a/src/cuda-sim/Makefile +++ b/src/cuda-sim/Makefile @@ -48,7 +48,6 @@ ifeq ($(DEBUG),1) endif OPT += -I$(CUDA_INSTALL_PATH)/include -I$(OUTPUT_DIR)/ -I. -I$(SIM_OBJ_FILES_DIR) OPT += -fPIC -OPT += std=c++11 ifeq ($(TRACE),1) OPT += -DTRACING_ON=1 @@ -56,10 +55,10 @@ endif CXX_OPT = $(OPT) ifeq ($(INTEL),1) - CXX_OPT += -std=c++0x + CXX_OPT += -std=c++11 else ifeq ($(GNUC_CPP0X),1) - CXX_OPT += -std=c++0x + CXX_OPT += -std=c++11 endif endif -- cgit v1.3 From 6f2d125c1a4f445f33f03b08d1e6d677329b1b35 Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sat, 4 Jul 2020 17:05:27 -0700 Subject: okay, old school --- src/cuda-sim/Makefile | 4 ++-- src/cuda-sim/instructions.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cuda-sim/Makefile b/src/cuda-sim/Makefile index 1ce6df0..85d1c8c 100644 --- a/src/cuda-sim/Makefile +++ b/src/cuda-sim/Makefile @@ -55,10 +55,10 @@ endif CXX_OPT = $(OPT) ifeq ($(INTEL),1) - CXX_OPT += -std=c++11 + CXX_OPT += -std=c++0x else ifeq ($(GNUC_CPP0X),1) - CXX_OPT += -std=c++11 + CXX_OPT += -std=c++0x endif endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 1090ba4..9bdd53a 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -6561,7 +6561,7 @@ void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, i case S32_TYPE: { // assert all operands are S32_TYPE: scalar_type = pI->get_scalar_type(); - for (auto scalar : scalar_type) + for (std::list::iterator scalar = scalar_type.begin(); scalar != scalar_type.end(); scalar++) { assert(scalar == S32_TYPE); } -- cgit v1.3 From 78f264024dd4542e731633b3e68d205a571b97b7 Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Sat, 4 Jul 2020 17:21:43 -0700 Subject: edit --- src/cuda-sim/instructions.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 9bdd53a..8936fa8 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -6563,7 +6563,7 @@ void video_mem_instruction(const ptx_instruction *pI, ptx_thread_info *thread, i scalar_type = pI->get_scalar_type(); for (std::list::iterator scalar = scalar_type.begin(); scalar != scalar_type.end(); scalar++) { - assert(scalar == S32_TYPE); + assert(*scalar == S32_TYPE); } assert(scalar_type.size() == 3); scalar_type.clear(); -- cgit v1.3