From 69f2911e04ffb1b19eef1fafb8c040af271f656e Mon Sep 17 00:00:00 2001 From: Tor Aamodt Date: Thu, 15 Jul 2010 18:09:46 -0800 Subject: creating branch for adding support for CUDA 3.x and Fermi [git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 6829] --- src/gpgpu-sim/Makefile | 116 ++ src/gpgpu-sim/addrdec.cc | 471 +++++ src/gpgpu-sim/addrdec.h | 96 + src/gpgpu-sim/cflogger.h | 121 ++ src/gpgpu-sim/delayqueue.cc | 468 +++++ src/gpgpu-sim/delayqueue.h | 122 ++ src/gpgpu-sim/dram.cc | 604 ++++++ src/gpgpu-sim/dram.h | 237 +++ src/gpgpu-sim/dram_sched.cc | 247 +++ src/gpgpu-sim/dram_sched.h | 99 + src/gpgpu-sim/dwf.cc | 2609 ++++++++++++++++++++++++++ src/gpgpu-sim/dwf.h | 118 ++ src/gpgpu-sim/gpu-cache.cc | 609 ++++++ src/gpgpu-sim/gpu-cache.h | 191 ++ src/gpgpu-sim/gpu-misc.cc | 96 + src/gpgpu-sim/gpu-misc.h | 106 ++ src/gpgpu-sim/gpu-sim.cc | 1752 ++++++++++++++++++ src/gpgpu-sim/gpu-sim.h | 116 ++ src/gpgpu-sim/histogram.h | 128 ++ src/gpgpu-sim/icnt_wrapper.cc | 97 + src/gpgpu-sim/icnt_wrapper.h | 100 + src/gpgpu-sim/l2cache.cc | 999 ++++++++++ src/gpgpu-sim/l2cache.h | 48 + src/gpgpu-sim/mem_fetch.h | 110 ++ src/gpgpu-sim/mem_latency_stat.h | 544 ++++++ src/gpgpu-sim/shader.cc | 3811 ++++++++++++++++++++++++++++++++++++++ src/gpgpu-sim/shader.h | 524 ++++++ src/gpgpu-sim/stack.cc | 127 ++ src/gpgpu-sim/stack.h | 90 + src/gpgpu-sim/stat-tool.cc | 1081 +++++++++++ src/gpgpu-sim/visualizer.cc | 602 ++++++ src/gpgpu-sim/warp_tracker.cc | 471 +++++ src/gpgpu-sim/warp_tracker.h | 100 + 33 files changed, 17010 insertions(+) create mode 100644 src/gpgpu-sim/Makefile create mode 100644 src/gpgpu-sim/addrdec.cc create mode 100644 src/gpgpu-sim/addrdec.h create mode 100644 src/gpgpu-sim/cflogger.h create mode 100644 src/gpgpu-sim/delayqueue.cc create mode 100644 src/gpgpu-sim/delayqueue.h create mode 100644 src/gpgpu-sim/dram.cc create mode 100644 src/gpgpu-sim/dram.h create mode 100644 src/gpgpu-sim/dram_sched.cc create mode 100644 src/gpgpu-sim/dram_sched.h create mode 100644 src/gpgpu-sim/dwf.cc create mode 100644 src/gpgpu-sim/dwf.h create mode 100644 src/gpgpu-sim/gpu-cache.cc create mode 100644 src/gpgpu-sim/gpu-cache.h create mode 100644 src/gpgpu-sim/gpu-misc.cc create mode 100644 src/gpgpu-sim/gpu-misc.h create mode 100644 src/gpgpu-sim/gpu-sim.cc create mode 100644 src/gpgpu-sim/gpu-sim.h create mode 100644 src/gpgpu-sim/histogram.h create mode 100644 src/gpgpu-sim/icnt_wrapper.cc create mode 100644 src/gpgpu-sim/icnt_wrapper.h create mode 100644 src/gpgpu-sim/l2cache.cc create mode 100644 src/gpgpu-sim/l2cache.h create mode 100644 src/gpgpu-sim/mem_fetch.h create mode 100644 src/gpgpu-sim/mem_latency_stat.h create mode 100644 src/gpgpu-sim/shader.cc create mode 100644 src/gpgpu-sim/shader.h create mode 100644 src/gpgpu-sim/stack.cc create mode 100644 src/gpgpu-sim/stack.h create mode 100644 src/gpgpu-sim/stat-tool.cc create mode 100644 src/gpgpu-sim/visualizer.cc create mode 100644 src/gpgpu-sim/warp_tracker.cc create mode 100644 src/gpgpu-sim/warp_tracker.h (limited to 'src/gpgpu-sim') diff --git a/src/gpgpu-sim/Makefile b/src/gpgpu-sim/Makefile new file mode 100644 index 0000000..2b1412a --- /dev/null +++ b/src/gpgpu-sim/Makefile @@ -0,0 +1,116 @@ +# Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, +# George L. Yuan, Ivan Sham and the +# University of British Columbia +# Vancouver, BC V6T 1Z4 +# All Rights Reserved. +# +# THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE +# TERMS AND CONDITIONS. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h +# are derived from the CUDA Toolset available from http://www.nvidia.com/cuda +# (property of NVIDIA). The files benchmarks/BlackScholes/* and +# benchmarks/template/* are derived from the CUDA SDK available from +# http://www.nvidia.com/cuda (also property of NVIDIA). The files +# src/gpgpusim_entrypoint.c and src/simplesim-3.0/* are derived from the +# SimpleScalar Toolset available from http://www.simplescalar.com/ +# (property of SimpleScalar LLC) and the files src/intersim/* are derived +# from Booksim (Simulator provided with the textbook "Principles and +# Practices of Interconnection Networks" available from +# http://cva.stanford.edu/books/ppin/). As such, those files are bound by +# the corresponding legal terms and conditions set forth separately (original +# copyright notices are left in files from these sources and where we have +# modified a file our copyright notice appears before the original copyright +# notice). +# +# Using this version of GPGPU-Sim requires a complete installation of CUDA +# version 1.1, which is distributed seperately by NVIDIA under separate terms +# and conditions. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the University of British Columbia nor the names of +# its contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. +# +# 5. No nonprofit user may place any restrictions on the use of this software, +# including as modified by the user, by any other authorized user. +# +# 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, +# Ali Bakhoda, George L. Yuan, at the University of British Columbia, +# Vancouver, BC V6T 1Z4 + +# GPGPU-Sim Makefile + +DEBUG=0 + +CXXFLAGS = -Wall -DDEBUG +CXXFLAGS_L2CACHE = -Wall -DDEBUG + +GNUC_CPP0X := $(shell gcc --version | perl -ne 'if (/gcc\s+\(.*\)\s+([0-9.]+)/){ if($$1 >= 4.3) {$$n=1} else {$$n=0;} } END { print $$n; }') +ifeq ($(GNUC_CPP0X), 1) + CXXFLAGS += -std=c++0x + CXXFLAGS_L2CACHE += -std=c++0x +endif + +ifneq ($(DEBUG),1) + OPTFLAGS += -O3 +else + CXXFLAGS_L2CACHE += -DGLIBCXX_DEBUG_PEDANTIC + CXXFLAGS += -D_GLIBCXX_DEBUG -DGLIBCXX_DEBUG_PEDANTIC +endif + +OPTFLAGS += -g3 -fPIC + +CPP = g++ $(SNOW) +OEXT = o + +SRCS = $(shell ls *.cc) +OBJS = $(SRCS:.cc=.$(OEXT)) + +libgpu_uarch_sim.a:$(OBJS) + ar rcs libgpu_uarch_sim.a $(OBJS) + +depend: + makedepend $(SRCS) 2> /dev/null + +l2cache.$(OEXT): l2cache.cc + $(CPP) $(OPTFLAGS) $(CXXFLAGS_L2CACHE) -o $*.$(OEXT) -c $*.cc + +.cc.$(OEXT): + $(CPP) $(OPTFLAGS) $(CXXFLAGS) -o $*.$(OEXT) -c $*.cc + +clean: + rm -f *.o core *~ *.a + +option_parser.$(OEXT): option_parser.h + +dram_sched.$(OEXT): ../cuda-sim/ptx.tab.h + +../cuda-sim/ptx.tab.h: + make -C ../cuda-sim/ ptx.tab.c + +# DO NOT DELETE + diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc new file mode 100644 index 0000000..333d6de --- /dev/null +++ b/src/gpgpu-sim/addrdec.cc @@ -0,0 +1,471 @@ +/* + * addrdec.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include "addrdec.h" +//#include "gpu-sim.h" +#include "../option_parser.h" + +int ADDR_CHIP_S = 10; +extern int gpgpu_mem_address_mask; + +long int powli( long int x, long int y ) // compute x to the y +{ + long int r = 1; + int i; + for (i = 0; i < y; ++i ) { + r *= x; + } + return r; +} + +void addrdec_display(addrdec_t *a) { + //printf("DRAM: unused:%x chip:%x row:%x col:%x bk:%x\n", + // a.dram.unused, a.dram.chip, GET_ROW(a), GET_COL(a), a.dram.bk); + + if (a->chip) printf("\tchip:%x ", a->chip); + if (a->row) printf("\trow:%x ", a->row); + if (a->col) printf("\tcol:%x ", a->col); + if (a->bk) printf("\tbk:%x ", a->bk); + if (a->burst) printf("\tburst:%x ", a->burst); +} + +unsigned long long int addrdec_packbits(unsigned long long int mask, + unsigned long long int val, + unsigned char high, unsigned char low) +{ + int i, pos; + unsigned long long int out; + out = 0; + pos = 0; + for (i=low;i> i) << pos; + pos++; + } + // printf("%02d: %016llx %d\n",i,out,pos); + } + + return out; +} + +unsigned long long int addrdec_mask[5] = { + 0x0000000000001C00, + 0x0000000000000300, + 0x000000000FFF0000, + 0x000000000000E0FF, + 0x000000000000000F +}; + +void addrdec_getmasklimit(unsigned long long int mask, unsigned char *high, unsigned char *low) +{ + *high = 64; + *low = 0; + int i; + int low_found = 0; + + for (i=0;i<64;i++) { + if ((mask & ((unsigned long long int)1<chip = addrdec_packbits(addrdec_mask[CHIP], addr, addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]); + tlx->bk = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], addrdec_mklow[BK]); + tlx->row = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]); + tlx->col = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], addrdec_mklow[COL]); + tlx->burst= addrdec_packbits(addrdec_mask[BURST], addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); + } else { + addr_for_chip= ( (addr>>ADDR_CHIP_S) % Nchips) << ADDR_CHIP_S; + rest_of_addr= ( (addr>>ADDR_CHIP_S) / Nchips) << ADDR_CHIP_S; + + tlx->chip = addrdec_packbits(addrdec_mask[CHIP], addr_for_chip, addrdec_mkhigh[CHIP], addrdec_mklow[CHIP]); + if (addrdec_mask[BK] > addrdec_mask[CHIP]) { + tlx->bk = addrdec_packbits(addrdec_mask[BK], rest_of_addr, addrdec_mkhigh[BK], addrdec_mklow[BK]); + } else { + tlx->bk = addrdec_packbits(addrdec_mask[BK], addr, addrdec_mkhigh[BK], addrdec_mklow[BK]); + } + if (addrdec_mask[ROW] > addrdec_mask[CHIP]) { + tlx->row = addrdec_packbits(addrdec_mask[ROW], rest_of_addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]); + } else { + tlx->row = addrdec_packbits(addrdec_mask[ROW], addr, addrdec_mkhigh[ROW], addrdec_mklow[ROW]); + } + tlx->col = addrdec_packbits(addrdec_mask[COL], addr, addrdec_mkhigh[COL], addrdec_mklow[COL]); + tlx->burst= addrdec_packbits(addrdec_mask[BURST], addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); + } +} + +unsigned int LOGB2_32( unsigned int v ) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; + shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; + shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; + shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; + shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + + return r; +} + + +static char *addrdec_option = NULL; +void addrdec_setoption(option_parser_t opp) +{ + option_parser_register(opp, "-gpgpu_mem_addr_mapping", OPT_CSTR, &addrdec_option, + "mapping memory address to dram model {dramid@;}", + NULL); +} + +void addrdec_parseoption(const char *option) +{ + unsigned int dramid_start = 0; + int dramid_parsed = sscanf(option, "dramid@%d", &dramid_start); + if (dramid_parsed == 1) { + ADDR_CHIP_S = dramid_start; + } else { + ADDR_CHIP_S = -1; + } + + const char *cmapping = strchr(option, ';'); + if (cmapping == NULL) { + cmapping = option; + } else { + cmapping += 1; + } + + addrdec_mask[CHIP] = 0x0; + addrdec_mask[BK] = 0x0; + addrdec_mask[ROW] = 0x0; + addrdec_mask[COL] = 0x0; + addrdec_mask[BURST]= 0x0; + + int ofs = 63; + while ((*cmapping) != '\0') { + switch (*cmapping) { + case 'D': case 'd': + assert(dramid_parsed != 1); addrdec_mask[CHIP] |= (1ULL << ofs); ofs--; break; + case 'B': case 'b': addrdec_mask[BK] |= (1ULL << ofs); ofs--; break; + case 'R': case 'r': addrdec_mask[ROW] |= (1ULL << ofs); ofs--; break; + case 'C': case 'c': addrdec_mask[COL] |= (1ULL << ofs); ofs--; break; + case 'S': case 's': addrdec_mask[BURST] |= (1ULL << ofs); addrdec_mask[COL] |= (1ULL << ofs); ofs--; break; + // ignore bit + case '0': ofs--; break; + // ignore character + case '|': + case ' ': + case '.': break; + default: + fprintf(stderr, "ERROR: Invalid address mapping character '%c' in option '%s'\n", *cmapping, option); + } + cmapping += 1; + } + + if (ofs != -1) { + fprintf(stderr, "ERROR: Invalid address mapping length (%d) in option '%s'\n", 63 - ofs, option); + assert(ofs == -1); + } +} + + +void addrdec_setnchip(unsigned int nchips) +{ + unsigned i; + unsigned long long int mask; + unsigned int nchipbits = LOGB2_32(nchips); + Nchips = nchips; + + gap = (nchips - powli(2,nchipbits)); + if (gap) { + nchipbits++; + } + switch (gpgpu_mem_address_mask) { + case 0: + //old, added 2row bits, use #define ADDR_CHIP_S 10 + ADDR_CHIP_S = 10; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000000300; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001CFF; + break; + case 1: + ADDR_CHIP_S = 13; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 2: + ADDR_CHIP_S = 11; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 3: + ADDR_CHIP_S = 11; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x000000000FFFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + + case 14: + ADDR_CHIP_S = 14; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 15: + ADDR_CHIP_S = 15; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 16: + ADDR_CHIP_S = 16; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 6: + ADDR_CHIP_S = 6; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 5: + ADDR_CHIP_S = 5; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 100: + ADDR_CHIP_S = 1; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000000003; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001FFC; + break; + case 103: + ADDR_CHIP_S = 3; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000000003; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001FFC; + break; + case 106: + ADDR_CHIP_S = 6; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000001800; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x00000000000007FF; + break; + case 160: + //old, added 2row bits, use #define ADDR_CHIP_S 10 + ADDR_CHIP_S = 6; + addrdec_mask[CHIP] = 0x0000000000000000; + addrdec_mask[BK] = 0x0000000000000300; + addrdec_mask[ROW] = 0x0000000007FFE000; + addrdec_mask[COL] = 0x0000000000001CFF; + + default: + break; + } + + if (addrdec_option != NULL) { + addrdec_parseoption(addrdec_option); + } + + if (ADDR_CHIP_S != -1) { + mask = ((unsigned long long int)1 << ADDR_CHIP_S) - 1; + addrdec_mask[BK] = ((addrdec_mask[BK] & ~mask) << nchipbits) | (addrdec_mask[BK] & mask); + addrdec_mask[ROW] = ((addrdec_mask[ROW] & ~mask) << nchipbits) | (addrdec_mask[ROW] & mask); + addrdec_mask[COL] = ((addrdec_mask[COL] & ~mask) << nchipbits) | (addrdec_mask[COL] & mask); + + for (i=ADDR_CHIP_S;i<(ADDR_CHIP_S+nchipbits);i++) { + mask = (unsigned long long int)1 << i; + addrdec_mask[CHIP] |= mask; + } + } else { + // make sure nchips is power of two when explicit dram id mask is used + assert((nchips & (nchips - 1)) == 0); + } + + addrdec_getmasklimit(addrdec_mask[CHIP], &addrdec_mkhigh[CHIP], &addrdec_mklow[CHIP] ); + addrdec_getmasklimit(addrdec_mask[BK], &addrdec_mkhigh[BK], &addrdec_mklow[BK] ); + addrdec_getmasklimit(addrdec_mask[ROW], &addrdec_mkhigh[ROW], &addrdec_mklow[ROW] ); + addrdec_getmasklimit(addrdec_mask[COL], &addrdec_mkhigh[COL], &addrdec_mklow[COL] ); + addrdec_getmasklimit(addrdec_mask[BURST], &addrdec_mkhigh[BURST], &addrdec_mklow[BURST]); + + printf("addr_dec_mask[CHIP] = %016llx \thigh:%d low:%d\n", addrdec_mask[CHIP], addrdec_mkhigh[CHIP], addrdec_mklow[CHIP] ); + printf("addr_dec_mask[BK] = %016llx \thigh:%d low:%d\n", addrdec_mask[BK], addrdec_mkhigh[BK], addrdec_mklow[BK] ); + printf("addr_dec_mask[ROW] = %016llx \thigh:%d low:%d\n", addrdec_mask[ROW], addrdec_mkhigh[ROW], addrdec_mklow[ROW] ); + printf("addr_dec_mask[COL] = %016llx \thigh:%d low:%d\n", addrdec_mask[COL], addrdec_mkhigh[COL], addrdec_mklow[COL] ); + printf("addr_dec_mask[BURST] = %016llx \thigh:%d low:%d\n", addrdec_mask[BURST], addrdec_mkhigh[BURST], addrdec_mklow[BURST]); +} + +#ifdef UNIT_TEST + +int main () { + unsigned int tb = 1; + unsigned pos; + addrdec_t_o tlx; + + printf("DRAM: %d %d %d %d %d %d %d\n", + D_COLL, D_BK, D_COLU, D_ROWL, D_CHIP, D_ROWU, D_UNUSED); + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + tlx.plain = tb; + addrdec_fetch(tlx); + addrdec_dram(tlx); + printf("\n"); + } + + unsigned long long int packed; + packed = addrdec_packbits(0xFFFF0000FFFF0000, 0x2244113322441133); + assert (packed == 0x22442244); + printf("%016llx\n", packed); + + packed = addrdec_packbits(0x5555555555555555, 0x3333333333333333); + assert (packed == 0x55555555); + printf("%016llx\n", packed); + + packed = addrdec_packbits(0x5555555555555555, 0x6363636363636363); + assert (packed == 0x99999999); + printf("%016llx\n", packed); + + addrdec_t tls; + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + } + + addrdec_setnchip(32); + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + } + addrdec_setnchip(16); + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + } + addrdec_setnchip(8); + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + } + addrdec_setnchip(7); + for (tb=1, pos=0; tb!=0; tb <<= 1, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + } +/* addrdec_setnchip(6); + for(tb=1, pos=0; tb!=0; tb ++, pos++) { + printf("%08lx|%02d =>", tb,pos); + addrdec_tlx(tb, &tls); + addrdec_display(&tls); + printf("\n"); + }*/ + return 0; +} + +#endif diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h new file mode 100644 index 0000000..ba28c05 --- /dev/null +++ b/src/gpgpu-sim/addrdec.h @@ -0,0 +1,96 @@ +/* + * addrdec.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung and the University of + * British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include +#include +#include "../option_parser.h" + +#ifndef ADDRDEC_H +#define ADDRDEC_H + +enum { + CHIP = 0, + BK = 1, + ROW = 2, + COL = 3, + BURST = 4, + N_ADDRDEC +}; + +typedef struct { + unsigned chip; + unsigned bk; + unsigned row; + unsigned col; + unsigned burst; +} addrdec_t; + +void addrdec_tlx(unsigned long long int addr, addrdec_t *tlx); +void addrdec_setnchip(unsigned int nchips); +void addrdec_setoption(option_parser_t opp); +void addrdec_parseoption(const char *option); + +#endif diff --git a/src/gpgpu-sim/cflogger.h b/src/gpgpu-sim/cflogger.h new file mode 100644 index 0000000..bdf57e5 --- /dev/null +++ b/src/gpgpu-sim/cflogger.h @@ -0,0 +1,121 @@ +/* + * cflogger.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef CFLOGGER_H +#define CFLOGGER_H + +void try_snap_shot (unsigned long long current_cycle); +void set_spill_interval (unsigned long long interval); +void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle); + +void create_thread_CFlogger( int n_loggers, int n_threads, int n_insn, address_type start_pc, unsigned long long logging_interval); +void destroy_thread_CFlogger( ); +void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ); +void cflog_snapshot( int logger_id, unsigned long long cycle ); +void cflog_print(FILE *fout); +void cflog_print_path_expression(FILE *fout); +void cflog_visualizer_print(FILE *fout); + + +void insn_warp_occ_create( int n_loggers, int simd_width, int n_insn ); +void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ ); +void insn_warp_occ_print( FILE *fout ); + + +void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long logging_interval ); +void shader_warp_occ_log( int logger_id, int warp_occ ); +void shader_warp_occ_snapshot( int logger_id, unsigned long long current_cycle ); +void shader_warp_occ_print( FILE *fout ); + + +void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long logging_interval ); +void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw ); +void shader_mem_acc_snapshot( int logger_id, unsigned long long current_cycle ); +void shader_mem_acc_print( FILE *fout ); + + +void shader_mem_lat_create( int n_loggers, unsigned long long logging_interval ); +void shader_mem_lat_log( int logger_id, int latency ); +void shader_mem_lat_snapshot( int logger_id, unsigned long long current_cycle ); +void shader_mem_lat_print( FILE *fout ); + + +int get_shader_normal_cache_id(); +int get_shader_texture_cache_id(); +int get_shader_constant_cache_id(); +void shader_cache_access_create( int n_loggers, int n_types, unsigned long long logging_interval ); +void shader_cache_access_log( int logger_id, int type, int miss); +void shader_cache_access_unlog( int logger_id, int type, int miss); +void shader_cache_access_print( FILE *fout ); + + +void shader_CTA_count_create( int n_shaders, unsigned long long logging_interval); +void shader_CTA_count_log( int shader_id, int nCTAadded ); +void shader_CTA_count_unlog( int shader_id, int nCTAdone ); +void shader_CTA_count_resetnow( ); +void shader_CTA_count_print( FILE *fout ); +void shader_CTA_count_visualizer_print( FILE *fout ); + +#endif /* CFLOGGER_H */ diff --git a/src/gpgpu-sim/delayqueue.cc b/src/gpgpu-sim/delayqueue.cc new file mode 100644 index 0000000..dc870d0 --- /dev/null +++ b/src/gpgpu-sim/delayqueue.cc @@ -0,0 +1,468 @@ +/* + * delayqueue.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * Ivan Sham, Henry Tran and the University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "delayqueue.h" +#include "gpu-misc.h" +#include "../intersim/statwraper.h" + +extern unsigned long long gpu_sim_cycle; //for stat collection + +unsigned char dq_full( delay_queue* dq ) +{ + if (dq->max_len && dq->length >= dq->max_len) + return 1; + return 0; +} + +unsigned char dq_empty(delay_queue* dq ) +{ + return(dq->head == NULL)?1:0; +} + +unsigned int dq_n_element(delay_queue* dq ) +{ + return(dq->n_element); +} + +unsigned char dq_push(delay_queue* dq, void* data) { + if (dq->max_len) assert(dq->length < dq->max_len); + if (dq->head) { + if (dq->tail->data || dq->length < dq->min_len) { + dq->tail->next = (delay_data*) malloc(sizeof(delay_data)); + dq->tail = dq->tail->next; + dq->length++; + dq->n_element++; + } + } else { + dq->head = dq->tail = (delay_data*) malloc(sizeof(delay_data)); + dq->length++; + dq->n_element++; + } + dq->tail->next = NULL; + dq->tail->time_elapsed = dq->latency; + dq->tail->data = (void*)data; + dq->tail->push_time = gpu_sim_cycle; + return 1; +} + +void* dq_top(delay_queue* dq) { + if (dq->head) { + return dq->head->data; + } else { + return NULL; + } +} + +void* dq_pop(delay_queue* dq) { + delay_data* next; + void* data; + if (dq->head) { + if (dq->head->time_elapsed) { + dq->head->time_elapsed--; + data = NULL; + } else { + next = dq->head->next; + data = dq->head->data; + StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - dq->head->push_time)); + if ( dq->head == dq->tail ) { + assert( next == NULL ); + dq->tail = NULL; + } + free(dq->head); + dq->head = next; + dq->length--; + if (dq->length == 0) { + assert( dq->head == NULL ); + dq->tail = dq->head; + } + dq->n_element--; + } + if (dq->min_len && dq->length < dq->min_len) { + dq_push(dq,NULL); + dq->n_element--; // uncount NULL elements inserted to create delays + } + } else { + data = NULL; + } + return data; +} + +void dq_set_min_length(delay_queue* dq, unsigned int new_min_len) { + if (new_min_len == dq->min_len) return; + + if (new_min_len > dq->min_len) { + dq->min_len = new_min_len; + while (dq->length < dq->min_len) { + dq_push(dq,NULL); + dq->n_element--; // uncount NULL elements inserted to create delays + } + } else { + // in this branch imply that the original min_len is larger then 0 + // ie. dq->head != 0 + assert(dq->head); + dq->min_len = new_min_len; + while ((dq->length > dq->min_len) && (dq->tail->data == 0)) { + delay_data *iter; + iter = dq->head; + while (iter && (iter->next != dq->tail)) + iter = iter->next; + if (!iter) { + // there is only one node, and that node is empty + assert(dq->head->data == 0); + dq_pop(dq); + } else { + // there are more than one node, and tail node is empty + assert(iter->next == dq->tail); + free(dq->tail); + dq->tail = iter; + dq->tail->next = 0; + dq->length--; + } + } + } +} + +void dq_remove(void* data, delay_queue* dq) +{ + // removes an item from the queue without deallocating the memory + delay_data* ptr = NULL; + delay_data* temp = NULL; + + assert(dq); + assert(data); + + ptr = dq->head; + if (ptr) { + if (ptr->data == data) { + StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - ptr->push_time)); + dq->head = ptr->next; + if ( dq->head == NULL ) + dq->tail = NULL; + dq->length--; + return; + } + while (ptr->next) { + if (ptr->next->data == data) { + temp = ptr->next; + StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - temp->push_time)); + if ( ptr->next == dq->tail ) { + dq->tail = ptr; + } + ptr->next = ptr->next->next; + dq->length--; + return; + } + ptr = ptr->next; + } + } +} + +void removeEntry(void* data, delay_queue** dqq, int size_dq) +{ + int i; + delay_data* ptr = NULL; + delay_queue* dq = NULL; + delay_data* temp = NULL; + + assert(dqq); + assert(data); + + + for (i = 0; ihead; + if (ptr) { + if (ptr->data == data) { + dq->head = ptr->next; + if ( dq->head == NULL ) + dq->tail = NULL; + StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - ptr->push_time)); + free(ptr); + dq->length--; + return; + } + while (ptr->next) { + if (ptr->next->data == data) { + temp = ptr->next; + if ( ptr->next == dq->tail ) { + dq->tail = ptr; + } + ptr->next = ptr->next->next; + StatAddSample(dq->lat_stat, LOGB2 (gpu_sim_cycle - temp->push_time)); + free(temp); + dq->length--; + return; + } + ptr = ptr->next; + } + } + + } +} + +static int dq_uid_counter = 0; + +delay_queue* dq_create(const char* name, unsigned int latency, unsigned int min_len, unsigned int max_len) { + unsigned i; + delay_queue* dq; + dq = (delay_queue*) malloc(sizeof(delay_queue)); + dq->name = name; + dq->latency = latency; + dq->min_len = min_len; + dq->max_len = max_len; + dq->length = 0; + dq->n_element = 0; + dq->head = NULL; + dq->tail = NULL; + for (i=0;iuid = dq_uid_counter; + dq_uid_counter++; + if (1) { + dq->lat_stat = StatCreate(dq->name,1,32); + } + dq->max_size_stat = 0; + dq->avg_size_stat =0.0 ; + return dq; +} + +void dq_print(delay_queue* dq) { + delay_data* ddp = dq->head; + printf("%s(%d): ", dq->name, dq->length); + while (ddp) { + printf("%p ", ddp->data); + ddp = ddp->next; + } + printf("\n"); +} + +void dq_free(delay_queue* dq) { + while (dq->head) { + dq->tail = dq->head; + dq->head = dq->head->next; + free(dq->tail); + } + free(dq); + dq = NULL; +} + +void dq_update_stat(delay_queue* dq) { + if (dq->n_element > dq->max_size_stat) { + dq->max_size_stat = dq->n_element; + } + dq->avg_size_stat = (dq->avg_size_stat*dq->n_stat_samples + dq->n_element)/(++dq->n_stat_samples); +} +void dq_print_stat(delay_queue* dq) { + printf("Max Length: %d, Average Length: %f\n",dq->max_size_stat,dq->avg_size_stat ); +} + + +#ifdef TEST_DQ + +void regresstion_test01() { + delay_queue *dqa, *dqb; + int i; + int a[7]; + for (i=0;i<7;i++) a[i]=i; + + dqa = dq_create("dqa", 0, 7, 0); + for (i=0;i<3;i++) dq_push(dqa, &a[i]); + + for (i=0;i<6;i++) { + dq_print(dqa); + assert(dq_pop(dqa) == 0); + } + dq_print(dqa); + assert(dq_pop(dqa) == &a[0]); + + // shortening queue + dq_print(dqa); + dq_set_min_length(dqa, 4); + // see if data in the queue still persist + dq_print(dqa); + assert(dq_pop(dqa) == &a[1]); + // see if the queue behave with min length = 4 + dq_push(dqa, &a[3]); + dq_print(dqa); + assert(dq_pop(dqa) == &a[2]); + for (i=0;i<2;i++) { + dq_print(dqa); + assert(dq_pop(dqa) == 0); + } + dq_print(dqa); + assert(dq_pop(dqa) == &a[3]); + + // lengthening queue + dq_set_min_length(dqa, 6); + dq_push(dqa, &a[4]); + dq_push(dqa, &a[5]); + for (i=0;i<5;i++) { + dq_print(dqa); + assert(dq_pop(dqa) == 0); + } + dq_print(dqa); + assert(dq_pop(dqa) == &a[4]); + + // queue with no min length + dq_set_min_length(dqa, 0); + dq_print(dqa); + assert(dq_pop(dqa) == &a[5]); + dq_print(dqa); + dq_push(dqa, &a[6]); + dq_print(dqa); + assert(dq_pop(dqa) == &a[6]); + + // lengthening the queue, then shorten it again, + // but with some data exceeding the new min length + // the data should retain. + dq_print(dqa); + dq_set_min_length(dqa, 7); + dq_print(dqa); + dq_push(dqa, &a[0]); + assert(dq_pop(dqa) == 0); + dq_print(dqa); + dq_set_min_length(dqa, 4); + dq_print(dqa); + assert(dq_pop(dqa) == 0); + assert(dq_pop(dqa) == 0); + assert(dq_pop(dqa) == 0); + assert(dq_pop(dqa) == 0); + assert(dq_pop(dqa) == 0); + // This is the 7th pop: min-length is obeyed + assert(dq_pop(dqa) == &a[0]); + dq_print(dqa); + + // Shortening a queue with null entry only + dq_set_min_length(dqa, 0); + assert(dqa->length == 0); + dq_print(dqa); + + // Lengthening + dq_set_min_length(dqa, 6); + assert(dqa->length == 6); + dq_print(dqa); + + // Shortening a queue with null entry only + dq_set_min_length(dqa, 3); + assert(dqa->length == 3); + dq_print(dqa); + + dq_free(dqa); + printf("regression test 01 passed!\n"); +} + +int regresstion_test00() { + delay_queue *dqa, *dqb, *dqc, *dqd; + int i; + int a[4]; + int *b; + for (i=0;i<4;i++) a[i]=i; + dqa = dq_create("dqa", 0, 4, 0); + dqb = dq_create("dqb", 0, 10, 0); + dq_print(dqa); + dq_print(dqb); + dq_push(dqa,a); + dq_print(dqa); + dq_pop(dqa); + dq_print(dqa); + dq_push(dqa,a); + dq_print(dqa); + dq_pop(dqa); + dq_print(dqa); + dq_pop(dqa); + dq_print(dqa); + b = dq_pop(dqa); + dq_print(dqa); + for (i=0;i<4;i++) printf("%d\n",b[i]); + dqc = dq_create("dqc", 0, 0, 3); + for (i=0;i<4;i++) { + if (!dq_push(dqc,&a[i])) printf("cannot push.\n"); + dq_print(dqc); + } + dqd = dq_create("dqd", 0, 2, 3); + if (!dq_push(dqd,&a[0])) printf("cannot push.\n"); + dq_print(dqd); + if (!dq_push(dqd,&a[1])) printf("cannot push.\n"); + dq_print(dqd); + if (!dq_push(dqd,&a[2])) printf("cannot push.\n"); + dq_print(dqd); + dq_pop(dqd); + if (!dq_push(dqd,&a[3])) printf("cannot push.\n"); + dq_print(dqd); + + dq_free(dqa); + dq_free(dqb); + dq_free(dqc); + dq_free(dqd); + + return 0; +} + +int main() { + regresstion_test01(); + return 0; +} + + +#endif diff --git a/src/gpgpu-sim/delayqueue.h b/src/gpgpu-sim/delayqueue.h new file mode 100644 index 0000000..baa3892 --- /dev/null +++ b/src/gpgpu-sim/delayqueue.h @@ -0,0 +1,122 @@ +/* + * delayqueue.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * Ivan Sham, Henry Tran and the University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include +#include + +#ifndef DELAYQUEUE_H +#define DELAYQUEUE_H + +#include "../util.h" + +typedef struct delay_data_t delay_data; +struct delay_data_t { + void *data; + unsigned int time_elapsed; + delay_data *next; + unsigned long long push_time; //for stat collection +}; + +typedef struct { + const char* name; + int uid; + + unsigned int latency; + unsigned int min_len; + unsigned int max_len; + unsigned int length; + unsigned int n_element; + + delay_data *head; + delay_data *tail; + + void* lat_stat; //a pointer to latency stats distribution structure + //occupancy stat + unsigned int max_size_stat; + unsigned int n_stat_samples; + float avg_size_stat; +} delay_queue; + +unsigned char dq_full(delay_queue* dq ); +unsigned char dq_empty(delay_queue* dq ); +unsigned int dq_n_element(delay_queue* dq ); +unsigned char dq_push(delay_queue* dq, void* data); +void* dq_pop(delay_queue* dq); +void dq_set_min_length(delay_queue* dq, unsigned int new_min_len); +void removeEntry(void* data, delay_queue** dq, int size_dq); +delay_queue* dq_create( const char* name, + unsigned int latency, + unsigned int min_len, + unsigned int max_len); +void dq_remove(void* data, delay_queue* dq); +void dq_print(delay_queue* dq); +void dq_free(delay_queue* dq); +void* dq_top(delay_queue* dq);//return the data in the head without poping the queue + +void dq_update_stat(delay_queue* dq); +void dq_print_stat(delay_queue* dq); + +#endif diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc new file mode 100644 index 0000000..9f76b32 --- /dev/null +++ b/src/gpgpu-sim/dram.cc @@ -0,0 +1,604 @@ +/* + * dram.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, George L. Yuan, + * Ivan Sham, Justin Kwong, Dan O'Connor and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +//#include "gpu-sim.h" +#include "gpu-misc.h" +//#include "shader.h" +#include "dram.h" + +extern unsigned long long gpu_sim_cycle; +extern signed long long gpu_tot_sim_cycle; +extern int gpgpu_memlatency_stat; +extern unsigned max_dq_latency; +extern unsigned dq_lat_table[24]; +extern unsigned max_dq_latency; +extern unsigned dq_lat_table[24]; +unsigned int gpu_n_warps; +extern unsigned int gpu_n_mem_per_ctrlr; +extern unsigned int recent_dram_util; + +#ifdef DRAM_VERIFY +int PRINT_CYCLE = 0; +#endif + +dram_t* dram_create( unsigned int id, unsigned int nbk, + unsigned int tCCD, unsigned int tRRD, + unsigned int tRCD, unsigned int tRAS, + unsigned int tRP, unsigned int tRC, + unsigned int CL, unsigned int WL, + unsigned int BL, unsigned int tWTR, + unsigned int busW, unsigned int queue_limit, + unsigned char scheduler_type ) +{ + dram_t *dm; + unsigned i; + + dm = (dram_t*)calloc(1,sizeof(dram_t)); + + dm->id = id; + + dm->nbk = nbk; + dm->tCCD = tCCD; + dm->tRRD = tRRD; + dm->tRCD = tRCD; + dm->tRCDWR = tRCD - (WL + 1); //formula given in datasheet + dm->tRAS = tRAS; + dm->tRP = tRP; + dm->tRC = tRC; + dm->CL = CL; + dm->WL = WL; + dm->BL = BL; + + dm->tRTW = (CL+(BL/2)+2-WL); //read to write time according to datasheet + dm->tWTR = tWTR; + + dm->busW = busW; + + dm->CCDc = 0; + dm->RRDc = 0; + dm->RTWc = 0; + dm->WTRc = 0; + + dm->rw = READ; //read mode is default + + dm->bk = (bank_t**) calloc(sizeof(bank_t*),dm->nbk); + dm->bk[0] = (bank_t*) calloc(sizeof(bank_t),dm->nbk); + for (i=1;inbk;i++) { + dm->bk[i] = dm->bk[0] + i; + } + for (i=0;inbk;i++) { + dm->bk[i]->state = BANK_IDLE; + } + dm->prio = 0; + dm->rwq = dq_create("rwq",0,dm->CL,dm->CL+1); + dm->mrqq = dq_create("mrqq",0,0,0); + dm->queue_limit = queue_limit; + + dm->returnq = dq_create("dramreturnq",0,0, queue_limit); + + dm->m_fast_ideal_scheduler = NULL; + if ( scheduler_type == DRAM_IDEAL_FAST ) + dm->m_fast_ideal_scheduler = alloc_fast_ideal_scheduler(dm); + + + dm->n_cmd = 0; + dm->n_activity = 0; + dm->n_nop = 0; + dm->n_act = 0; + dm->n_pre = 0; + dm->n_rd = 0; + dm->n_wr = 0; + dm->n_req = 0; + dm->max_mrqs_temp = 0; + + dm->bwutil = 0; + + dm->max_mrqs = 0; + + dm->scheduler_type = scheduler_type; + + dm->realistic_scheduler_mode = READ; //realistic scheduler defaults to read + for (i=0;i<10;i++) { + dm->dram_util_bins[i]=0; + dm->dram_eff_bins[i]=0; + } + dm->last_n_cmd = dm->last_n_activity = dm->last_bwutil = 0; + + dm->n_cmd_partial = 0; + dm->n_activity_partial = 0; + dm->n_nop_partial = 0; + dm->n_act_partial = 0; + dm->n_pre_partial = 0; + dm->n_req_partial = 0; + dm->ave_mrqs_partial = 0; + dm->bwutil_partial = 0; + return dm; +} + +void dram_free( dram_t *dm ) +{ + dq_free(dm->mrqq); + dq_free(dm->rwq); + dq_free( dm->returnq ); + + free(dm->bk[0]); + free(dm->bk); + free(dm); +} + +int dram_full( dram_t *dm ) +{ + int full = 0; + if ( dm->queue_limit == 0 ) return 0; + if ( dm->scheduler_type == DRAM_IDEAL_FAST ) { + unsigned nreqs = fast_scheduler_queue_length(dm) + dq_n_element(dm->mrqq); + full = (nreqs >= dm->queue_limit); + } else { + full = (dm->mrqq->length >= dm->queue_limit); + } + + return full; +} + +unsigned int dram_que_length( dram_t *dm ) +{ + unsigned nreqs = 0; + if (dm->scheduler_type == DRAM_IDEAL_FAST ) { + nreqs = fast_scheduler_queue_length(dm); + } else { + nreqs = dm->mrqq->length ; + } + return nreqs; +} + +void dram_push( dram_t *dm, unsigned int bank, + unsigned int row, unsigned int col, + unsigned int nbytes, unsigned int write, + unsigned int wid, + unsigned int sid, int cache_hits_waiting, unsigned long long addr, + void *data ) +{ + dram_req_t *mrq; + + if (bank>=dm->nbk) printf("ERROR: no such bank exist in DRAM %d\n", bank); + + mrq = (dram_req_t *) malloc(sizeof(dram_req_t)); + + mrq->bk = bank; + mrq->row = row; + mrq->col = col; + mrq->nbytes = nbytes; + mrq->txbytes = 0; + mrq->dqbytes = 0; + mrq->data = data; + mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle; + mrq->cache_hits_waiting = cache_hits_waiting; + mrq->addr = addr; + mrq->insertion_time = (unsigned) gpu_sim_cycle; + + if (!write) { + mrq->rw = READ; //request is a read + } else { + mrq->rw = WRITE; //request is a write + } + + dq_push(dm->mrqq,mrq); + dm->n_req += 1; + dm->n_req_partial += 1; + + if ( dm->scheduler_type == DRAM_IDEAL_FAST ) { + unsigned nreqs = fast_scheduler_queue_length(dm); + if ( nreqs > dm->max_mrqs_temp) + dm->max_mrqs_temp = nreqs; + } else { + dm->max_mrqs_temp = (dm->max_mrqs_temp > dm->mrqq->length)? dm->max_mrqs_temp : dm->mrqq->length; + } +} + +void scheduler_fifo(dram_t* dm) +{ + if (dm->mrqq->head) { + dram_req_t *head_mrqq; + unsigned int bkn; + head_mrqq = (dram_req_t *)dm->mrqq->head->data; + bkn = head_mrqq->bk; + if (!dm->bk[bkn]->mrq) { + dm->bk[bkn]->mrq = (dram_req_t*) dq_pop(dm->mrqq); + } + } +} + + +#define DEC2ZERO(x) x = (x)? (x-1) : 0; +#define SWAP(a,b) a ^= b; b ^= a; a ^= b; + +void dram_issueCMD (dram_t* dm) +{ + unsigned i,j,k; + unsigned char issued; + issued = 0; + + /* check if the upcoming request is on an idle bank */ + /* Should we modify this so that multiple requests are checked? */ + + switch (dm->scheduler_type) { + case DRAM_FIFO: + scheduler_fifo(dm); + break; + case DRAM_IDEAL_FAST: + fast_scheduler_ideal(dm); + break; + default: + printf("Error: Unknown DRAM scheduler type\n"); + assert(0); + } + if ( dm->scheduler_type == DRAM_IDEAL_FAST ) { + unsigned nreqs = fast_scheduler_queue_length(dm); + if ( nreqs > dm->max_mrqs) { + dm->max_mrqs = nreqs; + } + dm->ave_mrqs += nreqs; + dm->ave_mrqs_partial += nreqs; + } else { + if (dm->mrqq->length > dm->max_mrqs) { + dm->max_mrqs = dm->mrqq->length; + } + dm->ave_mrqs += dm->mrqq->length; + dm->ave_mrqs_partial += dm->mrqq->length; + } + k=dm->nbk; + // check if any bank is ready to issue a new read + for (i=0;inbk;i++) { + j = (i + dm->prio) % dm->nbk; + if (dm->bk[j]->mrq) { //if currently servicing a memory request + // correct row activated for a READ + if ( !issued && !dm->CCDc && !dm->bk[j]->RCDc && + (dm->bk[j]->curr_row == dm->bk[j]->mrq->row) && + (dm->bk[j]->mrq->rw == READ) && (dm->WTRc == 0 ) && + (dm->bk[j]->state == BANK_ACTIVE) && + !dq_full(dm->rwq) ) { + if (dm->rw==WRITE) { + dm->rw=READ; + dq_set_min_length(dm->rwq, dm->CL); + } + dq_push(dm->rwq,(void*)dm->bk[j]->mrq); //only push when rwq empty? + dm->bk[j]->mrq->txbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; //16 bytes + dm->CCDc = dm->tCCD; + dm->RTWc = dm->tRTW; + issued = 1; + dm->n_rd++; + //printf("\tn_rd++ Bank: %d Row: %d Col: %d\n", j, dm->bk[j]->mrq->row, dm->bk[j]->mrq->col); + dm->bwutil+= dm->BL/2; + dm->bwutil_partial += dm->BL/2; + dm->bk[j]->n_access++; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tRD Bk:%d Row:%03x Col:%03x \n", + j, dm->bk[j]->curr_row, + dm->bk[j]->mrq->col+dm->bk[j]->mrq->txbytes-dm->BL*dm->busW); +#endif + // transfer done + if ( !(dm->bk[j]->mrq->txbytes < dm->bk[j]->mrq->nbytes) ) { + dm->bk[j]->mrq = NULL; + } + } else + // correct row activated for a WRITE + if ( !issued && !dm->CCDc && !dm->bk[j]->RCDWRc && + (dm->bk[j]->curr_row == dm->bk[j]->mrq->row) && + (dm->bk[j]->mrq->rw == WRITE) && (dm->RTWc == 0 ) && + (dm->bk[j]->state == BANK_ACTIVE) && + !dq_full(dm->rwq) ) { + if (dm->rw==READ) { + dm->rw=WRITE; + dq_set_min_length(dm->rwq, dm->WL); + } + dq_push(dm->rwq,(void*)dm->bk[j]->mrq); + + dm->bk[j]->mrq->txbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; /*16 bytes*/ + dm->CCDc = dm->tCCD; + issued = 1; + dm->n_wr++; + dm->bwutil+=2; + dm->bwutil_partial += dm->BL/2; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tWR Bk:%d Row:%03x Col:%03x \n", + j, dm->bk[j]->curr_row, + dm->bk[j]->mrq->col+dm->bk[j]->mrq->txbytes-dm->BL*dm->busW); +#endif + // transfer done + if ( !(dm->bk[j]->mrq->txbytes < dm->bk[j]->mrq->nbytes) ) { + dm->bk[j]->mrq = NULL; + } + } + + else + // bank is idle + if ( !issued && !dm->RRDc && + (dm->bk[j]->state == BANK_IDLE) && + !dm->bk[j]->RPc && !dm->bk[j]->RCc ) { +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tACT BK:%d NewRow:%03x From:%03x \n", + j,dm->bk[j]->mrq->row,dm->bk[j]->curr_row); +#endif + // activate the row with current memory request + dm->bk[j]->curr_row = dm->bk[j]->mrq->row; + dm->bk[j]->state = BANK_ACTIVE; + dm->RRDc = dm->tRRD; + dm->bk[j]->RCDc = dm->tRCD; + dm->bk[j]->RCDWRc = dm->tRCDWR; + dm->bk[j]->RASc = dm->tRAS; + dm->bk[j]->RCc = dm->tRC; + dm->prio = (j + 1) % dm->nbk; + issued = 1; + dm->n_act_partial++; + dm->n_act++; + } + + else + // different row activated + if ( (!issued) && + (dm->bk[j]->curr_row != dm->bk[j]->mrq->row) && + (dm->bk[j]->state == BANK_ACTIVE) && + (!dm->bk[j]->RASc) ) { + //printf("\tRASc: %d \n", dm->bk[j]->RASc); + // make the bank idle again + dm->bk[j]->state = BANK_IDLE; + dm->bk[j]->RPc = dm->tRP; + dm->prio = (j + 1) % dm->nbk; + issued = 1; + dm->n_pre++; + dm->n_pre_partial++; +#ifdef DRAM_VERIFY + PRINT_CYCLE=1; + printf("\tPRE BK:%d Row:%03x \n", j,dm->bk[j]->curr_row); + //printf("\tRASc: %d \n", dm->bk[j]->RASc); +#endif + } + } else { + if (!dm->CCDc && !dm->RRDc && !dm->RTWc && !dm->WTRc && !dm->bk[j]->RCDc && !dm->bk[j]->RASc + && !dm->bk[j]->RCc && !dm->bk[j]->RPc && !dm->bk[j]->RCDWRc) k--; + dm->bk[i]->n_idle++; + } + } + if (!issued) { + dm->n_nop++; + dm->n_nop_partial++; +#ifdef DRAM_VIEWCMD + printf("\tNOP "); +#endif + } + if (k) { + dm->n_activity++; + dm->n_activity_partial++; + } + dm->n_cmd++; + dm->n_cmd_partial++; + + // decrements counters once for each time dram_issueCMD is called + DEC2ZERO(dm->RRDc); + DEC2ZERO(dm->CCDc); + DEC2ZERO(dm->RTWc); + DEC2ZERO(dm->WTRc); + for (j=0;jnbk;j++) { + DEC2ZERO(dm->bk[j]->RCDc); + DEC2ZERO(dm->bk[j]->RASc); + DEC2ZERO(dm->bk[j]->RCc); + DEC2ZERO(dm->bk[j]->RPc); + DEC2ZERO(dm->bk[j]->RCDWRc); + } + +#ifdef DRAM_VISUALIZE + dram_visualize(dm); +#endif +} + +//if mrq is being serviced by dram, gets popped after CL latency fulfilled +void* dram_pop( dram_t *dm ) +{ + dram_req_t *mrq; + void *data; + unsigned dq_latency; + + data = NULL; + mrq = (dram_req_t*)dq_pop(dm->rwq); + if (mrq) { + // data = mrq->data; +#ifdef DRAM_VIEWCMD + printf("\tDQ: BK%d Row:%03x Col:%03x", + mrq->bk, mrq->row, mrq->col + mrq->dqbytes); +#endif + mrq->dqbytes += dm->BL * dm->busW * gpu_n_mem_per_ctrlr; /*16 bytes*/ + if (mrq->dqbytes >= mrq->nbytes) { + + if (gpgpu_memlatency_stat) { + dq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - mrq->timestamp; + dq_lat_table[LOGB2(dq_latency)]++; + if (dq_latency > max_dq_latency) + max_dq_latency = dq_latency; + } + data = mrq->data; + + free(mrq); + } + } +#ifdef DRAM_VIEWCMD + printf("\n"); +#endif + + return data; +} + +// a hack to allow peeking into what memory request will be serviced. +void* dram_top( dram_t *dm ) +{ + dram_req_t *mrq; + void *data; + + data = NULL; + mrq = (dram_req_t*)dq_top(dm->rwq); + if (mrq) { + // number of bytes returned from dram if this is ever popped + unsigned tobe_dqbytes = mrq->dqbytes + dm->BL * dm->busW * gpu_n_mem_per_ctrlr; + if (tobe_dqbytes >= mrq->nbytes) { + data = mrq->data; + } + } + + return data; +} + +void dram_print( dram_t* dm, FILE* simFile) +{ + unsigned i; + fprintf(simFile,"DRAM[%d]: %d bks, busW=%d BL=%d CL=%d, ", + dm->id, dm->nbk, dm->busW, dm->BL, dm->CL ); + fprintf(simFile,"tRRD=%d tCCD=%d, tRCD=%d tRAS=%d tRP=%d tRC=%d\n", + dm->tCCD, dm->tRRD, dm->tRCD, dm->tRAS, dm->tRP, dm->tRC ); + fprintf(simFile,"n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g\n", + dm->n_cmd, dm->n_nop, dm->n_act, dm->n_pre, dm->n_req, dm->n_rd, dm->n_wr, + (float)dm->bwutil/dm->n_cmd); + fprintf(simFile,"n_activity=%d dram_eff=%.4g\n", + dm->n_activity, (float)dm->bwutil/dm->n_activity); + for (i=0;inbk;i++) { + fprintf(simFile, "bk%d: %da %di ",i,dm->bk[i]->n_access,dm->bk[i]->n_idle); + } + fprintf(simFile, "\n"); + fprintf(simFile, "dram_util_bins:"); + for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_util_bins[i]); + fprintf(simFile, "\ndram_eff_bins:"); + for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_eff_bins[i]); + fprintf(simFile, "\n"); + /* + { + delay_data* mrq; + mrq = dm->mrqq->head; + while (mrq) { + printf("%d",((dram_req_t*)mrq->data)->bk); + mrq = mrq->next; + } + printf("\n"); + } + */ + fprintf(simFile, "mrqq: max=%d avg=%g\n", dm->max_mrqs, (float)dm->ave_mrqs/dm->n_cmd); +} + +void dram_visualize( dram_t* dm ) +{ + unsigned i; + + printf("RRDc=%d CCDc=%d mrqq.Length=%d rwq.Length=%d\n", + dm->RRDc, dm->CCDc, dm->mrqq->length,dm->rwq->length); + for (i=0;inbk;i++) { + printf("BK%d: state=%c curr_row=%03x, %2d %2d %2d %2d %p ", + i, dm->bk[i]->state, dm->bk[i]->curr_row, + dm->bk[i]->RCDc, dm->bk[i]->RASc, + dm->bk[i]->RPc, dm->bk[i]->RCc, + dm->bk[i]->mrq ); + if (dm->bk[i]->mrq) + printf("txf: %d %d", dm->bk[i]->mrq->nbytes, dm->bk[i]->mrq->txbytes); + printf("\n"); + } + if ( dm->m_fast_ideal_scheduler ) { + dump_fast_ideal_scheduler( dm ); + } + +} + +void dram_print_stat( dram_t* dm, FILE* simFile ) +{ + int i; + fprintf(simFile,"DRAM (%d): n_cmd=%d n_nop=%d n_act=%d n_pre=%d n_req=%d n_rd=%d n_write=%d bw_util=%.4g ", + dm->id, dm->n_cmd, dm->n_nop, dm->n_act, dm->n_pre, dm->n_req, dm->n_rd, dm->n_wr, + (float)dm->bwutil/dm->n_cmd); + fprintf(simFile, "mrqq: %d %.4g mrqsmax=%d ", dm->max_mrqs, (float)dm->ave_mrqs/dm->n_cmd, dm->max_mrqs_temp); + fprintf(simFile, "\n"); + fprintf(simFile, "dram_util_bins:"); + for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_util_bins[i]); + fprintf(simFile, "\ndram_eff_bins:"); + for (i=0;i<10;i++) fprintf(simFile, " %d", dm->dram_eff_bins[i]); + fprintf(simFile, "\n"); + dm->max_mrqs_temp = 0; +} + + +unsigned dram_busy( dram_t* dm) +{ + unsigned busy = 0; + + switch (dm->scheduler_type) { + case DRAM_FIFO: + busy = (dm->mrqq->length > 0); + break; + case DRAM_IDEAL_FAST: + busy = (fast_scheduler_queue_length(dm) > 0) || (dm->mrqq->length > 0); + break; + } + + return busy; +} + diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h new file mode 100644 index 0000000..fff23f6 --- /dev/null +++ b/src/gpgpu-sim/dram.h @@ -0,0 +1,237 @@ +/* + * dram.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, George L. Yuan, + * Ivan Sham, Justin Kwong, Dan O'Connor and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files + * src/gpgpusim_entrypoint.c and src/simplesim-3.0/ are derived from the + * SimpleScalar Toolset available from http://www.simplescalar.com/ + * (property of SimpleScalar LLC) and the files src/intersim/ are derived + * from Booksim (Simulator provided with the textbook "Principles and + * Practices of Interconnection Networks" available from + * http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include + +#include "delayqueue.h" +#include "../cuda-sim/dram_callback.h" + +#ifndef DRAM_H +#define DRAM_H + +#define FIFO_AGE_LIMIT 50 //used for both BANK_CONF and REALISTIC schedulers +#define FIFO_NUM_WRITE_LIMIT 3 //used for both BANK_CONF and REALISTIC schedulers +#define LOOKAHEAD_VALUE 10 //used for REALISTIC scheduler ONLY + +enum { + DRAM_FIFO, + DRAM_IDEAL_FAST, + DRAM_NUM_HANDLES +}; + + +#define READ 'R' //define read and write states +#define WRITE 'W' +typedef struct { + unsigned int row; + unsigned int col; + unsigned int bk; + unsigned int nbytes; + unsigned int txbytes; + unsigned int dqbytes; + unsigned int age; + unsigned int timestamp; + unsigned char rw; //is the request a read or a write? + unsigned long long int addr; + unsigned int insertion_time; + void* data; + + int cache_hits_waiting; +} dram_req_t; + +#define BANK_IDLE 'I' +#define BANK_ACTIVE 'A' + +typedef struct { + unsigned int RCDc; + unsigned int RCDWRc; + unsigned int RASc; + unsigned int RPc; + unsigned int RCc; + + unsigned char rw; //is the bank reading or writing? + unsigned char state; //is the bank active or idle? + unsigned int curr_row; + + dram_req_t *mrq; + + unsigned int n_access; + unsigned int n_writes; + unsigned int n_idle; +} bank_t; + +typedef struct { + unsigned int id; + + unsigned int tCCD; //column to column delay + unsigned int tRRD; //minimal time required between activation of rows in different banks + unsigned int tRCD; //row to column delay - time required to activate a row before a read + unsigned int tRCDWR;//row to column delay for a write command + unsigned int tRAS; //time needed to activate row + unsigned int tRP; //row precharge ie. deactivate row + unsigned int tRC; //row cycle time ie. precharge current, then activate different row + + unsigned int CL; //CAS latency + unsigned int WL; //WRITE latency + unsigned int BL; //Burst Length in bytes (we're using 4? could be 8) + unsigned int tRTW; //time to switch from read to write + unsigned int tWTR; //time to switch from write to read 5? look in datasheet + unsigned int busW; + + unsigned int nbk; + bank_t **bk; + unsigned int prio; + + unsigned int RRDc; + unsigned int CCDc; + unsigned int RTWc; //read to write penalty applies across banks + unsigned int WTRc; //write to read penalty applies across banks + + unsigned char rw; //was last request a read or write? (important for RTW, WTR) + + unsigned int pending_writes; + unsigned char realistic_scheduler_mode; + + delay_queue *rwq; + delay_queue *mrqq; + //buffer to hold packets when DRAM processing is over + //should be filled with dram clock and popped with l2or icnt clock + delay_queue *returnq; + + + unsigned int dram_util_bins[10]; + unsigned int dram_eff_bins[10]; + unsigned int last_n_cmd, last_n_activity, last_bwutil; + + unsigned int queue_limit; + + unsigned int n_cmd; + unsigned int n_activity; + unsigned int n_nop; + unsigned int n_act; + unsigned int n_pre; + unsigned int n_rd; + unsigned int n_wr; + unsigned int n_req; + unsigned int max_mrqs_temp; + + unsigned int bwutil; + unsigned int max_mrqs; + unsigned int ave_mrqs; + unsigned char scheduler_type; + + void *m_fast_ideal_scheduler; + + void *m_L2cache; + + unsigned int n_cmd_partial; + unsigned int n_activity_partial; + unsigned int n_nop_partial; + unsigned int n_act_partial; + unsigned int n_pre_partial; + unsigned int n_req_partial; + unsigned int ave_mrqs_partial; + unsigned int bwutil_partial; + + void * req_hist; +} dram_t; + + +dram_t* dram_create( unsigned int id, unsigned int nbk, + unsigned int tCCD, unsigned int tRRD, + unsigned int tRCD, unsigned int tRAS, + unsigned int tRP, unsigned int tRC, + unsigned int CL, unsigned int WL, + unsigned int BL, unsigned int tWTR, + unsigned int busW, unsigned int queue_limit, + unsigned char scheduler_type ); +void dram_free( dram_t *dm ); +int dram_full( dram_t *dm ); +void dram_push( dram_t *dm, unsigned int bank, + unsigned int row, unsigned int col, + unsigned int nbytes, unsigned int write, + unsigned int wid, unsigned int sid, int cache_hits_waiting, unsigned long long addr, + void *data ); +void scheduler_fifo(dram_t* dm); +void dram_issueCMD (dram_t* dm); +void* dram_pop( dram_t *dm ); +void* dram_top( dram_t *dm ); +unsigned dram_busy( dram_t *dm); +void dram_print( dram_t* dm, FILE* simFile ); +void dram_visualize( dram_t* dm ); +void dram_print_stat( dram_t* dm, FILE* simFile ); +void fast_scheduler_ideal(dram_t* dm); +void* alloc_fast_ideal_scheduler(dram_t *dm); +void dump_fast_ideal_scheduler(dram_t *dm); +unsigned fast_scheduler_queue_length(dram_t *dm); + +//supposed to return the current queue length for all memory scheduler types. +unsigned int dram_que_length( dram_t *dm ); + +#endif /*DRAM_H*/ diff --git a/src/gpgpu-sim/dram_sched.cc b/src/gpgpu-sim/dram_sched.cc new file mode 100644 index 0000000..24b518f --- /dev/null +++ b/src/gpgpu-sim/dram_sched.cc @@ -0,0 +1,247 @@ +/* + * dram_sched.cc + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "dram_sched.h" +#include "gpu-misc.h" +#include "gpu-sim.h" +#include "../util.h" + +extern unsigned long long gpu_sim_cycle; +extern signed long long gpu_tot_sim_cycle; +extern unsigned max_mrq_latency; +extern unsigned mrq_lat_table[24]; +extern int gpgpu_memlatency_stat; +extern int gpgpu_dram_sched_queue_size; +extern unsigned int **concurrent_row_access; //concurrent_row_access[dram chip id][bank id] +extern unsigned int **row_access; //concurrent_row_access[dram chip id][bank id] +extern unsigned int **num_activates; //num_activates[dram chip id][bank id] +extern unsigned int **max_conc_access2samerow; //max_conc_access2samerow[dram chip id][bank id] +extern unsigned int **max_servicetime2samerow; + +ideal_dram_scheduler::ideal_dram_scheduler( dram_t *dm ) +{ + + m_num_pending = 0; + m_dram = dm; + m_queue = new std::list[dm->nbk]; + m_bins = new std::map::iterator> >[ dm->nbk ]; + m_last_row = new std::list::iterator>*[ dm->nbk ]; + curr_row_service_time = new unsigned[dm->nbk]; + row_service_timestamp = new unsigned[dm->nbk]; + for ( unsigned i=0; i < dm->nbk; i++ ) { + m_queue[i].clear(); + m_bins[i].clear(); + m_last_row[i] = NULL; + curr_row_service_time[i] = 0; + row_service_timestamp[i] = 0; + } + +} + +void ideal_dram_scheduler::add_req( dram_req_t *req ) +{ + m_num_pending++; + + m_queue[req->bk].push_front(req); + std::list::iterator ptr = m_queue[req->bk].begin(); + + m_bins[req->bk][req->row].push_front( ptr ); //newest reqs to the front + + +} + + +inline void ideal_dram_scheduler::data_collection(unsigned int bank) +{ + if (gpu_sim_cycle > row_service_timestamp[bank]) { + curr_row_service_time[bank] = gpu_sim_cycle - row_service_timestamp[bank]; + if (curr_row_service_time[bank] > max_servicetime2samerow[m_dram->id][bank]) + max_servicetime2samerow[m_dram->id][bank] = curr_row_service_time[bank]; + } + curr_row_service_time[bank] = 0; + row_service_timestamp[bank] = gpu_sim_cycle; + if (concurrent_row_access[m_dram->id][bank] > max_conc_access2samerow[m_dram->id][bank]) { + max_conc_access2samerow[m_dram->id][bank] = concurrent_row_access[m_dram->id][bank]; + } + concurrent_row_access[m_dram->id][bank] = 0; + num_activates[m_dram->id][bank]++; +} + +dram_req_t *ideal_dram_scheduler::schedule( unsigned bank, unsigned curr_row ) +{ + int row_hit = 0; + if ( m_last_row[bank] == NULL ) { + if ( m_queue[bank].empty() ) + return NULL; + + std::map::iterator> >::iterator bin_ptr = m_bins[bank].find( curr_row ); + if ( bin_ptr == m_bins[bank].end()) { + dram_req_t *req = m_queue[bank].back(); + bin_ptr = m_bins[bank].find( req->row ); + assert( bin_ptr != m_bins[bank].end() ); // where did the request go??? + m_last_row[bank] = &(bin_ptr->second); + data_collection(bank); + } else { + m_last_row[bank] = &(bin_ptr->second); + + } + } + row_hit=1; + std::list::iterator next = m_last_row[bank]->back(); + dram_req_t *req = (*next); + + concurrent_row_access[m_dram->id][bank]++; + row_access[m_dram->id][bank]++; + m_last_row[bank]->pop_back(); + + m_queue[bank].erase(next); + if ( m_last_row[bank]->empty() ) { + m_bins[bank].erase( req->row ); + m_last_row[bank] = NULL; + } +#ifdef DEBUG_FAST_IDEAL_SCHED + if ( req ) + printf("%08u : DRAM(%u) scheduling memory request to bank=%u, row=%u\n", + (unsigned)gpu_sim_cycle, m_dram->id, req->bk, req->row ); +#endif + assert( req != NULL && m_num_pending != 0 ); + m_num_pending--; + + return req; +} + + +void ideal_dram_scheduler::print( FILE *fp ) +{ + for ( unsigned b=0; b < m_dram->nbk; b++ ) { + printf(" %u: queue length = %u\n", b, (unsigned)m_queue[b].size() ); + } +} + +void* alloc_fast_ideal_scheduler(dram_t *dm) +{ + return new ideal_dram_scheduler(dm); +} + +void fast_scheduler_ideal(dram_t* dm) +{ + + + unsigned mrq_latency; + // replacement for scheduler_ideal() + + ideal_dram_scheduler *sched = reinterpret_cast( dm->m_fast_ideal_scheduler ); + while ( !dq_empty(dm->mrqq) && (!gpgpu_dram_sched_queue_size || sched->num_pending() < (unsigned) gpgpu_dram_sched_queue_size)) { + dram_req_t *req = (dram_req_t*)dq_pop(dm->mrqq); + sched->add_req(req); + } + + dram_req_t *req; + unsigned i; + for ( i=0; i < dm->nbk; i++ ) { + unsigned b = (i+dm->prio)%dm->nbk; + if ( !dm->bk[b]->mrq ) { + + req = sched->schedule(b, dm->bk[b]->curr_row); + + if ( req ) { + dm->prio = (dm->prio+1)%dm->nbk; + dm->bk[b]->mrq = req; + if (gpgpu_memlatency_stat) { + mrq_latency = gpu_sim_cycle + gpu_tot_sim_cycle - dm->bk[b]->mrq->timestamp; + dm->bk[b]->mrq->timestamp = gpu_tot_sim_cycle + gpu_sim_cycle; + mrq_lat_table[LOGB2(mrq_latency)]++; + if (mrq_latency > max_mrq_latency) { + max_mrq_latency = mrq_latency; + } + } + + break; + } + } + } +} + + + +void dump_fast_ideal_scheduler( dram_t *dm ) +{ + ideal_dram_scheduler *sched = reinterpret_cast( dm->m_fast_ideal_scheduler ); + sched->print(stdout); +} + +unsigned fast_scheduler_queue_length(dram_t *dm) +{ + if (dm->m_fast_ideal_scheduler ) { + ideal_dram_scheduler *sched = reinterpret_cast( dm->m_fast_ideal_scheduler ); + return sched->num_pending(); + } else { + printf("fast_scheduler_queue_length(): Where did the scheduler go?\n"); + exit(1); + } +} + + diff --git a/src/gpgpu-sim/dram_sched.h b/src/gpgpu-sim/dram_sched.h new file mode 100644 index 0000000..49cae22 --- /dev/null +++ b/src/gpgpu-sim/dram_sched.h @@ -0,0 +1,99 @@ +/* + * dram_sched.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef dram_sched_h_INCLUDED +#define dram_sched_h_INCLUDED + +#include "dram.h" +#include "shader.h" +#include "gpu-sim.h" +#include "gpu-misc.h" +#include +#include + +class ideal_dram_scheduler { +public: + ideal_dram_scheduler( dram_t *dm ); + void add_req( dram_req_t *req ); + std::list::iterator binarysort_VFTF(dram_req_t *req); + std::list::iterator sort_VFTF(dram_req_t *req); + inline void data_collection(unsigned bank); + dram_req_t *schedule( unsigned bank, unsigned curr_row ); + void print( FILE *fp ); + unsigned num_pending() const { return m_num_pending;} + +private: + + dram_t *m_dram; + unsigned m_num_pending; + std::list *m_queue; + std::map::iterator> > *m_bins; + std::list::iterator> **m_last_row; + unsigned *curr_row_service_time; //one set of variables for each bank. + unsigned *row_service_timestamp; //tracks when scheduler began servicing current row +}; + +#endif diff --git a/src/gpgpu-sim/dwf.cc b/src/gpgpu-sim/dwf.cc new file mode 100644 index 0000000..cd66084 --- /dev/null +++ b/src/gpgpu-sim/dwf.cc @@ -0,0 +1,2609 @@ +/* + * dwf.cc + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + + +#include "dwf.h" +#include "histogram.h" +#include +#include +#include +#include +#include + +using namespace std; + +unsigned int gpgpu_dwf_regbk = 1; +unsigned int gpgpu_dwf_heuristic = 0; +enum { + MAJORITY = 0, + MINORITY = 1, + FIFO = 2, + PDOMPRIO = 3, + PC = 4, + MAJORITY_MAXHEAP = 5, + N_DWFMODE +}; + +typedef struct warp_entry { + address_type pc; + int* tid; // thread id's + int occ; // occupancy vector + int pdom_prio; // pdom_priority + int pdom_occ; // pdom_priority's aux data + int next_warp; // index to next warp in an implicit queue + void* lut_ptr; // pointer to the warp lut entry that last update this warp (a hack), done to decouple warp lut and warp pool + int uid; // unique id of a warp +} warp_entry_t; + +class issue_warp_majority { +public: + + virtual void add_threads( address_type pc, int *tid) = 0; + virtual void push_warp( address_type pc, int idx) = 0; + virtual int pop_warp( ) = 0; + virtual void print( FILE *fout ) = 0; + virtual ~issue_warp_majority( ) {} +}; + +typedef struct maxheap_lut_entry { + address_type pc; // pc of the warps + int maxheap_idx; // index to the max heap +} maxheap_lut_entry_t; + +typedef struct maxheap_entry { + address_type pc; // pc of the warps + int n_thds; // number of threads with this pc (from lut) + int wpool_head; // the first warp with this pc + int wpool_tail; // the last warp with this pc + int lut_idx; // reverse index to the lut (for update in entry movement) +} maxheap_entry_t; + +class mh_lut_class { +private: + + maxheap_lut_entry_t *lut_data; + list *lru_stack; // front = LRU + int n_set; + int insn_size_lgb2; + +public: + + int size; + int assoc; + int n_read; + int n_write; + int n_read_per_cycle; + int n_write_per_cycle; + + int n_aliased; + static maxheap_lut_entry_t clean_entry; + + mh_lut_class (int size, int assoc, int n_read_per_cycle, int n_write_per_cycle ) { + int i; + + this->size = size; + this->assoc = assoc; + lut_data = new maxheap_lut_entry_t[size]; + + for (i=0; i[n_set]; + for (i=0; in_read_per_cycle = n_read_per_cycle; + this->n_write_per_cycle = n_write_per_cycle; + this->n_read = 0; + this->n_write = 0; + this->n_aliased = 0; + } + + ~mh_lut_class ( ) { + delete[] lut_data; + } + + // obtain entry at a known location + maxheap_lut_entry_t get( int lut_idx ) { + assert(lut_idx < size); + n_read++; + return lut_data[lut_idx]; + } + + // modify an entry at a known location + void set( int lut_idx, maxheap_lut_entry_t lut_entry ) { + n_write++; + lut_data[lut_idx] = lut_entry; + } + + // update a lut entry with a new index + void update_mh_idx( int lut_idx, int mh_idx ) { + n_write++; + lut_data[lut_idx].maxheap_idx = mh_idx; + } + + // lookup an entry with a pc + int lookup( address_type pc ) { + int i; + int lut_idx = -1; + int set_start_idx = get_set(pc) * assoc; + + // look for the matched entry within the set + for (i = set_start_idx; i < (set_start_idx + assoc); i++) { + if (lut_data[i].pc == pc) { + lut_idx = i; + } + } + + // update lru stack if hit + if (lut_idx != -1) { + int set_idx = set_start_idx / assoc; + list::iterator it; + it = find(lru_stack[set_idx].begin(), lru_stack[set_idx].end(), lut_idx); + + if (it != lru_stack[set_idx].end()) { + lru_stack[set_idx].erase(it); + lru_stack[set_idx].push_back(lut_idx); + } + } + + return lut_idx; + } + + void free(int lut_idx) { + set(lut_idx, clean_entry); + + int set_idx = lut_idx / assoc; + list::iterator it; + it = find(lru_stack[set_idx].begin(), lru_stack[set_idx].end(), lut_idx); + + if (it != lru_stack[set_idx].end()) { + lru_stack[set_idx].erase(it); + lru_stack[set_idx].push_front(lut_idx); + } + } + + // find the LRU entry to be replaced + int find_lru( maxheap_lut_entry_t lut_entry ) { + int set_idx = get_set(lut_entry.pc); + int lru_idx = lru_stack[set_idx].front(); + + return lru_idx; + } + + // actually replacing the LRU entry + int replace_lru( maxheap_lut_entry_t lut_entry ) { + int set_idx = get_set(lut_entry.pc); + int lru_idx = lru_stack[set_idx].front(); + lru_stack[set_idx].pop_front(); + + // counting the number of overwritten entries + if (lut_data[lru_idx].maxheap_idx != 0) n_aliased++; + + set(lru_idx, lut_entry); + lru_stack[set_idx].push_back(lru_idx); + + return lru_idx; + } + + // reset the number of accesses to zero + void reset_access( ) { + n_read = 0; + n_write = 0; + } + + // clear the number of accesses - done at the end of scheduler cycle + void clear_access( ) { + n_read -= n_read_per_cycle; + n_read = (n_read >= 0)? n_read : 0; + n_write -= n_write_per_cycle; + n_write = (n_write >= 0)? n_write : 0; + } + + // test if the structure is done with all the required accesses + int all_access_done( ) { + return(n_read == 0 && n_write == 0); + } + + void print_lut_e(FILE *fout, maxheap_lut_entry_t lut_e) { + fprintf(fout, "[%08x]mh%02d", + lut_e.pc, lut_e.maxheap_idx); + } + + void print(FILE *fout) { + int i, j; + for (i=0; i::iterator it = lru_stack[i].begin(); + for (; it != lru_stack[i].end(); it++) { + fprintf(fout, "%02d-", *it); + } + fprintf(fout, "\n"); + } + } + +private: + + inline int get_set(address_type pc) { + return((pc >> insn_size_lgb2) & (n_set - 1)); + } +}; + +maxheap_lut_entry_t mh_lut_class::clean_entry = {0xDEADBEEF, 0}; + +// A class tracking the number of accesses done to the maxheap structure +// and the index ranges from 1..n_entries with 1 being the root +class maxheap_class { +private: + + maxheap_entry_t *maxheap_data; + mh_lut_class *lut; + +public: + + int n_read; + int n_write; + int n_entries; + int size; + int n_read_per_cycle; + int n_write_per_cycle; + + int max_n_entries; + static maxheap_entry_t clean_entry; + + maxheap_class( int size, mh_lut_class *lut, int n_read_per_cycle, int n_write_per_cycle ) { + n_read = 0; + n_write = 0; + n_entries = 0; // index to the last element + this->size = size; + maxheap_data = new maxheap_entry_t[size]; + + for (int i=0; ilut = lut; + + this->n_read_per_cycle = n_read_per_cycle; + this->n_write_per_cycle = n_write_per_cycle; + this->n_read = 0; + this->n_write = 0; + this->max_n_entries = 0; + } + + ~maxheap_class( ) { + delete[] maxheap_data; + } + + // insert a new entry into the maxheap + // return: the index to the new entry + int insert( maxheap_entry_t mh_entry ) { + assert(n_entries + 1 < size); + n_write++; + n_entries++; + maxheap_data[n_entries] = mh_entry; + max_n_entries = (max_n_entries >= n_entries)? max_n_entries : n_entries; + return n_entries; + } + + // retrieve the max heap entry at index [mh_idx] + maxheap_entry_t get( int mh_idx ) { + assert(mh_idx > 0); + assert(mh_idx <= n_entries); + n_read++; + return maxheap_data[mh_idx]; + } + + // replace the max heap entry at index [mh_idx] + void set( int mh_idx, maxheap_entry_t mh_entry ) { + assert(mh_idx > 0); + assert(mh_idx <= n_entries); + n_write++; + maxheap_data[mh_idx] = mh_entry; + } + + // a special version of set that only reset the lut_idx + void remove_lut_idx( int mh_idx ) { + assert(mh_idx > 0); + assert(mh_idx <= n_entries); + n_write++; + maxheap_data[mh_idx].lut_idx = -1; + } + + // read both childrens of a given node, count as one read + // return the number of child read + int get_childof(int mh_idx, maxheap_entry_t *child) { + int child_idx = childof(mh_idx); + int child_read = 0; + + if (child_idx <= n_entries) { + n_read++; + child[0] = maxheap_data[child_idx]; + child_read++; + } + if (child_idx + 1 <= n_entries) { + child[1] = maxheap_data[child_idx + 1]; + child_read++; + } + + return child_read; + } + + // pop the root entry of max heap + maxheap_entry_t pop_root( ) { + maxheap_entry_t old_root = get(1); + maxheap_entry_t curr_mhe[3]; + curr_mhe[0] = get(n_entries); + + set(1, curr_mhe[0]); + if (curr_mhe[0].lut_idx >= 0) + lut->update_mh_idx(curr_mhe[0].lut_idx, 1); + + n_entries--; + + int curr_node = 1; + int n_child = 0; + + n_child = get_childof(curr_node, curr_mhe + 1); + while (n_child > 0) { + int max_child = 0; + int i; + for (i = 1; i < n_child + 1; i++) { + if (cmp_mh(curr_mhe[i], curr_mhe[max_child])) { + max_child = i; + } + } + + n_child = 0; + if (max_child > 0) { + int max_child_node = childof(curr_node) + max_child - 1; + set(curr_node, curr_mhe[max_child]); + set(max_child_node, curr_mhe[0]); + + // update the lut for this swap + if (curr_mhe[max_child].lut_idx >= 0) + lut->update_mh_idx(curr_mhe[max_child].lut_idx, curr_node); + if (curr_mhe[0].lut_idx >= 0) + lut->update_mh_idx(curr_mhe[0].lut_idx, max_child_node); + + // get the next child + curr_node = max_child_node; + n_child = get_childof(curr_node, curr_mhe + 1); + } + } + + return old_root; + } + + // probe if the maxheap is empty + int empty( ) { + return(n_entries == 0); + } + + // reset the number of accesses to zero + void reset_access( ) { + n_read = 0; + n_write = 0; + } + + // clear the number of accesses - done at the end of scheduler cycle + void clear_access( ) { + n_read -= n_read_per_cycle; + n_read = (n_read >= 0)? n_read : 0; + n_write -= n_write_per_cycle; + n_write = (n_write >= 0)? n_write : 0; + } + + // test if the structure is done with all the required accesses + int all_access_done( ) { + return(n_read == 0 && n_write == 0); + } + + // sort the max heap again starting from start_idx + // (this entry can only go up in the tree to the root) + void sort_bottomup(int start_idx) { + maxheap_entry_t mh_entry; + maxheap_entry_t mh_parent; + + if (start_idx == 1) return; // no need to resort if the root is incremented + + int curr_idx = start_idx; + int parent_idx = parentof(start_idx); + + int continue_sort = 1; + while (curr_idx > 1 && continue_sort) { + mh_entry = get(curr_idx); + mh_parent = get(parent_idx); + + // swap the entries if it is now larger than it's parent + if (cmp_mh(mh_entry, mh_parent)) { + set(parent_idx, mh_entry); + set(curr_idx, mh_parent); + + // update the lut for this swap + if (mh_entry.lut_idx >= 0) + lut->update_mh_idx(mh_entry.lut_idx, parent_idx); + if (mh_parent.lut_idx >= 0) + lut->update_mh_idx(mh_parent.lut_idx, curr_idx); + + // update index for next iteration + curr_idx = parent_idx; + parent_idx = parentof(curr_idx); + } else { + // swap did not happen, no need to sort anymore + continue_sort = 0; + } + } + } + + void print_mh_e(FILE *fout, maxheap_entry_t mh_e) { + fprintf(fout, "[%08x]%03d(H%03dT%03d)p%02d | ", + mh_e.pc, mh_e.n_thds, mh_e.wpool_head, mh_e.wpool_tail, mh_e.lut_idx); + } + + void print(FILE *fout) { + fprintf(fout, "MaxHeap: "); + fprintf(fout, "N_entries = %d\n", n_entries); + for (int i=0; i 0); + return(mh_idx / 2); + } + + static inline int childof(int mh_idx) { + return(mh_idx * 2); + } + + static inline int cmp_mh(maxheap_entry_t &a, maxheap_entry_t &b) { + if (a.n_thds > b.n_thds) return 1; + if (a.n_thds == b.n_thds) { + if (a.pc < b.pc) return 1; + } + return 0; + } + +}; + +maxheap_entry_t maxheap_class::clean_entry = {0, 0, -1, -1, -1}; + +typedef struct mh_update_struct { + int n_maxheap_read; + int n_maxheap_write; + int n_mhlut_read; + int n_mhlut_write; +} mh_update; + +// heap implementation of majority policy +class issue_warp_majority_heap : public issue_warp_majority { +public: + + mh_lut_class mh_lut; + maxheap_class maxheap; + + maxheap_lut_entry_t major_lut_e; + maxheap_entry_t major_mh_e; + + vector *warp_pool; + int simd_width; + + int n_stall_on_maxheap; + + queue update_queue; + static pow2_histogram n_pending_updates_histo; + + issue_warp_majority_heap (int simd_width = 0, vector *bp = NULL, + int lut_size = 32, int lut_assoc = 4, int maxheap_size = 128, + int n_read_lut = 4, int n_write_lut = 4, + int n_read_mh = 4, int n_write_mh = 4) + : mh_lut(lut_size, lut_assoc, n_read_lut, n_write_lut), + maxheap(maxheap_size, &mh_lut, n_read_mh, n_write_mh) + { + this->simd_width = simd_width; + this->warp_pool = bp; + + this->major_lut_e = mh_lut_class::clean_entry; + this->major_mh_e = maxheap_class::clean_entry; + + this->n_stall_on_maxheap = 0; + } + + // adding more threads to a specify pc + // these threads may end up in different warpes + void add_threads( address_type pc, int *tid) { + int i; + int n_thds = 0; + for (i=0; i= 0) n_thds++; + } + + // handle special case with adding threads to current majority pc + if (major_lut_e.pc == pc) { + assert(major_mh_e.pc == pc); + major_mh_e.n_thds += n_thds; + return; + } + + maxheap_lut_entry_t lut_e; + maxheap_entry_t mh_entry; + + // snapshot the current maxheap read/write demand + mh_update new_mh_update; + new_mh_update.n_maxheap_read = maxheap.n_read; + new_mh_update.n_maxheap_write = maxheap.n_write; + new_mh_update.n_mhlut_read = mh_lut.n_read; + new_mh_update.n_mhlut_write = mh_lut.n_write; + + int lut_idx = mh_lut.lookup(pc); + + int sort_from_idx = 0; + + if (lut_idx >= 0) { + // obtain the entry + lut_e = mh_lut.get(lut_idx); + + // get the maxheap entry and update its number of threads + mh_entry = maxheap.get(lut_e.maxheap_idx); + mh_entry.n_thds += n_thds; + maxheap.set(lut_e.maxheap_idx, mh_entry); + + // sort from this specific entry + sort_from_idx = lut_e.maxheap_idx; + } else { + // create a new lut entry + lut_e = mh_lut_class::clean_entry; + lut_e.pc = pc; + + // get index to the LRU lut entry in this set + lut_idx = mh_lut.find_lru(lut_e); + + // get the replaced lut entry and remove its link with the maxheap entry + maxheap_lut_entry_t lut_old = mh_lut.get(lut_idx); + if (lut_old.maxheap_idx > 0) maxheap.remove_lut_idx(lut_old.maxheap_idx); + + // create a new maxheap entry + mh_entry = maxheap_class::clean_entry; + mh_entry.pc = pc; + mh_entry.n_thds = n_thds; + mh_entry.lut_idx = lut_idx; + + // push the new entry into the maxheap and lut respectively + lut_e.maxheap_idx = maxheap.insert(mh_entry); + mh_lut.replace_lru(lut_e); + + // start sorting from the bottom? + sort_from_idx = lut_e.maxheap_idx; + } + + maxheap.sort_bottomup(sort_from_idx); + + // record the newly generated maxheap read/write demand from this update + new_mh_update.n_maxheap_read = maxheap.n_read - new_mh_update.n_maxheap_read; + new_mh_update.n_maxheap_write = maxheap.n_write - new_mh_update.n_maxheap_write; + new_mh_update.n_mhlut_read = mh_lut.n_read - new_mh_update.n_mhlut_read; + new_mh_update.n_mhlut_write = mh_lut.n_write - new_mh_update.n_mhlut_write; + + update_queue.push(new_mh_update); + } + + // call this when a new warp allocated for a specific pc + void push_warp( address_type pc, int idx) { + maxheap_entry_t *p_mh_e = NULL; + maxheap_entry_t mh_e; + maxheap_lut_entry_t lut_e = mh_lut_class::clean_entry; + int lut_idx = -1; + + if (major_mh_e.pc == pc) { + p_mh_e = &major_mh_e; + } else { + lut_idx = mh_lut.lookup(pc); + assert(lut_idx >= 0); // if it is a miss, a new entry should have been created already + lut_e = mh_lut.get(lut_idx); + mh_e = maxheap.get(lut_e.maxheap_idx); + p_mh_e = &mh_e; + + // discounting these 'gets' + // because they should be combined with the 'gets' in add_threads() + mh_lut.n_read--; + maxheap.n_read--; + } + + if (p_mh_e->wpool_head == -1) { + p_mh_e->wpool_head = idx; + p_mh_e->wpool_tail = idx; + } else { + (*warp_pool)[p_mh_e->wpool_tail].next_warp = idx; + p_mh_e->wpool_tail = idx; + } + + if (major_mh_e.pc == pc) { + } else { + maxheap.set(lut_e.maxheap_idx, mh_e); + // discounting this 'set' + // because it should be combined with the 'set' in add_threads() + maxheap.n_write--; + } + } + + // obtain a warp index from this issue logic + int pop_warp( ) { + int bidx = -1; + if (major_mh_e.wpool_head == -1 && !maxheap.empty()) { + if (this->all_access_done( )) { + // pop the majority PC from max heap + major_mh_e = maxheap.pop_root(); + + // pop its corresponding entry from the lut as well (if it exists) + if (major_mh_e.lut_idx >= 0) { + major_lut_e = mh_lut.get(major_mh_e.lut_idx); + mh_lut.free(major_mh_e.lut_idx); + } else { + major_lut_e = mh_lut_class::clean_entry; + } + } else { + n_stall_on_maxheap += 1; + bidx = -1; + return bidx; + } + } + + // just pop and entry to from the virtual queue (and set the head pointer to next warp) + bidx = major_mh_e.wpool_head; + if (bidx >= 0) { + major_mh_e.wpool_head = (*warp_pool)[major_mh_e.wpool_head].next_warp; + } + + return bidx; + } + + void reset_access( ) { + maxheap.reset_access(); + mh_lut.reset_access(); + + while (!update_queue.empty()) { + update_queue.pop(); + } + } + + inline void consume_access( int &req_acc, int &avl_acc) { + if (req_acc > avl_acc) { + req_acc -= avl_acc; + avl_acc = 0; + } else { + avl_acc -= req_acc; + req_acc = 0; + } + } + + void clear_access( ) { + maxheap.clear_access(); + mh_lut.clear_access(); + + int n_maxheap_read_bw = maxheap.n_read_per_cycle; + int n_maxheap_write_bw = maxheap.n_write_per_cycle; + int n_mhlut_read_bw = mh_lut.n_read_per_cycle; + int n_mhlut_write_bw = mh_lut.n_write_per_cycle; + + while ((n_maxheap_read_bw > 0 || n_maxheap_read_bw > 0 || + n_mhlut_read_bw > 0 || n_mhlut_write_bw > 0) && !update_queue.empty()) { + mh_update &c_update = update_queue.front(); + + consume_access (c_update.n_maxheap_read, n_maxheap_read_bw); + consume_access (c_update.n_maxheap_write, n_maxheap_write_bw); + consume_access (c_update.n_mhlut_read, n_mhlut_read_bw); + consume_access (c_update.n_mhlut_write, n_mhlut_write_bw); + + if (c_update.n_maxheap_read == 0 && c_update.n_maxheap_write == 0 && + c_update.n_mhlut_read == 0 && c_update.n_mhlut_write == 0) { + update_queue.pop(); + } else { + break; + } + } + + n_pending_updates_histo.add2bin(update_queue.size()); + } + + void print( FILE *fout ) { + fprintf(fout, "LUT: "); + mh_lut.print_lut_e(fout, major_lut_e); + fprintf(fout, " \tMH: "); + maxheap.print_mh_e(fout, major_mh_e); + fprintf(fout, "\n"); + mh_lut.print(fout); + maxheap.print(fout); + } + + static void print_stat( FILE *fout) { + fprintf(fout, "n_pending_maxheap_updates = "); + n_pending_updates_histo.fprint(fout); + fprintf(fout, "\n"); + } + +private: + + int all_access_done( ) { + return(maxheap.all_access_done() && mh_lut.all_access_done()); + + } +}; +pow2_histogram issue_warp_majority_heap::n_pending_updates_histo; + +class warp_queue { +public: + int m_pc; + int n_thds; + int simd_width; + deque idx_queue; + + warp_queue( address_type pc, int simd_width) { + this->m_pc = pc; + this->n_thds = 0; + this->simd_width = simd_width; + } + + // called right after a lut_entry is looked up + void add_threads( int *tid ) { + for (int i=0; i= 0) this->n_thds++; + } + } + + // called right after a warp is issued + void sub_threads( int *tid ) { + for (int i=0; i= 0) this->n_thds--; + } + } + + // if other warp queue should be ahead + bool operator<(const warp_queue& other) const { + if (n_thds == other.n_thds) { + return(m_pc > other.m_pc); // smaller pc first + } else { + return(n_thds < other.n_thds); + } + } + bool operator>(const warp_queue& other) const { + if (n_thds == other.n_thds) { + return(m_pc > other.m_pc); // smaller pc first + } else { + return(n_thds > other.n_thds); + } + } + + void print( FILE *fout ) { + fprintf(fout, "0x%08x(%03d)=[", m_pc, n_thds); + deque::iterator dit = idx_queue.begin(); + for (; dit != idx_queue.end(); dit++) { + fprintf(fout, "%03d ", *dit); + } + fprintf(fout, "]\n"); + } +}; + +bool minor_warp( const warp_queue* a, const warp_queue* b ) { + return(*a<*b); +} + +// queue implementation of majority scheduling policy +class issue_warp_majority_queue : public issue_warp_majority { +public: + map majority_map; + set warpq_set; + warp_queue* maj_warp; + + vector *warp_pool; + int simd_width; + + issue_warp_majority_queue(int simd_width = 0, vector *bp = NULL) { + this->maj_warp = NULL; + this->simd_width = simd_width; + this->warp_pool = bp; + } + + // adding more threads to a specify pc + // these threads may end up in different warps + void add_threads( address_type pc, int *tid) { + warp_queue* bq = majority_map[pc]; + if (bq == NULL) { + bq = new warp_queue(pc,simd_width); + warpq_set.insert(bq); + majority_map[pc] = bq; + } + bq->add_threads(tid); + } + + // call this when a new warp allocated for a specific pc + void push_warp( address_type pc, int idx) { + warp_queue* bq = majority_map[pc]; + assert(bq != NULL); + bool check_redundant_idx = false; + if (check_redundant_idx) { + deque::iterator dit = find(bq->idx_queue.begin(), bq->idx_queue.end(), idx); + assert(dit == bq->idx_queue.end()); + } + bq->idx_queue.push_back(idx); + } + + // obtain a warp index from this issue logic + int pop_warp( ) { + int bidx = -1; + + // find the new majority pc if it didn't exist + if (maj_warp == NULL && warpq_set.size()) { + maj_warp = *max_element(warpq_set.begin(), warpq_set.end(), minor_warp); + } + + // if a majority pc indeed exist + if (maj_warp) { + assert(!maj_warp->idx_queue.empty()); + bidx = maj_warp->idx_queue.front(); + maj_warp->idx_queue.pop_front(); + maj_warp->sub_threads((*warp_pool)[bidx].tid); + + // when the majority pc runs out of thread + if (maj_warp->n_thds == 0) { + // remove that warp queue + warpq_set.erase(maj_warp); + majority_map.erase(maj_warp->m_pc); + delete maj_warp; + maj_warp = NULL; + } + } + + return bidx; + } + + void print( FILE *fout ) { + fprintf(fout, "issue_warp_majority:\n"); + set::iterator dit = warpq_set.begin(); + for (; dit != warpq_set.end(); dit++) { + fprintf(fout, " %c ", ((*dit)==maj_warp)? 'M':' '); + (*dit)->print(fout); + } + } + + void check_consistency( ) { + set::iterator set_it = warpq_set.begin(); + for (; set_it != warpq_set.end(); set_it++) { + warp_queue* bq = (*set_it); + + int real_nthds = 0; + deque::iterator dit = bq->idx_queue.begin(); + for (; dit != bq->idx_queue.end(); dit++) { + int *tid = (*warp_pool)[*dit].tid; + for (int i = 0; i < simd_width; i++) { + real_nthds += (tid[i] >= 0)? 1 : 0; + } + } + + assert(real_nthds == bq->n_thds); + } + } +}; + +// pdom priority +class lesspdom_first { +public: + vector *warp_pool; + lesspdom_first( vector *bp=NULL ) { + this->warp_pool = bp; + } + bool operator() (const int &idx_a, const int &idx_b) const { + if ((*warp_pool)[idx_a].pdom_prio != (*warp_pool)[idx_b].pdom_prio) { + return((*warp_pool)[idx_a].pdom_prio < (*warp_pool)[idx_b].pdom_prio); + } else { + return((*warp_pool)[idx_a].occ > (*warp_pool)[idx_b].occ); + } + } +}; + + +class issue_warp_pdom_prio { +public: + vector *warp_pool; + int* thd_pdom_prio; + int simd_width; + int n_threads; + + int resort_needed; + list pdom_pqueue; //the queue holding all index + + lesspdom_first lesspdom_cmp; + + static set reconvgence_pt; //table holding all recvg pt + + issue_warp_pdom_prio (int simd_width = 0, vector *bp = NULL, + int n_threads = 0) + : lesspdom_cmp(bp) + { + this->simd_width = simd_width; + this->warp_pool = bp; + this->n_threads = n_threads; + this->thd_pdom_prio = new int[n_threads]; + memset(this->thd_pdom_prio, 0, sizeof(int)*n_threads); + this->resort_needed = 0; + } + + ~issue_warp_pdom_prio( ) { + delete[] this->thd_pdom_prio; + } + + void reinit( ) { + memset(this->thd_pdom_prio, 0, sizeof(int)*n_threads); + } + + // adding more threads to a warp + void add_threads( int idx, address_type pc) { + assert((*warp_pool)[idx].pc == pc); + + // check to see if this is a newly allocated warp + bool check_pdom = false; + if ((*warp_pool)[idx].pdom_prio == -1) { + check_pdom = true; + } + + // check for newly assigned threads to the warp + int pdom_occ = (*warp_pool)[idx].pdom_occ; + int *tid = (*warp_pool)[idx].tid; + for (int i=0; i= 0 && !(pdom_occ & (1<= 0) { + thd_pdom_prio[(*warp_pool)[bidx].tid[i]] = (*warp_pool)[bidx].pdom_prio; + } + } + } + + return bidx; + } + +}; + +set issue_warp_pdom_prio::reconvgence_pt = set(); +//*/ + + +class npc_tracker_class { +public: + map pc_count; + unsigned* acc_pc_count; + int simd_width; + static map histogram; + + npc_tracker_class( ) { + this->acc_pc_count = NULL; + this->simd_width = 0; + } + + npc_tracker_class(unsigned* acc_pc_count, int simd_width) { + this->acc_pc_count = acc_pc_count; + this->simd_width = simd_width; + } + + void add_threads( int *tid, address_type pc ) { + for (int i=0; i= 0); + if (pc_count[pc] == 0) pc_count.erase(pc); // manually erasing entries with 0 count + } + } + } + + void update_acc_count( ) { + (*acc_pc_count) += pc_count.size(); + histogram[pc_count.size()] += 1; + } + + unsigned count( ) { return pc_count.size();} + + static void histo_print( FILE* fout ) { + map::iterator i; + fprintf(fout, "DYHW nPC Histogram: "); + for (i = histogram.begin(); i != histogram.end(); i++) { + fprintf(fout, "%d:%d ", i->first, i->second); + } + fprintf(fout, "\n"); + } +}; + +map npc_tracker_class::histogram; + +class pc_tag { +private: + + address_type m_pc; + +public: + + pc_tag () { + this->reset(); + } + + pc_tag (const pc_tag& p) { this->m_pc = p.m_pc;} + pc_tag (const address_type& other_pc) { this->m_pc = other_pc;} + + pc_tag& operator=(const pc_tag& p) { m_pc = p.m_pc; return *this;} + pc_tag& operator=(const address_type& other_pc) { m_pc = other_pc; return *this;} + + inline bool operator==(const pc_tag& p) const { return(m_pc == p.m_pc);} + inline bool operator==(const address_type& other_pc) const { return(m_pc == other_pc);} + + inline bool operator!=(const pc_tag& p) const { return(m_pc != p.m_pc);} + inline bool operator!=(const address_type& other_pc) const { return(m_pc != other_pc);} + + inline bool operator<(const pc_tag& p) const { return(m_pc < p.m_pc);} + + inline void reset() { + m_pc = -1; + } + + inline address_type get_pc() const { return m_pc;} + + // the hash function to warp LUT + inline unsigned lut_hash( int insn_size_lgb2, int lut_nsets ) const { + return(m_pc >> insn_size_lgb2) & (lut_nsets - 1); + } + + inline void to_print(char *buffer, unsigned length) { + snprintf(buffer, length, "0x%08x", m_pc); + } +}; + +template +class tag2warp_entry_t { +public: + + Tag tag; + int idx; // pointing to warp pool + int occ; // occupancy vector + int accessed; // is the entry accessed this cycle + + tag2warp_entry_t () { + this->reset(); + } + + ~tag2warp_entry_t () {} + + tag2warp_entry_t (const tag2warp_entry_t& p) { + this->tag = p.tag; + this->idx = p.idx; + this->occ = p.occ; + this->accessed = p.accessed; + } + + tag2warp_entry_t& operator=(const tag2warp_entry_t& p) { + if (this != &p) { + tag = p.tag; + idx = p.idx; + occ = p.occ; + accessed = p.accessed; + } + return *this; + } + + inline bool operator==(const tag2warp_entry_t& p) const { + return(tag == p.tag); + } + + inline bool operator==(const Tag& test_tag) const { + return(tag == test_tag); + } + + inline bool operator()(const tag2warp_entry_t& p) const { + return(tag == p.tag); + } + + inline void reset() { + tag.reset(); + idx = 0; + occ = 0; + accessed = 0; + } + + void print( FILE *fout ) { + static char buffer[20]; + tag.to_print(buffer,20); + fprintf(fout, "\t%s->%03d (%02x)\n", buffer, idx, occ); + } + +}; + +template +class tag2warp_set { +public: + vector< tag2warp_entry_t > entry; + list< tag2warp_entry_t* > lru_stack; + + tag2warp_set(int assoc = 0) : entry(assoc) { + for (unsigned j=0; jentry.size(); j++) { + this->lru_stack.push_back(&(this->entry[j])); + } + } + + tag2warp_set(const tag2warp_set& other) : entry(other.entry.size()) { + for (unsigned j=0; jentry.size(); j++) { + this->lru_stack.push_back(&(this->entry[j])); + } + } + + tag2warp_set& operator=(const tag2warp_set& p) { + printf("tag2warp_set assignment operator called!\n"); + return *this; + } + + ~tag2warp_set() {} +}; + +template +class warp_lut { +public: + virtual ~warp_lut() {} + virtual tag2warp_entry_t* lookup_pc2warp( const Tag& tag, bool& lut_missed ) = 0; + virtual void invalidate_entry( tag2warp_entry_t* lut_entry, int warp_idx ) = 0; + virtual void clear_accessed( ) = 0; + virtual void print( FILE* fout) = 0; +}; + +template +class warp_lut_sa : public warp_lut { +private: + int lut_size; + int lut_assoc; + vector< tag2warp_set > tag2warp_lut; + int insn_size_lgb2; + + queue< tag2warp_entry_t* > lut_accessed_q; // store accessed lut entry for clear + + struct same_tag { + Tag tag; + bool operator()(tag2warp_entry_t* a) { + return(a->tag == tag); + } + }; + + static unsigned int lut_aliased; + +public: + warp_lut_sa(int lut_size, int lut_assoc, int insn_size) { + this->lut_size = lut_size; + this->lut_assoc = lut_assoc; + + // optimize for LUT hash function + insn_size_lgb2 = 0; + while ( (1 << insn_size_lgb2) < insn_size ) insn_size_lgb2++; + + // initialize the pc2warp LUT + // note: lut_size is the absolute size of LUT regardless of assoc. + this->tag2warp_lut.assign(lut_size/lut_assoc, tag2warp_set(lut_assoc)); + + // assert on #set in LUT to be power of 2 + int lut_nset_pow2 = 1; + while ( lut_nset_pow2 < (int)tag2warp_lut.size() ) lut_nset_pow2 <<= 1; + assert((int)tag2warp_lut.size() == lut_nset_pow2); + } + + tag2warp_entry_t* lookup_pc2warp( const Tag& tag, bool& lut_missed ); + void invalidate_entry( tag2warp_entry_t* lut_entry, int warp_idx ) { + if (lut_entry != NULL) { // check for warp lut entry invalidation + if (lut_entry->idx == warp_idx) { + lut_entry->reset(); + } + } + } + + void clear_accessed( ); + + void print( FILE* fout) { + for (unsigned i=0; i< tag2warp_lut.size(); i++) { + for (unsigned j=0; j< tag2warp_lut[i].entry.size(); j++) { + fprintf(fout, "lut%03d-%02d:", i, j); + tag2warp_lut[i].entry[j].print(fout); + } + } + } + + static void print_stats ( FILE* fout ) { + fprintf( fout, "lut_aliased = %d\n", lut_aliased); + } +}; +template unsigned int warp_lut_sa::lut_aliased = 0; + + +// lookup function in LUT +// may return an entry that has different PC for replacement +// or return a NULL pointer to indicate that the entry is accessed by another port +template +tag2warp_entry_t* warp_lut_sa::lookup_pc2warp( const Tag &tag, bool &lut_missed ) +{ + tag2warp_entry_t* lut_entry = NULL; + unsigned hashed_pc = tag.lut_hash(insn_size_lgb2, tag2warp_lut.size()); + list< tag2warp_entry_t* > &hashed_lru_stack = tag2warp_lut.at(hashed_pc).lru_stack; + struct same_tag same_tag_f; + + same_tag_f.tag = tag; + typename list< tag2warp_entry_t* >::iterator lut_it; + lut_it = find_if(hashed_lru_stack.begin(), + hashed_lru_stack.end(), + same_tag_f); + if (lut_it != hashed_lru_stack.end()) { + lut_entry = *lut_it; + lut_entry->accessed = 1; + lut_accessed_q.push(lut_entry); + hashed_lru_stack.splice(hashed_lru_stack.end(), hashed_lru_stack, lut_it); + assert(lut_entry == hashed_lru_stack.back()); + lut_missed = false; + } else { + assert(!hashed_lru_stack.empty()); + lut_entry = hashed_lru_stack.front(); + if (lut_entry->accessed) { + lut_entry = NULL; + } else { + lut_entry->accessed = 1; + lut_accessed_q.push(lut_entry); + hashed_lru_stack.splice(hashed_lru_stack.end(), hashed_lru_stack, hashed_lru_stack.begin()); + assert(lut_entry == hashed_lru_stack.back()); + lut_aliased++; + } + lut_missed = true; + } + assert(hashed_lru_stack.size() == tag2warp_lut[hashed_pc].entry.size()); + + return lut_entry; +} + +template +void warp_lut_sa::clear_accessed( ) { + while ( !lut_accessed_q.empty() ) { + lut_accessed_q.front()->accessed = 0; + lut_accessed_q.pop(); + } +} + +// a perfect warp lut that never misses. +template +class warp_lut_perfect : public warp_lut { +private: + typedef map< Tag, tag2warp_entry_t* > warp_map_t; + warp_map_t m_tag2entry_map; + + static unsigned int lut_max_size; +public: + warp_lut_perfect() {} + ~warp_lut_perfect() { + typename warp_map_t::iterator mit = m_tag2entry_map.begin(); + for (; mit != m_tag2entry_map.end(); mit++) { + delete mit->second; + } + } + + // idealistic implementation of lookup: the entry is never aliased, + // and a new one is created automatically if it does not exist + tag2warp_entry_t* lookup_pc2warp( const Tag& tag, bool& lut_missed ) { + typename warp_map_t::iterator mit = m_tag2entry_map.find(tag); + + tag2warp_entry_t* lut_entry = NULL; + if (mit != m_tag2entry_map.end()) { + lut_entry = mit->second; + assert(lut_entry->tag == tag); + } else { + lut_entry = new tag2warp_entry_t(); + m_tag2entry_map.insert(make_pair(tag, lut_entry)); + } + + lut_missed = false; + lut_max_size = (lut_max_size < m_tag2entry_map.size())? m_tag2entry_map.size() : lut_max_size; + + return lut_entry; + } + + void invalidate_entry( tag2warp_entry_t* lut_entry, int warp_idx ) { + if (lut_entry == NULL) return; + if (lut_entry->idx != warp_idx) return; + + typename warp_map_t::iterator mit = m_tag2entry_map.find(lut_entry->tag); + if (mit != m_tag2entry_map.end()) { + assert(mit->second == lut_entry); + mit->second->reset(); + delete mit->second; + m_tag2entry_map.erase(mit); + } + } + + void clear_accessed( ) {} + + void print( FILE* fout) { + typename warp_map_t::iterator mit = m_tag2entry_map.begin(); + for (; mit != m_tag2entry_map.end(); mit++) { + mit->second->print(fout); + } + } + + static void print_stats ( FILE* fout ) { + fprintf( fout, "lut_max_size = %d\n", lut_max_size); + } +}; +template unsigned int warp_lut_perfect::lut_max_size = 0; + + +typedef tag2warp_entry_t warplut_entry_t; +typedef pc_tag warp_tag_t; + +class dwf_hw_sche_class { +public: + int m_id; + warp_lut *warp_lut_pc; + vector warp_pool; + deque free_warp_q; // the warp allocator + int simd_width; + int regf_width; + int insn_size_lgb2; + bool just_resume; + + vector m_req; // request vector from incoming warp + vector m_occ_new; // occupancy vector of the new warp, double as conflict vector + vector m_occ_upd; // occupancy vector of the updated existing warp + vector m_occ_ext; // occupancy vector of the existing warp + + dwf_hw_sche_class( int lut_size, int lut_assoc, + int simd_width, int regf_width, + int n_threads, int insn_size, + int heuristic, int id, + char *policy_opt = NULL ); + ~dwf_hw_sche_class(); + + warplut_entry_t* lookup_pc2warp( const warp_tag_t& lookup_tag ); + int update_warp( int* tid, address_type pc ); + + // barrier handling + int m_nbarriers; + class dwf_barrier { + public: + bool m_release; // see if a barrier is to be released (ie. all warp in cta hit already) + deque m_queue; // queue storing warps currently hitting a barrier, skipping warplut and scheduler + + dwf_barrier() : m_release(false) {} + dwf_barrier(const dwf_barrier& that) + : m_release(that.m_release), m_queue(that.m_queue) {} + bool ready_to_issue() { + return(m_release && !m_queue.empty()); + } + }; + set< int > m_cta_released_barrier; // set of cta with released barrier + map< int, dwf_barrier > m_barrier; // map + int update_warp_at_barrier( int* tid, address_type pc, int cta_id, int barrier_num = 0 ); + void hit_barrier( int cta_id, int barrier_num = 0 ); + void release_barrier( int cta_id, int barrier_num = 0 ); + + int allocate_warp( address_type pc, bool update_scheduler = true ); + void free_warp( int idx, bool update_warplut = true ); + + void issue_warp( int *tid, address_type *pc ); + + void clear_accessed( ) { + warp_lut_pc->clear_accessed(); + } + + void init_cta(int start_thread, int cta_size, address_type start_pc); + + void print_pc2warp_lut( FILE *fout ); + void print_warp_pool( FILE *fout ); + void print_free_warp_q( FILE *fout ); + + int heuristic; + + // FIFO warp issue logic + queue issue_warp_FIFO_q; + + // PC warp issue logic + class pc_first { + public: + vector &warp_pool; + pc_first( vector &bp ) : warp_pool(bp) {} + bool operator() (const int &idx_a, const int &idx_b) const { + if (warp_pool[idx_a].pc != warp_pool[idx_b].pc) { + return(warp_pool[idx_a].pc > warp_pool[idx_b].pc); + } else { + return(warp_pool[idx_a].occ < warp_pool[idx_b].occ); + } + } + }; + pc_first mypc_first; + priority_queue, pc_first > issue_warp_PC_q; + + // Majority warp issue logic + issue_warp_majority *issue_warp_MAJ; + void clear_policy_access( ); + void reset_policy_access( ); + + // PDOM Priority issue logic + issue_warp_pdom_prio issue_warp_pdom; + + // statistics + npc_tracker_class npc_tracker; + int max_warppool_occ; + int *warppool_occ_histo; // histogram of warppool occupancy + static unsigned int lut_realmiss; + static unsigned int uid_cnt; + static unsigned int warp_fragmentation; + static unsigned int warp_merge_conflict; + static void print_stats ( FILE* fout ) { + warp_lut_perfect::print_stats( fout ); + warp_lut_sa::print_stats( fout ); + fprintf( fout, "lut_realmiss = %d\n", lut_realmiss); + fprintf( fout, "warp_fragmentation = %d\n", warp_fragmentation); + fprintf( fout, "warp_merge_conflict = %d\n", warp_merge_conflict); + } +}; + +unsigned int dwf_hw_sche_class::lut_realmiss = 0; +unsigned int dwf_hw_sche_class::uid_cnt = 0; +unsigned int dwf_hw_sche_class::warp_fragmentation = 0; +unsigned int dwf_hw_sche_class::warp_merge_conflict = 0; + + +dwf_hw_sche_class::dwf_hw_sche_class( int lut_size, int lut_assoc, + int simd_width, int regf_width, + int n_threads, int insn_size, + int heuristic, int id, + char *policy_opt ) +: m_id(id), +// WarpLUT w/ pc tag +warp_lut_pc( (lut_size == 0)? (warp_lut *) new warp_lut_perfect() : + (warp_lut *) new warp_lut_sa(lut_size, lut_assoc, insn_size) ), +m_nbarriers(1), // for barrier +mypc_first( warp_pool ), issue_warp_PC_q( mypc_first ), // DPC +issue_warp_pdom(simd_width, &warp_pool, n_threads), // DPdPri +npc_tracker( NULL, simd_width ) +{ + unsigned i; + + this->simd_width = simd_width; + this->regf_width = regf_width; + this->m_req.resize(regf_width); + this->m_occ_new.resize(regf_width); + this->m_occ_upd.resize(regf_width); + this->m_occ_ext.resize(regf_width); + + // initialize the warp pool + // (make sure the thread id's are init to -1) + this->warp_pool.resize(n_threads); + for (i=0; iheuristic = heuristic; + switch (heuristic) { + case MAJORITY: + issue_warp_MAJ = new issue_warp_majority_queue(simd_width, &warp_pool); + break; + case MAJORITY_MAXHEAP: { + int mh_lut_size = 32; + int mh_lut_assoc = 4; + int n_reads_per_cycle_lut = 4; + int n_writes_per_cycle_lut = 4; + int mh_size = 128; + int n_reads_per_cycle_mh = 4; + int n_writes_per_cycle_mh = 4; + if (policy_opt != NULL) { + sscanf(policy_opt, ";LUT=%d:%dr%dw%d;MH=%dr%dw%d", + &mh_lut_size, &mh_lut_assoc, &n_reads_per_cycle_lut, &n_writes_per_cycle_lut, + &mh_size, &n_reads_per_cycle_mh, &n_writes_per_cycle_mh); + } + issue_warp_MAJ = new issue_warp_majority_heap(simd_width, &warp_pool, + mh_lut_size, mh_lut_assoc, mh_size, + n_reads_per_cycle_lut, n_writes_per_cycle_lut, + n_reads_per_cycle_mh, n_writes_per_cycle_mh); + } + break; + } + + this->just_resume = false; + + this->max_warppool_occ = 0; + this->warppool_occ_histo = new int[n_threads]; + memset(this->warppool_occ_histo, 0, n_threads*sizeof(int)); +} + +// should never be called (only at exit?) +dwf_hw_sche_class::~dwf_hw_sche_class( ) +{ + unsigned i; + + for (i=0; iwarppool_occ_histo; + + delete warp_lut_pc; +} + +// allocate a new warp in warp pool +int dwf_hw_sche_class::allocate_warp( address_type pc, bool update_scheduler ) +{ + int idx; + assert(!free_warp_q.empty()); + idx = free_warp_q.front(); + free_warp_q.pop_front(); + warp_pool[idx].uid = uid_cnt; + uid_cnt++; + warp_pool[idx].pc = pc; + warp_pool[idx].next_warp = -1; + warp_pool[idx].lut_ptr = NULL; + + if (update_scheduler) { + if (heuristic == FIFO) issue_warp_FIFO_q.push(idx); + if (heuristic == PC) issue_warp_PC_q.push(idx); + if (heuristic == MAJORITY || heuristic == MAJORITY_MAXHEAP) + issue_warp_MAJ->push_warp(pc, idx); + if (heuristic == PDOMPRIO) issue_warp_pdom.push_warp(pc, idx); + } + + return idx; +} + +// free a warp in warp pool +// it will reset the content of the warp entry as well +void dwf_hw_sche_class::free_warp( int idx, bool update_warplut ) +{ + bool redundant_idx_check = false; + if (redundant_idx_check) { + deque::iterator dit = find(free_warp_q.begin(), free_warp_q.end(), idx); + assert(dit == free_warp_q.end()); + } + + warp_pool[idx].pc = -1; + memset(warp_pool[idx].tid, -1, sizeof(int)*simd_width); + warp_pool[idx].occ = 0; + warp_pool[idx].next_warp = -1; + if (update_warplut) { + warp_lut_pc->invalidate_entry( (warplut_entry_t*)warp_pool[idx].lut_ptr, idx ); + } + + free_warp_q.push_back(idx); + assert(free_warp_q.size() <= warp_pool.size()); +} + +warplut_entry_t* dwf_hw_sche_class::lookup_pc2warp( const warp_tag_t& lookup_tag ) +{ + bool lut_missed = false; + + warplut_entry_t* lut_entry; + lut_entry = warp_lut_pc->lookup_pc2warp( lookup_tag, lut_missed ); + + if (!lut_missed) { + if (lut_entry->tag != warp_pool[lut_entry->idx].pc) lut_missed = true; + } + + if (lut_missed) { + if (npc_tracker.pc_count.find(lookup_tag.get_pc()) != npc_tracker.pc_count.end()) { + lut_realmiss++; // ie. the incoming warp lost an opportunity to merge + } + } + + return lut_entry; +} + + +void fill_all (vector& container, const char& value) +{ + fill(container.begin(), container.end(), value); +} + +int regfile_hash(signed istream_number, unsigned simd_size, unsigned n_banks); +int dwf_hw_sche_class::update_warp( int *tid, address_type pc ) +{ + int i; + bool newwarp = false; + bool newwarp_alloc = false; + warplut_entry_t* lut_entry; + warp_tag_t warp_tag(pc); + lut_entry = lookup_pc2warp(warp_tag); + + // no LUT entry returned, stall + if (!lut_entry) { + assert(0); + } + + if (heuristic == MAJORITY || heuristic == MAJORITY_MAXHEAP) { + issue_warp_MAJ->add_threads(pc, tid); + } + + npc_tracker.add_threads( tid, pc ); + + // if the pc of the LUT entry does not match, + // allocate a new entry + if (lut_entry->tag != warp_tag) { + lut_entry->idx = allocate_warp(pc); + lut_entry->tag = warp_tag; + lut_entry->occ = 0; + assert(warp_pool[lut_entry->idx].pc == pc); + newwarp = true; + newwarp_alloc = true; + } + + // create the request vector + bool tid_has_valid_entry = false; + fill_all(m_req, 0); + for (i = 0; iidx; + + // create the conflict vector + fill_all(m_occ_ext, 0); + int regf_mask = regf_width - 1; + for (i = 0; iocc & (1 << i)) == 0)? 0 : 1; + } + fill_all(m_occ_upd, 0); + fill_all(m_occ_new, 0); + int n_regf_slot = simd_width / regf_width; + bool conflict = false; + for (i = 0; i n_regf_slot) { + m_occ_new[i] = m_occ_ext[i] + m_req[i] - n_regf_slot; + m_occ_upd[i] = n_regf_slot - m_occ_ext[i]; + conflict = true; + } else { + m_occ_upd[i] = m_req[i]; + } + } + + // if the pc of the warp mismatch with lut, + // set conflict vector to all one. + // that force all threads to the newly allocated warp + if (warp_pool[old_idx].pc != pc) { + conflict = true; + for (i = 0; iidx = new_idx; + lut_entry->occ = 0; //update the lut_entry + assert(warp_pool[new_idx].pc == pc); + + int total_occ = 0; + for (i = 0; i < regf_width; i++) + total_occ += m_occ_ext[i] + m_req[i]; + if (total_occ <= simd_width) warp_fragmentation += 1; + warp_merge_conflict += 1; + + newwarp_alloc = true; + } + + // update the warp as indicated by the LUT + // if the lane is conflicted, or the old warp is just not + // write to the new warp + int new_occ = 0; + fill_all(m_occ_new, 0); + for (i = 0; iocc |= (1<occ = new_occ; + issue_warp_pdom.add_threads(new_idx, pc); + } + + warp_pool[lut_entry->idx].lut_ptr = lut_entry; // link up the lut entry and warp + + bool scheduler_consistency_check = false; + if (scheduler_consistency_check && heuristic == MAJORITY) { + ((issue_warp_majority_queue*)issue_warp_MAJ)->check_consistency(); + } + + return 1; +} + +// called AFTER threads hit a barrier to insert them into the barrier queue +// ASSUME: threads from released barrier are not hitting second barrier right away +int dwf_hw_sche_class::update_warp_at_barrier( int* tid, address_type pc, int cta_id, int barrier_num ) +{ + assert(barrier_num < m_nbarriers); + assert(cta_id >= 0); + + int i; + int warp_index = 0xDEADBEEF; + + npc_tracker.add_threads( tid, pc ); + + // always allocate new warp + warp_index = allocate_warp(pc, false); + assert(warp_pool[warp_index].pc == pc); + + // no need to create the request vector + // no need to create the conflict vector + + // assign threads into the new warp + fill_all(m_occ_ext, 0); + int max_nthreads_per_rfbank = simd_width / regf_width; + for (i = 0; i= 0); + + m_barrier[cta_id].m_release = false; +} + +// called at decode stage when all thread in cta hit the barrier +// ASSUME: threads from released barrier are not hitting second barrier right away +void dwf_hw_sche_class::release_barrier( int cta_id, int barrier_num ) +{ + assert(barrier_num < m_nbarriers); + assert(cta_id >= 0); + + map::iterator i_barrier = m_barrier.find(cta_id); + assert(i_barrier != m_barrier.end()); // barrier has to exists in the first place! + i_barrier->second.m_release = true; +} + +void dwf_hw_sche_class::issue_warp( int *tid, address_type *pc ) +{ + int i; + bool warp_issued = false; + + // scan the released barriers for ready warp + // TODO: arbitrate between different queues? + set::iterator i_ctabar = m_cta_released_barrier.begin(); + for (; i_ctabar != m_cta_released_barrier.end(); ++i_ctabar) { + int cta_id = *i_ctabar; + map::iterator i_barrier = m_barrier.find(cta_id); + + if ( i_barrier->second.ready_to_issue() ) { + int warp_idx = i_barrier->second.m_queue.front(); + + for (i = 0; i < simd_width; i++) { + tid[i] = warp_pool[warp_idx].tid[i]; + } + *pc = warp_pool[warp_idx].pc; + + i_barrier->second.m_queue.pop_front(); + free_warp(warp_idx, false); // don't update warplut as the warp is not linked to it + + // remove cta from checking list if the queue is emptied + // (if the last threads haven't made it back to scheduler in time, + // update_warp_at_barrier will insert the cta id again) + if (i_barrier->second.m_queue.empty()) { + m_cta_released_barrier.erase(i_ctabar); + } + + warp_issued = true; + + break; + } + } + + if (!warp_issued) { + switch (heuristic) { + case FIFO: + // Oldest warp are issued first + if (!issue_warp_FIFO_q.empty()) { + int idx = issue_warp_FIFO_q.front(); + for (i = 0; i < simd_width; i++) { + tid[i] = warp_pool[idx].tid[i]; + } + *pc = warp_pool[idx].pc; + + issue_warp_FIFO_q.pop(); + free_warp(idx); + } else { + memset(tid, -1, sizeof(int)*simd_width); + *pc = -1; + } + break; + case PC: + // lowest PC warp are issued first + if (!issue_warp_PC_q.empty()) { + int idx = issue_warp_PC_q.top(); + for (i = 0; i < simd_width; i++) { + tid[i] = warp_pool[idx].tid[i]; + } + *pc = warp_pool[idx].pc; + + issue_warp_PC_q.pop(); + free_warp(idx); + } else { + memset(tid, -1, sizeof(int)*simd_width); + *pc = -1; + } + break; + case MAJORITY: + case MAJORITY_MAXHEAP: + // issue the most common PC first + { + int idx = issue_warp_MAJ->pop_warp(); + if (idx >= 0) { + for (i = 0; i < simd_width; i++) { + tid[i] = warp_pool[idx].tid[i]; + } + *pc = warp_pool[idx].pc; + free_warp(idx); + } else { + memset(tid, -1, sizeof(int)*simd_width); + *pc = -1; + } + } + break; + case PDOMPRIO: + // issue the warp with lowest PDOM count + { + int idx = issue_warp_pdom.front_warp(); + if (idx >= 0) { + issue_warp_pdom.pop_warp(); + + for (i = 0; i < simd_width; i++) { + tid[i] = warp_pool[idx].tid[i]; + } + *pc = warp_pool[idx].pc; + free_warp(idx); + + just_resume = false; + } else { + memset(tid, -1, sizeof(int)*simd_width); + *pc = -1; + } + } + break; + default: + printf("Unsupported Heuristics!\n"); + abort(); + break; + } + } + + npc_tracker.sub_threads( tid, *pc ); + + int warppool_occ = warp_pool.size() - free_warp_q.size(); + if (max_warppool_occ < warppool_occ) { + max_warppool_occ = warppool_occ; + } + warppool_occ_histo[warppool_occ] += 1; +} + +void dwf_hw_sche_class::init_cta(int start_thread, int cta_size, address_type start_pc) +{ + assert((start_thread % simd_width) == 0); // thread id starting at a warp + + int n_warp_2assign = cta_size / simd_width; + n_warp_2assign += (cta_size % simd_width)? 1 : 0; // round up + + static int *thd_id = NULL; + if (thd_id == NULL) thd_id = new int[simd_width]; + + for (int w = 0; w < n_warp_2assign; w++) { + // generate the warp update register for each warp + fill_n(thd_id, simd_width, -1); + int warp_start_tid = start_thread + w * simd_width; + for (int i = 0; (i < simd_width) && (warp_start_tid + i) < (start_thread + cta_size); i++) { + thd_id[i] = warp_start_tid + i; + } + + // push these warps into DWF scheduler + update_warp( thd_id, start_pc ); + } +} + +void dwf_hw_sche_class::print_free_warp_q( FILE *fout ) +{ + fprintf(fout, "free_node_q (%zd)= ", free_warp_q.size() ); + deque::iterator dit = free_warp_q.begin(); + for (; dit != free_warp_q.end(); dit++) { + fprintf(fout, "%03d ", *dit); + } + fprintf(fout, "\n"); +} + +void print_warp( FILE *fout, warp_entry_t warp_e, int simd_width ) +{ + fprintf(fout, "\t%02d 0x%08x: (", warp_e.pdom_prio, warp_e.pc ); + for (int i=0;iclear_access( ); + } +} + +void dwf_hw_sche_class::reset_policy_access( ) { + if (heuristic == MAJORITY_MAXHEAP) { + ((issue_warp_majority_heap*)issue_warp_MAJ)->reset_access( ); + } +} + +/////////////////////////////////////////////////////////////////////////// +// c-wrapper interface +/////////////////////////////////////////////////////////////////////////// + +int dwf_hw_n_sche = 0; +dwf_hw_sche_class **dwf_hw_sche; +unsigned *acc_dyn_pcs = NULL; + +void create_dwf_schedulers( int n_shaders, + int lut_size, int lut_assoc, + int simd_width, int regf_width, + int n_threads, int insn_size, + int heuristic, + char *policy_opt ) +{ + dwf_hw_n_sche = n_shaders; + dwf_hw_sche = new dwf_hw_sche_class*[n_shaders]; + for (int i=0; inpc_tracker.acc_pc_count = &acc_dyn_pcs[i]; + } +} + +int dwf_update_warp( int shd_id, int* tid, address_type pc ) +{ + return dwf_hw_sche[shd_id]->update_warp( tid, pc ); +} + +int dwf_update_warp_at_barrier( int shd_id, int* tid, address_type pc, int cta_id ) +{ + return dwf_hw_sche[shd_id]->update_warp_at_barrier( tid, pc, cta_id); +} + +void dwf_hit_barrier( int shd_id, int cta_id ) +{ + dwf_hw_sche[shd_id]->hit_barrier( cta_id ); +} + +void dwf_release_barrier( int shd_id, int cta_id ) +{ + dwf_hw_sche[shd_id]->release_barrier( cta_id ); +} + +void dwf_issue_warp( int shd_id, int *tid, address_type *pc ) +{ + dwf_hw_sche[shd_id]->issue_warp( tid, pc ); +} + +void dwf_clear_accessed( int shd_id ) +{ + dwf_hw_sche[shd_id]->clear_accessed( ); +} + +void dwf_clear_policy_access( int shd_id ) +{ + dwf_hw_sche[shd_id]->clear_policy_access( ); +} + +void dwf_reset_policy_access( int shd_id ) +{ + dwf_hw_sche[shd_id]->reset_policy_access( ); +} + +void dwf_init_CTA(int shd_id, int start_thread, int cta_size, address_type start_pc) +{ + dwf_hw_sche[shd_id]->init_cta(start_thread, cta_size, start_pc); + dwf_hw_sche[shd_id]->clear_accessed( ); + dwf_hw_sche[shd_id]->clear_policy_access( ); +} + +void dwf_print_stat( FILE* fout ) +{ + dwf_hw_sche_class::print_stats( fout ); + npc_tracker_class::histo_print( fout ); + fprintf(fout, "max_warppool_occ = "); + for (int i=0; imax_warppool_occ); + } + fprintf(fout, "\n"); + for (int i=0; iwarppool_occ_histo[j]); + } + fprintf(fout, "\n"); + } + if (dwf_hw_sche[0]->heuristic == MAJORITY_MAXHEAP) { + fprintf(fout, "n_stall_on_maxheap = "); + for (int i=0; iissue_warp_MAJ)->n_stall_on_maxheap); + } + fprintf(fout, "\n"); + fprintf(fout, "maxheap_n_entries = "); + for (int i=0; iissue_warp_MAJ)->maxheap.max_n_entries); + } + fprintf(fout, "\n"); + fprintf(fout, "maxheap_lut_n_aliased = "); + for (int i=0; iissue_warp_MAJ)->mh_lut.n_aliased); + } + fprintf(fout, "\n"); + issue_warp_majority_heap::print_stat(fout); + } +} + +void dwf_reset_reconv_pt() +{ + issue_warp_pdom_prio::reconvgence_pt.clear(); +} + +void dwf_insert_reconv_pt(address_type pc) +{ + issue_warp_pdom_prio::reconvgence_pt.insert(pc); +} + +void dwf_reinit_schedulers( int n_shaders ) +{ + for (int i=0; iissue_warp_pdom.reinit(); + } +} + +void dwf_update_statistics( int shader_id ) +{ + dwf_hw_sche[shader_id]->npc_tracker.update_acc_count(); +} + +void g_print_dmaj_scheduler(int sid) { + dwf_hw_sche[sid]->issue_warp_MAJ->print(stdout); +} + +void g_print_warp_lut(int sid) { + dwf_hw_sche[sid]->warp_lut_pc->print(stdout); +} + +void g_print_free_warp_q(int sid) { + dwf_hw_sche[sid]->print_free_warp_q(stdout); +} + +void g_print_warp_pool(int sid) { + dwf_hw_sche[sid]->print_warp_pool(stdout); +} + +void g_print_max_heap(int sid) { + dwf_hw_sche[sid]->issue_warp_MAJ->print(stdout); +} + +#ifdef UNIT_TEST + + #undef UNIT_TEST + #include "stat-tool.cc" + +unsigned gpgpu_thread_swizzling = 0; +unsigned long long gpu_sim_cycle = 0; + +int regfile_hash(signed istream_number, unsigned simd_size, unsigned n_banks) { + if (gpgpu_thread_swizzling) { + signed warp_ID = istream_number / simd_size; + return((istream_number + warp_ID) % n_banks); + } else { + return(istream_number % n_banks); + } +} + +int log2i(int n) { + int lg; + lg = -1; + while (n) { + n>>=1;lg++; + } + return lg; +} + +int test_FIFO() +{ + dwf_hw_sche_class *dwf_sche; + int i; + int tid[6][4] = { + { 0, 1, 2, 3}, + { 4, 5, 6, 7}, + { 8,-1,10,-1}, + {-1, 1,-1, 3}, + { 4, 9,-1,11}, + {-1,13,14,-1} + }; + + int expect_out[12][4] = { + { 0, 1, 2, 3}, + { 0, 1, 2, 3}, + { 0, 1, 2, 3}, + { 4, 5, 6, 7}, + { 8, 1,10, 3}, + { 4, 9,14,11}, + {-1,13,-1,-1}, + { 4, 9,-1,11}, + {-1,13,14,-1}, + { 8,-1,10,-1}, + { 4, 9,14,11}, + { 8,13,10,-1} + }; + + int tid_out[4]; + address_type pc_out; + + dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, FIFO); + + // same threads - different pc + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[0], 0x409a80); + dwf_sche->update_warp(tid[0], 0x409a88); + + // different threads - different pc + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[0], 0x409a90); + dwf_sche->update_warp(tid[1], 0x409a80); + + // different threads - same pc + // expect two warp to merge into one as there is no lane conflict + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[2], 0x409a90); + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[3], 0x409a90); + + // same as above, but with lane conflict + // expect a new warp allocated, + // but only the conflicting threads goes to new warp + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[4], 0x409a80); + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[5], 0x409a80); + + // different threads - different pc + // purposely try to alias an existing mapping + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[4], 0x410a80); + dwf_sche->update_warp(tid[5], 0x411a80); + + // going back to that mapping + // a new warp should be allocated (despite lack of conflict) + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[2], 0x409a80); + + // testing the occupancy vector + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[4], 0x409aa0); + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[5], 0x409aa0); + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[2], 0x409aa0); + + // fill the warp pool up + for (i=12; i<64; ) { + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[1], 0x409a80 + 8 * i++); + dwf_sche->update_warp(tid[4], 0x409a80 + 8 * i++); + } + // issue all the warp (do some auto checking on the way) + for (i=0; i<64; i++) { + dwf_sche->issue_warp(tid_out, &pc_out); + printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]); + if (i<12) { + if ( memcmp(tid_out, expect_out[i], 4*sizeof(int) ) ) { + printf("%d warp mismatches\n", i); + assert(0); + } + } + } + + // now that all warpes are issue, no entries in the lut is valid + // updating warp with an old address that remains in the lut + // to see if detects the invalid lut entry + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[2], 0x409a80 + 8 * 63); + dwf_sche->update_warp(tid[3], 0x409a80 + 8 * 62); + dwf_sche->issue_warp(tid_out, &pc_out); + assert(!memcmp(tid_out, tid[2], 4*sizeof(int) )); + dwf_sche->issue_warp(tid_out, &pc_out); + assert(!memcmp(tid_out, tid[3], 4*sizeof(int) )); + + dwf_sche->print_warp_pool(stdout); + dwf_sche->warp_lut_pc->print(stdout); + dwf_hw_sche_class::print_stats(stdout); + + delete dwf_sche; + + return 0; +} + +int test_PC () +{ + dwf_hw_sche_class *dwf_sche; + int i; + int tid[4][4] = { + { 0, 1, 2, 3}, + { 4, 5, 6, 7}, + { 8,-1,10,-1}, + {-1,13,14,-1} + }; + + int tid_out[4]; + address_type pc_out; + + dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, PC); + + // fill the warp pool up in reverse PC order + for (i=0; i<4; i++) { + for (int j=0; j<4; j++) { + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[j], 0x409a80 - 8 * i); + } + } + + // issue the warps, expect them to be in PC order, with higher occ warp issued first + printf("PC Issue Logic:\n"); + for (i=0; i<4; i++) { + for (int j=0; j<4; j++) { + dwf_sche->issue_warp(tid_out, &pc_out); + printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]); + } + } + +} + +int test_MAJ () +{ + dwf_hw_sche_class *dwf_sche; + int i; + int tid[4][4] = { + { 0, 1, 2, 3}, + { 4, 5, 6, 7}, + { 8,-1,10,-1}, + {-1,13,14,-1} + }; + + int tid_out[4]; + address_type pc_out; + + dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, MAJORITY); + + // fill the warp pool up in reverse PC order + for (i=0; i<4; i++) { + for (int j=0; j<(4-i); j++) { + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[j], 0x409a80 - 8 * i); + } + } + + // issue the warps, expect them to be in PC order, with higher occ warp issued first + printf("Majority Issue Logic:\n"); + for (i=0; i<4; i++) { + for (int j=0; j<4; j++) { + dwf_sche->issue_warp(tid_out, &pc_out); + printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]); + } + } +} + +int test_MAJ_HEAP () +{ + printf("\ntest_MAJ_HEAP:\n"); + dwf_hw_sche_class *dwf_sche; + int i; + int tid[4][4] = { + { 0, 1, 2, 3}, + { 4, 5, 6, 7}, + { 8,-1,10,-1}, + {-1,13,14,-1} + }; + + int tid_out[4]; + address_type pc_out; + + dwf_sche = new dwf_hw_sche_class(16, 2, 4, 4, 16, 1, MAJORITY_MAXHEAP); + + // fill the warp pool up in reverse PC order + for (i=0; i<4; i++) { + for (int j=0; j<(i+1); j++) { + dwf_sche->clear_accessed(); + dwf_sche->update_warp(tid[j], 0x409a80 + 8 * i); + } + } + + dwf_sche->reset_policy_access(); + dwf_sche->issue_warp_MAJ->print(stdout); + + // issue the warps, expect them to be in PC order, with higher occ warp issued first + printf("Majority (Max Heap) Issue Logic:\n"); + for (i=0; i<4; i++) { + for (int j=0; j<4; j++) { + dwf_sche->issue_warp(tid_out, &pc_out); + printf("0x%08x [%d %d %d %d]\n", pc_out, tid_out[0], tid_out[1], tid_out[2], tid_out[3]); + } + dwf_sche->reset_policy_access(); + } +} + +void test_warp_lut_pc () +{ + printf("\ntest_warp_lut_pc:\n"); + warp_lut_sa warp_lut_pc(16, // size + 4, // assoc + 1); // insn_size + + address_type pc_value[] = {0, 4, 0, 8, 12, 16, 20, 8, 8, 0}; + int n_entry = sizeof(pc_value) / sizeof(address_type); + vector pc_stream(pc_value, pc_value + n_entry); + + int misses = 0; + for (int n = 0; n < n_entry * 100; n++) { + int i = n % n_entry; + tag2warp_entry_t *lut_entry = NULL; + bool lut_miss = false; + + lut_entry = warp_lut_pc.lookup_pc2warp(pc_stream[i], lut_miss); + + if (lut_entry->tag != pc_stream[i]) { + lut_entry->tag = pc_stream[i]; + lut_entry->occ = 1; + misses += 1; + } + warp_lut_pc.clear_accessed(); + lut_entry->accessed = 0; + } + + printf("Number of Miss = %d\n", misses); +} + +int main () { + //test_FIFO(); + //test_PC(); + //test_MAJ(); + test_MAJ_HEAP(); + test_warp_lut_pc(); + return 0; +} + +#endif diff --git a/src/gpgpu-sim/dwf.h b/src/gpgpu-sim/dwf.h new file mode 100644 index 0000000..6328f1a --- /dev/null +++ b/src/gpgpu-sim/dwf.h @@ -0,0 +1,118 @@ +/* + * dwf.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef dwf_h_INCLUDED +#define dwf_h_INCLUDED + +#ifdef __cplusplus + + #include + #include + #include + #include + #include + #include + #include + +#endif + +#include "../util.h" + +extern unsigned *acc_dyn_pcs; + +void create_dwf_schedulers( int n_shaders, + int lut_size, int lut_assoc, + int simd_width, int regf_width, + int n_threads, int insn_size, + int heuristic, + char *policy_opt ); + +int dwf_update_warp( int shd_id, int* tid, address_type pc ); + +void dwf_issue_warp( int shd_id, int *tid, address_type *pc ); + +void dwf_clear_accessed( int shd_id ); + +void dwf_clear_policy_access( int shd_id ); +void dwf_reset_policy_access( int shd_id ); + +int dwf_update_warp_at_barrier( int shd_id, int* tid, address_type pc, int cta_id ); +void dwf_hit_barrier( int shd_id, int cta_id ); +void dwf_release_barrier( int shd_id, int cta_id ); + +void dwf_init_CTA(int shd_id, int start_thread, int cta_size, address_type start_pc); + +void dwf_print_stat( FILE* fout ); + +void dwf_reset_reconv_pt(); +void dwf_insert_reconv_pt(address_type pc); + +void dwf_reinit_schedulers( int n_shaders ); + +void dwf_set_accPC( int n_shaders, unsigned *acc_pc_count ); + +void dwf_update_statistics( int shader_id ); + +#endif diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc new file mode 100644 index 0000000..679d1d2 --- /dev/null +++ b/src/gpgpu-sim/gpu-cache.cc @@ -0,0 +1,609 @@ +/* + * gpu-cache.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "gpu-cache.h" +#include "gpu-misc.h" +#include "addrdec.h" +#include +#include + +// both shd_cache_access and shd_cache_probe functions use +// shd_cache_access_internal to access/probe cache +shd_cache_line_t* shd_cache_access_internal( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle, + unsigned int real_access); + +shd_cache_t * shd_cache_create( char *name, + unsigned int nset, + unsigned int assoc, + unsigned int line_sz, + unsigned char policy, + unsigned int hit_latency, + unsigned long long int bank_mask, + enum cache_write_policy wp) { + + shd_cache_t *cp; + unsigned int nlines; + + unsigned int i; + + if (!nset || !assoc) { + printf("Creating non-existing cache!\n"); + return 0; + } + + nlines = nset * assoc; + cp = (shd_cache_t*) calloc(1, sizeof(shd_cache_t)); + + cp->bank_mask = bank_mask; + cp->name = (char*) malloc(sizeof(char) * (strlen(name) + 1)); + strcpy(cp->name, name); + cp->nset = nset; + cp->nset_log2 = LOGB2(nset); + cp->assoc = assoc; + cp->line_sz = line_sz; + cp->line_sz_log2 = LOGB2(line_sz); + cp->policy = policy; + cp->hit_latency = hit_latency; + cp->lines = (shd_cache_line_t*) calloc(nlines, sizeof(shd_cache_line_t)); + cp->write_policy = wp; + + for (i=0; ilines[i].line_sz = line_sz; + cp->lines[i].status = 0; + } + + // don't hook up with any logger + cp->core_id = -1; + cp->type_id = -1; + + // initialize snapshot counters for visualizer + cp->prev_snapshot_access = 0; + cp->prev_snapshot_miss = 0; + cp->prev_snapshot_merge_hit = 0; + +// printf("%s: %d(%x) x %d x %d(%x) %c, %d\n", +// cp->name, cp->nset, cp->nset_log2, cp->assoc, cp->line_sz, +// cp->line_sz_log2, cp->policy, nlines); + + return cp; +} + +void shd_cache_destroy( shd_cache_t* cp ) { + + free(cp->lines); + free(cp); +} + +extern void shader_cache_miss_log( int logger_id, int type ); +// hook up with shader core logger +void shd_cache_bind_logger(shd_cache_t* cp, int core_id, int type_id) { + cp->core_id = core_id; + cp->type_id = type_id; +} + +extern unsigned long long int addrdec_packbits(unsigned long long int mask, + unsigned long long int val, + unsigned char high, unsigned char low); +extern void shader_cache_access_log( int logger_id, int type, int miss); +extern void shader_cache_access_unlog( int logger_id, int type, int miss); + +shd_cache_line_t* shd_cache_access_internal( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle, + unsigned int real_access) { + //if real_access==0 then its only a cache probe stats and LRU tags should not be updated + unsigned int i; + unsigned int set; + unsigned long long int tag; + unsigned long long int packed_addr; + shd_cache_line_t *pline; + + if (cp->bank_mask) + packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0); + else + packed_addr = addr; + + set = (packed_addr >> cp->line_sz_log2) & ( (1<nset_log2) - 1 ); + tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2); + + if (real_access) { + cp->access++; + shader_cache_access_log(cp->core_id, cp->type_id, 0); + } + + for (i=0; iassoc; i++) { + pline = &(cp->lines[set*cp->assoc+i] ); + if (pline->status & VALID) { + if (pline->tag == tag) { + //printf("Cache Hit! Addr=%08x Set=%x Way=%x Tag=%x\n", packed_addr, set, i, tag); + if (real_access) { + pline->last_used = sim_cycle; + if (write) { + pline->status |= DIRTY; + } + } + return pline; + } + } + } + if (real_access) { + cp->miss++; + shader_cache_access_log(cp->core_id, cp->type_id, 1); + } + return 0; +} + +shd_cache_line_t* shd_cache_access( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle ) +{ + return shd_cache_access_internal(cp,addr,nbytes,write,sim_cycle,1/*this is a real access*/); +} + +extern int gpgpu_cache_wt_through; +shd_cache_t *test = NULL; +enum cache_request_status shd_cache_access_wb( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle, address_type *wb_address) +{ + unsigned int i; + unsigned int set; + unsigned long long int tag; + unsigned long long int packed_addr; + shd_cache_line_t *pline; + + unsigned already_reserved = 0; + unsigned all_reserved = 1; + shd_cache_line_t *free_line = NULL; + + if (cp->bank_mask) + packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0); + else + packed_addr = addr; + + set = (packed_addr >> cp->line_sz_log2) & ( (1<nset_log2) - 1 ); + tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2); + + cp->access++; + shader_cache_access_log(cp->core_id, cp->type_id, 0); + + for (i=0; iassoc; i++) { + pline = &(cp->lines[set*cp->assoc+i] ); + if (pline->tag == tag) { + if (pline->status & RESERVED) { + already_reserved = 1; + break; + } else if (pline->status & VALID) { + //printf("Cache Hit! Addr=%08x Set=%x Way=%x Tag=%x\n", packed_addr, set, i, tag); + pline->last_used = sim_cycle; + if (write) { + pline->status |= DIRTY; + } + //return pline; + if (cp->write_policy == write_through) return HIT_W_WT; + return HIT; + } + } + if (!(pline->status & RESERVED)) { + all_reserved = 0; + if (!(pline->status & VALID)) { + free_line = pline; + } + } + } + cp->miss++; + shader_cache_access_log(cp->core_id, cp->type_id, 1); + + if (already_reserved || cp->write_policy != write_back || write) { + //not in cache yet, but no wb as place is reserved. + //or wt caches never nead to worry about it (as do no_write caches) + //or is a write + if (already_reserved && write) { + //write the data into the cache line, make it dirty + //up to mshrs to save write mask to not overwrite this data when the read returns + pline->status |= DIRTY; + return WB_HIT_ON_MISS; + } else if (already_reserved) { + return WB_HIT_ON_MISS; + } + return MISS_NO_WB; + } + + //if not in cache, and a write back cache, and not already allocated, need to allocate a place for this request + + if (all_reserved) { + //cannot service this request, because we can't garantee that we have room for the line when it comes back + return RESERVATION_FAIL; + } + + //printf("RESRV %d\n",tag); + + if (free_line) { + //reserve fo this request + free_line->status |= RESERVED; + free_line->tag = tag; + //no writeback + return MISS_NO_WB; + } + + // need to kick a line out to reserve a spot + shd_cache_line_t *rline = NULL; + + for (i=0; iassoc; i++) { + pline = &(cp->lines[set*cp->assoc+i] ); + if (pline->status & VALID && !(pline->status & RESERVED)) { + if (!rline) { + rline = pline; //select first available for ejection for later comparison + continue; + } + switch (cp->policy) { + case LRU: + if (pline->last_used < rline->last_used) + rline = pline; + break; + case FIFO: + if (pline->fetch_time < rline->fetch_time) + rline = pline; + break; + default: + rline = pline; //pick one, ie. the last valied one. + } + } + } + assert(rline); //ensure we actually found one. + + unsigned needs_wb = (rline->status & (DIRTY|VALID)) == (DIRTY|VALID); + /* Set the replaced cache line address */ + if (needs_wb) { + *wb_address = rline->addr; + } + + /* reserve this new line */ + rline->status |= RESERVED; + rline->status &= ~VALID; + rline->status &= ~DIRTY; + rline->tag = tag; + + /* printf("Fetching! Addr=%08x ReplAddr=%08x(%d) Set=%x Tag=%x\n", + packed_addr, repl_addr, nofreeslot, set, tag); + */ + if (needs_wb) { + return MISS_W_WB; + } else { + return MISS_NO_WB; + } +} + +shd_cache_line_t* shd_cache_probe( shd_cache_t *cp, + unsigned long long int addr) +{ + return shd_cache_access_internal(cp,addr, + 1,0,0, /*do not matter*/ + 0/*this is just a probe*/); +} + +void shd_cache_undo_stats( shd_cache_t *cp, int miss ) +{ + if (miss) { + cp->miss--; + shader_cache_access_unlog(cp->core_id, cp->type_id, 1); + } + cp->access--; + shader_cache_access_unlog(cp->core_id, cp->type_id, 0); +} + +// Obtain the windowed cache miss rate for visualizer +float shd_cache_windowed_cache_miss_rate( shd_cache_t *cp, int minus_merge_hit ) +{ + unsigned int n_access = cp->access - cp->prev_snapshot_access; + unsigned int n_miss = cp->miss - cp->prev_snapshot_miss; + unsigned int n_merge_hit = cp->merge_hit - cp->prev_snapshot_merge_hit; + + if (minus_merge_hit) { + n_miss -= n_merge_hit; + } + float missrate = 0.0f; + if (n_access != 0) { + missrate = (float) n_miss / n_access; + } + + return missrate; +} + +// start a new sampling window +void shd_cache_new_window( shd_cache_t *cp ) +{ + cp->prev_snapshot_access = cp->access; + cp->prev_snapshot_miss = cp->miss; + cp->prev_snapshot_merge_hit = cp->merge_hit; +} + +unsigned long long int L2_shd_cache_fill( shd_cache_t *cp, + unsigned long long int addr, + unsigned int sim_cycle ) { + unsigned long long int result = shd_cache_fill(cp, addr, sim_cycle); + return result; +} + +static unsigned int _n_line_existed = 0; // debug counter + +// Fetch requested data into cache line. +// Returning address on the replaced line if it is dirty, or -1 if it is clean +// Assume the line is filled all at once. +unsigned long long int shd_cache_fill( shd_cache_t *cp, + unsigned long long int addr, + unsigned int sim_cycle ) { + + unsigned int i; + unsigned int set; + unsigned long long int tag; + unsigned long long int packed_addr; + unsigned long long int repl_addr; + + unsigned char nofreeslot; + unsigned char line_exists; + unsigned int base = 0 ; + unsigned int maxway = cp->assoc ; + + shd_cache_line_t *pline, *cline; + + if (cp->bank_mask) + packed_addr = addrdec_packbits(cp->bank_mask, addr, 64, 0); + else + packed_addr = addr; + set = (packed_addr >> cp->line_sz_log2) & ( (1<nset_log2) - 1 ); + tag = packed_addr >> (cp->line_sz_log2 + cp->nset_log2); + + if (cp->write_policy == write_back) { + //this request must have a reserved spot + cline = NULL; + for (i=base; ilines[set*cp->assoc+i] ); + if ((pline->tag == tag) && (pline->status & RESERVED)) { + cline = pline; + break; + } + if ((pline->tag == tag) && (pline->status & VALID)) { + //A second fill has returned to a line in the cache + //discard it as line in cache may have been modified, or is the same + _n_line_existed++; + + return -1; + } + } + + //if (cline) printf("FOUND %d\n",tag); + //else printf("UNFOUND!!! %d\n", tag); + + if (!cline) printf("----!!! about to abort - this probably happened because global memory msrh merging is not enabled with a writeback cache !!!----\n"); + + assert(cline); //error if it doesn't have a reserved space + + /* Fetch data into block */ + cline->status &= ~RESERVED; + cline->status |= VALID; + //cline->status &= ~DIRTY; Don't clear dirty bit, as might be dirty from write. + cline->tag = tag; + cline->addr = addr; + cline->last_used = sim_cycle; + cline->fetch_time = sim_cycle; + + // no wb, already handled. + return -1; + } + + //behavior unchanged for write through cache... probably not all necessary. + + // Look for any free slots and the possibility that the line is in the cache already + nofreeslot = 1; + line_exists = 0; + for (i=base; ilines[set*cp->assoc+i] ); + if (!(pline->status & VALID)) { + cline = pline; + nofreeslot = 0; + break; + } else if (pline->tag == tag) { + cline = pline; + line_exists = 1; + break; + } + } + + if (line_exists) { + _n_line_existed += 1; + return -1; // don't need to spill any line, nor it needs to be filled + } + + if (nofreeslot) { + cline = &(cp->lines[set*cp->assoc+base] ); + for (i=1+base; ilines[set*cp->assoc+i] ); + if (pline->status & VALID) { + switch (cp->policy) { + case LRU: + if (pline->last_used < cline->last_used) + cline = pline; + break; + case FIFO: + if (pline->fetch_time < cline->fetch_time) + cline = pline; + break; + default: + break; + } + } + } + } + + /* Set the replaced cache line address */ + if ((cline->status & (DIRTY|VALID)) == (DIRTY|VALID)) { + repl_addr = cline->addr; + } else { + repl_addr = -1; + } + + /* Fetch data into block */ + cline->status |= VALID; + cline->status &= ~DIRTY; + cline->tag = tag; + cline->addr = addr; + cline->last_used = sim_cycle; + cline->fetch_time = sim_cycle; + +/* printf("Fetching! Addr=%08x ReplAddr=%08x(%d) Set=%x Tag=%x\n", + packed_addr, repl_addr, nofreeslot, set, tag); + */ + return repl_addr; +} + +void shd_cache_mergehit( shd_cache_t *cp, unsigned long long int addr ) +{ + cp->merge_hit += 1; +} + +void shd_cache_print( shd_cache_t *cp, FILE *stream) { + fprintf( stream, "Cache %s:\t", cp->name); + fprintf( stream, "Size = %d B (%d Set x %d-way x %d byte line)\n", + cp->line_sz * cp->nset * cp->assoc, + cp->nset, cp->assoc, cp->line_sz ); + fprintf( stream, "\t\tAccess = %d, Miss = %d (%.3g), -MgHts = %d (%.3g)\n", + cp->access, cp->miss, (float) cp->miss / cp->access, + cp->miss - cp->merge_hit, (float) (cp->miss - cp->merge_hit) / cp->access); +} + +#ifdef UNIT_TEST + +int main() { + shd_cache_t *cp[3]; + unsigned int addr, i; + unsigned int cachenum; + unsigned int sim_cycle; + + unsigned int test_addrs[8] = { 0x100, 0x200, 0x300, 0x400, + 0x104, 0x204, 0x500, 0x100}; + unsigned int repl_addr[8] = {0,0,0,0,0,0,0,0}; + unsigned int rdwr[8] = {0,1,0,0,0,0,0,0}; + + sim_cycle = 0; + cp[0] = shd_cache_create ("cp1", 16, 4, 16, LRU, 1); + cp[1] = shd_cache_create ("cp2", 16, 4, 16, FIFO, 1); + + for (cachenum = 0; cachenum<2; cachenum++) + for (i=0; i<8; i++) { + if ( !shd_cache_access(cp[cachenum], test_addrs[i], 4, rdwr[i], sim_cycle) ) { + repl_addr[i] = shd_cache_fill(cp[cachenum], test_addrs[i], sim_cycle); + shd_cache_access(cp[cachenum], test_addrs[i], 4, rdwr[i], sim_cycle); + } + sim_cycle++; + } + + printf("replaced address:"); + for (i=0; i<8; i++) { + printf("0x%x ", repl_addr[i]); + } + printf("\n"); + shd_cache_print(cp[0],stdout); + shd_cache_print(cp[1],stdout); + + shd_cache_fill(cp[0], 0x104b3ecb0, sim_cycle); + printf("Accessing 64-bit address tag: %d\n", + shd_cache_access(cp[0], 0x104b3ecb2, 4, 0, sim_cycle)); + printf("Accessing 64-bit address tag: %d\n", + shd_cache_access(cp[0], 0x103433330, 4, 0, sim_cycle)); + + + shd_set_coherency_policy(2); + cp[2] = shd_cache_create("cp2", 16, 4, 16, LRU, 1); + shd_cache_fill(cp[2], 0x12345000, 0); + shd_cache_access(cp[2], 0x12345000, 4, 1, 0); + shd_cache_access(cp[2], 0x12345004, 4, 0, 0); + shd_cache_access(cp[2], 0x12345008, 4, 0, 0); + shd_cache_access(cp[2], 0x1234500C, 4, 1, 0); + printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0xf, + shd_cache_linedirty(cp[2], 0x12345000, 0xf), 1 ); + printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x6, + shd_cache_linedirty(cp[2], 0x12345000, 0x6), 0 ); + printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x1, + shd_cache_linedirty(cp[2], 0x12345000, 0x1), 1 ); + printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x8, + shd_cache_linedirty(cp[2], 0x12345000, 0x8), 1 ); + printf("Checking Dirty Vector %x, Result = %d (Expect %d)\n", 0x9, + shd_cache_linedirty(cp[2], 0x12345000, 0x9), 1 ); + +} + +#endif diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h new file mode 100644 index 0000000..e82ad05 --- /dev/null +++ b/src/gpgpu-sim/gpu-cache.h @@ -0,0 +1,191 @@ +/* + * gpu-cache.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include +#include "../util.h" + +#ifndef GPU_CACHE_H +#define GPU_CACHE_H + +#define VALID 0x01 +#define DIRTY 0x02 +#define RESERVED 0x04 + +enum cache_request_status { + HIT, + HIT_W_WT, /* Hit, but write through cache, still needs to send to memory */ + MISS_NO_WB, /* miss, but witeback not necessary*/ + MISS_W_WB, /* miss, must do writeback */ + WB_HIT_ON_MISS, /* request hit on a reservation in wb cache*/ + RESERVATION_FAIL, + NUM_CACHE_REQUEST_STATUS +}; + + +typedef struct { + unsigned long long int tag; + unsigned long long int addr; + unsigned int set; + unsigned int line_sz; /* bytes */ + unsigned int fetch_time; + unsigned int last_used; + unsigned char status; /* valid, dirty... etc */ +} shd_cache_line_t; + + +#define LRU 'L' +#define FIFO 'F' +#define RANDOM 'R' + +enum cache_write_policy{ + no_writes, //line replacement when new line arrives + write_back, //line replacement when new line arrives + write_through //reservation based, use much handle reservation full error. +}; + +typedef struct { + + char *name; + + shd_cache_line_t *lines; /* nset x assoc lines in total */ + unsigned int nset; + unsigned int nset_log2; + unsigned int assoc; + unsigned int line_sz; // bytes + unsigned int line_sz_log2; + enum cache_write_policy write_policy; + unsigned char policy; + unsigned int hit_latency; + + unsigned int access; + unsigned int miss; + unsigned int merge_hit; // number of cache miss that hit the same line (and merged as a result) + + // performance counters for calculating the amount of misses within a time window + unsigned int prev_snapshot_access; + unsigned int prev_snapshot_miss; + unsigned int prev_snapshot_merge_hit; + + int core_id; // which shader core is using this + int type_id; // what kind of cache is this (normal, texture, constant) + + unsigned long long int bank_mask; + +} shd_cache_t; + +shd_cache_t * shd_cache_create( char *name, + unsigned int nset, + unsigned int assoc, + unsigned int line_sz, + unsigned char policy, + unsigned int hit_latency, + unsigned long long int bank_mask, + enum cache_write_policy wp); + +void shd_cache_destroy( shd_cache_t* cp ); + +// hook up with shader core logger +void shd_cache_bind_logger(shd_cache_t* cp, int core_id, int type_id); + +//depercated, use _wb +shd_cache_line_t* shd_cache_access( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle ); + +//cache check checks for wb and forwards information over. +enum cache_request_status shd_cache_access_wb( shd_cache_t *cp, + unsigned long long int addr, + unsigned int nbytes, + unsigned char write, + unsigned int sim_cycle, + address_type *wb_address); + + +//just probe the tag array to see if addr is in the cache or not +//does not update LRU or stats... +shd_cache_line_t* shd_cache_probe( shd_cache_t *cp, + unsigned long long int addr); + +// undo the statistic record when the memory access is stalled/squashed and will try again next cycle +void shd_cache_undo_stats( shd_cache_t *cp, int miss ); + +void shd_cache_mergehit( shd_cache_t *cp, unsigned long long int addr ); + +unsigned long long int shd_cache_fill( shd_cache_t *cp, + unsigned long long int addr, + unsigned int sim_cycle ); + +unsigned long long int L2_shd_cache_fill( shd_cache_t *cp, + unsigned long long int addr, + unsigned int sim_cycle ); + +void shd_cache_print( shd_cache_t *cp, FILE *stream); + + +#endif diff --git a/src/gpgpu-sim/gpu-misc.cc b/src/gpgpu-sim/gpu-misc.cc new file mode 100644 index 0000000..06a0f68 --- /dev/null +++ b/src/gpgpu-sim/gpu-misc.cc @@ -0,0 +1,96 @@ +/* + * gpu-misc.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "gpu-misc.h" + +unsigned int LOGB2( unsigned int v ) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; + shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; + shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; + shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; + shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; + + return r; +} + +unsigned int MAX2NUM( unsigned int a, unsigned int b ) { + if (a > b) { + return a; + } else + return b; +} + +unsigned int MIN2NUM( unsigned int a, unsigned int b ) { + if (a < b) { + return a; + } else + return b; +} diff --git a/src/gpgpu-sim/gpu-misc.h b/src/gpgpu-sim/gpu-misc.h new file mode 100644 index 0000000..3d07d77 --- /dev/null +++ b/src/gpgpu-sim/gpu-misc.h @@ -0,0 +1,106 @@ +/* + * gpu-misc.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef GPU_MISC_H +#define GPU_MISC_H + +#define CONSTC 100 +#define DCACHE 200 +#define TEXTC 300 +#define SHD_CACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1cache->line_sz - 1))) +#define SHD_TEXCACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1texcache->line_sz - 1))) +#define SHD_CONSTCACHE_TAG(x,shdr) ((x) & (~((unsigned long long int)shdr->L1constcache->line_sz - 1))) +#define CACHE_TAG_OF(x,cache) ((x) & (~((unsigned long long int)cache->line_sz - 1))) +#define CACHE_TAG_OF_64(x) ((x) & (~((unsigned long long int)64 - 1))) + +#define ispowerof2(x) ((((x) - 1) & (x)) == 0) +#define powerof2(x) (1 << (x)) + + +enum mem_space { //used for cudasim + SHARED_SPACE, + CONST_SPACE, + GLOBAL_SPACE, + LOCAL_SPACE, + TEX_SPACE +}; +//enables a verbose printout of all L1 cache misses and all MSHR status changes +//good for a single shader configuration +#define DEBUGL1MISS 0 + +unsigned int LOGB2( unsigned int v ); + +unsigned int MAX2NUM( unsigned int a, unsigned int b ); + +unsigned int MIN2NUM( unsigned int a, unsigned int b ); + + +#define gs_max2(a,b) (((a)>(b))?(a):(b)) +#define gs_min2(a,b) (((a)<(b))?(a):(b)) +#define min3(x,y,z) (((x)<(y) && (x)<(z))?(x):(gs_min2((y),(z)))) + +#endif + diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc new file mode 100644 index 0000000..6ae7430 --- /dev/null +++ b/src/gpgpu-sim/gpu-sim.cc @@ -0,0 +1,1752 @@ +/* + * gpu-sim.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan, Ivan Sham, Henry Wong, Dan O'Connor and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "gpu-sim.h" + +#include +#include "gpu-cache.h" +#include "gpu-misc.h" +#include "delayqueue.h" +#include "shader.h" +#include "icnt_wrapper.h" +#include "dram.h" +#include "addrdec.h" +#include "dwf.h" +#include "warp_tracker.h" +#include "cflogger.h" +#include "l2cache.h" + +#include "../cuda-sim/ptx-stats.h" +#include "../intersim/statwraper.h" +#include "../abstract_hardware_model.h" + +#include +#include +#define MAX(a,b) (((a)>(b))?(a):(b)) + +extern unsigned L2_write_miss; +extern unsigned L2_write_hit; +extern unsigned L2_read_hit; +extern unsigned L2_read_miss; +unsigned made_read_mfs = 0; +unsigned made_write_mfs = 0; +unsigned freed_read_mfs = 0; +unsigned freed_L1write_mfs = 0; +unsigned freed_L2write_mfs = 0; +unsigned freed_dummy_read_mfs = 0; +unsigned long long gpu_sim_cycle = 0; +unsigned long long gpu_sim_insn = 0; +unsigned long long gpu_sim_insn_no_ld_const = 0; +unsigned long long gpu_sim_prev_insn = 0; +unsigned long long gpu_sim_insn_last_update = 0; +unsigned long long gpu_tot_sim_cycle = 0; +unsigned long long gpu_tot_sim_insn = 0; +unsigned long long gpu_last_sim_cycle = 0; +unsigned long long gpu_completed_thread = 0; +unsigned long long gpu_tot_issued_cta = 0; +unsigned long long gpu_tot_completed_thread = 0; + +unsigned int **concurrent_row_access; //concurrent_row_access[dram chip id][bank id] +unsigned int **num_activates; //num_activates[dram chip id][bank id] +unsigned int **row_access; //row_access[dram chip id][bank id] +unsigned int **max_conc_access2samerow; //max_conc_access2samerow[dram chip id][bank id] +unsigned int **max_servicetime2samerow; //max_servicetime2samerow[dram chip id][bank id] +unsigned int mergemiss = 0; +unsigned int L1_read_miss = 0; +unsigned int L1_write_miss = 0; +unsigned int L1_write_hit_on_miss = 0; +unsigned int L1_writeback = 0; +unsigned int L1_texture_miss = 0; +unsigned int L1_const_miss = 0; +unsigned int gpgpu_n_sent_writes = 0; +unsigned int gpgpu_n_processed_writes = 0; +unsigned int *max_return_queue_length; + +// performance counter for stalls due to congestion. +unsigned int gpu_stall_shd_mem = 0; +unsigned int gpu_stall_wr_back = 0; +unsigned int gpu_stall_dramfull = 0; +unsigned int gpu_stall_icnt2sh = 0; +unsigned int gpu_stall_by_MSHRwb = 0; + +//shader cannot send to icnt because icnt buffer is full +//Note: it is accumulative for all shaders and is never reset +//so it might increase 8 times in a cycle if we have 8 shaders +unsigned int gpu_stall_sh2icnt = 0; +// performance counters to account for instruction distribution +extern unsigned int gpgpu_n_load_insn; +extern unsigned int gpgpu_n_store_insn; +extern unsigned int gpgpu_n_shmem_insn; +extern unsigned int gpgpu_n_tex_insn; +extern unsigned int gpgpu_n_const_insn; +extern unsigned int gpgpu_multi_unq_fetches; +char *gpgpu_runtime_stat; +int gpu_stat_sample_freq = 10000; +int gpu_runtime_stat_flag = 0; +extern int gpgpu_warpdistro_shader; + +// GPGPU options +unsigned long long gpu_max_cycle = 0; +unsigned long long gpu_max_insn = 0; +int gpu_max_cycle_opt = 0; +int gpu_max_insn_opt = 0; +int gpu_max_cta_opt = 0; +int gpu_deadlock_detect = 0; +int gpu_deadlock = 0; +static unsigned long long last_gpu_sim_insn = 0; +int gpgpu_dram_scheduler = DRAM_FIFO; +int g_save_embedded_ptx = 0; +int gpgpu_simd_model = 0; +int gpgpu_no_dl1 = 0; +char *gpgpu_cache_texl1_opt; +char *gpgpu_cache_constl1_opt; +char *gpgpu_cache_dl1_opt; +char *gpgpu_cache_dl2_opt; +extern int gpgpu_l2_readoverwrite; +int gpgpu_partial_write_mask = 0; + +int gpgpu_perfect_mem = FALSE; +char *gpgpu_shader_core_pipeline_opt; +extern unsigned int *requests_by_warp; +unsigned int gpgpu_dram_buswidth = 4; +unsigned int gpgpu_dram_burst_length = 4; +int gpgpu_dram_sched_queue_size = 0; +char * gpgpu_dram_timing_opt; +int gpgpu_flush_cache = 0; +int gpgpu_mem_address_mask = 0; +unsigned int recent_dram_util = 0; + +int gpgpu_cflog_interval = 0; + +unsigned int finished_trace = 0; + +unsigned g_next_request_uid = 1; + +extern struct regs_t regs; + +extern long int gpu_reads; + +void ptx_dump_regs( void *thd ); + +int g_nthreads_issued; +int g_total_cta_left; + + +unsigned ptx_kernel_program_size(); +void visualizer_printstat(); +void time_vector_create(int ld_size,int st_size); +void time_vector_print(void); +void time_vector_update(unsigned int uid,int slot ,long int cycle,int type); +void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type); +void node_req_hist_clear(void *p); +void node_req_hist_dump(void *p); +void node_req_hist_update(void * p,int node, long long cycle); + +/* functionally simulated memory */ +extern struct mem_t *mem; + +/* Defining Clock Domains +basically just the ratio is important */ + +#define CORE 0x01 +#define L2 0x02 +#define DRAM 0x04 +#define ICNT 0x08 + +double core_time=0; +double icnt_time=0; +double dram_time=0; +double l2_time=0; + +#define MhZ *1000000 +double core_freq=2 MhZ; +double icnt_freq=2 MhZ; +double dram_freq=2 MhZ; +double l2_freq=2 MhZ; + +double core_period = 1 /( 2 MhZ); +double icnt_period = 1 /( 2 MhZ); +double dram_period = 1 /( 2 MhZ); +double l2_period = 1 / (2 MhZ); + +char * gpgpu_clock_domains; + +/* GPU uArch parameters */ +unsigned int gpu_n_mem = 8; +unsigned int gpu_mem_n_bk = 4; +unsigned int gpu_n_mem_per_ctrlr = 1; +unsigned int gpu_n_shader = 8; +int gpu_concentration = 1; +int gpu_n_tpc = 8; +unsigned int gpu_n_mshr_per_shader; +unsigned int gpu_n_thread_per_shader = 128; +unsigned int gpu_n_warp_per_shader; +unsigned int gpu_n_mshr_per_thread = 1; + +extern int gpgpu_interwarp_mshr_merge ; + +extern unsigned int gpgpu_shmem_size; +extern unsigned int gpgpu_shader_registers; +extern unsigned int gpgpu_shader_cta; +extern int gpgpu_shmem_bkconflict; +extern int gpgpu_cache_bkconflict; +extern int gpgpu_n_cache_bank; +extern unsigned int warp_size; +extern int pipe_simd_width; +extern unsigned int gpgpu_dwf_heuristic; +extern unsigned int gpgpu_dwf_regbk; +int gpgpu_reg_bankconflict = FALSE; +extern int gpgpu_shmem_port_per_bank; +extern int gpgpu_cache_port_per_bank; +extern int gpgpu_const_port_per_bank; +extern int gpgpu_shmem_pipe_speedup; +extern int gpgpu_reg_bank_conflict_model; +extern int gpgpu_num_reg_banks; + +extern unsigned int gpu_max_cta_per_shader; +extern unsigned int gpu_padded_cta_size; +extern int gpgpu_local_mem_map; + +unsigned int gpgpu_pre_mem_stages = 0; +unsigned int gpgpu_no_divg_load = 0; +char *gpgpu_dwf_hw_opt; +unsigned int gpgpu_thread_swizzling = 0; +unsigned int gpgpu_strict_simd_wrbk = 0; + +int pdom_sched_type = 0; +int n_pdom_sc_orig_stat = 0; //the selected pdom schedular is used +int n_pdom_sc_single_stat = 0; //only a single warp is ready to go in that cycle. +int *num_warps_issuable; +int *num_warps_issuable_pershader; + +// Thread Dispatching Unit option +int gpgpu_cuda_sim = 1; +int gpgpu_spread_blocks_across_cores = 1; + +/* GPU uArch structures */ +shader_core_ctx_t **sc; +dram_t **dram; +unsigned int common_clock = 0; +unsigned int more_thread = 1; +extern unsigned int n_regconflict_stall; +unsigned int warp_conflict_at_writeback = 0; +unsigned int gpgpu_commit_pc_beyond_two = 0; +extern int g_network_mode; +int gpgpu_cache_wt_through = 0; + + +//memory access classification +int gpgpu_n_mem_read_local = 0; +int gpgpu_n_mem_write_local = 0; +int gpgpu_n_mem_texture = 0; +int gpgpu_n_mem_const = 0; +int gpgpu_n_mem_read_global = 0; +int gpgpu_n_mem_write_global = 0; + +#define MEM_LATENCY_STAT_IMPL +#include "mem_latency_stat.h" + +unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid ); +unsigned char fq_push(unsigned long long int addr, int bsize, unsigned char write, partial_write_mask_t partial_write_mask, + int sid, int wid, mshr_entry* mshr, int cache_hits_waiting, + enum mem_access_type mem_acc, address_type pc); +int issue_mf_from_fq(mem_fetch_t *mf); +unsigned char single_check_icnt_has_buffer(int chip, int sid, unsigned char is_write ); +unsigned char fq_pop(int tpc_id); +void fill_shd_L1_with_new_line(shader_core_ctx_t * sc, mem_fetch_t * mf); + +void set_option_gpgpu_spread_blocks_across_cores(int option); +void set_param_gpgpu_num_shaders(int num_shaders); +unsigned ptx_sim_grid_size(); +void icnt_init_grid(); +void interconnect_stats(); +void icnt_overal_stat(); +unsigned ptx_sim_cta_size(); +unsigned ptx_sim_init_thread( void** thread_info, int sid, unsigned tid,unsigned threads_left,unsigned num_threads, core_t *core, unsigned hw_cta_id, unsigned hw_warp_id ); + +void gpu_sim_loop( int grid_num ); + +void print_shader_cycle_distro( FILE *fout ) ; +void find_reconvergence_points(); +void dwf_process_reconv_pts(); + +extern int gpgpu_ptx_instruction_classification ; +extern int g_ptx_sim_mode; + +extern int gpgpu_coalesce_arch; + +#define CREATELOG 111 +#define SAMPLELOG 222 +#define DUMPLOG 333 +void L2c_log(int task); +void dram_log(int task); + +void visualizer_options(option_parser_t opp); +void gpu_reg_options(option_parser_t opp) +{ + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", + "0"); + option_parser_register(opp, "-gpgpu_simd_model", OPT_INT32, &gpgpu_simd_model, + "0 = no recombination, 1 = post-dominator, 2 = MIMD, 3 = dynamic warp formation", "0"); + option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &gpgpu_dram_scheduler, + "0 = fifo (default), 1 = fast ideal", "0"); + + option_parser_register(opp, "-gpgpu_max_cycle", OPT_INT32, &gpu_max_cycle_opt, + "terminates gpu simulation early (0 = no limit)", + "0"); + option_parser_register(opp, "-gpgpu_max_insn", OPT_INT32, &gpu_max_insn_opt, + "terminates gpu simulation early (0 = no limit)", + "0"); + option_parser_register(opp, "-gpgpu_max_cta", OPT_INT32, &gpu_max_cta_opt, + "terminates gpu simulation early (0 = no limit)", + "0"); + + option_parser_register(opp, "-gpgpu_tex_cache:l1", OPT_CSTR, &gpgpu_cache_texl1_opt, + "per-shader L1 texture cache (READ-ONLY) config, i.e., {:::|none}", + "512:64:2:L"); + + option_parser_register(opp, "-gpgpu_const_cache:l1", OPT_CSTR, &gpgpu_cache_constl1_opt, + "per-shader L1 constant memory cache (READ-ONLY) config, i.e., {:::|none}", + "64:64:2:L"); + + option_parser_register(opp, "-gpgpu_no_dl1", OPT_BOOL, &gpgpu_no_dl1, + "no dl1 cache (voids -gpgpu_cache:dl1 option)", + "0"); + + option_parser_register(opp, "-gpgpu_cache:dl1", OPT_CSTR, &gpgpu_cache_dl1_opt, + "shader L1 data cache config, i.e., {:::|none}", + "256:128:1:L"); + + option_parser_register(opp, "-gpgpu_cache:dl2", OPT_CSTR, &gpgpu_cache_dl2_opt, + "unified banked L2 data cache config, i.e., {:::|none}; disabled by default", + NULL); + + option_parser_register(opp, "-gpgpu_perfect_mem", OPT_BOOL, &gpgpu_perfect_mem, + "enable perfect memory mode (no cache miss)", + "0"); + + option_parser_register(opp, "-gpgpu_shader_core_pipeline", OPT_CSTR, &gpgpu_shader_core_pipeline_opt, + "shader core pipeline config, i.e., {::}", + "256:32:32"); + + option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, + "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", + "8192"); + + option_parser_register(opp, "-gpgpu_shader_cta", OPT_UINT32, &gpgpu_shader_cta, + "Maximum number of concurrent CTAs in shader (default 8)", + "8"); + + option_parser_register(opp, "-gpgpu_n_shader", OPT_UINT32, &gpu_n_shader, + "number of shaders in gpu", + "8"); + option_parser_register(opp, "-gpgpu_n_mem", OPT_UINT32, &gpu_n_mem, + "number of memory modules (e.g. memory controllers) in gpu", + "8"); + option_parser_register(opp, "-gpgpu_n_mem_per_ctrlr", OPT_UINT32, &gpu_n_mem_per_ctrlr, + "number of memory chips per memory controller", + "1"); + option_parser_register(opp, "-gpgpu_runtime_stat", OPT_CSTR, &gpgpu_runtime_stat, + "display runtime statistics such as dram utilization {:}", + "10000:0"); + + option_parser_register(opp, "-gpgpu_dwf_heuristic", OPT_UINT32, &gpgpu_dwf_heuristic, + "DWF scheduling heuristic: 0 = majority, 1 = minority, 2 = timestamp, 3 = pdom priority, 4 = pc-based, 5 = max-heap", + "0"); + + option_parser_register(opp, "-gpgpu_reg_bankconflict", OPT_BOOL, &gpgpu_reg_bankconflict, + "Check for bank conflict in the pipeline", + "0"); + + option_parser_register(opp, "-gpgpu_dwf_regbk", OPT_BOOL, (int*)&gpgpu_dwf_regbk, + "Have dwf scheduler to avoid bank conflict", + "1"); + + option_parser_register(opp, "-gpgpu_memlatency_stat", OPT_INT32, &gpgpu_memlatency_stat, + "track and display latency statistics 0x2 enables MC, 0x4 enables queue logs", + "0"); + + option_parser_register(opp, "-gpu_n_mshr_per_shader", OPT_UINT32, &gpu_n_mshr_per_shader, + "Number of MSHRs per shader", + "64"); + + option_parser_register(opp, "-gpgpu_interwarp_mshr_merge", OPT_INT32, &gpgpu_interwarp_mshr_merge, + "interwarp coalescing", + "0"); + + option_parser_register(opp, "-gpgpu_dram_sched_queue_size", OPT_INT32, &gpgpu_dram_sched_queue_size, + "0 = unlimited (default); # entries per chip", + "0"); + + option_parser_register(opp, "-gpgpu_dram_buswidth", OPT_UINT32, &gpgpu_dram_buswidth, + "default = 4 bytes (8 bytes per cycle at DDR)", + "4"); + + option_parser_register(opp, "-gpgpu_dram_burst_length", OPT_UINT32, &gpgpu_dram_burst_length, + "Burst length of each DRAM request (default = 4 DDR cycle)", + "4"); + + option_parser_register(opp, "-gpgpu_dram_timing_opt", OPT_CSTR, &gpgpu_dram_timing_opt, + "DRAM timing parameters = {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tWTR}", + "4:2:8:12:21:13:34:9:4:5"); + + + option_parser_register(opp, "-gpgpu_mem_address_mask", OPT_INT32, &gpgpu_mem_address_mask, + "0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits", + "0"); + + option_parser_register(opp, "-gpgpu_flush_cache", OPT_BOOL, &gpgpu_flush_cache, + "Flush cache at the end of each kernel call", + "0"); + + option_parser_register(opp, "-gpgpu_pre_mem_stages", OPT_UINT32, &gpgpu_pre_mem_stages, + "default = 0 pre-memory pipeline stages", + "0"); + + option_parser_register(opp, "-gpgpu_no_divg_load", OPT_BOOL, (int*)&gpgpu_no_divg_load, + "Don't allow divergence on load", + "0"); + + option_parser_register(opp, "-gpgpu_dwf_hw", OPT_CSTR, &gpgpu_dwf_hw_opt, + "dynamic warp formation hw config, i.e., {<#LUT_entries>:|none}", + "32:2"); + + option_parser_register(opp, "-gpgpu_thread_swizzling", OPT_BOOL, (int*)&gpgpu_thread_swizzling, + "Thread Swizzling (1=on, 0=off)", + "0"); + + option_parser_register(opp, "-gpgpu_strict_simd_wrbk", OPT_BOOL, (int*)&gpgpu_strict_simd_wrbk, + "Applying Strick SIMD WriteBack Stage (1=on, 0=off)", + "0"); + + option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, + "Size of shared memory per shader core (default 16kB)", + "16384"); + + option_parser_register(opp, "-gpgpu_shmem_bkconflict", OPT_BOOL, &gpgpu_shmem_bkconflict, + "Turn on bank conflict check for shared memory", + "0"); + + option_parser_register(opp, "-gpgpu_shmem_pipe_speedup", OPT_INT32, &gpgpu_shmem_pipe_speedup, + "Number of groups each warp is divided for shared memory bank conflict check", + "2"); + + option_parser_register(opp, "-gpgpu_cache_wt_through", OPT_BOOL, &gpgpu_cache_wt_through, + "L1 cache become write through (1=on, 0=off)", + "0"); + + option_parser_register(opp, "-gpgpu_deadlock_detect", OPT_BOOL, &gpu_deadlock_detect, + "Stop the simulation at deadlock (1=on (default), 0=off)", + "1"); + + option_parser_register(opp, "-gpgpu_cache_bkconflict", OPT_BOOL, &gpgpu_cache_bkconflict, + "Turn on bank conflict check for L1 cache access", + "0"); + + option_parser_register(opp, "-gpgpu_n_cache_bank", OPT_INT32, &gpgpu_n_cache_bank, + "Number of banks in L1 cache, also for memory coalescing stall", + "1"); + + option_parser_register(opp, "-gpgpu_warpdistro_shader", OPT_INT32, &gpgpu_warpdistro_shader, + "Specify which shader core to collect the warp size distribution from", + "-1"); + + + option_parser_register(opp, "-gpgpu_pdom_sched_type", OPT_INT32, &pdom_sched_type, + "0 = first ready warp found, 1 = random, 8 = loose round robin", + "8"); + + option_parser_register(opp, "-gpgpu_spread_blocks_across_cores", OPT_BOOL, + &gpgpu_spread_blocks_across_cores, + "Spread block-issuing across all cores instead of filling up core by core (do NOT disable)", + "1"); + + option_parser_register(opp, "-gpgpu_cuda_sim", OPT_BOOL, &gpgpu_cuda_sim, + "use PTX instruction set", + "1"); + option_parser_register(opp, "-gpgpu_ptx_instruction_classification", OPT_INT32, + &gpgpu_ptx_instruction_classification, + "if enabled will classify ptx instruction types per kernel (Max 255 kernels now)", + "0"); + option_parser_register(opp, "-gpgpu_ptx_sim_mode", OPT_INT32, &g_ptx_sim_mode, + "Select between Performance (default) or Functional simulation (1)", + "0"); + option_parser_register(opp, "-gpgpu_clock_domains", OPT_CSTR, &gpgpu_clock_domains, + "Clock Domain Frequencies in MhZ {:::}", + "500.0:2000.0:2000.0:2000.0"); + + option_parser_register(opp, "-gpgpu_shmem_port_per_bank", OPT_INT32, &gpgpu_shmem_port_per_bank, + "Number of access processed by a shared memory bank per cycle (default = 2)", + "2"); + option_parser_register(opp, "-gpgpu_cache_port_per_bank", OPT_INT32, &gpgpu_cache_port_per_bank, + "Number of access processed by a cache bank per cycle (default = 2)", + "2"); + option_parser_register(opp, "-gpgpu_const_port_per_bank", OPT_INT32, &gpgpu_const_port_per_bank, + "Number of access processed by a constant cache bank per cycle (default = 2)", + "2"); + option_parser_register(opp, "-gpgpu_cflog_interval", OPT_INT32, &gpgpu_cflog_interval, + "Interval between each snapshot in control flow logger", + "0"); + option_parser_register(opp, "-gpgpu_partial_write_mask", OPT_INT32, &gpgpu_partial_write_mask, + "use partial write mask to filter memory requests <1>No extra reads(use this!)<2>extra reads generated for partial chunks", + "0"); + option_parser_register(opp, "-gpu_concentration", OPT_INT32, &gpu_concentration, + "Number of shader cores per interconnection port (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_local_mem_map", OPT_INT32, &gpgpu_local_mem_map, + "Mapping from local memory space address to simulated GPU physical address space (default = 1)", + "1"); + option_parser_register(opp, "-gpgpu_reg_bank_conflict_model", OPT_BOOL, &gpgpu_reg_bank_conflict_model, + "Turn on register bank conflict model (default = off)", + "0"); + option_parser_register(opp, "-gpgpu_num_reg_banks", OPT_INT32, &gpgpu_num_reg_banks, + "Number of register banks (default = 8)", + "8"); + option_parser_register(opp, "-gpgpu_coalesce_arch", OPT_INT32, &gpgpu_coalesce_arch, + "Coalescing arch (default = 13, anything else is off for now)", + "13"); + addrdec_setoption(opp); + L2c_options(opp); + visualizer_options(opp); + ptx_file_line_stats_options(opp); +} + +///////////////////////////////////////////////////////////////////////////// + +inline int mem2device(int memid) { + return memid + gpu_n_tpc; +} + +///////////////////////////////////////////////////////////////////////////// + + +/* Allocate memory for uArch structures */ +void init_gpu () +{ + int i; + + gpu_max_cycle = gpu_max_cycle_opt; + gpu_max_insn = gpu_max_insn_opt; + + i = sscanf(gpgpu_shader_core_pipeline_opt,"%d:%d:%d", + &gpu_n_thread_per_shader, &warp_size, &pipe_simd_width); + gpu_n_warp_per_shader = gpu_n_thread_per_shader / warp_size; + num_warps_issuable = (int*) calloc(gpu_n_warp_per_shader+1, sizeof(int)); + num_warps_issuable_pershader = (int*) calloc(gpu_n_shader, sizeof(int)); + if (i == 2) { + pipe_simd_width = warp_size; + } else if (i == 3) { + assert(warp_size % pipe_simd_width == 0); + } + + sscanf(gpgpu_runtime_stat, "%d:%x", + &gpu_stat_sample_freq, &gpu_runtime_stat_flag); + + sc = (shader_core_ctx_t**) calloc(gpu_n_shader, sizeof(shader_core_ctx_t*)); + int mshr_que = gpu_n_mshr_per_thread; + for (i=0;(unsigned)inot_completed = 0; + shader_reinit(sc[i],0,sc[i]->n_threads); + } + if (gpu_max_cta_opt != 0) { + g_total_cta_left = gpu_max_cta_opt; + } else { + g_total_cta_left = ptx_sim_grid_size(); + } + if (gpu_max_cta_opt != 0) { + // the maximum number of CTA has been reached, stop any further simulation + if (gpu_tot_issued_cta >= (unsigned)gpu_max_cta_opt) { + return 0; + } + } + + if (gpu_max_cycle && (gpu_tot_sim_cycle + gpu_sim_cycle) >= gpu_max_cycle) { + return gpu_sim_cycle; + } + if (gpu_max_insn && (gpu_tot_sim_insn + gpu_sim_insn) >= gpu_max_insn) { + return gpu_sim_cycle; + } + + // refind the diverge/reconvergence pairs + dwf_reset_reconv_pt(); + find_reconvergence_points(); + + dwf_process_reconv_pts(); + dwf_reinit_schedulers(gpu_n_shader); + + // initialize the control-flow, memory access, memory latency logger + create_thread_CFlogger( gpu_n_shader, gpu_n_thread_per_shader, ptx_kernel_program_size(), 0, gpgpu_cflog_interval ); + shader_CTA_count_create( gpu_n_shader, gpgpu_cflog_interval); + if (gpgpu_cflog_interval != 0) { + insn_warp_occ_create( gpu_n_shader, warp_size, ptx_kernel_program_size() ); + shader_warp_occ_create( gpu_n_shader, warp_size, gpgpu_cflog_interval); + shader_mem_acc_create( gpu_n_shader, gpu_n_mem, 4, gpgpu_cflog_interval); + shader_mem_lat_create( gpu_n_shader, gpgpu_cflog_interval); + shader_cache_access_create( gpu_n_shader, 3, gpgpu_cflog_interval); + set_spill_interval (gpgpu_cflog_interval * 40); + } + + // calcaulte the max cta count and cta size for local memory address mapping + gpu_max_cta_per_shader = max_cta_per_shader(sc[0]); + //gpu_max_cta_per_shader is limited by number of CTAs if not enough + if (ptx_sim_grid_size() < gpu_max_cta_per_shader*gpu_n_shader) { + gpu_max_cta_per_shader = (ptx_sim_grid_size() / gpu_n_shader); + if (ptx_sim_grid_size() % gpu_n_shader) + gpu_max_cta_per_shader++; + } + unsigned int gpu_cta_size = ptx_sim_cta_size(); + gpu_padded_cta_size = (gpu_cta_size%32) ? 32*((gpu_cta_size/32)+1) : gpu_cta_size; + + if (g_network_mode) { + icnt_init_grid(); + } + last_gpu_sim_insn = 0; + // add this condition as well? (gpgpu_n_processed_writes < gpgpu_n_sent_writes) + while (not_completed || mem_busy || icnt2mem_busy) { + gpu_sim_loop(grid_num); + + not_completed = 0; + for (unsigned i=0;inot_completed; + } + // dram_busy just check the request queue length into the dram + // to make sure all the memory requests (esp the writes) are done + mem_busy = 0; + for (unsigned i=0;i= gpu_max_cycle) { + break; + } + if (gpu_max_insn && (gpu_tot_sim_insn + gpu_sim_insn) >= gpu_max_insn) { + break; + } + if (gpu_deadlock_detect && gpu_deadlock) { + break; + } + + } + memlatstat_lat_pw(); + gpu_tot_sim_cycle += gpu_sim_cycle; + gpu_tot_sim_insn += gpu_sim_insn; + gpu_tot_completed_thread += gpu_completed_thread; + + ptx_file_line_stats_write_file(); + + printf("stats for grid: %d\n", grid_num); + gpu_print_stat(); + if (g_network_mode) { + interconnect_stats(); + printf("----------------------------Interconnect-DETAILS---------------------------------" ); + icnt_overal_stat(); + printf("----------------------------END-of-Interconnect-DETAILS-------------------------" ); + } + if (gpgpu_memlatency_stat & GPU_MEMLATSTAT_QUEUELOGS ) { + dramqueue_latency_log_dump(); + dram_log(DUMPLOG); + if (gpgpu_cache_dl2_opt) { + L2c_log(DUMPLOG); + L2c_latency_log_dump(); + } + } + +#define DEADLOCK 0 + if (gpu_deadlock_detect && gpu_deadlock) { + fflush(stdout); + printf("ERROR ** deadlock detected: last writeback @ gpu_sim_cycle %u (+ gpu_tot_sim_cycle %u) (%u cycles ago)\n", + (unsigned) gpu_sim_insn_last_update, (unsigned) (gpu_tot_sim_cycle-gpu_sim_cycle), + (unsigned) (gpu_sim_cycle - gpu_sim_insn_last_update )); + fflush(stdout); + assert(DEADLOCK); + } + return gpu_sim_cycle; +} + +extern void ** g_inst_classification_stat; +extern void ** g_inst_op_classification_stat; +extern int g_ptx_kernel_count; // used for classification stat collection purposes + +extern unsigned get_max_mshr_used(shader_core_ctx_t* shader); + +void gpu_print_stat() +{ + unsigned i; + int j,k; + + printf("gpu_sim_cycle = %lld\n", gpu_sim_cycle); + printf("gpu_sim_insn = %lld\n", gpu_sim_insn); + printf("gpu_sim_no_ld_const_insn = %lld\n", gpu_sim_insn_no_ld_const); + printf("gpu_ipc = %12.4f\n", (float)gpu_sim_insn / gpu_sim_cycle); + printf("gpu_completed_thread = %lld\n", gpu_completed_thread); + printf("gpu_tot_sim_cycle = %lld\n", gpu_tot_sim_cycle); + printf("gpu_tot_sim_insn = %lld\n", gpu_tot_sim_insn); + printf("gpu_tot_ipc = %12.4f\n", (float)gpu_tot_sim_insn / gpu_tot_sim_cycle); + printf("gpu_tot_completed_thread = %lld\n", gpu_tot_completed_thread); + printf("gpu_tot_issued_cta = %lld\n", gpu_tot_issued_cta); + printf("gpgpu_n_sent_writes = %d\n", gpgpu_n_sent_writes); + printf("gpgpu_n_processed_writes = %d\n", gpgpu_n_processed_writes); + + // performance counter for stalls due to congestion. + printf("gpu_stall_by_MSHRwb= %d\n", gpu_stall_by_MSHRwb); + printf("gpu_stall_shd_mem = %d\n", gpu_stall_shd_mem ); + printf("gpu_stall_wr_back = %d\n", gpu_stall_wr_back ); + printf("gpu_stall_dramfull = %d\n", gpu_stall_dramfull); + printf("gpu_stall_icnt2sh = %d\n", gpu_stall_icnt2sh ); + printf("gpu_stall_sh2icnt = %d\n", gpu_stall_sh2icnt ); + // performance counter that are not local to one shader + shader_print_accstats(stdout); + + memlatstat_print(); + printf("max return queue length = "); + for (unsigned i=0;iL1cache,stdout); + j+=sc[i]->L1cache->miss; + k+=sc[i]->L1cache->access; + } + printf("L1 Data Cache Total Miss Rate = %0.3f\n", (float)j/k); + + for (i=0,j=0,k=0;iL1texcache,stdout); + j+=sc[i]->L1texcache->miss; + k+=sc[i]->L1texcache->access; + } + printf("L1 Texture Cache Total Miss Rate = %0.3f\n", (float)j/k); + + for (i=0,j=0,k=0;iL1constcache,stdout); + j+=sc[i]->L1constcache->miss; + k+=sc[i]->L1constcache->access; + } + printf("L1 Const Cache Total Miss Rate = %0.3f\n", (float)j/k); + + if (gpgpu_cache_dl2_opt) { + L2c_print_cache_stat(); + } + printf("n_regconflict_stall = %d\n", n_regconflict_stall); + + if (gpgpu_simd_model == DWF) { + dwf_print_stat(stdout); + } + + if (gpgpu_simd_model == POST_DOMINATOR) { + printf("num_warps_issuable:"); + for (unsigned i=0;i<(gpu_n_warp_per_shader+1);i++) { + printf("%d ", num_warps_issuable[i]); + } + printf("\n"); + } + if (gpgpu_strict_simd_wrbk) { + printf("warp_conflict_at_writeback = %d\n", warp_conflict_at_writeback); + } + + printf("gpgpu_commit_pc_beyond_two = %d\n", gpgpu_commit_pc_beyond_two); + + print_shader_cycle_distro( stdout ); + + print_thread_pc_histogram( stdout ); + + if (gpgpu_cflog_interval != 0) { + spill_log_to_file (stdout, 1, gpu_sim_cycle); + insn_warp_occ_print(stdout); + } + if ( gpgpu_ptx_instruction_classification ) { + StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp( g_inst_op_classification_stat[g_ptx_kernel_count]); + } + time_vector_print(); + + fflush(stdout); +} + +//////////////////////////////////////////////////////////////////////////////////// +// Wrapper function for shader cores' memory system: +//////////////////////////////////////////////////////////////////////////////////// + +// a hack to make the size of a packet discrete multiples of the interconnect's flit_size. +static inline +unsigned int fill_to_next_flit(unsigned int size) +{ + assert (g_network_mode == INTERSIM); + return size; +} + + + +unsigned char check_icnt_has_buffer(unsigned long long int *addr, int *bsize, + int n_addr, int sid ) +{ + addrdec_t tlx; + static unsigned int *req_buffer = NULL; + //the req_buf size can be equal to gpu_n_mem ; gpu_n_shader is added to make it compatible + //with the case where a mem controller is sending to shd + if (!req_buffer) req_buffer = (unsigned int*)malloc((gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int)); + memset(req_buffer, 0, (gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int)); + + // aggregate all buffer requirement of all memory accesses by dram chips + for (int i=0; i< n_addr; i++) { + addrdec_tlx(addr[i],&tlx); + req_buffer[tlx.chip] += fill_to_next_flit(bsize[i]); + } + + int tpc_id = sid / gpu_concentration; + + return icnt_has_buffer(tpc_id, req_buffer); +} + +unsigned char single_check_icnt_has_buffer(int chip, int sid, unsigned char is_write ) +{ + static unsigned int *req_buffer = NULL; + //the req_buf size can be equal to gpu_n_mem ; gpu_n_shader is added to make it compatible + //with the case where a mem controller is sending to shd + if (!req_buffer) req_buffer = (unsigned int*)malloc((gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int)); + memset(req_buffer, 0, (gpu_n_mem+gpu_n_tpc)*sizeof(unsigned int)); + + // aggregate all buffer requirement of all memory accesses by dram chips + + int b_size; + if (is_write) + b_size = sc[sid]->L1cache->line_sz; + else + b_size = READ_PACKET_SIZE; + req_buffer[chip] += fill_to_next_flit(b_size); + + int tpc_id = sid / gpu_concentration; + + return icnt_has_buffer(tpc_id, req_buffer); +} + +int max_n_addr = 0; + +// Check the memory system for buffer availability +unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid ) +{ + //requests should be single always now + int rsize = bsize; + //maintain similar functionality with fq_push, if its a read, bsize is the load size, not the request's size + if (!write) { + rsize = READ_PACKET_SIZE; + } + return check_icnt_has_buffer(&addr, &rsize, 1, sid); +} + +// Takes in memory address and their parameters and pushes to the fetch queue +unsigned char fq_push(unsigned long long int addr, int bsize, unsigned char write, partial_write_mask_t partial_write_mask, + int sid, int wid, mshr_entry* mshr, int cache_hits_waiting, + enum mem_access_type mem_acc, address_type pc) +{ + mem_fetch_t *mf; + + mf = (mem_fetch_t*) calloc(1,sizeof(mem_fetch_t)); + mf->request_uid = g_next_request_uid++; + mf->addr = addr; + mf->nbytes_L1 = bsize; + mf->sid = sid; + mf->source_node = sid / gpu_concentration; + mf->wid = wid; + mf->cache_hits_waiting = cache_hits_waiting; + mf->txbytes_L1 = 0; + mf->rxbytes_L1 = 0; + mf->mshr = mshr; + if (mshr) mshr->mf = (void*)mf; // for debugging + mf->write = write; + + if (write) + made_write_mfs++; + else + made_read_mfs++; + memlatstat_start(mf); + addrdec_tlx(addr,&mf->tlx); + mf->bank = mf->tlx.bk; + mf->chip = mf->tlx.chip; + if (gpgpu_cache_dl2_opt) + mf->nbytes_L2 = L2c_get_linesize( dram[mf->tlx.chip] ); + else + mf->nbytes_L2 = 0; + mf->txbytes_L2 = 0; + mf->rxbytes_L2 = 0; + + mf->write_mask = partial_write_mask; + if (!write) assert(partial_write_mask == NO_PARTIAL_WRITE); + + // stat collection codes + mf->mem_acc = mem_acc; + mf->pc = pc; + + switch (mem_acc) { + case CONST_ACC_R: gpgpu_n_mem_const++; break; + case TEXTURE_ACC_R: gpgpu_n_mem_texture++; break; + case GLOBAL_ACC_R: gpgpu_n_mem_read_global++; break; + case GLOBAL_ACC_W: gpgpu_n_mem_write_global++; break; + case LOCAL_ACC_R: gpgpu_n_mem_read_local++; break; + case LOCAL_ACC_W: gpgpu_n_mem_write_local++; break; + default: assert(0); + } + + return(issue_mf_from_fq(mf)); + +} + +int issue_mf_from_fq(mem_fetch_t *mf){ + int destination; // where is the next level of memory? + destination = mf->tlx.chip; + int tpc_id = mf->sid / gpu_concentration; + + if (mf->mshr) mshr_update_status(mf->mshr,IN_ICNT2MEM); + if (!mf->write) { + mf->type = RD_REQ; + assert( mf->timestamp == (gpu_sim_cycle+gpu_tot_sim_cycle) ); + time_vector_update(mf->mshr->insts[0].uid, MR_ICNT_PUSHED, gpu_sim_cycle+gpu_tot_sim_cycle, mf->type ); + icnt_push(tpc_id, mem2device(destination), (void*)mf, READ_PACKET_SIZE); + } else { + mf->type = WT_REQ; + icnt_push(tpc_id, mem2device(destination), (void*)mf, mf->nbytes_L1); + gpgpu_n_sent_writes++; + assert( mf->timestamp == (gpu_sim_cycle+gpu_tot_sim_cycle) ); + time_vector_update(mf->request_uid, MR_ICNT_PUSHED, gpu_sim_cycle+gpu_tot_sim_cycle, mf->type ) ; + } + + return 0; +} + +extern void mshr_return_from_mem(shader_core_ctx_t * shader, mshr_entry_t* mshr); + +inline void fill_shd_L1_with_new_line(shader_core_ctx_t * sc, mem_fetch_t * mf) { + unsigned long long int repl_addr = -1; + // When the data arrives, it flags all the appropriate MSHR + // entries accordingly (by checking the address in each entry ) + memlatstat_read_done(mf); + + mshr_return_from_mem(sc, mf->mshr); + + if (mf->mshr->istexture) { + shd_cache_fill(sc->L1texcache,mf->addr,sc->gpu_cycle); + repl_addr = -1; + } else if (mf->mshr->isconst) { + shd_cache_fill(sc->L1constcache,mf->addr,sc->gpu_cycle); + repl_addr = -1; + } else { + if (!gpgpu_no_dl1) { + //if we are doing a writeback cache we may have marked off a mask in the mshr + //only write into the cache unmasked bytes. + //since this doesn't affect timing we don't actually do it. + repl_addr = shd_cache_fill(sc->L1cache,mf->addr,sc->gpu_cycle); + } + } + + freed_read_mfs++; + free(mf); +} + +unsigned char fq_pop(int tpc_id) +{ + mem_fetch_t *mf; + + mf = (mem_fetch_t*) icnt_pop(tpc_id); + + // if there is a memory fetch request coming back, forward it to the proper shader core + if (mf) { + assert(mf->type == REPLY_DATA); + time_vector_update(mf->mshr->insts[0].uid ,MR_2SH_FQ_POP,gpu_sim_cycle+gpu_tot_sim_cycle, mf->type ) ; + fill_shd_L1_with_new_line(sc[mf->sid], mf); + } + return 0; +} + +//////////////////////////////////////////////////////////////////////////////////////////////// + +int issue_block2core( shader_core_ctx_t *shdr, int grid_num ) +{ + int tid, nthreads_2beissued, more_threads; + int nthreads_in_block= 0; + int start_thread = 0; + int end_thread = shdr->n_threads; + int cta_id=-1; + int cta_size=0; + int padded_cta_size; + + cta_size = ptx_sim_cta_size(); + padded_cta_size = cta_size; + + assert(gpgpu_spread_blocks_across_cores); //should be if muliple CTA per shader supported + + for (unsigned i=0;icta_status[i]==0) { // + cta_id=i; + break; + } + } + assert( cta_id!=-1);//must have found a CTA to run + if (padded_cta_size%warp_size) { + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + } + start_thread = cta_id * padded_cta_size; + end_thread = start_thread + cta_size; + shader_reinit(shdr,start_thread, end_thread); + + // issue threads in blocks (if it is specified) + warp_set_t warps; + for (int i = start_thread; ithread[i].cta_id = cta_id; + nthreads_in_block += ptx_sim_init_thread(&shdr->thread[i].ptx_thd_info,shdr->sid,i,cta_size-(i-start_thread),shdr->n_threads/*cta_size*/,shdr,cta_id,warp_id); + warps.set( warp_id ); + } + shdr->allocate_barrier( cta_id, warps ); + + shader_init_CTA(shdr, start_thread, end_thread); + nthreads_2beissued = nthreads_in_block; + shdr->cta_status[cta_id]+=nthreads_2beissued; + assert( nthreads_2beissued ); //we should have not reached this point if there is no more thread to - + + assert( (unsigned) nthreads_2beissued <= shdr->n_threads); //confirm threads to be issued is less than or equal to number of threads supported by microarchitecture + + int n_cta_issued= nthreads_2beissued/cta_size ;//+ nthreads_2beissued%cta_size; + shdr->n_active_cta += n_cta_issued; + shader_CTA_count_log(shdr->sid, n_cta_issued); + g_total_cta_left-= n_cta_issued; + + more_threads = 1; + if (gpgpu_spread_blocks_across_cores) { + nthreads_2beissued += start_thread; + } + printf("Shader %d initializing CTA #%d with hw tids from %d to %d @(%lld,%lld)", + shdr->sid, cta_id, start_thread, nthreads_2beissued, gpu_sim_cycle, gpu_tot_sim_cycle ); + printf(" shdr->not_completed = %d\n", shdr->not_completed); + + for (tid=start_thread;tidnot_completed += 1; + assert( shdr->warp[tid/warp_size].n_completed > 0 ); + assert( shdr->warp[tid/warp_size].n_completed <= warp_size); + shdr->warp[tid/warp_size].n_completed--; + + // set avail4fetch flag to ready + shdr->thread[tid].avail4fetch = 1; + assert( shdr->warp[tid/warp_size].n_avail4fetch < warp_size ); + shdr->warp[tid/warp_size].n_avail4fetch++; + + g_nthreads_issued++; + } + + if (!nthreads_in_block) more_threads = 0; + return more_threads; //if there are no more threads to be issued, return 0 +} + +/////////////////////////////////////////////////////////////////////////////////////////// +// wrapper code to to create an illusion of a memory controller with L2 cache. +// +int mem_ctrl_full( int mc_id ) +{ + if (gpgpu_cache_dl2_opt) { + return L2c_full( dram[mc_id] ); + } else { + return( gpgpu_dram_sched_queue_size && dram_full(dram[mc_id]) ); + } +} + +//#define DEBUG_PARTIAL_WRITES +void mem_ctrl_push( int mc_id, mem_fetch_t* mf ) +{ + if (gpgpu_cache_dl2_opt) { + L2c_push(dram[mc_id], mf); + } else { + addrdec_t tlx; + addrdec_tlx(mf->addr, &tlx); +#if 0 //old chunking no longer valid. + if (gpgpu_partial_write_mask && mf->write) { + assert( gpgpu_no_dl1 ); // gpgpu_partial_write_mask is not supported with caches for now + } +#endif //#if 0 //old chunking no longer valid + dram_push(dram[mc_id], + tlx.bk, tlx.row, tlx.col, + mf->nbytes_L1, mf->write, + mf->wid, mf->sid, mf->cache_hits_waiting, mf->addr, mf); + memlatstat_dram_access(mf, mc_id, tlx.bk); + if (mf->mshr) mshr_update_status(mf->mshr,IN_DRAM_REQ_QUEUE); + } +} + +void* mem_ctrl_pop( int mc_id ) +{ + mem_fetch_t* mf; + if (gpgpu_cache_dl2_opt) { + mf = L2c_pop(dram[mc_id]); + if (mf && mf->mshr && mf->mshr->insts[0].callback.function) { + dram_callback_t* cb = &(mf->mshr->insts[0].callback); + cb->function(cb->instruction, cb->thread); + } + return mf; + } else { + mf = static_cast (dq_pop(dram[mc_id]->returnq)); //dram_pop(dram[mc_id]); + if (mf) mf->type = REPLY_DATA; + if (mf && mf->mshr && mf->mshr->insts[0].callback.function) { + dram_callback_t* cb = &(mf->mshr->insts[0].callback); + cb->function(cb->instruction, cb->thread); + } + return mf; + } +} + +void* mem_ctrl_top( int mc_id ) +{ + mem_fetch_t* mf; + if (gpgpu_cache_dl2_opt) { + return L2c_top(dram[mc_id]); + } else { + mf = static_cast (dq_top(dram[mc_id]->returnq));//dram_top(dram[mc_id]); + if (mf) mf->type = REPLY_DATA; + return mf ;//dram_top(dram[mc_id]); + } +} + +void get_dram_output ( dram_t* dram_p ) +{ + mem_fetch_t* mf; + mem_fetch_t* mf_top; + mf_top = (mem_fetch_t*) dram_top(dram_p); //test + if (mf_top) { + if (mf_top->type == DUMMY_READ) { + dram_pop(dram_p); + free(mf_top); + freed_dummy_read_mfs++; + return; + } + } + if (gpgpu_cache_dl2_opt) { + L2c_get_dram_output( dram_p ); + } else { + if ( dq_full(dram_p->returnq) ) return; + mf = (mem_fetch_t*) dram_pop(dram_p); + assert (mf_top==mf ); + if (mf) { + dq_push(dram_p->returnq, mf); + if (mf->mshr) mshr_update_status(mf->mshr,IN_DRAMRETURN_Q); + } + } +} + +void dram_log (int task ) { + static void ** mrqq_Dist; //memory request queue inside DRAM + if (task == CREATELOG) { + mrqq_Dist = (void **) calloc(gpu_n_mem,sizeof(void*)); + for (unsigned i=0;iqueue_limit) + mrqq_Dist[i] = StatCreate("mrqq_length",1,dram[i]->queue_limit); + else //queue length is unlimited; + mrqq_Dist[i] = StatCreate("mrqq_length",1,64); //track up to 64 entries + } + } else if (task == SAMPLELOG) { + for (unsigned i=0;imrqq->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i);StatDisp(dram[i]->rwq->lat_stat); + } +} + +//Find next clock domain and increment its time +inline int next_clock_domain(void) +{ + double smallest = min3(core_time,icnt_time,dram_time); + int mask = 0x00; + if (gpgpu_cache_dl2_opt //when no-L2 it will never be L2's turn + && ( l2_time <= smallest) ) { + smallest = l2_time; + mask |= L2 ; + l2_time += l2_period; + } + if ( icnt_time <= smallest ) { + mask |= ICNT; + icnt_time += icnt_period; + } + if ( dram_time <= smallest ) { + mask |= DRAM; + dram_time += dram_period; + } + if ( core_time <= smallest ) { + mask |= CORE; + core_time += core_period; + } + return mask; +} + +extern time_t simulation_starttime; +void gpu_sim_loop( int grid_num ) +{ + int clock_mask = next_clock_domain(); + + // shader core loading (pop from ICNT into shader core) follows CORE clock + if (clock_mask & CORE ) { + for (int i=0;isource_node = mem2device(i); + assert( mf->type != RD_REQ && mf->type != WT_REQ ); // never should a request come out from L2 or dram + if (!mf->write) { + int return_dev = -1; + return_dev = mf->sid / gpu_concentration; + assert(return_dev != -1); + // check icnt resource for READ data return + rt_size[return_dev] = mf->nbytes_L1; + if ( icnt_has_buffer( mem2device(i), rt_size) ) { + if (mf->mshr) mshr_update_status(mf->mshr,IN_ICNT2SHADER); + memlatstat_icnt2sh_push(mf); + time_vector_update(mf->mshr->insts[0].uid ,MR_2SH_ICNT_PUSHED,gpu_sim_cycle+gpu_tot_sim_cycle,RD_REQ); + icnt_push( mem2device(i), return_dev, mf, mf->nbytes_L1); + mem_ctrl_pop(i); + } else { + gpu_stall_icnt2sh++; + } + rt_size[return_dev] = 0; // clean up for the next dram_pop + } else { + time_vector_update(mf->request_uid ,MR_2SH_ICNT_PUSHED,gpu_sim_cycle+gpu_tot_sim_cycle,WT_REQ ) ; + mem_ctrl_pop(i); + free(mf); + freed_L1write_mfs++; + gpgpu_n_processed_writes++; + } + } + } + } + + if (clock_mask & DRAM) { + for (unsigned i=0;itype==RD_REQ) { + time_vector_update(mf->mshr->insts[0].uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ; + } else { + time_vector_update(mf->request_uid ,MR_DRAMQ,gpu_sim_cycle+gpu_tot_sim_cycle,mf->type ) ; + } + memlatstat_icnt2mem_pop(mf); + mem_ctrl_push( i, mf ); + } + } + icnt_transfer( ); + } + + if (clock_mask & CORE) { + // L1 cache + shader core pipeline stages + for (unsigned i=0;inot_completed || more_thread) + shader_cycle(sc[i], i, grid_num); + sc[i]->gpu_cycle++; + } + gpu_sim_cycle++; + + for (unsigned i=0;in_active_cta + cta_issue_count) <= max_cta_per_shader(sc[i]) ) + && g_total_cta_left ) { + int j; + for (j=0;j -1 ); + } + } else { + if (!(sc[i]->not_completed)) + more_thread = issue_block2core(sc[i], grid_num); + } + } + + + // Flush the caches once all of threads are completed. + if (gpgpu_flush_cache) { + int all_threads_complete = 1 ; + for (unsigned i=0;inot_completed == 0) { + shader_cache_flush(sc[i]); + } else { + all_threads_complete = 0 ; + } + } + if (all_threads_complete) { + printf("Flushed L1 caches...\n"); + if (gpgpu_cache_dl2_opt) { + int dlc = 0; + for (unsigned i=0;i= gpu_n_shader ) { + printf("shader %u is out of range\n",sid); + return; + } + if ( tid >= gpu_n_thread_per_shader ) { + printf("thread %u is out of range\n",tid); + return; + } + + shader_core_ctx_t *s = sc[sid]; + + ptx_dump_regs( s->thread[tid].ptx_thd_info ); +} + +int ptx_thread_done( void *thr ); + +void shader_dump_istream_state(shader_core_ctx_t *shader, FILE *fout ) +{ + fprintf( fout, "\n"); + for (unsigned t=0; t < gpu_n_thread_per_shader/warp_size; t++ ) { + int tid = t*warp_size; + if ( shader->warp[t].n_completed < warp_size ) { + fprintf( fout, " %u:%3u fetch state = c:%u a4f:%u bw:%u (completed: ", shader->sid, tid, + shader->warp[t].n_completed, + shader->warp[t].n_avail4fetch, + shader->warp[t].n_waiting_at_barrier ); + + for (unsigned i = tid; i < (t+1)*warp_size; i++ ) { + if ( ptx_thread_done(shader->thread[i].ptx_thd_info) ) { + fprintf(fout,"1"); + } else { + fprintf(fout,"0"); + } + if ( (((i+1)%4) == 0) && (i+1) < (t+1)*warp_size ) { + fprintf(fout,","); + } + } + fprintf(fout,")\n"); + } + } +} + +void dump_pipeline_impl( int mask, int s, int m ) +{ +/* + You may want to use this function while running GPGPU-Sim in gdb. + One way to do that is add the following to your .gdbinit file: + + define dp + call dump_pipeline_impl((0x40|0x4|0x1),$arg0,0) + end + + Then, typing "dp 3" will show the contents of the pipeline for shader core 3. +*/ + + printf("Dumping pipeline state...\n"); + if(!mask) mask = 0xFFFFFFFF; + for (unsigned i=0;i +#include +#include +#include "zlib.h" + +#include "../option_parser.h" +#include "shader.h" +#include "dram.h" +#include "mem_fetch.h" + +#ifndef GPU_SIM_H +#define GPU_SIM_H + +#define NUM_SHADERS 8 +#define STREAMS_PER_FILE 128 + +//unsigned int run_gpu_sim(int grid_num); + +unsigned int get_converge_point(unsigned int pc, void *thd); + +#define GPU_RSTAT_SHD_INFO 0x1 +#define GPU_RSTAT_BW_STAT 0x2 +#define GPU_RSTAT_WARP_DIS 0x4 +#define GPU_RSTAT_DWF_MAP 0x8 + +//gpgpu_interwarp_mshr_merge +#define TEX_MSHR_MERGE 0x4 +#define CONST_MSHR_MERGE 0x2 +#define GLOBAL_MSHR_MERGE 0x1 + +//Prints out a verbose L1 miss rate per thread for shader 0 +#define GPU_RSTAT_L1MISS 0x10 +#define GPU_RSTAT_PDOM 0x20 +#define GPU_RSTAT_SCHED 0x40 + +//options for gpgpu_memlatency_stat +#define GPU_MEMLATSTAT_MC 0x2 +#define GPU_MEMLATSTAT_QUEUELOGS 0x4 + +//void gpu_reg_options(option_parser_t opp); +//void init_gpu(); +void gpu_print_stat(); + +int mem_ctrl_full( int mc_id ); + +void dramqueue_latency_log_dump(); + +#endif diff --git a/src/gpgpu-sim/histogram.h b/src/gpgpu-sim/histogram.h new file mode 100644 index 0000000..2d39410 --- /dev/null +++ b/src/gpgpu-sim/histogram.h @@ -0,0 +1,128 @@ +/* + * histogram.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef HISTOGRAM_H +#define HISTOGRAM_H + +#ifdef __cplusplus + +#include + +class binned_histogram { +protected: + + std::string m_name; + int m_nbins; + int *m_bins; // bin boundaries + int *m_bin_cnts; // counters + int m_maximum; // the maximum sample + +public: + + binned_histogram (std::string name = "", int nbins = 32, int* bins = NULL); + + binned_histogram (const binned_histogram& other); + + void reset_bins (); + + void add2bin (int sample); + + void fprint (FILE *fout); + + virtual ~binned_histogram (); + +}; + +class pow2_histogram : public binned_histogram { + +public: + + pow2_histogram ( std::string name = "", int nbins = 32, int* bins = NULL); + + ~pow2_histogram() {} + + void add2bin (int sample); + +}; + +class linear_histogram : public binned_histogram { + +private: + + int m_stride; + +public: + + linear_histogram (int stride = 1, const char *name = NULL, int nbins = 32, int* bins = NULL); + + ~linear_histogram() {} + + void add2bin (int sample); + +}; + +#endif + +#endif /* HISTOGRAM_H */ diff --git a/src/gpgpu-sim/icnt_wrapper.cc b/src/gpgpu-sim/icnt_wrapper.cc new file mode 100644 index 0000000..3e38269 --- /dev/null +++ b/src/gpgpu-sim/icnt_wrapper.cc @@ -0,0 +1,97 @@ +/* + * icnt_wrapper.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "icnt_wrapper.h" +#include +#include "../intersim/interconnect_interface.h" + +icnt_has_buffer_p icnt_has_buffer; +icnt_push_p icnt_push; +icnt_pop_p icnt_pop; +icnt_transfer_p icnt_transfer; +icnt_busy_p icnt_busy; + +extern int g_network_mode; +extern char* g_network_config_filename; + +void icnt_init( unsigned int n_shader, unsigned int n_mem ) +{ + switch (g_network_mode) { + + case INTERSIM: + init_interconnect(g_network_config_filename ,n_shader, n_mem); + icnt_has_buffer = interconnect_has_buffer; + icnt_push = interconnect_push; + icnt_pop = interconnect_pop; + icnt_transfer = advance_interconnect; + icnt_busy = interconnect_busy; + break; + + default: + assert(0); + break; + } +} diff --git a/src/gpgpu-sim/icnt_wrapper.h b/src/gpgpu-sim/icnt_wrapper.h new file mode 100644 index 0000000..998a4f3 --- /dev/null +++ b/src/gpgpu-sim/icnt_wrapper.h @@ -0,0 +1,100 @@ +/* + * icnt_wrapper.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef ICNT_WRAPPER_H +#define ICNT_WRAPPER_H + +// functional interface to the interconnect +typedef int (*icnt_has_buffer_p)(unsigned int input, unsigned int *size); +typedef void (*icnt_push_p)(unsigned int input, unsigned int output, void* data, unsigned int size); +typedef void* (*icnt_pop_p)(unsigned int output); +typedef void (*icnt_transfer_p)( ); +typedef unsigned (*icnt_busy_p)( ); +typedef void (*icnt_drain_p)( ); + + +extern icnt_has_buffer_p icnt_has_buffer; +extern icnt_push_p icnt_push; +extern icnt_pop_p icnt_pop; +extern icnt_transfer_p icnt_transfer; +extern icnt_busy_p icnt_busy; +extern icnt_drain_p icnt_drain; + +/* +// definition of valid gpu network mode. +extern enum { + INTERSIM = 1, + N_NETWORK_MODE +} gpu_network_mode; +*/ +enum network_mode { + INTERSIM = 1, + N_NETWORK_MODE +}; + +void icnt_init( unsigned int n_shader, unsigned int n_mem ); + +#endif diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc new file mode 100644 index 0000000..aef837b --- /dev/null +++ b/src/gpgpu-sim/l2cache.cc @@ -0,0 +1,999 @@ +#include +#include +#include + +#include +#include +#include "../tr1_hash_map.h" // for unordered_map failback + +#include "../option_parser.h" +#include "mem_fetch.h" +#include "dram.h" +#include "gpu-cache.h" +#include "histogram.h" +#include "l2cache.h" +#include "../intersim/statwraper.h" + +class L2c_mshr; +class L2c_miss_tracker; +class L2c_access_locality; + +mem_fetch_t* g_debug_mf = NULL; + +// L2 cache block (include the cache model + flow controls) +struct L2cacheblk +{ + shd_cache_t *L2cache; + + delay_queue *cbtoL2queue; //latency 10 + delay_queue *cbtoL2writequeue; + delay_queue *dramtoL2queue; //latency 10 + delay_queue *dramtoL2writequeue; + delay_queue *L2todramqueue; //latency 0 + delay_queue *L2todram_wbqueue; + delay_queue *L2tocbqueue; //latency 0 + + mem_fetch_t *L2request; //request currently being serviced by the L2 Cache + + L2c_mshr *m_mshr; // mshr model + L2c_miss_tracker *m_missTracker; // tracker observing for redundant misses + L2c_access_locality *m_accessLocality; // tracking true locality of L2 Cache access + + L2cacheblk(size_t linesize); + ~L2cacheblk(); +}; + +// external dependencies +extern unsigned long long int addrdec_mask[5]; +extern dram_t **dram; +extern int gpgpu_dram_sched_queue_size; +extern unsigned int gpu_n_shader; +extern unsigned int gpu_n_mem; +extern unsigned long long gpu_sim_cycle; +extern unsigned made_write_mfs; +extern unsigned freed_L1write_mfs; +extern unsigned freed_L2write_mfs; +extern unsigned int gpgpu_n_sent_writes; +extern unsigned int gpgpu_n_processed_writes; +extern unsigned g_next_request_uid; + +void memlatstat_icnt2sh_push(mem_fetch_t *mf); +void memlatstat_dram_access(mem_fetch_t *mf, unsigned dram_id, unsigned bank); +void memlatstat_start(mem_fetch_t *mf); +unsigned memlatstat_done(mem_fetch_t *mf); + +// option +char *gpgpu_L2_queue_config; +int gpgpu_l2_readoverwrite = 0; +int l2_ideal = 0; + +void L2c_options(option_parser_t opp) +{ + option_parser_register(opp, "-gpgpu_L2_queue", OPT_CSTR, &gpgpu_L2_queue_config, + "L2 data cache queue length and latency config", + "0:0:0:0:0:0:10:10"); + + option_parser_register(opp, "-gpgpu_l2_readoverwrite", OPT_BOOL, &gpgpu_l2_readoverwrite, + "Prioritize read requests over write requests for L2", + "0"); + + option_parser_register(opp, "-l2_ideal", OPT_BOOL, &l2_ideal, + "Use a ideal L2 cache that always hit", + "0"); +} + +// stats +unsigned L2_write_miss = 0; +unsigned L2_write_hit = 0; +unsigned L2_read_hit = 0; +unsigned L2_read_miss = 0; +unsigned int *L2_cbtoL2length; +unsigned int *L2_cbtoL2writelength; +unsigned int *L2_L2tocblength; +unsigned int *L2_dramtoL2length; +unsigned int *L2_dramtoL2writelength; +unsigned int *L2_L2todramlength; + +//////////////////////////////////////////////// +// L2 MSHR model + +class L2c_mshr +{ +private: + typedef std::list mem_fetch_list; + typedef tr1_hash_map L2missGroup; + L2missGroup m_L2missgroup; // structure tracking redundant dram access + + struct active_chain { + address_type cacheTag; + mem_fetch_list *list; + active_chain() : cacheTag(0xDEADBEEF), list(NULL) { } + }; + active_chain m_active_mshr_chain; + size_t m_linesize; // L2 cache line size + + const size_t m_n_entries; // total number of entries available + size_t m_entries_used; // number of entries in use + + int m_n_miss; + int m_n_miss_serviced_by_dram; + int m_n_mshr_hits; + size_t m_max_entries_used; + + address_type cache_tag(const mem_fetch_t *mf) const + { + // return mf->addr; + return (mf->addr & ~(m_linesize - 1)); + } + +public: + L2c_mshr(size_t linesize, size_t n_entries = 64) + : m_linesize(linesize), m_n_entries(n_entries), m_entries_used(0), + m_n_miss(0), m_n_miss_serviced_by_dram(0), m_n_mshr_hits(0), m_max_entries_used(0) { } + + // add a cache miss to MSHR, return true if this access is hit another existing entry and merges with it + bool new_miss(const mem_fetch_t *mf); + + // notify MSHR that a new cache line has been fetched, activate the associated MSHR chain + void miss_serviced(const mem_fetch_t *mf); + + // probe if there are pending hits left in this MSHR chain + bool mshr_chain_empty(); + + // peek the first entry in the active MSHR chain + mem_fetch_t *mshr_chain_top(); + + // pop the first entry in the active MSHR chain + void mshr_chain_pop(); + + void print(FILE *fout = stdout); + void print_stat(FILE *fout = stdout); +}; + +bool L2c_mshr::new_miss(const mem_fetch_t *mf) +{ + address_type cacheTag = cache_tag(mf); + mem_fetch_list &missGroup = m_L2missgroup[cacheTag]; + + bool mshr_hit = not missGroup.empty(); + + missGroup.push_front(mf); + + m_n_miss += 1; + if (mshr_hit) + m_n_mshr_hits += 1; + m_entries_used += 1; + m_max_entries_used = std::max(m_max_entries_used, m_entries_used); + + return mshr_hit; +} + +void L2c_mshr::miss_serviced(const mem_fetch_t *mf) +{ + assert(m_active_mshr_chain.list == NULL); + address_type cacheTag = cache_tag(mf); + L2missGroup::iterator missGroup = m_L2missgroup.find(cacheTag); + if (missGroup == m_L2missgroup.end() || mf->type == L2_WTBK_DATA) { + assert(mf->type == L2_WTBK_DATA); // only this returning mem req can be missed by the MSHR + return; + } + assert(missGroup->first == cacheTag); + + m_active_mshr_chain.cacheTag = cacheTag; + m_active_mshr_chain.list = &(missGroup->second); + + m_n_miss_serviced_by_dram += 1; +} + +bool L2c_mshr::mshr_chain_empty() +{ + return (m_active_mshr_chain.list == NULL); +} + +mem_fetch_t *L2c_mshr::mshr_chain_top() +{ + const mem_fetch_t *mf = m_active_mshr_chain.list->back(); + assert(cache_tag(mf) == m_active_mshr_chain.cacheTag); + + return const_cast(mf); +} + +void L2c_mshr::mshr_chain_pop() +{ + m_entries_used -= 1; + m_active_mshr_chain.list->pop_back(); + if (m_active_mshr_chain.list->empty()) { + address_type cacheTag = m_active_mshr_chain.cacheTag; + m_L2missgroup.erase(cacheTag); + m_active_mshr_chain.list = NULL; + } +} + +void L2c_mshr::print(FILE *fout) +{ + fprintf(fout, "L2c MSHR: n_entries_used = %zu\n", m_entries_used); + L2missGroup::iterator missGroup; + for (missGroup = m_L2missgroup.begin(); missGroup != m_L2missgroup.end(); ++missGroup) { + fprintf(fout, "%#08x: ", missGroup->first); + mem_fetch_list &mf_list = missGroup->second; + for (mem_fetch_list::iterator imf = mf_list.begin(); imf != mf_list.end(); ++imf) { + fprintf(fout, "%p:%d ", *imf, (*imf)->request_uid); + } + fprintf(fout, "\n"); + } +} + +void L2c_mshr::print_stat(FILE *fout) +{ + fprintf(fout, "L2c MSHR: max_entry = %zu, n_miss = %d, n_mshr_hits = %d, n_serviced_by_dram %d\n", + m_max_entries_used, m_n_miss, m_n_mshr_hits, m_n_miss_serviced_by_dram); +} + +//////////////////////////////////////////////// +// track redundant dram access generated by L2 cache +class L2c_miss_tracker +{ +private: + typedef std::set mem_fetch_set; + typedef tr1_hash_map L2missGroup; + L2missGroup m_L2missgroup; // structure tracking redundant dram access + size_t m_linesize; // L2 cache line size + + typedef tr1_hash_map L2redundantCnt; + L2redundantCnt m_L2redundantCnt; + + int m_totalL2redundantAcc; + + address_type cache_tag(const mem_fetch_t *mf) const + { + // return mf->addr; + return (mf->addr & ~(m_linesize - 1)); + } + +public: + L2c_miss_tracker(size_t linesize) : m_linesize(linesize), m_totalL2redundantAcc(0) { } + void new_miss(mem_fetch_t *mf); + void miss_serviced(mem_fetch_t *mf); + + void print(FILE *fout, bool brief = true); + void print_stat(FILE *fout, bool brief = true); + +}; + +void L2c_miss_tracker::new_miss(mem_fetch_t *mf) +{ + address_type cacheTag = cache_tag(mf); + mem_fetch_set &missGroup = m_L2missgroup[cacheTag]; + + if (missGroup.size() != 0) { + m_L2redundantCnt[cacheTag] += 1; + m_totalL2redundantAcc += 1; + } + + missGroup.insert(mf); +} + +void L2c_miss_tracker::miss_serviced(mem_fetch_t *mf) +{ + address_type cacheTag = cache_tag(mf); + L2missGroup::iterator iMissGroup = m_L2missgroup.find(cacheTag); + if (iMissGroup == m_L2missgroup.end()) return; // this is possible for write miss + mem_fetch_set &missGroup = iMissGroup->second; + + missGroup.erase(mf); + + // remove the miss group if it goes empty + if (missGroup.empty()) { + m_L2missgroup.erase(iMissGroup); + } +} + +void L2c_miss_tracker::print(FILE *fout, bool brief) +{ + L2missGroup::iterator iMissGroup; + for (iMissGroup = m_L2missgroup.begin(); iMissGroup != m_L2missgroup.end(); ++iMissGroup) { + fprintf(fout, "%#08x: ", iMissGroup->first); + for (mem_fetch_set::iterator iMemSet = iMissGroup->second.begin(); iMemSet != iMissGroup->second.end(); ++iMemSet) { + fprintf(fout, "%p ", *iMemSet); + } + fprintf(fout, "\n"); + } +} + +void L2c_miss_tracker::print_stat(FILE *fout, bool brief) +{ + fprintf(fout, "RedundantMiss = %d\n", m_totalL2redundantAcc); + if (brief == true) return; + fprintf(fout, " Detail:"); + for (L2redundantCnt::iterator iL2rc = m_L2redundantCnt.begin(); iL2rc != m_L2redundantCnt.end(); ++iL2rc) { + fprintf(fout, "%#08x:%d ", iL2rc->first, iL2rc->second); + } + fprintf(fout, "\n"); +} + +//////////////////////////////////////////////// +// track all locality of L2 cache access +class L2c_access_locality +{ +private: + size_t m_linesize; // L2 cache line size + + typedef tr1_hash_map L2accCnt; + L2accCnt m_L2accCnt; + + int m_totalL2accAcc; + + address_type cache_tag(const mem_fetch_t *mf) const + { + // return mf->addr; + return (mf->addr & ~(m_linesize - 1)); + } + +public: + L2c_access_locality(size_t linesize) : m_linesize(linesize), m_totalL2accAcc(0) { } + void access(mem_fetch_t *mf); + + void print_stat(FILE *fout, bool brief = true); + +}; + +void L2c_access_locality::access(mem_fetch_t *mf) +{ + address_type cacheTag = cache_tag(mf); + m_L2accCnt[cacheTag] += 1; + m_totalL2accAcc += 1; +} + +void L2c_access_locality::print_stat(FILE *fout, bool brief) +{ + float access_locality = (float) m_totalL2accAcc / m_L2accCnt.size(); + fprintf(fout, "Access Locality = %d / %zu (%f) \n", m_totalL2accAcc, m_L2accCnt.size(), access_locality); + if (brief == true) return; + fprintf(fout, " Detail:"); + pow2_histogram locality_histo(" Hits"); + for (L2accCnt::iterator iL2rc = m_L2accCnt.begin(); iL2rc != m_L2accCnt.end(); ++iL2rc) { + locality_histo.add2bin(iL2rc->second); + // fprintf(fout, "%#08x:%d\n", iL2rc->first, iL2rc->second); + } + locality_histo.fprint(fout); + fprintf(fout, "\n"); +} + +L2cacheblk::L2cacheblk(size_t linesize) +: m_mshr(new L2c_mshr(linesize)), + m_missTracker(new L2c_miss_tracker(linesize)), + m_accessLocality(new L2c_access_locality(linesize)) +{ } + +L2cacheblk::~L2cacheblk() +{ + delete m_mshr; + delete m_missTracker; + delete m_accessLocality; +} + + +//////////////////////////////////////////////// +// L2 access functions + +// L2 Cache Creation +void L2c_create ( dram_t* dram_p, const char* cache_opt ) +{ + unsigned int shd_n_set; + unsigned int shd_linesize; + unsigned int shd_n_assoc; + unsigned char shd_policy; + + unsigned int L2c_cb_L2_length; + unsigned int L2c_cb_L2w_length; + unsigned int L2c_L2_dm_length; + unsigned int L2c_dm_L2_length; + unsigned int L2c_dm_L2w_length; + unsigned int L2c_L2_cb_length; + unsigned int L2c_L2_cb_minlength; + unsigned int L2c_L2_dm_minlength; + + sscanf(cache_opt,"%d:%d:%d:%c", + &shd_n_set, &shd_linesize, &shd_n_assoc, &shd_policy); + + L2cacheblk *p_L2c = new L2cacheblk(shd_linesize); + + char L2c_name[32]; + snprintf(L2c_name, 32, "L2c_%03d", dram_p->id); + p_L2c->L2cache = shd_cache_create(L2c_name, + shd_n_set, shd_n_assoc, shd_linesize, + shd_policy, 16, ~addrdec_mask[CHIP], + write_through); //write_through maintains old behavior for now + + sscanf(gpgpu_L2_queue_config,"%d:%d:%d:%d:%d:%d:%d:%d", + &L2c_cb_L2_length, &L2c_cb_L2w_length, &L2c_L2_dm_length, + &L2c_dm_L2_length, &L2c_dm_L2w_length, &L2c_L2_cb_length, + &L2c_L2_cb_minlength, &L2c_L2_dm_minlength ); + //(,,,) + p_L2c->cbtoL2queue = dq_create("cbtoL2queue", 0,0,L2c_cb_L2_length); + p_L2c->cbtoL2writequeue = dq_create("cbtoL2writequeue", 0,0,L2c_cb_L2w_length); + p_L2c->L2todramqueue = dq_create("L2todramqueue", 0,L2c_L2_dm_minlength,L2c_L2_dm_length); + p_L2c->dramtoL2queue = dq_create("dramtoL2queue", 0,0,L2c_dm_L2_length); + p_L2c->dramtoL2writequeue = dq_create("dramtoL2writequeue",0,0,L2c_dm_L2w_length); + p_L2c->L2tocbqueue = dq_create("L2tocbqueue", 0,L2c_L2_cb_minlength,L2c_L2_cb_length); + + p_L2c->L2todram_wbqueue = dq_create("L2todram_wbqueue", 0,L2c_L2_dm_minlength, + L2c_L2_dm_minlength + gpgpu_dram_sched_queue_size + L2c_dm_L2_length); + + p_L2c->L2request = NULL; + + assert(dram_p->m_L2cache == NULL); + dram_p->m_L2cache = reinterpret_cast(p_L2c); +} + +unsigned L2c_get_linesize( dram_t *dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + return p_L2c->L2cache->line_sz; +} + +int L2c_full( dram_t *dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + return(dq_full(p_L2c->cbtoL2queue) || dq_full(p_L2c->cbtoL2writequeue)); +} + +void L2c_push( dram_t *dram_p, mem_fetch_t *mf ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + if (gpgpu_l2_readoverwrite && mf->write) + dq_push(p_L2c->cbtoL2writequeue, mf); + else + dq_push(p_L2c->cbtoL2queue, mf); + p_L2c->m_accessLocality->access(mf); + if (mf->mshr) mshr_update_status(mf->mshr, IN_CBTOL2QUEUE); +} + +mem_fetch_t* L2c_pop( dram_t *dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + mem_fetch_t *mf; + mf = (mem_fetch_t*)dq_pop(p_L2c->L2tocbqueue); + + return mf; +} + +mem_fetch_t* L2c_top( dram_t *dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + return (mem_fetch_t*)dq_top(p_L2c->L2tocbqueue); +} + +void L2c_qlen ( dram_t *dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + printf("\n"); + printf("cb->L2{%d}\tcb->L2w{%d}\tL2->cb{%d}\n", + p_L2c->cbtoL2queue->length, + p_L2c->cbtoL2writequeue->length, + p_L2c->L2tocbqueue->length); + printf("dm->L2{%d}\tdm->L2w{%d}\tL2->dm{%d}\tL2->wb_dm{%d}\n", + p_L2c->dramtoL2queue->length, + p_L2c->dramtoL2writequeue->length, + p_L2c->L2todramqueue->length, + p_L2c->L2todram_wbqueue->length); +} + +// service memory request in icnt-to-L2 queue, writing to L2 as necessary +// (if L2 writeback miss, writeback to memory) +void L2c_service_mem_req ( dram_t* dram_p, int dm_id ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + mem_fetch_t* mf; + + if (!p_L2c->L2request) { + //if not servicing L2 cache request.. + p_L2c->L2request = (mem_fetch_t*) dq_pop(p_L2c->cbtoL2queue); //..then get one + if (!p_L2c->L2request) { + p_L2c->L2request = (mem_fetch_t*) dq_pop(p_L2c->cbtoL2writequeue); + } + } + + mf = p_L2c->L2request; + + if (!mf) return; + + switch (mf->type) { + case RD_REQ: + case WT_REQ: { + shd_cache_line_t *hit_cacheline = shd_cache_access(p_L2c->L2cache, + mf->addr, + 4, mf->write, + gpu_sim_cycle); + + if (hit_cacheline || l2_ideal) { //L2 Cache Hit; reads are sent as a single command and need to be stored + if (!mf->write) { //L2 Cache Read + if ( dq_full(p_L2c->L2tocbqueue) ) { + p_L2c->L2cache->access--; + } else { + mf->type = REPLY_DATA; + dq_push(p_L2c->L2tocbqueue, mf); + // at this point, should first check if earlier L2 miss is ready to be serviced + // if so, service earlier L2 miss first + p_L2c->L2request = NULL; //finished servicing + L2_read_hit++; + memlatstat_icnt2sh_push(mf); + if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TOCBQUEUE_HIT); + } + } else { //L2 Cache Write aka servicing L1 Writeback + p_L2c->L2request = NULL; + L2_write_hit++; + freed_L1write_mfs++; + free(mf); //writeback from L1 successful + gpgpu_n_processed_writes++; + } + } else { + // L2 Cache Miss; issue commands accordingly + if ( dq_full(p_L2c->L2todramqueue) ) { + p_L2c->L2cache->miss--; + p_L2c->L2cache->access--; + } else { + // if a miss hit the mshr, that means there is another inflight request for the same data + // this miss just need to access the cache later when this request is serviced + bool mshr_hit = p_L2c->m_mshr->new_miss(mf); + if (not mshr_hit) { + if (!mf->write) { + dq_push(p_L2c->L2todramqueue, mf); + } else { + // if request is writeback from L1 and misses, + // then redirect mf writes to dram (no write allocate) + mf->nbytes_L2 = mf->nbytes_L1 - READ_PACKET_SIZE; + dq_push(p_L2c->L2todramqueue, mf); + } + } + if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TODRAMQUEUE); + p_L2c->L2request = NULL; + } + } + } + break; + default: assert(0); + } +} + +// service memory request in L2todramqueue, pushing to dram +void L2c_push_miss_to_dram ( dram_t* dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + mem_fetch_t* mf; + + if ( gpgpu_dram_sched_queue_size && dram_full(dram_p) ) return; + + mf = (mem_fetch_t*) dq_pop(p_L2c->L2todram_wbqueue); //prioritize writeback + if (!mf) mf = (mem_fetch_t*) dq_pop(p_L2c->L2todramqueue); + if (mf) { + if (mf->write) { + L2_write_miss++; + } else { + L2_read_miss++; + } + p_L2c->m_missTracker->new_miss(mf); + memlatstat_dram_access(mf, dram_p->id, mf->tlx.bk); + dram_push(dram_p, + mf->tlx.bk, mf->tlx.row, mf->tlx.col, + mf->nbytes_L2, mf->write, + mf->wid, mf->sid, mf->cache_hits_waiting, mf->addr, mf); + if (mf->mshr) mshr_update_status(mf->mshr, IN_DRAM_REQ_QUEUE); + } +} + +//Service writes that are finished in Dram +//only updates the stats and frees the mf +void dramtoL2_service_write(mem_fetch_t * mf) { + freed_L2write_mfs++; + free(mf); + gpgpu_n_processed_writes++; +} + +// pop completed memory request from dram and push it to dram-to-L2 queue +void L2c_get_dram_output ( dram_t* dram_p ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + mem_fetch_t* mf; + mem_fetch_t* mf_top; + if ( dq_full(p_L2c->dramtoL2queue) || dq_full(p_L2c->dramtoL2writequeue) ) return; + mf_top = (mem_fetch_t*) dram_top(dram_p); //test + mf = (mem_fetch_t*) dram_pop(dram_p); + assert (mf_top==mf ); + if (mf) { + if (gpgpu_l2_readoverwrite && mf->write) + dq_push(p_L2c->dramtoL2writequeue, mf); + else + dq_push(p_L2c->dramtoL2queue, mf); + if (mf->mshr) mshr_update_status(mf->mshr, IN_DRAMTOL2QUEUE); + } +} + +// service memory request in dramtoL2queue, writing to L2 as necessary +// (may cause cache eviction and subsequent writeback) +void L2c_process_dram_output ( dram_t* dram_p, int dm_id ) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + static mem_fetch_t **L2dramout = NULL; + static unsigned long long int *wb_addr = NULL; + if (!L2dramout) L2dramout = (mem_fetch_t**)calloc(gpu_n_mem, sizeof(mem_fetch_t*)); + if (!wb_addr) { + wb_addr = (unsigned long long int*)calloc(gpu_n_mem, sizeof(unsigned long long int)); + for (unsigned i = 0; i < gpu_n_mem; i++) wb_addr[i] = -1; + } + + if (L2dramout[dm_id] == NULL) { + // pop from mshr chain if it is not empty, otherwise, pop a new cacheline from dram output queue + if (p_L2c->m_mshr->mshr_chain_empty() == false) { + L2dramout[dm_id] = p_L2c->m_mshr->mshr_chain_top(); + p_L2c->m_mshr->mshr_chain_pop(); + } else { + L2dramout[dm_id] = (mem_fetch_t*) dq_pop(p_L2c->dramtoL2queue); + if (!L2dramout[dm_id]) L2dramout[dm_id] = (mem_fetch_t*) dq_pop(p_L2c->dramtoL2writequeue); + + if (L2dramout[dm_id] != NULL) { + p_L2c->m_mshr->miss_serviced(L2dramout[dm_id]); + + if (p_L2c->m_mshr->mshr_chain_empty() == false) { // possible if this is a L2 writeback + L2dramout[dm_id] = p_L2c->m_mshr->mshr_chain_top(); + p_L2c->m_mshr->mshr_chain_pop(); + } + } + } + } + + mem_fetch_t* mf = L2dramout[dm_id]; + if (mf) { + if (!mf->write) { //service L2 read miss + + // it is a pre-fill dramout mf + if (wb_addr[dm_id] == (unsigned long long int)-1) { + if ( dq_full(p_L2c->L2tocbqueue) ) goto RETURN; + + if (mf->mshr) mshr_update_status(mf->mshr, IN_L2TOCBQUEUE_MISS); + + //only transfer across icnt once the whole line has been received by L2 cache + mf->type = REPLY_DATA; + dq_push(p_L2c->L2tocbqueue, mf); + + assert(mf->sid <= (int)gpu_n_shader); + shd_cache_line_t *fetch_line_exist = shd_cache_probe(p_L2c->L2cache, mf->addr); + if (fetch_line_exist == NULL) { + wb_addr[dm_id] = L2_shd_cache_fill(p_L2c->L2cache, mf->addr, gpu_sim_cycle ); + } + } + // only perform a write on cache eviction (write-back policy) + // it is the 1st or nth time trial to writeback + if (wb_addr[dm_id] != (unsigned long long int)-1) { + // performing L2 writeback (no false sharing for memory-side cache) + int wb_succeed = L2c_write_back(wb_addr[dm_id], p_L2c->L2cache->line_sz, dm_id ); + if (!wb_succeed) goto RETURN; //try again next cycle + } + + p_L2c->m_missTracker->miss_serviced(mf); + L2dramout[dm_id] = NULL; + wb_addr[dm_id] = -1; + } else { //service L2 write miss + p_L2c->m_missTracker->miss_serviced(mf); + dramtoL2_service_write(mf); + L2dramout[dm_id] = NULL; + wb_addr[dm_id] = -1; + } + } + RETURN: + assert (L2dramout[dm_id] || wb_addr[dm_id] == (unsigned long long int)-1); +} + +// Writeback from L2 to DRAM: +// - Takes in memory address and their parameters and pushes to dram request queue +// - This is used only for L2 writeback +unsigned char L2c_write_back(unsigned long long int addr, int bsize, int dram_id ) +{ + addrdec_t tlx; + addrdec_tlx(addr,&tlx); + + assert(dram[dram_id]->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram[dram_id]->m_L2cache); + + if ( dq_full(p_L2c->L2todram_wbqueue) ) return 0; + + mem_fetch_t *mf; + + mf = (mem_fetch_t*) malloc(sizeof(mem_fetch_t)); + made_write_mfs++; + mf->request_uid = g_next_request_uid++; + mf->addr = addr; + mf->nbytes_L1 = bsize + READ_PACKET_SIZE; + mf->txbytes_L1 = 0; + mf->rxbytes_L1 = 0; + mf->nbytes_L2 = bsize; + mf->sid = gpu_n_shader; // (gpu_n_shader+1); + mf->wid = 0; + mf->txbytes_L2 = 0; + mf->rxbytes_L2 = 0; + mf->mshr = NULL; + mf->pc = -1; // disable ptx_file_line_stats + mf->write = 1; // it is writeback + mf->mem_acc = L2_WRBK_ACC; + memlatstat_start(mf); + mf->tlx = tlx; + mf->bank = mf->tlx.bk; + mf->chip = mf->tlx.chip; + + + //writeback + mf->type = L2_WTBK_DATA; + if (!dq_push(p_L2c->L2todram_wbqueue, mf)) assert(0); + gpgpu_n_sent_writes++; + return 1; +} + +unsigned int L2c_cache_flush ( dram_t* dram_p) { + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + shd_cache_t *cp = p_L2c->L2cache; + int dirty_lines_flushed = 0 ; + for (unsigned i = 0; i < cp->nset * cp->assoc ; i++) { + if ( (cp->lines[i].status & (DIRTY|VALID)) == (DIRTY|VALID) ) { + dirty_lines_flushed++; + } + cp->lines[i].status &= ~VALID; + cp->lines[i].status &= ~DIRTY; + } + return dirty_lines_flushed; +} + +void L2c_init_stat() +{ + L2_cbtoL2length = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); + L2_cbtoL2writelength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); + L2_L2tocblength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); + L2_dramtoL2length = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); + L2_dramtoL2writelength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); + L2_L2todramlength = (unsigned int*) calloc(gpu_n_mem, sizeof(unsigned int)); +} + +void L2c_update_stat( dram_t* dram_p) +{ + assert(dram_p->m_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram_p->m_L2cache); + + int i = dram_p->id; + + if (p_L2c->cbtoL2queue->length > L2_cbtoL2length[i]) + L2_cbtoL2length[i] = p_L2c->cbtoL2queue->length; + if (p_L2c->cbtoL2writequeue->length > L2_cbtoL2writelength[i]) + L2_cbtoL2writelength[i] = p_L2c->cbtoL2writequeue->length; + if (p_L2c->L2tocbqueue->length > L2_L2tocblength[i]) + L2_L2tocblength[i] = p_L2c->L2tocbqueue->length; + if (p_L2c->dramtoL2queue->length > L2_dramtoL2length[i]) + L2_dramtoL2length[i] = p_L2c->dramtoL2queue->length; + if (p_L2c->dramtoL2writequeue->length > L2_dramtoL2writelength[i]) + L2_dramtoL2writelength[i] = p_L2c->dramtoL2writequeue->length; + if (p_L2c->L2todramqueue->length > L2_L2todramlength[i]) + L2_L2todramlength[i] = p_L2c->L2todramqueue->length; +} + +void L2c_print_stat( ) +{ + unsigned i; + + printf(" "); + for (i=0;im_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram[i]->m_L2cache); + + shd_cache_print(p_L2c->L2cache,stdout); + j += p_L2c->L2cache->miss; + k += p_L2c->L2cache->access; + p_L2c->m_mshr->print_stat(stdout); + p_L2c->m_missTracker->print_stat(stdout); + p_L2c->m_accessLocality->print_stat(stdout, false); + } + printf("L2 Cache Total Miss Rate = %0.3f\n", (float)j/k); +} + +void L2c_print_debug( ) +{ + unsigned i; + + printf(" "); + for (i=0;icbtoL2queue->length); + } + printf("\n"); + + printf("cbtoL2 write queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->cbtoL2writequeue->length); + } + printf("\n"); + + printf("L2tocb queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->L2tocbqueue->length); + } + printf("\n"); + + printf("dramtoL2 queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->dramtoL2queue->length); + } + printf("\n"); + + printf("dramtoL2 write queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->dramtoL2writequeue->length); + } + printf("\n"); + + printf("L2todram queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->L2todramqueue->length); + } + printf("\n"); + + printf("L2todram writeback queue length ="); + for (i=0;i(dram[i]->m_L2cache); + printf("%8d", p_L2c->L2todram_wbqueue->length); + } + printf("\n"); +} + +#define CREATELOG 111 +#define SAMPLELOG 222 +#define DUMPLOG 333 + +void L2c_log(int task) +{ + unsigned i; + static void ** cbtol2_Dist ; + static void ** cbtoL2wr_Dist ; + static void ** L2tocb_Dist ; + static void ** dramtoL2_Dist ; + static void ** dramtoL2wr_Dist ; + static void ** L2todram_Dist ; + static void ** L2todram_wb_Dist ; + if (task == CREATELOG) { + cbtol2_Dist = (void **) calloc(gpu_n_mem,sizeof(void*)); + cbtoL2wr_Dist = (void **) calloc(gpu_n_mem,sizeof(void*)); + L2tocb_Dist = (void **) calloc(gpu_n_mem,sizeof(void*)); + dramtoL2_Dist = (void **)calloc(gpu_n_mem,sizeof(void*)); + dramtoL2wr_Dist = (void **)calloc(gpu_n_mem,sizeof(void*)); + L2todram_Dist = (void **)calloc(gpu_n_mem,sizeof(void*)); + L2todram_wb_Dist = (void **)calloc(gpu_n_mem,sizeof(void*)); + + for (i=0;im_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram[i]->m_L2cache); + + cbtol2_Dist[i] = StatCreate("cbtoL2",1,p_L2c->cbtoL2queue->max_len); + cbtoL2wr_Dist[i] = StatCreate("cbtoL2write",1,p_L2c->cbtoL2writequeue->max_len); + L2tocb_Dist[i] = StatCreate("L2tocb",1,p_L2c->L2tocbqueue->max_len); + dramtoL2_Dist[i] = StatCreate("dramtoL2",1,p_L2c->dramtoL2queue->max_len); + dramtoL2wr_Dist[i] = StatCreate("dramtoL2write",1,p_L2c->dramtoL2writequeue->max_len); + L2todram_Dist[i] = StatCreate("L2todram",1,p_L2c->L2todramqueue->max_len); + L2todram_wb_Dist[i] = StatCreate("L2todram_wb",1,p_L2c->L2todram_wbqueue->max_len); + } + } else if (task == SAMPLELOG) { + for (i=0;im_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram[i]->m_L2cache); + + StatAddSample(cbtol2_Dist[i], p_L2c->cbtoL2queue->length); + StatAddSample(cbtoL2wr_Dist[i], p_L2c->cbtoL2writequeue->length); + StatAddSample(L2tocb_Dist[i], p_L2c->L2tocbqueue->length); + StatAddSample(dramtoL2_Dist[i], p_L2c->dramtoL2queue->length); + StatAddSample(dramtoL2wr_Dist[i], p_L2c->dramtoL2writequeue->length); + StatAddSample(L2todram_Dist[i], p_L2c->L2todramqueue->length); + StatAddSample(L2todram_wb_Dist[i], p_L2c->L2todram_wbqueue->length); + } + } else if (task == DUMPLOG) { + for (i=0;im_L2cache != NULL); + L2cacheblk *p_L2c = reinterpret_cast(dram[i]->m_L2cache); + + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->cbtoL2queue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->cbtoL2writequeue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2tocbqueue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->dramtoL2queue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->dramtoL2writequeue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2todramqueue->lat_stat); + printf ("(LOGB2)Latency DRAM[%d] ",i); StatDisp(p_L2c->L2todram_wbqueue->lat_stat); + } +} + + diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h new file mode 100644 index 0000000..449d47a --- /dev/null +++ b/src/gpgpu-sim/l2cache.h @@ -0,0 +1,48 @@ +#pragma once + +#include "dram.h" + +// L2 Cache Creation +void L2c_create ( dram_t* dram_p, const char* cache_opt ); + +void L2c_qlen ( dram_t *dram_p ); + +// service memory request in icnt-to-L2 queue, writing to L2 as necessary +// (if L2 writeback miss, writeback to memory) +void L2c_service_mem_req ( dram_t* dram_p, int dm_id ); + +// service memory request in L2todramqueue, pushing to dram +void L2c_push_miss_to_dram ( dram_t* dram_p ); + +// pop completed memory request from dram and push it to dram-to-L2 queue +void L2c_get_dram_output ( dram_t* dram_p ); + +// service memory request in dramtoL2queue, writing to L2 as necessary +// (may cause cache eviction and subsequent writeback) +void L2c_process_dram_output ( dram_t* dram_p, int dm_id ); + +// Writeback from L2 to DRAM: +// - Takes in memory address and their parameters and pushes to dram request queue +// - This is used only for L2 writeback +unsigned char L2c_write_back(unsigned long long int addr, int bsize, int dram_id ); + +unsigned int L2c_cache_flush ( dram_t* dram_p); + +unsigned L2c_get_linesize( dram_t *dram_p ); + +// probe L2 cache for fullness +int L2c_full( dram_t *dram_p ); +void L2c_push( dram_t *dram_p, mem_fetch_t *mf ); +mem_fetch_t* L2c_pop( dram_t *dram_p ); +mem_fetch_t* L2c_top( dram_t *dram_p ); + +void L2c_init_stat(); +void L2c_update_stat( dram_t* dram_p); +void L2c_print_stat(); +void L2c_print_cache_stat(); +void L2c_print_debug(); +void L2c_log(int task); +void L2c_latency_log_dump(); + +void L2c_options(option_parser_t opp); + diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h new file mode 100644 index 0000000..0e12a2d --- /dev/null +++ b/src/gpgpu-sim/mem_fetch.h @@ -0,0 +1,110 @@ +/* + * mem_fetch.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef MEM_FETCH_H +#define MEM_FETCH_H + +#include "shader.h" +#include "addrdec.h" + +enum mf_type { + RD_REQ = 0, + WT_REQ, + REPLY_DATA, // send to shader + L2_WTBK_DATA, + DUMMY_READ, //used in write mask + N_MF_TYPE +}; + +typedef struct { + unsigned request_uid; + unsigned long long int addr; + int nbytes_L1; + int txbytes_L1; + int rxbytes_L1; + int nbytes_L2; + int txbytes_L2; + int rxbytes_L2; + int sid; //shader core id + int wid; //warp id + int cache_hits_waiting; + mshr_entry* mshr; + address_type pc; + unsigned char write; + enum mem_access_type mem_acc; + unsigned int timestamp; //set to gpu_sim_cycle at struct creation + unsigned int timestamp2; //set to gpu_sim_cycle when pushed onto icnt to shader; only used for reads + unsigned int icnt_receive_time; //set to gpu_sim_cycle + interconnect_latency when fixed icnt latency mode is enabled + unsigned char bank; + unsigned char chip; + addrdec_t tlx; + enum mf_type type; + partial_write_mask_t write_mask; + int source_node; //memory node id when sending from mem to shader + //same as sid when sending from shader 2 mem +} mem_fetch_t; + +#endif diff --git a/src/gpgpu-sim/mem_latency_stat.h b/src/gpgpu-sim/mem_latency_stat.h new file mode 100644 index 0000000..2fa6b1c --- /dev/null +++ b/src/gpgpu-sim/mem_latency_stat.h @@ -0,0 +1,544 @@ +/* + * mem_latency_stat.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + + +#ifndef MEM_LATENCY_STAT_H +#define MEM_LATENCY_STAT_H + +extern unsigned long long gpu_sim_cycle; +extern unsigned int gpu_n_mem; +extern unsigned int gpu_n_shader; +extern int gpgpu_dram_sched_queue_size; +extern int gpgpu_dram_scheduler; +extern unsigned int gpu_mem_n_bk; +#ifdef MEM_LATENCY_STAT_IMPL + #define EXTERN_DEF +#else + #define EXTERN_DEF extern +#endif + +EXTERN_DEF int gpgpu_memlatency_stat = FALSE; + +EXTERN_DEF unsigned max_mrq_latency; +EXTERN_DEF unsigned max_dq_latency; +EXTERN_DEF unsigned max_mf_latency; +EXTERN_DEF unsigned max_icnt2mem_latency; +EXTERN_DEF unsigned max_icnt2sh_latency; +EXTERN_DEF unsigned mrq_lat_table[32]; +EXTERN_DEF unsigned dq_lat_table[32]; +EXTERN_DEF unsigned mf_lat_table[32]; +EXTERN_DEF unsigned icnt2mem_lat_table[24]; +EXTERN_DEF unsigned icnt2sh_lat_table[24]; +EXTERN_DEF unsigned mf_lat_pw_table[32]; //table storing values of mf latency Per Window +EXTERN_DEF unsigned mf_num_lat_pw; +EXTERN_DEF unsigned mf_tot_lat_pw; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window +EXTERN_DEF unsigned long long int mf_total_lat; +EXTERN_DEF unsigned long long int ** mf_total_lat_table; //mf latency sums[dram chip id][bank id] +EXTERN_DEF unsigned ** mf_max_lat_table; //mf latency sums[dram chip id][bank id] +EXTERN_DEF unsigned num_mfs; +EXTERN_DEF unsigned int ***bankwrites; //bankwrites[shader id][dram chip id][bank id] +EXTERN_DEF unsigned int ***bankreads; //bankreads[shader id][dram chip id][bank id] +EXTERN_DEF unsigned int **totalbankwrites; //bankwrites[dram chip id][bank id] +EXTERN_DEF unsigned int **totalbankreads; //bankreads[dram chip id][bank id] +EXTERN_DEF unsigned int **totalbankaccesses; //bankaccesses[dram chip id][bank id] +EXTERN_DEF unsigned int *requests_by_warp; +EXTERN_DEF unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp +EXTERN_DEF unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache +EXTERN_DEF unsigned int *position_of_mrq_chosen; //position of mrq in m_queue chosen +EXTERN_DEF unsigned *mf_num_lat_pw_perwarp; +EXTERN_DEF unsigned *mf_tot_lat_pw_perwarp; //total latency summed up per window per warp. divide by mf_num_lat_pw_perwarp to obtain average latency Per Window +EXTERN_DEF unsigned long long int *mf_total_lat_perwarp; +EXTERN_DEF unsigned *num_mfs_perwarp; +EXTERN_DEF unsigned *acc_mrq_length; + +EXTERN_DEF unsigned ***mem_access_type_stats; // dram access type classification + + +void memlatstat_init( ) +{ + unsigned i,j; + + max_mrq_latency = 0; + max_dq_latency = 0; + max_mf_latency = 0; + max_icnt2mem_latency = 0; + max_icnt2sh_latency = 0; + memset(mrq_lat_table, 0, sizeof(unsigned)*32); + memset(dq_lat_table, 0, sizeof(unsigned)*32); + memset(mf_lat_table, 0, sizeof(unsigned)*32); + memset(icnt2mem_lat_table, 0, sizeof(unsigned)*24); + memset(icnt2sh_lat_table, 0, sizeof(unsigned)*24); + memset(mf_lat_pw_table, 0, sizeof(unsigned)*32); + mf_num_lat_pw = 0; + mf_num_lat_pw_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int)); + mf_tot_lat_pw_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int)); + mf_total_lat_perwarp = (unsigned long long int *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned long long int)); + num_mfs_perwarp = (unsigned *) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int)); + acc_mrq_length = (unsigned *) calloc(gpu_n_mem, sizeof(unsigned int)); + mf_tot_lat_pw = 0; //total latency summed up per window. divide by mf_num_lat_pw to obtain average latency Per Window + mf_total_lat = 0; + num_mfs = 0; + printf("*** Initializing Memory Statistics ***\n"); + requests_by_warp = (unsigned int*) calloc((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1, sizeof(unsigned int)); + totalbankreads = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*)); + totalbankwrites = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*)); + totalbankaccesses = (unsigned int**) calloc(gpu_n_mem, sizeof(unsigned int*)); + mf_total_lat_table = (unsigned long long int **) calloc(gpu_n_mem, sizeof(unsigned long long *)); + mf_max_lat_table = (unsigned **) calloc(gpu_n_mem, sizeof(unsigned *)); + bankreads = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**)); + bankwrites = (unsigned int***) calloc(gpu_n_shader, sizeof(unsigned int**)); + MCB_accesses = (unsigned int*) calloc(gpu_n_mem*4, sizeof(unsigned int)); + num_MCBs_accessed = (unsigned int*) calloc(gpu_n_mem*4+1, sizeof(unsigned int)); + if (gpgpu_dram_sched_queue_size) { + position_of_mrq_chosen = (unsigned int*) calloc(gpgpu_dram_sched_queue_size, sizeof(unsigned int)); + } else + position_of_mrq_chosen = (unsigned int*) calloc(1024, sizeof(unsigned int)); + for (i=0;itimestamp = gpu_sim_cycle + gpu_tot_sim_cycle; + mf->timestamp2 = 0; +} + +// recorder the total latency +unsigned memlatstat_done(mem_fetch_t *mf) +{ + unsigned mf_latency; + unsigned wid = mf->sid*gpu_n_warp_per_shader + mf->wid; + mf_latency = (gpu_sim_cycle+gpu_tot_sim_cycle) - mf->timestamp; + mf_num_lat_pw++; + mf_num_lat_pw_perwarp[wid]++; + mf_tot_lat_pw_perwarp[wid] += mf_latency; + mf_tot_lat_pw += mf_latency; + check_time_vector_update(mf->mshr->insts[0].uid,MR_2SH_FQ_POP,mf_latency, mf->type ) ; + mf_lat_table[LOGB2(mf_latency)]++; + shader_mem_lat_log(mf->sid, mf_latency); + mf_total_lat_table[mf->chip][mf->bank] += mf_latency; + if (mf_latency > max_mf_latency) + max_mf_latency = mf_latency; + return mf_latency; +} + +void memlatstat_icnt2sh_push(mem_fetch_t *mf) +{ + mf->timestamp2 = gpu_sim_cycle+gpu_tot_sim_cycle; +} + +void memlatstat_read_done(mem_fetch_t *mf) +{ + if (gpgpu_memlatency_stat) { + unsigned mf_latency = memlatstat_done(mf); + + if (mf_latency > mf_max_lat_table[mf->chip][mf->bank]) { + mf_max_lat_table[mf->chip][mf->bank] = mf_latency; + } + + unsigned icnt2sh_latency; + icnt2sh_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->timestamp2; + icnt2sh_lat_table[LOGB2(icnt2sh_latency)]++; + if (icnt2sh_latency > max_icnt2sh_latency) + max_icnt2sh_latency = icnt2sh_latency; + } +} + +void memlatstat_dram_access(mem_fetch_t *mf, unsigned dram_id, unsigned bank) +{ + assert(dram_id < gpu_n_mem); + assert(bank < gpu_mem_n_bk); + if (gpgpu_memlatency_stat) { + if (mf->write) { + if ( (unsigned) mf->sid < gpu_n_shader ) { //do not count L2_writebacks here + bankwrites[mf->sid][dram_id][bank]++; + shader_mem_acc_log( mf->sid, dram_id, bank, 'w'); + } + totalbankwrites[dram_id][bank]++; + } else { + bankreads[mf->sid][dram_id][bank]++; + shader_mem_acc_log( mf->sid, dram_id, bank, 'r'); + totalbankreads[dram_id][bank]++; + } + + if (mf->pc != (unsigned) -1) { + ptx_file_line_stats_add_dram_traffic(mf->pc, 1); + } + + mem_access_type_stats[mf->mem_acc][dram_id][bank]++; + } +} + +void memlatstat_icnt2mem_pop(mem_fetch_t *mf) +{ + if (gpgpu_memlatency_stat) { + unsigned icnt2mem_latency; + icnt2mem_latency = (gpu_tot_sim_cycle+gpu_sim_cycle) - mf->timestamp; + icnt2mem_lat_table[LOGB2(icnt2mem_latency)]++; + if (icnt2mem_latency > max_icnt2mem_latency) + max_icnt2mem_latency = icnt2mem_latency; + } +} + +void memlatstat_lat_pw( ) +{ + unsigned i; + if (mf_num_lat_pw && gpgpu_memlatency_stat) { + assert(mf_tot_lat_pw); + mf_total_lat = mf_tot_lat_pw; + num_mfs = mf_num_lat_pw; + mf_lat_pw_table[LOGB2(mf_tot_lat_pw/mf_num_lat_pw)]++; + mf_tot_lat_pw = 0; + mf_num_lat_pw = 0; + } + for (i=0;i < ((gpu_n_shader * gpu_n_thread_per_shader / warp_size)+1); i++) { + if (mf_num_lat_pw_perwarp[i] && gpgpu_memlatency_stat) { + assert(mf_tot_lat_pw_perwarp[i]); + mf_total_lat_perwarp[i] += mf_tot_lat_pw_perwarp[i]; + num_mfs_perwarp[i] += mf_num_lat_pw_perwarp[i]; + //mf_lat_pw_table[LOGB2(mf_tot_lat_pw/mf_num_lat_pw)]++; + mf_tot_lat_pw_perwarp[i] = 0; + mf_num_lat_pw_perwarp[i] = 0; + } + } +} + + +void memlatstat_print( ) +{ + unsigned i,j,k,l,m; + unsigned max_bank_accesses, min_bank_accesses, max_chip_accesses, min_chip_accesses; + + if (gpgpu_memlatency_stat) { + printf("maxmrqlatency = %d \n", max_mrq_latency); + printf("maxdqlatency = %d \n", max_dq_latency); + printf("maxmflatency = %d \n", max_mf_latency); + if (num_mfs) { + printf("averagemflatency = %lld \n", mf_total_lat/num_mfs); + } + printf("max_icnt2mem_latency = %d \n", max_icnt2mem_latency); + printf("max_icnt2sh_latency = %d \n", max_icnt2sh_latency); + printf("mrq_lat_table:"); + for (i=0; i< 32; i++) { + printf("%d \t", mrq_lat_table[i]); + } + printf("\n"); + printf("dq_lat_table:"); + for (i=0; i< 32; i++) { + printf("%d \t", dq_lat_table[i]); + } + printf("\n"); + printf("mf_lat_table:"); + for (i=0; i< 32; i++) { + printf("%d \t", mf_lat_table[i]); + } + printf("\n"); + printf("icnt2mem_lat_table:"); + for (i=0; i< 24; i++) { + printf("%d \t", icnt2mem_lat_table[i]); + } + printf("\n"); + printf("icnt2sh_lat_table:"); + for (i=0; i< 24; i++) { + printf("%d \t", icnt2sh_lat_table[i]); + } + printf("\n"); + printf("mf_lat_pw_table:"); + for (i=0; i< 32; i++) { + printf("%d \t", mf_lat_pw_table[i]); + } + printf("\n"); + + /*MAXIMUM CONCURRENT ACCESSES TO SAME ROW*/ + printf("maximum concurrent accesses to same row:\n"); + for (i=0;i max_bank_accesses) + max_bank_accesses = l; + k += l; + m += l; + printf("%9d ",l); + } + if (m < min_chip_accesses) + min_chip_accesses = m; + if (m > max_chip_accesses) + max_chip_accesses = m; + m = 0; + printf("\n"); + } + printf("total accesses: %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); + + /*READ ACCESSES*/ + k = 0; + l = 0; + m = 0; + max_bank_accesses = 0; + max_chip_accesses = 0; + min_bank_accesses = 0xFFFFFFFF; + min_chip_accesses = 0xFFFFFFFF; + printf("number of total read accesses:\n"); + for (i=0;i max_bank_accesses) + max_bank_accesses = l; + k += l; + m += l; + printf("%9d ",l); + } + if (m < min_chip_accesses) + min_chip_accesses = m; + if (m > max_chip_accesses) + max_chip_accesses = m; + m = 0; + printf("\n"); + } + printf("total reads: %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); + + /*WRITE ACCESSES*/ + k = 0; + l = 0; + m = 0; + max_bank_accesses = 0; + max_chip_accesses = 0; + min_bank_accesses = 0xFFFFFFFF; + min_chip_accesses = 0xFFFFFFFF; + printf("number of total write accesses:\n"); + for (i=0;i max_bank_accesses) + max_bank_accesses = l; + k += l; + m += l; + printf("%9d ",l); + } + if (m < min_chip_accesses) + min_chip_accesses = m; + if (m > max_chip_accesses) + max_chip_accesses = m; + m = 0; + printf("\n"); + } + printf("total reads: %d\n", k); + if (min_bank_accesses) + printf("bank skew: %d/%d = %4.2f\n", max_bank_accesses, min_bank_accesses, (float)max_bank_accesses/min_bank_accesses); + else + printf("min_bank_accesses = 0!\n"); + if (min_chip_accesses) + printf("chip skew: %d/%d = %4.2f\n", max_chip_accesses, min_chip_accesses, (float)max_chip_accesses/min_chip_accesses); + else + printf("min_chip_accesses = 0!\n"); + + + /*AVERAGE MF LATENCY PER BANK*/ + printf("average mf latency per bank:\n"); + for (i=0;i +#include "shader.h" +#include "gpu-sim.h" +#include "addrdec.h" +#include "dram.h" +#include "dwf.h" +#include "warp_tracker.h" +#include "cflogger.h" +#include "gpu-misc.h" +#include "../cuda-sim/ptx_sim.h" +#include "../cuda-sim/ptx-stats.h" +#include "../cuda-sim/dram_callback.h" +#include "mem_fetch.h" +#include + +#define PRIORITIZE_MSHR_OVER_WB 1 +#define MAX(a,b) (((a)>(b))?(a):(b)) + +extern unsigned int mergemiss; +extern unsigned int L1_write_miss; +extern unsigned int L1_read_miss; +extern unsigned int L1_write_hit_on_miss; +extern unsigned int L1_writeback; +extern unsigned int L1_texture_miss; +extern unsigned int L1_const_miss; + +extern unsigned int finished_trace; +extern int gpgpu_perfect_mem; +extern int gpgpu_no_dl1; +extern char *gpgpu_cache_texl1_opt; +extern char *gpgpu_cache_constl1_opt; +extern char *gpgpu_cache_dl1_opt; +extern unsigned int gpu_n_thread_per_shader; +extern unsigned int gpu_n_mshr_per_shader; +extern unsigned int gpu_n_shader; +extern unsigned int gpu_n_mem; +extern int gpgpu_reg_bankconflict; +extern int gpgpu_dram_sched_queue_size; +extern int gpgpu_memlatency_stat; +extern dram_t **dram; +extern int *num_warps_issuable; +extern int *num_warps_issuable_pershader; + +extern unsigned long long gpu_sim_insn; +extern unsigned long long gpu_sim_insn_no_ld_const; +extern unsigned long long gpu_sim_insn_last_update; +extern unsigned long long gpu_completed_thread; +extern unsigned long long gpu_sim_cycle; +extern shader_core_ctx_t **sc; +extern unsigned int gpgpu_pre_mem_stages; +extern int gpgpu_no_divg_load; +extern unsigned int gpgpu_thread_swizzling; +extern unsigned int gpgpu_strict_simd_wrbk; +extern unsigned int warp_conflict_at_writeback; +extern unsigned int gpgpu_commit_pc_beyond_two; +extern int gpgpu_spread_blocks_across_cores; +extern int gpgpu_cflog_interval; + +extern unsigned int gpu_stall_by_MSHRwb; +extern unsigned int gpu_stall_shd_mem; +extern unsigned int gpu_stall_sh2icnt; + +enum mem_stage_access_type { + C_MEM, + T_MEM, + S_MEM, + G_MEM_LD, + L_MEM_LD, + G_MEM_ST, + L_MEM_ST, + N_MEM_STAGE_ACCESS_TYPE +}; + +enum mem_stage_stall_type { + NO_RC_FAIL = 0, + BK_CONF, + MSHR_RC_FAIL, + ICNT_RC_FAIL, + COAL_STALL, + WB_ICNT_RC_FAIL, + WB_CACHE_RSRV_FAIL, + N_MEM_STAGE_STALL_TYPE +}; +unsigned int gpu_stall_shd_mem_breakdown[N_MEM_STAGE_ACCESS_TYPE][N_MEM_STAGE_STALL_TYPE] = { {0} }; + +unsigned warp_size = 4; +int pipe_simd_width; +extern unsigned int **totalbankaccesses; //bankaccesses[shader id][dram chip id][bank id] +extern unsigned int *MCB_accesses; //upon cache miss, tracks which memory controllers accessed by a warp +extern unsigned int *num_MCBs_accessed; //tracks how many memory controllers are accessed whenever any thread in a warp misses in cache +extern unsigned int *max_return_queue_length; + +unsigned int *shader_cycle_distro; + +unsigned int g_waiting_at_barrier = 0; + +unsigned int gpgpu_shmem_size = 16384; +unsigned int gpgpu_shader_registers = 8192; +unsigned int gpgpu_shader_cta = 8; + +unsigned int gpgpu_n_load_insn = 0; +unsigned int gpgpu_n_store_insn = 0; +unsigned int gpgpu_n_shmem_insn = 0; +unsigned int gpgpu_n_tex_insn = 0; +unsigned int gpgpu_n_const_insn = 0; +unsigned int gpgpu_n_param_insn = 0; +unsigned int gpgpu_multi_unq_fetches = 0; + +extern int gpgpu_cache_wt_through; + +int gpgpu_shmem_bkconflict = 0; +unsigned int gpgpu_n_shmem_bkconflict = 0; +int gpgpu_n_shmem_bank = 16; + +int gpgpu_cache_bkconflict = 0; +unsigned int gpgpu_n_cache_bkconflict = 0; +unsigned int gpgpu_n_cmem_portconflict = 0; +int gpgpu_n_cache_bank = 16; + +extern int gpu_runtime_stat_flag; +int gpgpu_warpdistro_shader = -1; + +int gpgpu_interwarp_mshr_merge = 0; +int gpgpu_n_intrawarp_mshr_merge = 0; + +extern int gpgpu_partial_write_mask; +int gpgpu_n_partial_writes = 0; + +extern int gpgpu_n_mem_write_local; +extern int gpgpu_n_mem_write_global; + +#ifndef MhZ + #define MhZ *1000000 +#endif +extern double core_freq; +extern double icnt_freq; +extern double dram_freq; +extern double l2_freq; + +int gpgpu_shmem_port_per_bank = 4; +int gpgpu_cache_port_per_bank = 4; +int gpgpu_const_port_per_bank = 4; +int gpgpu_shmem_pipe_speedup = 2; + +unsigned int gpu_max_cta_per_shader = 8; +unsigned int gpu_padded_cta_size = 32; +int gpgpu_local_mem_map = 1; + + +extern int pdom_sched_type; +extern int n_pdom_sc_orig_stat; +extern int n_pdom_sc_single_stat; +extern int gpgpu_cuda_sim; +extern unsigned long long gpu_tot_sim_cycle; + +extern unsigned g_max_regs_per_thread; +void ptx_decode_inst( void *thd, unsigned *op, int *i1, int *i2, int *i3, int *i4, int *o1, int *o2, int *o3, int *o4, int *vectorin, int *vectorout, int *arch_reg ); +unsigned ptx_get_inst_op( void *thd); +void ptx_exec_inst( void *thd, address_type *addr, unsigned *space, unsigned *data_size, dram_callback_t* callback, unsigned warp_active_mask); +int ptx_branch_taken( void *thd ); +void ptx_sim_free_sm( void** thread_info ); +unsigned ptx_sim_init_thread( void** thread_info, int sid, unsigned tid,unsigned threads_left,unsigned num_threads, core_t *core, unsigned hw_cta_id, unsigned hw_warp_id); +unsigned ptx_sim_cta_size(); +const struct gpgpu_ptx_sim_kernel_info* ptx_sim_kernel_info(); +void set_option_gpgpu_spread_blocks_across_cores(int option); +int ptx_thread_done( void *thr ); +unsigned ptx_thread_donecycle( void *thr ); +int ptx_thread_get_next_pc( void *thd ); +void* ptx_thread_get_next_finfo( void *thd ); +int ptx_thread_at_barrier( void *thd ); +int ptx_thread_all_at_barrier( void *thd ); +unsigned long long ptx_thread_get_cta_uid( void *thd ); +void ptx_thread_reset_barrier( void *thd ); +void ptx_thread_release_barrier( void *thd ); +void ptx_print_insn( address_type pc, FILE *fp ); +int ptx_set_tex_cache_linesize( unsigned linesize); +void time_vector_update(unsigned int uid,int slot ,long int cycle,int type); + + + +///////////////////////////////////////////////////////////////////////////// +/*-------------------------------------------------------------------------*/ +/*-------------------------------------------------------------------------*/ + +static const char* MSHR_Status_str[] = { + "INITIALIZED", + "IN_ICNT2MEM", + "IN_ICNTOL2QUEUE", + "IN_L2TODRAMQUEUE", + "IN_DRAM_REQ_QUEUE", + "IN_DRAMTOL2QUEUE", + "IN_L2TOICNTQUEUE_HIT", + "IN_L2TOICNTQUEUE_MISS", + "IN_ICNT2SHADER", + "FETCHED", +}; + +// a helper function that deduce if a mshr contains an atomic operation +bool isatomic(mshr_entry_t *mshr) +{ + return (mshr->insts[0].callback.function != NULL); +} + +#include +#include +#include +// a class that speeds up mshr lookup with a C++ multimap +class mshr_lookup { +private: + typedef std::multimap mshr_lut_t; + mshr_lut_t m_lut; // multiple mshr entries can have the same tag +private: + void insert(mshr_entry* mshr) + { + using namespace std; + unsigned long long int tag_addr = mshr->addr; + m_lut.insert(make_pair(tag_addr, mshr)); + } + + mshr_entry* lookup(unsigned long long int addr) const + { + using namespace std; + // mshr_lut_t::const_iterator i_lut = m_lut.find(tag_addr); + pair i_range = m_lut.equal_range(addr); + if (i_range.first == i_range.second) { + return NULL; + } else { + mshr_lut_t::const_iterator i_lut = i_range.first; + mshr_entry* mshr_hit = i_lut->second; + //follow match to end of merge chain: + //this won't really work for different sized requests, ie can't merge a 64b request to a 32b + while (mshr_hit->merged_requests) { + mshr_hit = mshr_hit->merged_requests; + } + return mshr_hit; + } + } + + void remove(mshr_entry* mshr) + { + using namespace std; + std::pair i_range = m_lut.equal_range(mshr->addr); + + assert(i_range.first != i_range.second); + + for (mshr_lut_t::iterator i_lut = i_range.first; i_lut != i_range.second; ++i_lut) { + if (i_lut->second == mshr) { + m_lut.erase(i_lut); + break; + } + } + } +public: + //checks if we should do mshr merging for this mshr + bool can_merge(mshr_entry_t * mshr) + { + if (mshr->iswrite) return false; // can't merge a write + if (isatomic(mshr)) return false; // can't merge a atomic operation + bool interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & GLOBAL_MSHR_MERGE; + if (mshr->istexture) { + interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & TEX_MSHR_MERGE; + } else if (mshr->isconst) { + interwarp_mshr_merge = gpgpu_interwarp_mshr_merge & CONST_MSHR_MERGE; + } + return interwarp_mshr_merge; + } + + void mshr_fast_lookup_insert(mshr_entry* mshr) + { + if (!can_merge(mshr)) return; + insert(mshr); + } + + void mshr_fast_lookup_remove(mshr_entry* mshr) + { + if (!can_merge(mshr)) return; + remove(mshr); + } + + mshr_entry* shader_get_mergeable_mshr(mshr_entry_t* mshr) + { + if (!can_merge(mshr)) return NULL; + return lookup(mshr->addr); + } +}; + +class mem_access_t; +int is_tex ( int space ); +int is_const ( int space ); +int is_local ( int space ); +#include +class mshr_shader_unit{ +public: + mshr_shader_unit(unsigned max_mshr): m_max_mshr(max_mshr), m_max_mshr_used(0){ + m_mshrs.resize(max_mshr); + for (std::vector::iterator i = m_mshrs.begin(); i != m_mshrs.end(); i++) m_free_list.push_back(i); + } + bool has_mshr(unsigned num){return (num <= m_free_list.size());} + unsigned mshr_used(){ return m_max_mshr - m_free_list.size();} + mshr_entry_t* add_mshr(mem_access_t &access, inst_t* warp); + + //return queue access; (includes texture pipeline return) + mshr_entry_t* return_head(){ + if (has_return()) + return &(*(choose_return_queue().front())); + else + return NULL; + } + //return queue pop; (includes texture pipeline return) + void pop_return_head() { + free_mshr(return_head()->this_mshr); + choose_return_queue().pop_front(); + } + + static void mshr_update_status(mshr_entry *mshr, enum mshr_status new_status ); + void mshr_return_from_mem(mshr_entry *mshr); + void check_mshr(mshr_entry *mshr){ + assert(find(m_free_list.begin(),m_free_list.end(),mshr->this_mshr)==m_free_list.end()); + assert(mshr->insts.size()); + } + unsigned get_max_mshr_used(){return m_max_mshr_used;} + void print(FILE* fp, shader_core_ctx_t* shader); +private: + typedef std::vector mshr_storage_type;//list might be less complicated, but slower? + mshr_storage_type m_mshrs; + std::deque< mshr_storage_type::iterator > m_free_list; + std::deque< mshr_storage_type::iterator > m_mshr_return_queue; + std::deque< mshr_storage_type::iterator > m_texture_mshr_pipeline; + unsigned m_max_mshr; + unsigned m_max_mshr_used; + mshr_lookup m_mshr_lookup; + + mshr_entry_t *alloc_free_mshr(bool istexture){ + assert(!m_free_list.empty()); + std::vector::iterator i = m_free_list.back(); + m_free_list.pop_back(); + i->this_mshr = i; + if (istexture) { + //put in texture pipeline + m_texture_mshr_pipeline.push_back(i); + } + if (mshr_used() > m_max_mshr_used) m_max_mshr_used = mshr_used(); + return &(*i); + } + void free_mshr(std::vector::iterator &i){ + //clean up up for next time, since not reallocating memory. + m_mshr_lookup.mshr_fast_lookup_remove(&(*i)); //need to remove before clearing insts, as they are accessed + i->insts.clear(); //add expects this to be clear + m_free_list.push_back(i); + } + bool has_return() { return (not m_mshr_return_queue.empty()) or ((not m_texture_mshr_pipeline.empty()) and m_texture_mshr_pipeline.front()->fetched());} + std::deque< std::vector::iterator > & choose_return_queue() { + //prioritize a ready texture over a global/const... + if ((not m_texture_mshr_pipeline.empty()) and m_texture_mshr_pipeline.front()->fetched()) return m_texture_mshr_pipeline; + assert(!m_mshr_return_queue.empty()); + return m_mshr_return_queue; + } +}; + + + +void mshr_shader_unit::mshr_update_status(mshr_entry *mshr, enum mshr_status new_status ) { + mshr->status = new_status; +#if DEBUGL1MISS + printf("cycle %d Addr %x %d \n",gpu_sim_cycle,CACHE_TAG_OF_64(mshr->addr),new_status); +#endif + mshr_entry * merged_req = mshr->merged_requests; + while (merged_req) { + merged_req->status = new_status; + merged_req = merged_req->merged_requests; + } +} + +inline void mshr_shader_unit::mshr_return_from_mem(mshr_entry *mshr){ + mshr_update_status(mshr, FETCHED); + if (not mshr->istexture) { + //place in return queue + m_mshr_return_queue.push_back(mshr->this_mshr); + //place all merged requests in return queue + mshr_entry * merged_req = mshr->merged_requests; + while (merged_req) { + m_mshr_return_queue.push_back(merged_req->this_mshr); + merged_req = merged_req->merged_requests; + } + } +} + +void mshr_return_from_mem(shader_core_ctx_t * shader, mshr_entry_t* mshr){ + shader->mshr_unit->mshr_return_from_mem(mshr); +} + +unsigned get_max_mshr_used(shader_core_ctx_t * shader){ + return shader->mshr_unit->get_max_mshr_used(); +} + + +void mshr_print(FILE* fp, shader_core_ctx_t *shader) { + shader->mshr_unit->print(fp, shader); +} + +void mshr_shader_unit::print(FILE* fp, shader_core_ctx_t* shader){ + for (mshr_storage_type::iterator it = m_mshrs.begin(); it != m_mshrs.end(); it++) { + //valid if not in free list; + if (find(m_free_list.begin(),m_free_list.end(), it) == m_free_list.end()) { + mshr_entry *mshr = &(*it); + fprintf(fp, "MSHR(%d): %s Addr:0x%llx Fetched:%d Merged:%d Status:%s\n", + shader->sid, + (mshr->iswrite)? "=>" : "<=", + mshr->addr, mshr->fetched(), + (mshr->merged_requests != NULL or mshr->merged_on_other_reqest), MSHR_Status_str[mshr->status]); + for (unsigned i = 0; i < mshr->insts.size(); i++) { + fprintf(fp,"\tthread: UID:%d HW:%d ReqAddr:0x%llx\n", mshr->insts[i].uid, mshr->insts[i].hw_thread_id, mshr->insts[i].memreqaddr); + } + } + } +} + +void mshr_update_status(mshr_entry* mshr, enum mshr_status new_status) { + mshr_entry *merged_req; + mshr->status = new_status; +#if DEBUGL1MISS + printf("cycle %d Addr %x %d \n",gpu_sim_cycle,CACHE_TAG_OF_64(mshr->addr),new_status); +#endif + merged_req = mshr->merged_requests; + while (merged_req) { + merged_req->status = new_status; + merged_req = merged_req->merged_requests; + } +} + +///////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////// +/*-------------------------------------------------------------------------*/ + +inst_t create_nop_inst() // just because C++ does not have designated initializer list.... +{ + inst_t nop_inst; + nop_inst.pc = 0; + nop_inst.op=NO_OP; + memset(nop_inst.out, 0, sizeof(nop_inst.out)); + memset(nop_inst.in, 0, sizeof(nop_inst.in)); + nop_inst.is_vectorin=0; + nop_inst.is_vectorout=0; + nop_inst.memreqaddr=0; + nop_inst.reg_bank_access_pending=0; + nop_inst.reg_bank_conflict_stall_checked=0, + nop_inst.hw_thread_id=-1; + nop_inst.wlane=-1; + nop_inst.uid = (unsigned)-1; + nop_inst.priority = (unsigned)-1; + nop_inst.ptx_thd_info = NULL; + nop_inst.warp_active_mask = 0; + nop_inst.ts_cycle = 0; + nop_inst.id_cycle = 0; + nop_inst.ex_cycle = 0; + nop_inst.mm_cycle = 0; + return nop_inst; +} + +static inst_t nop_inst = create_nop_inst(); + +int log2i(int n) { + int lg; + lg = -1; + while (n) { + n>>=1;lg++; + } + return lg; +} + +extern unsigned int gpu_n_warp_per_shader; + +shader_core_ctx_t* shader_create( const char *name, int sid, + unsigned int n_threads, + unsigned int n_mshr, + fq_push_t fq_push, + fq_has_buffer_t fq_has_buffer, + unsigned int model ) +{ + shader_core_ctx_t *sc; + int i; + unsigned int shd_n_set; + unsigned int shd_linesize; + unsigned int shd_n_assoc; + unsigned char shd_policy; + + unsigned int l1tex_cache_n_set; //L1 texture cache parameters + unsigned int l1tex_cache_linesize; + unsigned int l1tex_cache_n_assoc; + unsigned char l1tex_cache_policy; + + unsigned int l1const_cache_n_set; //L1 constant cache parameters + unsigned int l1const_cache_linesize; + unsigned int l1const_cache_n_assoc; + unsigned char l1const_cache_policy; + + if ( gpgpu_cuda_sim ) { + unsigned cta_size = ptx_sim_cta_size(); + if ( cta_size > n_threads ) { + printf("Execution error: Shader kernel CTA (block) size is too large for microarch config.\n"); + printf(" This can cause problems with applications that use __syncthreads.\n"); + printf(" CTA size (x*y*z) = %u, n_threads = %u\n", cta_size, n_threads ); + printf(" => either change -gpgpu_shader argument in gpgpusim.config file or\n"); + printf(" modify the CUDA source to decrease the kernel block size.\n"); + abort(); + } + } + + sc = new shader_core_ctx_t( gpu_n_warp_per_shader, gpgpu_shader_cta ); + + sc->name = name; + sc->sid = sid; + + sc->RR_k = 0; + + sc->model = model; + + sc->pipeline_reg = (inst_t**) calloc(N_PIPELINE_STAGES, sizeof(inst_t*)); + for (int j = 0; jpipeline_reg[j] = (inst_t*) calloc(warp_size, sizeof(inst_t)); + for (unsigned i=0; ipipeline_reg[j][i] = nop_inst; + } + } + + if (gpgpu_pre_mem_stages) { + sc->pre_mem_pipeline = (inst_t**) calloc(gpgpu_pre_mem_stages+1, sizeof(inst_t*)); + for (unsigned j = 0; j<=gpgpu_pre_mem_stages; j++) { + sc->pre_mem_pipeline[j] = (inst_t*) calloc(pipe_simd_width, sizeof(inst_t)); + for (int i=0; ipre_mem_pipeline[j][i] = nop_inst; + } + } + } + sc->n_threads = n_threads; + sc->thread = (thread_ctx_t*) calloc(sizeof(thread_ctx_t), n_threads); + sc->not_completed = 0; + + unsigned n_warp = (n_threads/warp_size) + ((n_threads%warp_size)?1:0); + sc->warp.resize(n_warp, shd_warp_t(warp_size)); + for (unsigned j = 0; j < n_warp; j++) { + sc->warp[j].wid = j; + } + + sc->n_active_cta = 0; + for (i = 0; icta_status[i]=0; + } + //Warp variable initializations + sc->next_warp = 0; + sc->branch_priority = 0; + sc->max_branch_priority = (int*) malloc(sizeof(int)*n_threads); + + + for (unsigned i = 0; imax_branch_priority[i] = INT_MAX; + sc->thread[i].id = i; + + sc->thread[i].warp_priority = sc->max_branch_priority[i]; + sc->thread[i].avail4fetch = 0; + sc->thread[i].m_waiting_at_barrier = 0; + + sc->thread[i].ptx_thd_info = NULL; + sc->thread[i].cta_id = -1; + } + + sscanf(gpgpu_cache_dl1_opt,"%d:%d:%d:%c", + &shd_n_set, &shd_linesize, &shd_n_assoc, &shd_policy); + sscanf(gpgpu_cache_texl1_opt,"%d:%d:%d:%c", + &l1tex_cache_n_set, &l1tex_cache_linesize, &l1tex_cache_n_assoc, &l1tex_cache_policy); + sscanf(gpgpu_cache_constl1_opt,"%d:%d:%d:%c", + &l1const_cache_n_set, &l1const_cache_linesize, &l1const_cache_n_assoc, &l1const_cache_policy); +#define STRSIZE 32 + char L1c_name[STRSIZE]; + char L1texc_name[STRSIZE]; + char L1constc_name[STRSIZE]; + snprintf(L1c_name, STRSIZE, "L1c_%03d", sc->sid); + sc->L1cache = shd_cache_create(L1c_name,shd_n_set,shd_n_assoc,shd_linesize,shd_policy,1,0, + gpgpu_cache_wt_through?write_through:write_back); + shd_cache_bind_logger(sc->L1cache, sc->sid, get_shader_normal_cache_id()); + snprintf(L1texc_name, STRSIZE, "L1texc_%03d", sc->sid); + sc->L1texcache = shd_cache_create(L1texc_name,l1tex_cache_n_set,l1tex_cache_n_assoc,l1tex_cache_linesize,l1tex_cache_policy,1,0, no_writes ); + shd_cache_bind_logger(sc->L1texcache, sc->sid, get_shader_texture_cache_id()); + snprintf(L1constc_name, STRSIZE, "L1constc_%03d", sc->sid); + sc->L1constcache = shd_cache_create(L1constc_name,l1const_cache_n_set,l1const_cache_n_assoc,l1const_cache_linesize,l1const_cache_policy,1,0, no_writes ); + shd_cache_bind_logger(sc->L1constcache, sc->sid, get_shader_constant_cache_id()); + //at this point, should set the parameters used by addressing schemes of all textures + ptx_set_tex_cache_linesize(l1tex_cache_linesize); + + sc->mshr_unit = new mshr_shader_unit(gpu_n_mshr_per_shader); + + sc->fq_push = fq_push; + sc->fq_has_buffer = fq_has_buffer; + + sc->pdom_warp = (pdom_warp_ctx_t*)calloc(n_threads / warp_size, sizeof(pdom_warp_ctx_t)); + for (unsigned i = 0; i < n_threads / warp_size; ++i) { + sc->pdom_warp[i].m_stack_top = 0; + sc->pdom_warp[i].m_pc = (address_type*)calloc(warp_size * 2, sizeof(address_type)); + sc->pdom_warp[i].m_calldepth = (unsigned int*)calloc(warp_size * 2, sizeof(unsigned int)); + sc->pdom_warp[i].m_active_mask = (unsigned int*)calloc(warp_size * 2, sizeof(unsigned int)); + sc->pdom_warp[i].m_recvg_pc = (address_type*)calloc(warp_size * 2, sizeof(address_type)); + sc->pdom_warp[i].m_branch_div_cycle = (unsigned long long *)calloc(warp_size * 2, sizeof(unsigned long long )); + + memset(sc->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type)); + memset(sc->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int)); + memset(sc->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int)); + memset(sc->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type)); + } + + sc->waiting_at_barrier = 0; + + sc->last_issued_thread = sc->n_threads - 1; + + sc->using_dwf = (sc->model == DWF); + + sc->using_rrstage = (sc->model == DWF); + + sc->using_commit_queue = (sc->model == DWF + || sc->model == POST_DOMINATOR || sc->model == NO_RECONVERGE); + + if (sc->using_commit_queue) { + sc->thd_commit_queue = dq_create("thd_commit_queue", 0, 0, 0); + } + + sc->shmem_size = gpgpu_shmem_size; + sc->n_registers = gpgpu_shader_registers; + sc->n_cta = gpgpu_shader_cta; + + sc->shader_memory_new_instruction_processed = false; + + return sc; +} + + +unsigned shader_reinit(shader_core_ctx_t *sc, int start_thread, int end_thread ) +{ + int i; + unsigned result=0; + + if ( gpgpu_cuda_sim ) { + unsigned cta_size = ptx_sim_cta_size(); + if ( cta_size > sc->n_threads ) { + printf("Execution error: Shader kernel CTA (block) size is too large for microarch config.\n"); + printf(" This can cause problems with applications that use __syncthreads.\n"); + printf(" CTA size (x*y*z) = %u, n_threads = %u\n", cta_size, sc->n_threads ); + printf(" => either change -gpgpu_shader argument in gpgpusim.config file or\n"); + printf(" modify the CUDA source to decrease the kernel block size.\n"); + abort(); + } + } + + sc->next_warp = 0; + sc->branch_priority = 0; + + for (i = start_thread; ithread[i].ptx_thd_info); + + for (i = start_thread; imax_branch_priority[i] = INT_MAX; + sc->thread[i].warp_priority = sc->max_branch_priority[i]; + sc->thread[i].n_insn = 0; + sc->thread[i].cta_id = -1; + } + + for (unsigned i = start_thread / warp_size; i < end_thread / warp_size; ++i) { + sc->warp[i].reset(warp_size); + sc->pdom_warp[i].m_stack_top = 0; + memset(sc->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type)); + memset(sc->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int)); + memset(sc->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int)); + memset(sc->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type)); + memset(sc->pdom_warp[i].m_branch_div_cycle, 0, warp_size * 2 * sizeof(unsigned long long )); + } + + sc->waiting_at_barrier = 0; + sc->last_issued_thread = end_thread - 1; + + if (sc->using_commit_queue) { + if (!gpgpu_spread_blocks_across_cores) //assertion no longer holds with multiple blocks per core + assert(dq_empty(sc->thd_commit_queue)); + } + sc->pending_shmem_bkacc = 0; + sc->pending_cache_bkacc = 0; + sc->pending_cmem_acc = 0; + + //do not reset this here, shader memory may be in the middle of processing another cta's instruction. + //sc->shader_memory_new_instruction_processed = false; + + return result; +} + +// initialize a CTA in the shader core, currently only useful for PDOM and DWF + +void shader_init_CTA(shader_core_ctx_t *shader, int start_thread, int end_thread) +{ + int i; + int n_thread = end_thread - start_thread; + address_type start_pc = ptx_thread_get_next_pc(shader->thread[start_thread].ptx_thd_info); + if (shader->model == POST_DOMINATOR) { + int start_warp = start_thread / warp_size; + int end_warp = end_thread / warp_size + ((end_thread % warp_size)? 1 : 0); + for (i = start_warp; i < end_warp; ++i) { + shader->pdom_warp[i].m_stack_top = 0; + memset(shader->pdom_warp[i].m_pc, -1, warp_size * 2 * sizeof(address_type)); + memset(shader->pdom_warp[i].m_calldepth, 0, warp_size * 2 * sizeof(unsigned int)); + memset(shader->pdom_warp[i].m_active_mask, 0, warp_size * 2 * sizeof(unsigned int)); + memset(shader->pdom_warp[i].m_recvg_pc, -1, warp_size * 2 * sizeof(address_type)); + memset(shader->pdom_warp[i].m_branch_div_cycle, 0, warp_size * 2 * sizeof(unsigned long long )); + shader->pdom_warp[i].m_pc[0] = start_pc; + shader->pdom_warp[i].m_calldepth[0] = 1; + int t = 0; + for (t = 0; t < (int)warp_size; t++) { + if ( i * (int)warp_size + t < end_thread ) { + shader->pdom_warp[i].m_active_mask[0] |= (1 << t); + } + } + } + } else if (shader->model == DWF) { + dwf_init_CTA(shader->sid, start_thread, n_thread, start_pc); + } + + for (i = start_thread; ithread[i].in_scheduler = 1; + } +} + + + + +// register id for unused register slot in instruction +#define DNA (0) + +unsigned g_next_shader_inst_uid=1; + +// check to see if the fetch stage need to be stalled +int shader_fetch_stalled(shader_core_ctx_t *shader) +{ + int n_warp_parts = warp_size/pipe_simd_width; + + if (shader->warp_part2issue < n_warp_parts) { + return 1; + } + + for (unsigned i=0; ipipeline_reg[TS_IF][i].hw_thread_id != -1 ) { + return 1; // stalled + } + } + for (int i=0; ipipeline_reg[IF_ID][i].hw_thread_id != -1 ) { + return 1; // stalled + } + } + + shader->warp_part2issue = 0; // reset pointer to first warp part + shader->new_warp_TS = 1; + + return 0; // not stalled +} + +// initalize the pipeline stage register to nops +void shader_clear_stage_reg(shader_core_ctx_t *shader, int stage) +{ + for (unsigned i=0; ipipeline_reg[stage][i] = nop_inst; + } +} + +// return the next pc of a thread +address_type shader_thread_nextpc(shader_core_ctx_t *shader, int tid) +{ + assert( gpgpu_cuda_sim ); + address_type pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info ); + return pc; +} + +// issue thread to the warp +// tid - thread id, warp_id - used by PDOM, wlane - position in warp +void shader_issue_thread(shader_core_ctx_t *shader, int tid, int wlane, unsigned active_mask ) +{ + if ( gpgpu_cuda_sim ) { + shader->pipeline_reg[TS_IF][wlane].hw_thread_id = tid; + shader->pipeline_reg[TS_IF][wlane].wlane = wlane; + shader->pipeline_reg[TS_IF][wlane].pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info ); + shader->pipeline_reg[TS_IF][wlane].ptx_thd_info = shader->thread[tid].ptx_thd_info; + shader->pipeline_reg[TS_IF][wlane].memreqaddr = 0; + shader->pipeline_reg[TS_IF][wlane].reg_bank_conflict_stall_checked = 0; + shader->pipeline_reg[TS_IF][wlane].reg_bank_access_pending = 0; + shader->pipeline_reg[TS_IF][wlane].uid = g_next_shader_inst_uid++; + shader->pipeline_reg[TS_IF][wlane].warp_active_mask = active_mask; + shader->pipeline_reg[TS_IF][wlane].ts_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + } + assert( shader->thread[tid].avail4fetch > 0 ); + shader->thread[tid].avail4fetch--; + assert( shader->warp[wid_from_hw_tid(tid,warp_size)].n_avail4fetch > 0 ); + shader->warp[wid_from_hw_tid(tid,warp_size)].n_avail4fetch--; +} + +void update_max_branch_priority(shader_core_ctx_t *shader, unsigned warp_hw_id, unsigned grid_num ) +{ + int temp_max = 0; + // This means that a group of threads has completed, + // hence need to update max_priority + for (unsigned i = 0; ithread[hw_tid_from_wid(warp_hw_id,warp_size,i)].ptx_thd_info ) ) { + if (shader->thread[hw_tid_from_wid(warp_hw_id,warp_size,i)].warp_priority>=temp_max) { + temp_max = shader->thread[hw_tid_from_wid(warp_hw_id,warp_size,i)].warp_priority; + } + } + } + for (unsigned i = 0; imax_branch_priority[hw_tid_from_wid(warp_hw_id,warp_size,i)] = temp_max; + } +} + +void shader_fetch_simd_no_reconverge(shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) +{ + int i; + int tid; + int new_tid = 0; + address_type pc = 0; + int warp_ok = 0; + int n_warp = shader->n_threads/warp_size; + int complete = 0; + + assert(gpgpu_cuda_sim); + + // First, check to see if entire program is completed, + // if it is, then break out of loop + for (unsigned i=0; in_threads; i++) { + if (!ptx_thread_done( shader->thread[i].ptx_thd_info )) { + complete = 0; + break; + } else { + complete = 1; + } + } + if (complete) { + // printf("Shader has completed program.\n"); + return; + } + + if (shader_fetch_stalled(shader)) { + return; + } + shader_clear_stage_reg(shader, TS_IF); + + // Finds a warp where all threads in it are available for fetching + // simultaneously(all threads are not yet in pipeline, or, the ones + // that are not available, are completed already + for (i=0; iwarp[shader->next_warp].n_completed; + int n_avail4fetch = shader->warp[shader->next_warp].n_avail4fetch; + if (((n_completed) == (int)warp_size) || + ((n_completed + n_avail4fetch) < (int)warp_size) ) { + //All threads in this warp have completed, hence go to next warp + //Or, some of the threads are still in pipeline + warp_ok = 0; // hey look, it's a silent register update / store instruction! (this operation is redundant) + shader->next_warp = (shader->next_warp+1)%n_warp; + } else { + int n_waiting_at_barrier = shader->warp[shader->next_warp].n_waiting_at_barrier; + if ( n_waiting_at_barrier >= (int)warp_size ) { + warp_ok = 0; // hey look, it's a silent register update / store instruction! (this operation is redundant) + continue; + } + warp_ok = 1; + break; + } + } + // None of the instructions from inside the warp can be scheduled -> should + // probably just stall, ie nops into pipeline + if (!warp_ok) { + shader_clear_stage_reg(shader, TS_IF); // NOTE: is this needed? + shader->next_warp = (shader->next_warp+1)%n_warp; // NOTE: this is not round-robin. + return; + } + + tid = warp_size*shader->next_warp; + + for (i = 0; i<(int)warp_size; i++) { + if (shader->thread[tid+i].warp_priority == shader->max_branch_priority[tid+i]) { + pc = shader_thread_nextpc(shader, tid+i); + new_tid = tid+i; + break; + } + } + //Determine which instructions inside this 'warp' will be scheduled together at this run + //If they are cannot be scheduled together then 'save' their branch priority + for (i = 0; i<(int)warp_size; i++) { + if (!ptx_thread_done( shader->thread[tid+i].ptx_thd_info )) { + address_type next_pc; + next_pc = shader_thread_nextpc(shader, tid+i); + if (next_pc != pc || + shader->thread[tid+i].warp_priority != shader->max_branch_priority[tid+i] || + shader->thread[tid+i].m_waiting_at_barrier) { + if (!ptx_thread_done( shader->thread[tid+i].ptx_thd_info )) { + if ( !shader->thread[tid + i].m_waiting_at_barrier ) { + shader->thread[tid + i].warp_priority = shader->branch_priority; + } + } + } else { + shader_issue_thread(shader, tid+i, i,(unsigned)-1); + } + } + } + shader->branch_priority++; + + shader->next_warp = (shader->next_warp+1)%n_warp; +} + +int pdom_sched_find_next_warp (shader_core_ctx_t *shader,int pdom_sched_policy, int* ready_warps + , int ready_warp_count, int* last_warp, int w_comp_c, int w_pipe_c, int w_barr_c) +{ + int n_warp = shader->n_threads/warp_size; + int i=0; + int selected_warp = ready_warps[0]; + int found =0; + + switch (pdom_sched_policy) { + case 0: + selected_warp = ready_warps[0]; //first ok warp found + found=1; + break; + case 1 ://random + selected_warp = ready_warps[rand()%ready_warp_count]; + found=1; + break; + case 8 :// execute the first available warp which is after the warp execued last time + found=0; + selected_warp = (last_warp[shader->sid] + 1 ) % n_warp; + while (!found) { + for (i=0;in_threads/warp_size; + int complete = 0; + int tmp_warp; + int warp_id; + + address_type check_pc = -1; + + assert(gpgpu_cuda_sim); + + // First, check to see if entire program is completed, + // if it is, then break out of loop + for (unsigned i=0; in_threads; i++) { + if (!ptx_thread_done( shader->thread[i].ptx_thd_info )) { + complete = 0; + break; + } else { + complete = 1; + } + } + if (complete) { + return; + } + + if (shader_fetch_stalled(shader)) { + return; + } + shader_clear_stage_reg(shader, TS_IF); + + int ready_warp_count = 0; + int w_comp_c = 0 ; + int w_pipe_c = 0 ; + int w_barr_c = 0 ; + static int * ready_warps = NULL; + static int * tmp_ready_warps = NULL; + if (!ready_warps) { + ready_warps = (int*)calloc(n_warp,sizeof(int)); + } + if (!tmp_ready_warps) { + tmp_ready_warps = (int*)calloc(n_warp,sizeof(int)); + } + for (i=0; iwarp[shader->next_warp].n_completed; + int n_avail4fetch = shader->warp[shader->next_warp].n_avail4fetch; + + if ((n_completed) == (int)warp_size) { + //All threads in this warp have completed + w_comp_c++; + } else if ((n_completed+n_avail4fetch) < (int)warp_size) { + //some of the threads are still in pipeline + w_pipe_c++; + } else if ( shader->warp_waiting_at_barrier(shader->next_warp) ) { + w_barr_c++; + } else { + // A valid warp is found at this point + tmp_ready_warps[ready_warp_count] = shader->next_warp; + ready_warp_count++; + } + shader->next_warp = (shader->next_warp + 1) % n_warp; + } + for (i=0;isid]+= ready_warp_count; + + if (ready_warp_count) { + tmp_warp = pdom_sched_find_next_warp (shader, pdom_sched_type ,ready_warps + , ready_warp_count, last_warp, w_comp_c, w_pipe_c ,w_barr_c); + if (tmp_warp != -1) { + shader->next_warp = tmp_warp; + warp_ok=1; + } + } + + static int no_warp_issued; + // None of the instructions from inside the warp can be scheduled -> should + // probably just stall, ie nops into pipeline + if (!warp_ok) { + shader_clear_stage_reg(shader, TS_IF); + shader->next_warp = (shader->next_warp+1) % n_warp; + no_warp_issued = 1 ; + return; + } + + /************************************************************/ + //at this point we have a warp to execute which is pointed to by + //shader->next_warp + + warp_id = shader->next_warp; + last_warp[shader->sid] = warp_id; + int wtid = warp_size*shader->next_warp; + + pdom_warp_ctx_t *scheduled_warp = &(shader->pdom_warp[warp_id]); + + int stack_top = scheduled_warp->m_stack_top; + + address_type top_pc = scheduled_warp->m_pc[stack_top]; + unsigned int top_active_mask = scheduled_warp->m_active_mask[stack_top]; + address_type top_recvg_pc = scheduled_warp->m_recvg_pc[stack_top]; + + assert(top_active_mask != 0); + + const address_type null_pc = 0; + int warp_diverged = 0; + address_type new_recvg_pc = null_pc; + while (top_active_mask != 0) { + + // extract a group of threads with the same next PC among the active threads in the warp + address_type tmp_next_pc = null_pc; + unsigned int tmp_active_mask = 0; + void *first_active_thread=NULL; + for (i = warp_size - 1; i >= 0; i--) { + unsigned int mask = (1 << i); + if ((top_active_mask & mask) == mask) { // is this thread active? + if (ptx_thread_done( shader->thread[wtid+i].ptx_thd_info )) { + top_active_mask &= ~mask; // remove completed thread from active mask + } else if (tmp_next_pc == null_pc) { + first_active_thread = shader->thread[wtid+i].ptx_thd_info; + tmp_next_pc = shader_thread_nextpc(shader, wtid+i); + tmp_active_mask |= mask; + top_active_mask &= ~mask; + } else if (tmp_next_pc == shader_thread_nextpc(shader, wtid+i)) { + tmp_active_mask |= mask; + top_active_mask &= ~mask; + } + } + } + + // discard the new entry if its PC matches with reconvergence PC + // that automatically reconverges the entry + if (tmp_next_pc == top_recvg_pc) continue; + + // this new entry is not converging + // if this entry does not include thread from the warp, divergence occurs + if (top_active_mask != 0 && warp_diverged == 0) { + warp_diverged = 1; + // modify the existing top entry into a reconvergence entry in the pdom stack + new_recvg_pc = get_converge_point(top_pc,first_active_thread); + if (new_recvg_pc != top_recvg_pc) { + scheduled_warp->m_pc[stack_top] = new_recvg_pc; + scheduled_warp->m_branch_div_cycle[stack_top] = gpu_sim_cycle; + stack_top += 1; + scheduled_warp->m_branch_div_cycle[stack_top] = 0; + } + } + + // discard the new entry if its PC matches with reconvergence PC + if (warp_diverged && tmp_next_pc == new_recvg_pc) continue; + + // update the current top of pdom stack + scheduled_warp->m_pc[stack_top] = tmp_next_pc; + scheduled_warp->m_active_mask[stack_top] = tmp_active_mask; + if (warp_diverged) { + scheduled_warp->m_calldepth[stack_top] = 0; + scheduled_warp->m_recvg_pc[stack_top] = new_recvg_pc; + } else { + scheduled_warp->m_recvg_pc[stack_top] = top_recvg_pc; + } + stack_top += 1; // set top to next entry in the pdom stack + } + scheduled_warp->m_stack_top = stack_top - 1; + + assert(scheduled_warp->m_stack_top >= 0); + assert(scheduled_warp->m_stack_top < (int)warp_size * 2); + + // schedule threads according to active mask on the top of pdom stack + for (i = 0; i < (int)warp_size; i++) { + unsigned int mask = (1 << i); + if ((scheduled_warp->m_active_mask[scheduled_warp->m_stack_top] & mask) == mask) { + assert (!ptx_thread_done( shader->thread[wtid+i].ptx_thd_info )); + shader_issue_thread(shader, wtid+i, i, scheduled_warp->m_active_mask[scheduled_warp->m_stack_top]); + } + } + shader->next_warp = (shader->next_warp+1)%n_warp; + + // check if all issued threads have the same pc + for (i = 0; i < (int) warp_size; i++) { + if ( shader->pipeline_reg[TS_IF][i].hw_thread_id != -1 ) { + if ( check_pc == (unsigned)-1 ) { + check_pc = shader->pipeline_reg[TS_IF][i].pc; + } else { + assert( check_pc == shader->pipeline_reg[TS_IF][i].pc ); + } + } + } +} + + +void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ) +{ + unsigned warp_id = tid/warp_size; + pdom_warp_ctx_t *warp_info = &(sc[sid]->pdom_warp[warp_id]); + unsigned idx = warp_info->m_stack_top; + *pc = warp_info->m_pc[idx]; + *rpc = warp_info->m_recvg_pc[idx]; +} + +void shader_fetch_mimd( shader_core_ctx_t *shader, unsigned int shader_number ) +{ + unsigned int last_issued_thread = 0; + + if (shader_fetch_stalled(shader)) { + return; + } + shader_clear_stage_reg(shader, TS_IF); + + // some form of barrel processing: + // - checking availability from the thread after the last issued thread + for (int i=0, j=0;i<(int)shader->n_threads && j< (int) warp_size;i++) { + int thd_id = (i + shader->last_issued_thread + 1) % shader->n_threads; + if (shader->thread[thd_id].avail4fetch && !shader->thread[thd_id].m_waiting_at_barrier ) { + shader_issue_thread(shader, thd_id, j,(unsigned)-1); + last_issued_thread = thd_id; + j++; + } + } + shader->last_issued_thread = last_issued_thread; +} + +// seperate the incoming warp into multiple warps with seperate pcs +int split_warp_by_pc(int *tid_in, shader_core_ctx_t *shader, int **tid_split, address_type *pc) { + unsigned n_pc = 0; + static int *pc_cnt = NULL; // count the number of threads with the same pc + + assert(tid_in); + assert(tid_split); + assert(pc); + memset(pc,0,sizeof(address_type)*warp_size); + + if (!pc_cnt) pc_cnt = (int*) malloc(sizeof(int)*warp_size); + memset(pc_cnt,0,sizeof(int)*warp_size); + + // go through each thread in the given warp + for (unsigned i=0; i< warp_size; i++) { + if (tid_in[i] < 0) continue; + int matched = 0; + address_type thd_pc; + thd_pc = shader_thread_nextpc(shader, tid_in[i]); + + // check to see if the pc has occured before + for (unsigned j=0; jthread[tid_in[i]].m_reached_barrier) { + reached_barrier = 1; + break; + } + } + return reached_barrier; +} + +// seperate the incoming warp into multiple warps with seperate pcs and cta +int split_warp_by_cta(int *tid_in, shader_core_ctx_t *shader, int **tid_split, address_type *pc, int *cta) { + unsigned n_pc = 0; + static int *pc_cnt = NULL; // count the number of threads with the same pc + + assert(tid_in); + assert(tid_split); + assert(pc); + memset(pc,0,sizeof(address_type)*warp_size); + + if (!pc_cnt) pc_cnt = (int*) malloc(sizeof(int)*warp_size); + memset(pc_cnt,0,sizeof(int)*warp_size); + + // go through each thread in the given warp + for (unsigned i=0; ithread[tid_in[i]].ptx_thd_info ); + + // check to see if the pc has occured before + for (unsigned j=0; jthd_commit_queue) && warpupdate_bw > 0) { + // grab a committed warp, split it into multiple BRUs (tid_split) by PC + int *tid_commit = (int*)dq_pop(shader->thd_commit_queue); + memset(tid_split[0], -1, sizeof(int)*warp_size*warp_size); + memset(thd_pc, 0, sizeof(address_type)*warp_size); + memset(thd_cta, -1, sizeof(int)*warp_size); + + int reached_barrier = warp_reached_barrier(tid_commit, shader); + + unsigned n_warp_update; + if (reached_barrier) { + n_warp_update = split_warp_by_cta(tid_commit, shader, tid_split, thd_pc, thd_cta); + } else { + n_warp_update = split_warp_by_pc(tid_commit, shader, tid_split, thd_pc); + } + + if (n_warp_update > 2) gpgpu_commit_pc_beyond_two++; + warpupdate_bw -= n_warp_update; + // put the splitted warp updates into the DWF scheduler + for (unsigned i=0;ithread[tid_split[i][j]].avail4fetch); + assert(!shader->thread[tid_split[i][j]].in_scheduler); + shader->thread[tid_split[i][j]].in_scheduler = 1; + } + dwf_clear_accessed(shader->sid); + if (reached_barrier) { + dwf_update_warp_at_barrier(shader->sid, tid_split[i], thd_pc[i], thd_cta[i]); + } else { + dwf_update_warp(shader->sid, tid_split[i], thd_pc[i]); + } + } + + free_commit_warp(tid_commit); + } + + // Track the #PC right after the warps are input to the scheduler + dwf_update_statistics(shader->sid); + dwf_clear_policy_access(shader->sid); + + if (shader_fetch_stalled(shader)) { + return; + } + shader_clear_stage_reg(shader, TS_IF); + + address_type scheduled_pc; + dwf_issue_warp(shader->sid, tid_out, &scheduled_pc); + + for (unsigned i=0; i= 0) { + shader_issue_thread(shader, issue_tid, i, (unsigned)-1); + shader->thread[issue_tid].in_scheduler = 0; + shader->thread[issue_tid].m_reached_barrier = 0; + shader->last_issued_thread = issue_tid; + assert(shader->pipeline_reg[TS_IF][i].pc == scheduled_pc); + } + } +} + +void print_shader_cycle_distro( FILE *fout ) +{ + fprintf(fout, "Warp Occupancy Distribution:\n"); + fprintf(fout, "Stall:%d\t", shader_cycle_distro[0]); + fprintf(fout, "W0_Idle:%d\t", shader_cycle_distro[1]); + fprintf(fout, "W0_Mem:%d", shader_cycle_distro[2]); + for (unsigned i = 3; i < warp_size + 3; i++) { + fprintf(fout, "\tW%d:%d", i-2, shader_cycle_distro[i]); + } + fprintf(fout, "\n"); +} + +void inflight_memory_insn_add( shader_core_ctx_t *shader, inst_t *mem_insn) +{ + if (enable_ptx_file_line_stats) { + ptx_file_line_stats_add_inflight_memory_insn(shader->sid, mem_insn->pc); + } +} + +void inflight_memory_insn_sub( shader_core_ctx_t *shader, inst_t *mem_insn) +{ + if (enable_ptx_file_line_stats) { + ptx_file_line_stats_sub_inflight_memory_insn(shader->sid, mem_insn->pc); + } +} + +void report_exposed_memory_latency( shader_core_ctx_t *shader ) +{ + if (enable_ptx_file_line_stats) { + ptx_file_line_stats_commit_exposed_latency(shader->sid, 1); + } +} + +static int gpgpu_warp_occ_detailed = 0; +static int **warp_occ_detailed = NULL; + +void check_stage_pcs( shader_core_ctx_t *shader, unsigned stage ); +void check_pm_stage_pcs( shader_core_ctx_t *shader, unsigned stage ); + +void shader_fetch( shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) +{ + assert(shader->model < NUM_SIMD_MODEL); + int n_warp_parts = warp_size/pipe_simd_width; + + // check if decode stage is stalled + int decode_stalled = 0; + for (int i = 0; i < pipe_simd_width; i++) { + if (shader->pipeline_reg[IF_ID][i].hw_thread_id != -1 ) + decode_stalled = 1; + } + + if (shader->gpu_cycle % n_warp_parts == 0) { + + if (shader->model == POST_DOMINATOR || shader->model == NO_RECONVERGE) { + int warpupdate_bw = 1; // number of warps to be unlocked per scheduler cycle + while (!dq_empty(shader->thd_commit_queue) && warpupdate_bw > 0) { + // grab a committed warp and unlock it here + int *tid_commit = (int*)dq_pop(shader->thd_commit_queue); + for (unsigned i = 0; i < warp_size; i++) { + if (tid_commit[i] >= 0) { + shader->thread[tid_commit[i]].avail4fetch++; + assert(shader->thread[tid_commit[i]].avail4fetch <= 1); + assert( shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch < (unsigned)warp_size ); + shader->warp[wid_from_hw_tid(tid_commit[i],warp_size)].n_avail4fetch++; + } + } + warpupdate_bw -= 1; + free_commit_warp(tid_commit); + } + } + + switch (shader->model) { + case NO_RECONVERGE: + shader_fetch_simd_no_reconverge(shader, shader_number, grid_num ); + break; + case POST_DOMINATOR: + shader_fetch_simd_postdominator(shader, shader_number, grid_num); + break; + case MIMD: + shader_fetch_mimd(shader, shader_number); + break; + case DWF: + shader_fetch_simd_dwf(shader, shader_number); + break; + default: + fprintf(stderr, "Unknown scheduler: %d\n", shader->model); + assert(0); + break; + } + + static int *tid_out = NULL; + if (!tid_out) { + tid_out = (int*) malloc(sizeof(int) * warp_size); + } + memset(tid_out, -1, sizeof(int)*warp_size); + + if (!shader_cycle_distro) { + shader_cycle_distro = (unsigned int*) calloc(warp_size + 3, sizeof(unsigned int)); + } + + if (gpgpu_no_divg_load && shader->new_warp_TS && !decode_stalled) { + int n_thd_in_warp = 0; + address_type pc_out = 0xDEADBEEF; + for (unsigned i=0; ipipeline_reg[TS_IF][i].hw_thread_id; + if (tid_out[i] >= 0) { + n_thd_in_warp += 1; + pc_out = shader->pipeline_reg[TS_IF][i].pc; + } + } + wpt_register_warp(tid_out, shader); + if (gpu_runtime_stat_flag & GPU_RSTAT_DWF_MAP) { + track_thread_pc( shader->sid, tid_out, pc_out ); + } + if (gpgpu_cflog_interval != 0) { + insn_warp_occ_log( shader->sid, pc_out, n_thd_in_warp); + shader_warp_occ_log( shader->sid, n_thd_in_warp); + } + if ( gpgpu_warpdistro_shader < 0 || shader->sid == gpgpu_warpdistro_shader ) { + shader_cycle_distro[n_thd_in_warp + 2] += 1; + if (n_thd_in_warp == 0) { + if (shader->pending_mem_access == 0) shader_cycle_distro[1]++; + } + } + shader->new_warp_TS = 0; + + if (enable_ptx_file_line_stats && n_thd_in_warp > 0) { + //ptx_file_line_stats_add_warp_issued(pc_out); + //ptx_file_line_stats_add_warp_occ_total(pc_out, n_thd_in_warp); + } + + if ( gpgpu_warp_occ_detailed && + n_thd_in_warp && (shader->model == POST_DOMINATOR) ) { + int n_warp = gpu_n_thread_per_shader / warp_size; + if (!warp_occ_detailed) { + warp_occ_detailed = (int**) malloc(sizeof(int*) * gpu_n_shader * n_warp); + warp_occ_detailed[0] = (int*) calloc(sizeof(int), gpu_n_shader * n_warp * warp_size); + for (unsigned i = 0; i < n_warp * gpu_n_shader; i++) { + warp_occ_detailed[i] = warp_occ_detailed[0] + i * warp_size; + } + } + + int wid = -1; + for (unsigned i=0; i= 0) wid = tid_out[i] / warp_size; + } + assert(wid != -1); + warp_occ_detailed[shader->sid * n_warp + wid][n_thd_in_warp - 1] += 1; + + if (shader->sid == 0 && wid == 16 && 0) { + printf("wtrace[%08x] ", pc_out); + for (unsigned i=0; isid == gpgpu_warpdistro_shader ) { + shader_cycle_distro[0] += 1; + } + } + + if (!decode_stalled) { + for (unsigned i = 0; i < warp_size; i++) { + int tid_tsif = shader->pipeline_reg[TS_IF][i].hw_thread_id; + address_type pc_out = shader->pipeline_reg[TS_IF][i].pc; + cflog_update_thread_pc(shader->sid, tid_tsif, pc_out); + } + } + + if (enable_ptx_file_line_stats && !decode_stalled) { + int TS_stage_empty = 1; + for (unsigned i = 0; i < warp_size; i++) { + if (shader->pipeline_reg[TS_IF][i].hw_thread_id >= 0) { + TS_stage_empty = 0; + break; + } + } + if (TS_stage_empty) { + report_exposed_memory_latency(shader); + } + } + } + + // if not, send the warp part to decode stage + if (!decode_stalled && shader->warp_part2issue < n_warp_parts) { + check_stage_pcs(shader,TS_IF); + for (int i = 0; i < pipe_simd_width; i++) { + int wlane_idx = shader->warp_part2issue * pipe_simd_width + i; + shader->pipeline_reg[IF_ID][i] = shader->pipeline_reg[TS_IF][wlane_idx]; + shader->pipeline_reg[IF_ID][i].if_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + shader->pipeline_reg[TS_IF][wlane_idx] = nop_inst; + } + shader->warp_part2issue += 1; + } +} + +inline int is_load ( op_type op ) { + return op == LOAD_OP; +} + +inline int is_store ( op_type op ) { + return op == STORE_OP; +} + +inline int is_tex ( int space ) { + return((space) == TEX_DIRECTIVE); +} + +inline int is_const ( int space ) { + return((space) == CONST_DIRECTIVE || (space) == PARAM_DIRECTIVE); +} + +inline int is_local ( int space ) { + return((space) == LOCAL_DIRECTIVE); +} + +inline int is_param ( int space ) { + return((space) == PARAM_DIRECTIVE); +} + +inline int is_shared ( int space ) { + return((space) == SHARED_DIRECTIVE); +} + +inline int shmem_bank ( address_type addr ) { + return((int)(addr/((address_type)WORD_SIZE)) % gpgpu_n_shmem_bank); +} + +inline int cache_bank ( address_type addr, shader_core_ctx_t *shader ) { + return(int)( addr >> (address_type)shader->L1cache->line_sz_log2 ) & ( gpgpu_n_cache_bank - 1 ); +} + +inline address_type coalesced_segment(address_type addr, unsigned segment_size_lg2bytes) +{ + return (addr >> segment_size_lg2bytes); +} + + +inline address_type translate_local_memaddr(address_type localaddr, shader_core_ctx_t *shader, int tid) +{ + // During functional execution, each thread sees its own memory space for local memory, but these + // need to be mapped to a shared address space for timing simulation. We do that mapping here. + localaddr -= 0x100; + localaddr /=4; + if (gpgpu_local_mem_map) { + // Dnew = D*nTpC*nCpS*nS + nTpC*C + T%nTpC + // C = S + nS*(T/nTpC) + // D = data index; T = thread; C = CTA; S = shader core; p = per + // keep threads in a warp contiguous + // then distribute across memory space by CTAs from successive shader cores first, + // then by successive CTA in same shader core + localaddr *= gpu_padded_cta_size * gpu_max_cta_per_shader * gpu_n_shader; + localaddr += gpu_padded_cta_size * (shader->sid + gpu_n_shader * (tid / gpu_padded_cta_size)); + localaddr += tid % gpu_padded_cta_size; + } else { + // legacy mapping that maps the same address in the local memory space of all threads + // to a single contiguous address region + localaddr *= gpu_n_shader * gpu_n_thread_per_shader; + localaddr += (gpu_n_thread_per_shader*shader->sid) + tid; + } + localaddr *= 4; + localaddr += 0x100; + + return localaddr; +} + +///////////////////////////////////////////////////////////////////////////////////////// +// Register Bank Conflict Structures + +int gpgpu_reg_bank_conflict_model = 0; + +#define MAX_REG_BANKS 32 +unsigned int gpgpu_num_reg_banks=8; // this needs to be less than MAX_REG_BANKS + +#define MAX_BANK_CONFLICT 8 /* tex can have four source and four destination regs */ + +class reg_bank_access { +public: + reg_bank_access():tot(0),rd(0),wr(0){ + for (unsigned i = 0; i < 4; i++) rd_regs[i] = -1; + } + unsigned tot; + unsigned rd; + unsigned wr; + int rd_regs[4]; +}; + +reg_bank_access g_reg_bank_access[MAX_REG_BANKS]; + +// just to use as "shorthand" for clearing accesses each cycle +static const struct reg_bank_access empty_reg_bank_access; + +unsigned int gpu_reg_bank_conflict_stalls = 0; +///////////////////////////////////////////////////////////////////////////////////////// + +void shader_decode( shader_core_ctx_t *shader, + unsigned int shader_number, + unsigned int grid_num ) { + + address_type addr; + dram_callback_t callback; + op_type op = NO_OP; + register int is_write; + int tid; + int i1, i2, i3, i4, o1, o2, o3, o4; //4 outputs needed for texture fetches in cuda-sim + int i; + int touched_priority=0; + int warp_tid=0; + unsigned space, data_size; + int vectorin, vectorout; + int arch_reg[MAX_REG_OPERANDS] = { -1 }; + address_type regs_regs_PC = 0xDEADBEEF; + address_type warp_current_pc = 0x600DBEEF; + address_type warp_next_pc = 0x600DBEEF; + int warp_diverging = 0; + unsigned warp_id = -1; + unsigned cta_id = -1; + + // stalling for register bank conflict + if ( gpgpu_reg_bank_conflict_model ) { + for (i=0; ipipeline_reg[IF_ID][i].reg_bank_conflict_stall_checked ) { + if ( shader->pipeline_reg[IF_ID][i].reg_bank_access_pending > 0 ) { + assert( shader->pipeline_reg[IF_ID][i].reg_bank_access_pending <= 8 ); + shader->pipeline_reg[IF_ID][i].reg_bank_access_pending--; + gpu_reg_bank_conflict_stalls++; + return; + } + } + } + } + + for (i=0; iusing_rrstage)? ID_RR:ID_EX; + if (shader->pipeline_reg[next_stage][i].hw_thread_id != -1 ) { + return; /* stalled */ + } + } + + check_stage_pcs(shader,IF_ID); + + // decode the instruction + int first_valid_thread = -1; + for (i=0; ipipeline_reg[IF_ID][i].hw_thread_id == -1 ) + continue; /* bubble */ + + /* get the next instruction to execute from fetch stage */ + tid = shader->pipeline_reg[IF_ID][i].hw_thread_id; + if (first_valid_thread == -1) { + first_valid_thread = i; + warp_id = tid/warp_size; + assert( !shader->warp_waiting_at_barrier(warp_id) ); + cta_id = shader->thread[tid].cta_id; + } + + if ( gpgpu_cuda_sim ) { + ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg); + shader->pipeline_reg[IF_ID][i].op = op; + shader->pipeline_reg[IF_ID][i].pc = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info ); + shader->pipeline_reg[IF_ID][i].ptx_thd_info = shader->thread[tid].ptx_thd_info; + + } else { + abort(); + } + // put the info into the shader instruction structure + // - useful in tracking instruction dependency (not needed for now) + shader->pipeline_reg[IF_ID][i].in[0] = i1; + shader->pipeline_reg[IF_ID][i].in[1] = i2; + shader->pipeline_reg[IF_ID][i].in[2] = i3; + shader->pipeline_reg[IF_ID][i].in[3] = i4; + shader->pipeline_reg[IF_ID][i].out[0] = o1; + shader->pipeline_reg[IF_ID][i].out[1] = o2; + shader->pipeline_reg[IF_ID][i].out[2] = o3; + shader->pipeline_reg[IF_ID][i].out[3] = o4; + + } + + // checking for register bank conflict and stall accordingly + if ( gpgpu_reg_bank_conflict_model && + first_valid_thread != -1 && + !shader->pipeline_reg[first_valid_thread][IF_ID].reg_bank_conflict_stall_checked ) + { + for (i = 4; i < 8; i++) { + if( arch_reg[i] != -1 ) { + assert( arch_reg[i] >=0 ); + assert( gpgpu_num_reg_banks <= MAX_REG_BANKS ); + int skip = 0; + int bank = arch_reg[i] % gpgpu_num_reg_banks; + int opndreg = shader->pipeline_reg[first_valid_thread][IF_ID].in[i-4]; + assert(opndreg >= 0); + int j; + for (j = 0; j < 4; j++) { + if (g_reg_bank_access[bank].rd_regs[j] == -1) + break; + else if (g_reg_bank_access[bank].rd_regs[j] == opndreg) { + // two operands reading from same register in same bank, can be merged into a single read + skip = 1; + break; + } + } + if (!skip) { + g_reg_bank_access[bank].tot++; + g_reg_bank_access[bank].rd++; + g_reg_bank_access[bank].rd_regs[j] = opndreg; + } + } + } + + unsigned max_access=0; + inst_t* conflict_inst = &shader->pipeline_reg[first_valid_thread][IF_ID]; + for(unsigned r = 0; r < gpgpu_num_reg_banks; r++ ) { + if( g_reg_bank_access[r].tot > max_access ) + max_access = g_reg_bank_access[r].tot; + g_reg_bank_access[r] = empty_reg_bank_access; + } + if( max_access >= 1 ) { + assert( max_access <= MAX_REG_OPERANDS ); + conflict_inst->reg_bank_access_pending = max_access - 1; + if( max_access > 1 ) { + conflict_inst->reg_bank_conflict_stall_checked = 1; + return; // stall pipeline + } + } + shader->pipeline_reg[first_valid_thread][IF_ID].reg_bank_conflict_stall_checked = 1; + } + + // execute the instruction functionally + for (i=0; ipipeline_reg[IF_ID][i].hw_thread_id == -1 ) + continue; /* bubble */ + /* get the next instruction to execute from fetch stage */ + tid = shader->pipeline_reg[IF_ID][i].hw_thread_id; + if ( gpgpu_cuda_sim ) { + int arch_reg[MAX_REG_OPERANDS]; + ptx_decode_inst( shader->thread[tid].ptx_thd_info, (unsigned*)&op, &i1, &i2, &i3, &i4, &o1, &o2, &o3, &o4, &vectorin, &vectorout, arch_reg ); + ptx_exec_inst( shader->thread[tid].ptx_thd_info, &addr, &space, &data_size, &callback, shader->pipeline_reg[IF_ID][i].warp_active_mask ); + shader->pipeline_reg[IF_ID][i].callback = callback; + shader->pipeline_reg[IF_ID][i].space = space; + if (is_local(space) && (is_load(op) || is_store(op))) { + addr = translate_local_memaddr(addr, shader, tid); + } + shader->pipeline_reg[IF_ID][i].is_vectorin = vectorin; + shader->pipeline_reg[IF_ID][i].is_vectorout = vectorout; + shader->pipeline_reg[IF_ID][i].data_size = data_size; + warp_current_pc = shader->pipeline_reg[IF_ID][i].pc; + memcpy( shader->pipeline_reg[IF_ID][i].arch_reg, arch_reg, sizeof(arch_reg) ); + regs_regs_PC = ptx_thread_get_next_pc( shader->thread[tid].ptx_thd_info ); + } + + shader->pipeline_reg[IF_ID][i].memreqaddr = addr; + if ( op == LOAD_OP ) { + shader->pipeline_reg[IF_ID][i].inst_type = LOAD_OP; + } else if ( op == STORE_OP ) { + shader->pipeline_reg[IF_ID][i].inst_type = STORE_OP; + } + + if ( gpgpu_cuda_sim && ptx_thread_at_barrier( shader->thread[tid].ptx_thd_info ) ) { + if (shader->model == DWF) { + shader->thread[tid].m_waiting_at_barrier=1; + shader->thread[tid].m_reached_barrier=1; // not reset at barrier release, but at the issue after that + shader->warp[wid_from_hw_tid(tid,warp_size)].n_waiting_at_barrier++; + shader->waiting_at_barrier++; + int cta_uid = ptx_thread_get_cta_uid( shader->thread[tid].ptx_thd_info ); + dwf_hit_barrier( shader->sid, cta_uid ); + + int release = ptx_thread_all_at_barrier( shader->thread[tid].ptx_thd_info ); //test if all threads arrived at the barrier + if ( release ) { //All threads arrived at barrier...releasing + int cta_uid = ptx_thread_get_cta_uid( shader->thread[tid].ptx_thd_info ); + for ( unsigned t=0; t < gpu_n_thread_per_shader; ++t ) { + if ( !ptx_thread_at_barrier( shader->thread[t].ptx_thd_info ) ) + continue; + int other_cta_uid = ptx_thread_get_cta_uid( shader->thread[t].ptx_thd_info ); + if ( other_cta_uid == cta_uid ) { //reseting @barrier tracking info + shader->warp[wid_from_hw_tid(t,warp_size)].n_waiting_at_barrier=0; + shader->thread[t].m_waiting_at_barrier=0; + ptx_thread_reset_barrier( shader->thread[t].ptx_thd_info ); + shader->waiting_at_barrier--; + } + } + if (shader->model == DWF) { + dwf_release_barrier( shader->sid, cta_uid ); + } + ptx_thread_release_barrier( shader->thread[tid].ptx_thd_info ); + } + } + } else { + assert( !shader->thread[tid].m_waiting_at_barrier ); + } + + // put the info into the shader instruction structure + // - useful in tracking instruction dependency (not needed for now) + shader->pipeline_reg[IF_ID][i].in[0] = i1; + shader->pipeline_reg[IF_ID][i].in[1] = i2; + shader->pipeline_reg[IF_ID][i].in[2] = i3; + shader->pipeline_reg[IF_ID][i].in[3] = i4; + shader->pipeline_reg[IF_ID][i].out[0] = o1; + shader->pipeline_reg[IF_ID][i].out[1] = o2; + shader->pipeline_reg[IF_ID][i].out[2] = o3; + shader->pipeline_reg[IF_ID][i].out[3] = o4; + + if ( op == STORE_OP ) { + is_write = TRUE; + } + + if ( op == BRANCH_OP ) { + int taken=0; + assert( gpgpu_cuda_sim ); + taken = ptx_branch_taken(shader->thread[tid].ptx_thd_info); + } + + // go to the next instruction + // - done implicitly in ptx_exec_inst() + + // branch divergence detection + if (warp_next_pc != regs_regs_PC) { + if (warp_next_pc == 0x600DBEEF) { + warp_next_pc = regs_regs_PC; + } else { + warp_diverging = 1; + } + } + + // direct the instruction to the appropriate next stage (config dependent) + int next_stage = (shader->using_rrstage)? ID_RR:ID_EX; + shader->pipeline_reg[next_stage][i] = shader->pipeline_reg[IF_ID][i]; + shader->pipeline_reg[next_stage][i].id_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + shader->pipeline_reg[IF_ID][i] = nop_inst; + } + + if( op == BARRIER_OP ) { + shader->set_at_barrier(cta_id,warp_id); + } + + if ( shader->model == NO_RECONVERGE && touched_priority ) { + update_max_branch_priority(shader,warp_tid,grid_num); + } + shader->n_diverge += warp_diverging; + if (warp_diverging == 1) { + assert(warp_current_pc != 0x600DBEEF); // guard against empty warp causing warp divergence + ptx_file_line_stats_add_warp_divergence(warp_current_pc, 1); + } +} + +unsigned int n_regconflict_stall = 0; + + +int regfile_hash(signed thread_number, unsigned simd_size, unsigned n_banks) { + if (gpgpu_thread_swizzling) { + signed warp_ID = thread_number / simd_size; + return((thread_number + warp_ID) % n_banks); + } else { + return(thread_number % n_banks); + } +} + +int gpgpu_n_reg_banks = 8; +void shader_preexecute( shader_core_ctx_t *shader, + unsigned int shader_number ) { + int i; + static int *thread_warp = NULL; + int n_access_per_cycle = pipe_simd_width / gpgpu_n_reg_banks; + + if (!thread_warp) { + thread_warp = (int*) malloc(sizeof(int) * pipe_simd_width); + } + + for (i=0; ipipeline_reg[RR_EX][i].hw_thread_id != -1 ) { + //stalled, but can still service a register read + if (shader->RR_k) { + shader->RR_k--; + } + return; // stalled + } + } + + // if there is still register read to service, stall + if (shader->RR_k > 1) { + shader->RR_k--; + return; + } + + // if RR_k == 1, it was stalled previously and the register read is now done + if (!shader->RR_k && gpgpu_reg_bankconflict) { + int max_reg_bank_acc = 0; + for (i=0; ipipeline_reg[ID_RR][i].hw_thread_id != -1 ) + thread_warp[regfile_hash(shader->pipeline_reg[ID_RR][i].hw_thread_id, + warp_size, gpgpu_n_reg_banks)]++; + } + for (i=0; i max_reg_bank_acc ) { + max_reg_bank_acc = thread_warp[i]; + } + } + // calculate the number of cycles needed for each register bank to fulfill all accesses + shader->RR_k = (max_reg_bank_acc / n_access_per_cycle) + ((max_reg_bank_acc % n_access_per_cycle)? 1 : 0); + } + + // if there are more than one access cycle needed at a bank, stall + if (shader->RR_k > 1) { + n_regconflict_stall++; + shader->RR_k--; + return; + } + + check_stage_pcs(shader,ID_RR); + + shader->RR_k = 0; //setting RR_k to 0 to indicate RF conflict check next cycle + for (i=0; ipipeline_reg[ID_RR][i].hw_thread_id == -1 ) + continue; //bubble + shader->pipeline_reg[ID_EX][i] = shader->pipeline_reg[ID_RR][i]; + shader->pipeline_reg[ID_RR][i] = nop_inst; + } + +} + + +void shader_execute( shader_core_ctx_t *shader, + unsigned int shader_number ) { + + int i; + + for (i=0; ipre_mem_pipeline[0][i].hw_thread_id != -1 ) { + //printf("stalled in shader_execute\n"); + return; // stalled + } + } else { + if (shader->pipeline_reg[EX_MM][i].hw_thread_id != -1 ) + return; // stalled + } + } + + check_stage_pcs(shader,ID_EX); + + for (i=0; ipipeline_reg[ID_EX][i].hw_thread_id == -1 ) + continue; // bubble + if (gpgpu_pre_mem_stages) { + shader->pre_mem_pipeline[0][i] = shader->pipeline_reg[ID_EX][i]; + shader->pre_mem_pipeline[0][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + } else { + shader->pipeline_reg[EX_MM][i] = shader->pipeline_reg[ID_EX][i]; + shader->pipeline_reg[EX_MM][i].ex_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + } + shader->pipeline_reg[ID_EX][i] = nop_inst; + } + + if (!gpgpu_pre_mem_stages) { + // inform memory stage that a new instruction has arrived + shader->shader_memory_new_instruction_processed = 0; + } +} + +void shader_pre_memory( shader_core_ctx_t *shader, + unsigned int shader_number ) { + int i,j; + + + for (j = gpgpu_pre_mem_stages; j > 0; j--) { + for (i=0; ipre_mem_pipeline[j][i].hw_thread_id != -1 ) { + return; + } + } + check_pm_stage_pcs(shader,j-1); + for (i=0; ipre_mem_pipeline[j][i] = shader->pre_mem_pipeline[j - 1][i]; + shader->pre_mem_pipeline[j - 1][i] = nop_inst; + } + } + check_pm_stage_pcs(shader,gpgpu_pre_mem_stages); + for (i=0;ipipeline_reg[EX_MM][i] = shader->pre_mem_pipeline[gpgpu_pre_mem_stages][i]; + + // inform memory stage that a new instruction has arrived + shader->shader_memory_new_instruction_processed = 0; + + if (gpgpu_pre_mem_stages) { + for (i=0; ipre_mem_pipeline[0][i] = nop_inst; + } +} + +int gpgpu_coalesce_arch = 13; + +enum memory_path { + NO_MEM_PATH = 0, + SHARED_MEM_PATH, + GLOBAL_MEM_PATH, + TEXTURE_MEM_PATH, + CONSTANT_MEM_PATH, + NUM_MEM_PATHS //not a mem path +}; + +static unsigned next_access_uid = 0; + +class mem_access_t{ +public: + mem_access_t(): uid(next_access_uid++),addr(0),req_size(0),order(0),_quarter_count_all(0),warp_indices(),space(0),path(NO_MEM_PATH),isatomic(false),cache_hit(false),cache_checked(false),recheck_cache(false),iswrite(false),need_wb(false),wb_addr(0),reserved_mshr(NULL){}; + bool operator<(const mem_access_t &other) const {return (order > other.order);}//this is reverse + unsigned uid; + address_type addr; //address of the segment to load. + unsigned req_size; //bytes + unsigned order; // order of accesses, based on banks. + union{ + unsigned _quarter_count_all; + char quarter_count[4]; //access counts to each quarter of segment, for compaction; + }; + std::vector warp_indices; //warp indicies for this request. + unsigned space; + memory_path path; + bool isatomic; + bool cache_hit; + bool cache_checked; + bool recheck_cache; + bool iswrite; + bool need_wb; + address_type wb_addr; //address to wb too if necessary. + mshr_entry_t* reserved_mshr; +}; + +mshr_entry_t* mshr_shader_unit::add_mshr(mem_access_t &access, inst_t* warp) +{ + static unsigned next_request_uid = 1; + mshr_entry_t* mshr = alloc_free_mshr(is_tex(access.space)); + //note no constructor was called, all entries must be reinitialized! + mshr->request_uid = next_request_uid++; + mshr->status = INITIALIZED; + mshr->addr = access.addr; + mshr->mf = NULL; + mshr->merged_on_other_reqest = false; + mshr->merged_requests =NULL; + mshr->iswrite = access.iswrite; + assert(access.warp_indices.size()); //code assumes at least one instruction attached to mshr. + for (unsigned i = 0; i < access.warp_indices.size(); i++) { + mshr->insts.push_back(warp[access.warp_indices[i]]); + } + mshr->islocal = is_local(access.space); + mshr->isconst = is_const(access.space); + mshr->istexture = is_tex(access.space); + if (gpgpu_interwarp_mshr_merge) { + mshr_entry_t* mergehit = m_mshr_lookup.shader_get_mergeable_mshr(mshr); + if (mergehit) { + //merge this request; + mergehit->merged_requests = mshr; + mshr->merged_on_other_reqest = true; + if (mergehit->fetched()) mshr_return_from_mem(mshr); + } + m_mshr_lookup.mshr_fast_lookup_insert(mshr); + } + return mshr; +} + + +inline address_type line_size_based_tag_func(address_type address, unsigned line_size) +{ + return ((address) & (~((address_type)line_size - 1))); +} + +inline address_type null_tag_func(address_type address, unsigned line_size){ + return address; //no modification: each address is its own tag. Equivalent to line_size_based_tag_func(address,1), but line_size ignored. +} + +// only 1 bank +inline int null_bank_func(address_type add, unsigned line_size) +{ + return 1; +} + +inline int shmem_bank_func(address_type add, unsigned line_size) +{ + return shmem_bank(add); +} + +inline int dcache_bank_func(address_type add, unsigned line_size) +{ + if (gpgpu_no_dl1) return 1; //no banks + else return (add / line_size) & (gpgpu_n_cache_bank - 1); +} + +#include +void check_accessq( shader_core_ctx_t *shader, std::vector &accessq ){ + std::bitset<32> check = 0; + for (unsigned i = 0; i < accessq.size(); i++) { + if (shader) { + std::cout << shader->sid << ":" << i << " space " << accessq[i].space << " " << gpu_sim_cycle << std::endl; + assert(accessq[i].space == (unsigned) shader->pipeline_reg[EX_MM][accessq[i].warp_indices[0]].space); + } + for (unsigned j = 0; j < accessq[i].warp_indices.size(); j++) { + if (check[accessq[i].warp_indices[j]]) { + std::cout << "OOOPS" << std::endl; //good line for breakpoint + }else{check[accessq[i].warp_indices[j]] = true;} + } + } +} + +// This speciallized function calculates the list of independant memory accesses, sorted by access order +// Acesses to same tag line are coalesced. +// will neither coalesce nor overlap bank accesses accross warp parts. +template < int (*bank_func)(address_type add, unsigned line_size), address_type (*tag_func)(address_type add, unsigned line_size) > +inline void get_memory_access_list(inst_t* insns, unsigned char* paths, memory_path path, unsigned warp_parts, unsigned line_size, bool limit_broadcast,std::vector &accessq) +{ + // calculates the memory accesses for a generic cache with banks and tags. + // can be used for coalesescing + + //tracks bank accesses for sorting into generations; + static std::map bank_accs; + bank_accs.clear(); + //keep track of broadcasts with unique orders if limit_broadcast + //the normally calculated orders will never be greater than pipe_simd_width; + unsigned broadcast_order = pipe_simd_width; + + unsigned qbegin = accessq.size(); + unsigned qpartbegin = qbegin; + unsigned mem_pipe_size = pipe_simd_width / warp_parts; + for (unsigned part = 0; part < (unsigned)pipe_simd_width; part += mem_pipe_size) { + for (unsigned i = part; i < part + mem_pipe_size; i++) { + if (paths[i] != path) continue; //skip instructions from other memory paths + address_type segment = (*tag_func)(insns[i].memreqaddr, line_size); + unsigned quarter=0; + if ( line_size>=4 ) { + quarter = (insns[i].memreqaddr / (line_size/4)) & 3; + } + //check if we are already loading this segment. + bool isatomic = (insns[i].callback.function != NULL); + unsigned match = 0; + if (not isatomic) { //atomics must have own request + for (unsigned j = qpartbegin; j < accessq.size(); j++) { + if (segment == accessq[j].addr and not accessq[j].isatomic) { + //match + accessq[j].quarter_count[quarter]++; + accessq[j].warp_indices.push_back(i); + if (limit_broadcast) accessq[j].order = ++broadcast_order; //do proadcast in its own cycle. + match = 1; + break; + } + } + } + if (!match) { + //needs its own request + accessq.push_back(mem_access_t()); + accessq.back().addr = segment; + accessq.back().space = insns[i].space; + accessq.back().path = path; + accessq.back().isatomic = isatomic; + accessq.back().iswrite = is_store(insns[i].op); + accessq.back().req_size = line_size; + accessq.back().quarter_count[quarter]++; + accessq.back().warp_indices.push_back(i); + + //Determine Bank Conflicts. + unsigned bank = (*bank_func)(insns[i].memreqaddr, line_size); + //ensure no concurrent bank access accross warp parts. + // ie. order will be less than part for all previous loads in previous parts, so: + if (bank_accs[bank] < part) bank_accs[bank]=part; + accessq.back().order = bank_accs[bank]; + bank_accs[bank]++; + } + } + qpartbegin = accessq.size(); //don't coalesce accross warp parts + } + //sort requests into order accorting to order (orders will not necessarily be consequtive if multiple parts) + std::stable_sort(accessq.begin()+qbegin,accessq.end()); //this is a reverse sort, least order last, but doesn't really matter where consumed. +} + + +void shader_memory_shared_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector &accessq) +{ + get_memory_access_list<&shmem_bank_func, &null_tag_func>(shader->pipeline_reg[EX_MM], paths, SHARED_MEM_PATH, + gpgpu_shmem_pipe_speedup, + 1, //shared memory doesn't care about line_size, needs to be at least 1; + true, //limit broadcasts to single cycle. + accessq); + //thats it :) +} + +void shader_memory_const_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector &accessq) +{ + unsigned qbegin = accessq.size(); + get_memory_access_list<&null_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, CONSTANT_MEM_PATH, + 1, //warp parts + shader->L1constcache->line_sz, + false, //no broadcast limit. + accessq); + //do cache checks here for each request, could be done later for more accurate timing of cache accesses, but probably uneccesary; + for (unsigned i = qbegin; i < accessq.size(); i++) { + if (is_param(accessq[i].space)) { + accessq[i].cache_hit = true; + } else { + cache_request_status status = shd_cache_access_wb(shader->L1constcache, + accessq[i].addr, + WORD_SIZE, //this field is ingored. + 0, //should always be a read + shader->gpu_cycle, + NULL/*should never writeback*/); + accessq[i].cache_hit = (status == HIT); + if (gpgpu_perfect_mem) accessq[i].cache_hit = true; + if (accessq[i].cache_hit) L1_const_miss++; + } + accessq[i].cache_checked = true; + } +} + +void shader_memory_texture_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector &accessq) +{ + unsigned qbegin = accessq.size(); + get_memory_access_list<&null_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, TEXTURE_MEM_PATH, + 1, //warp parts + shader->L1texcache->line_sz, + false, //no broadcast limit. + accessq); + //do cache checks here for each request, could be done later for more accurate timing of cache accesses, but probably uneccesary; + for (unsigned i = qbegin; i < accessq.size(); i++) { + cache_request_status status = shd_cache_access_wb(shader->L1texcache, + accessq[i].addr, + WORD_SIZE, //this field is ignored. + 0, //should always be a read + shader->gpu_cycle, + NULL /*should never writeback*/); + accessq[i].cache_hit = (status == HIT); + if (gpgpu_perfect_mem) accessq[i].cache_hit = true; + if (accessq[i].cache_hit) L1_texture_miss++; + accessq[i].cache_checked = true; + } +} + +void shader_memory_global_process_inst(shader_core_ctx_t * shader, unsigned char* paths, std::vector &accessq) +{ + unsigned qbegin = accessq.size(); + unsigned warp_parts = 1; + unsigned line_size = shader->L1cache->line_sz; + if (gpgpu_coalesce_arch == 13) { + warp_parts = 2; + if(gpgpu_no_dl1) { + int valindex = -1; + for (int i = 0; i < pipe_simd_width; i++) { + if (paths[i] == GLOBAL_MEM_PATH) { + valindex = i; + break; + } + } + assert(valindex != -1); + // line size is dependant on instruction; + //assume first valid thread instruction is the same as the rest. + switch (shader->pipeline_reg[EX_MM][valindex].data_size) { + case 1: + line_size = 32; + break; + case 2: + line_size = 64; + break; + case 4: + case 8: + line_size = 128; + break; + default: + assert(0); + } + } + } + get_memory_access_list<&dcache_bank_func, &line_size_based_tag_func>(shader->pipeline_reg[EX_MM], paths, GLOBAL_MEM_PATH, + warp_parts, //warp parts + line_size, + false, //no broadcast limit. + accessq); + + for (unsigned i = qbegin; i < accessq.size(); i++) { + if (gpgpu_coalesce_arch == 13 and gpgpu_no_dl1) { + //if there is no l1 cache it makes sense to do coalescing here. + //reduce memory request sizes. + char* quarter_counts = accessq[i].quarter_count; + bool low = quarter_counts[0] or quarter_counts[1]; + bool high = quarter_counts[2] or quarter_counts[3]; + if (accessq[i].req_size == 128) { + if (low xor high) { //can reduce size + accessq[i].req_size = 64; + if (high) accessq[i].addr += 64; + low = quarter_counts[0] or quarter_counts[2]; //set low and high for next pass + high = quarter_counts[1] or quarter_counts[3]; + } + } + if (accessq[i].req_size == 64) { + if (low xor high) { //can reduce size + accessq[i].req_size = 32; + if (high) accessq[i].addr += 32; + } + } + } + } +} + + + +mem_stage_stall_type send_mem_request(shader_core_ctx_t *shader, mem_access_t &access){ + inst_t* warp = shader->pipeline_reg[EX_MM]; + inst_t* req_head = warp + access.warp_indices[0]; + + if (access.need_wb) { + //fill out and send a writeback + unsigned req_size = shader->L1cache->line_sz + WRITE_PACKET_SIZE; + if (!(shader->fq_has_buffer(access.wb_addr, req_size, true, shader->sid))) { + gpu_stall_sh2icnt++; + return WB_ICNT_RC_FAIL; + } + + shader->fq_push( access.wb_addr, + req_size, + true, NO_PARTIAL_WRITE, shader->sid, -1, NULL, + 0, + is_local(access.space)?LOCAL_ACC_W:GLOBAL_ACC_W, //space of cache line is same as new request + -1); + L1_writeback++; + access.need_wb = false; + } + + bool requires_mshr = (shader->model != MIMD) and (not access.iswrite); + + //this decoding here might belong elsewhere + unsigned code; + mem_access_type access_type; + switch(access.space) { + case CONST_DIRECTIVE: + case PARAM_DIRECTIVE: + code = CONSTC; + access_type = CONST_ACC_R; + break; + case TEX_DIRECTIVE: + code = TEXTC; + access_type = TEXTURE_ACC_R; + break; + case GLOBAL_DIRECTIVE: + code = DCACHE; + access_type = (access.iswrite)? GLOBAL_ACC_W: GLOBAL_ACC_R; + break; + case LOCAL_DIRECTIVE: + code = DCACHE; + access_type = (access.iswrite)? LOCAL_ACC_W: LOCAL_ACC_R; + break; + default: + assert(0); // NOT A MEM SPACE; + break; + } + + //reserve mshr + if (requires_mshr and not access.reserved_mshr) { + + // can allocate mshr? + if (not shader->mshr_unit->has_mshr(1)) { + //no mshr available; + return MSHR_RC_FAIL; + } + + access.reserved_mshr = shader->mshr_unit->add_mshr(access, warp); + access.recheck_cache = false; //we have an mshr now, so have checked cache in same cycle as checking mshrs, so have merged if necessary. + } + + //require inct if access is this far without reserved mshr, or has and mshr but not merged with another request + bool requires_icnt = (not access.reserved_mshr) or (not access.reserved_mshr->merged_on_other_reqest); + + if (requires_icnt) { + + //calculate request size for icnt check (and later send); + unsigned request_size = access.req_size; + if (access.iswrite) { + if (requires_mshr) { + //needs information for a load back into cache. + request_size += READ_PACKET_SIZE + WRITE_MASK_SIZE; + } else { + //plain write + request_size += WRITE_PACKET_SIZE + WRITE_MASK_SIZE; + } + } + + + // can allocate icnt? + //unsigned char fq_has_buffer(unsigned long long int addr, int bsize, bool write, int sid); + if (!(shader->fq_has_buffer(access.addr, request_size, access.iswrite, shader->sid))) { + gpu_stall_sh2icnt++; + //std::cout<< "failed to push " << request_size << " bytes" << std::endl; + return ICNT_RC_FAIL; + } + + //send over interconnect + + unsigned cache_hits_waiting = 0; //fixme do we really want to be passing this in? + + partial_write_mask_t write_mask = NO_PARTIAL_WRITE; + if (access.iswrite) { + for (unsigned i=0;i < access.warp_indices.size();i++) { + unsigned w = access.warp_indices[i]; + int data_offset = warp[w].memreqaddr & ((unsigned long long int)access.req_size - 1); + for (unsigned b = data_offset; b < data_offset + warp[w].data_size; b++) write_mask.set(b); + } + if (write_mask.count() != access.req_size) { + gpgpu_n_partial_writes++; + } + } + + //typedef unsigned char (*fq_push_t)(unsigned long long int addr, int bsize, unsigned char readwrite, + // unsigned long long int partial_write_mask, + // int sid, int wid, mshr_entry* mshr, int cache_hits_waiting, + // enum mem_access_type mem_acc, address_type pc); + shader->fq_push( access.addr, + request_size, + access.iswrite, write_mask, shader->sid, req_head->hw_thread_id/warp_size, access.reserved_mshr, + cache_hits_waiting, access_type, req_head->pc); + + } + + + //book keeping for mshr since this request is done (sent/accounted for) at this point; + if (requires_mshr) { + + for (unsigned i = 0; i < access.warp_indices.size(); i++) { + unsigned o = access.warp_indices[i]; + shader->pending_mem_access++; + inflight_memory_insn_add(shader, &warp[o]); + +#if 0 //old stats + if (i > 0) { //maintain old stats (yes/no?) + shader->thread[warp[o].hw_thread_id].n_l1_mrghit_ac++; + shd_cache_mergehit(shader->L1texcache, warp[o].memreqaddr); //fixme; + } +#endif + } + + if (not access.iswrite) { + // set the pipeline instructions in this request to noops, they all wait for memory; + for (unsigned i = 0; i < access.warp_indices.size(); i++) { + unsigned o = access.warp_indices[i]; + shader->pipeline_reg[EX_MM][o] = nop_inst; + + } + } + } + + return NO_RC_FAIL; +} + + +bool shader_memory_shared_cycle( shader_core_ctx_t *shader, std::vector &accessq, + mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){ + //consume port number orders from the top of the queue; + for (unsigned i = 0; i < (unsigned) gpgpu_shmem_port_per_bank; i++) { + if (accessq.empty()) break; + unsigned current_order = accessq.back().order; + //consume all requests of the same order (concurrent bank requests) + while ((not accessq.empty()) and accessq.back().order == current_order) accessq.pop_back(); + } + if (not accessq.empty()) { + rc_fail = BK_CONF; + fail_type = S_MEM; + gpgpu_n_shmem_bkconflict++; + } + return accessq.empty(); //done if empty. +} + +//generic memory access queue processing, accessq must be sorted by order +//--that is, requests of similar order are expected to be contiguous in the queueu. +//if you want to use this for shared memory, make sure they are marked as cashe hits (not default) +// cycle_exec may be called multiple times if memory fails. typically used for cache checks +template < mem_stage_stall_type (*cycle_exec)(shader_core_ctx_t*, mem_access_t&) > +inline mem_stage_stall_type shader_memory_generic_process_queue( shader_core_ctx_t *shader, + unsigned ports_per_bank, unsigned memory_send_max, + std::vector &accessq ){ + mem_stage_stall_type rc_fail = NO_RC_FAIL; + // number of requests to sent to memory this cycle + unsigned mem_req_count = 0; + //consume port number orders from the top of the queue; + for (unsigned i = 0; i < ports_per_bank; i++) { + if (accessq.empty()) break; + unsigned current_order = accessq.back().order; + //consume all requests of the same order (concurrent bank requests) + //stop when things that go to memory exceed a per cycle limit. + while ((not accessq.empty()) and accessq.back().order == current_order and rc_fail == NO_RC_FAIL) { + rc_fail = (*cycle_exec)(shader, accessq.back()); + if (rc_fail != NO_RC_FAIL) break; //can't complete this request this cycle. + if (not accessq.back().cache_hit){ + if (mem_req_count < memory_send_max) { + mem_req_count++; + rc_fail = send_mem_request(shader, accessq.back()); //try to get mshr, icnt, send; + } + else { + rc_fail = COAL_STALL; //not really a coal stall, its a too many memory request stall; + } + if (rc_fail != NO_RC_FAIL) break; //can't complete this request this cycle. + } + accessq.pop_back(); + } + } + if (not accessq.empty() and rc_fail == NO_RC_FAIL) { + //no resource failed so must be a bank comflict. + rc_fail = BK_CONF; + } + return rc_fail; +} + +mem_stage_stall_type ccache_check(shader_core_ctx_t *shader, mem_access_t& access){ /*done in process queue*/ return NO_RC_FAIL;} + +bool shader_memory_constant_cycle( shader_core_ctx_t *shader, std::vector &accessq, + mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){ + + mem_stage_stall_type fail = shader_memory_generic_process_queue( shader, gpgpu_const_port_per_bank, + 1, //memory send max per cycle + accessq ); + if (fail != NO_RC_FAIL){ + rc_fail = fail; //keep other fails if this didn't fail. + fail_type = C_MEM; + if (rc_fail == BK_CONF or rc_fail == COAL_STALL) { + gpgpu_n_cmem_portconflict++; //coal stalls aren't really a bank conflict, but this maintains previous behavior. + } + } + return accessq.empty(); //done if empty. +} + +mem_stage_stall_type tcache_check(shader_core_ctx_t *shader, mem_access_t& access){ /*done in process queue*/ return NO_RC_FAIL;} + +bool shader_memory_texture_cycle( shader_core_ctx_t *shader, std::vector &accessq, + mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){ + + mem_stage_stall_type fail = shader_memory_generic_process_queue(shader, 1, //how is tex memory banked? + 1, //memory send max per cycle + accessq ); + if (fail != NO_RC_FAIL){ + rc_fail = fail; //keep other fails if this didn't fail. + fail_type = T_MEM; + } + return accessq.empty(); //done if empty. +} + + +mem_stage_stall_type dcache_check(shader_core_ctx_t *shader, mem_access_t& access){ + if (access.cache_checked and not access.recheck_cache) return NO_RC_FAIL; + if (!gpgpu_no_dl1 && !gpgpu_perfect_mem) { + //check cache + cache_request_status status = shd_cache_access_wb(shader->L1cache, + access.addr, + WORD_SIZE, //this field is ignored. + access.iswrite, + shader->gpu_cycle, + &access.wb_addr); + if (status == RESERVATION_FAIL) { + access.cache_checked = false; + return WB_CACHE_RSRV_FAIL; + } + access.cache_hit = (status == HIT); //if HIT_W_WT then still send to memory so "MISS" + if (status == MISS_W_WB) access.need_wb = true; + if (status == WB_HIT_ON_MISS and access.iswrite) + { + //write has hit a reserved cache line + //it has writen its data into the cache line, so no need to go to memory + access.cache_hit = true; + L1_write_hit_on_miss++; + // here we would search the MSHRs for the originating read, + // and mask off the writen bytes, so they are not overwritten in the cache when it comes back + // --- don't actually do this since we are pretending. + // MSHR will still forward the unmasked value to its dependant reads. + // if doing stall on use, must stall this thread after this write (otherwise, inproper values may be forwarded to future reads). + } + if (status == WB_HIT_ON_MISS and not access.iswrite) { + //read has hit on a reserved cache line, + //we need to make sure cache check happens on same cycle as a mshr merge happens, otherwise we might miss it coming back + access.recheck_cache = true; + } + access.cache_checked = true; + } else { + access.cache_hit = false; + } + + if (gpgpu_perfect_mem) access.cache_hit = true; + + //atomics always go to memory + if (access.isatomic) { + if (!gpgpu_perfect_mem) { + access.cache_hit = false; + } else { + //unless perfect mem, in which case, the callback can only be done here + dram_callback_t &atom_exec = shader->pipeline_reg[EX_MM][access.warp_indices[0]].callback; + atom_exec.function(atom_exec.instruction, atom_exec.thread); + } + } + + if (!access.cache_hit) { + if (access.iswrite) L1_write_miss++; + else L1_read_miss++; + } + return NO_RC_FAIL; +} + +bool shader_memory_global_cycle( shader_core_ctx_t *shader, std::vector &accessq, + mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type){ + mem_stage_stall_type fail = shader_memory_generic_process_queue<&dcache_check>(shader, gpgpu_cache_port_per_bank, + 1, //memory send max per cycle + accessq ); + if (fail != NO_RC_FAIL) { + rc_fail = fail; //keep other fails if this didn't fail. + //need to determine load/store, local/global: + bool iswrite = accessq.back().iswrite; + if (is_local(accessq.back().space)) { + fail_type = (iswrite)?L_MEM_ST:L_MEM_LD; + } else { + fail_type = (iswrite)?G_MEM_ST:G_MEM_LD; + } + + if (rc_fail == BK_CONF or rc_fail == COAL_STALL) { + gpgpu_n_cache_bkconflict++; + } + } + return accessq.empty(); //done if empty. +} + +inline void mem_instruction_stats(inst_t* warp){ + //there must be a better way to count these + for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) { + if (warp[i].hw_thread_id == -1) continue; //bubble + //this breaks some encapsulation: the is_[space] functions, if you change those, change this. + bool store = is_store(warp[i].op); + switch (warp[i].space) { + case SHARED_DIRECTIVE: + gpgpu_n_shmem_insn++; + break; + case CONST_DIRECTIVE: + gpgpu_n_const_insn++; + break; + case PARAM_DIRECTIVE: + gpgpu_n_param_insn++; + break; + case TEX_DIRECTIVE: + gpgpu_n_tex_insn++; + break; + case GLOBAL_DIRECTIVE: + case LOCAL_DIRECTIVE: + if (store){ + gpgpu_n_store_insn++; + } else { + gpgpu_n_load_insn++; + } + break; + default: + //assert(0); //unknown mem space. + break; //not a mem instruction + } + } +} + +struct shader_queues_t{ + std::vector shared; + std::vector constant; + std::vector texture; + std::vector global; +}; + +void shader_memory_queue(shader_core_ctx_t *shader, shader_queues_t *accessqs) +{ + //classify memory according to type; + static unsigned char *path = NULL; + if (!path) path = (unsigned char*)malloc(pipe_simd_width * sizeof(unsigned char)); + memset(path, 0, pipe_simd_width * sizeof(unsigned char)); + //static std::vector path; + //path.clear(); path.resize(p, NO_MEM_PATH); + + static unsigned type_counts[NUM_MEM_PATHS]; + memset(type_counts, 0, sizeof(type_counts)); + //static std::vector type_counts; + //type_counts.clear(); type_counts.resize(NUM_MEM_PATHS); + + for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) { + if (shader->pipeline_reg[EX_MM][i].hw_thread_id == -1) continue; //bubble + //this breaks some encapsulation: the is_[space] functions; if you change those, change this. + switch (shader->pipeline_reg[EX_MM][i].space) { + case SHARED_DIRECTIVE: + path[i] = SHARED_MEM_PATH; + type_counts[SHARED_MEM_PATH]++; + break; + case CONST_DIRECTIVE: + case PARAM_DIRECTIVE: + path[i] = CONSTANT_MEM_PATH; + type_counts[CONSTANT_MEM_PATH]++; + break; + case TEX_DIRECTIVE: + path[i] = TEXTURE_MEM_PATH; + type_counts[TEXTURE_MEM_PATH]++; + break; + case GLOBAL_DIRECTIVE: + case LOCAL_DIRECTIVE: + path[i] = GLOBAL_MEM_PATH; + type_counts[GLOBAL_MEM_PATH]++; + break; + default: + //path[i] = NO_MEM_PATH; + break; //not a mem instruction + } + } + + //instruction counting: + mem_instruction_stats(shader->pipeline_reg[EX_MM]); + + + if (type_counts[SHARED_MEM_PATH]) shader_memory_shared_process_inst(shader, path, accessqs->shared); + if (type_counts[CONSTANT_MEM_PATH]) shader_memory_const_process_inst(shader, path, accessqs->constant); + if (type_counts[TEXTURE_MEM_PATH]) shader_memory_texture_process_inst(shader, path, accessqs->texture); + if (type_counts[GLOBAL_MEM_PATH]) shader_memory_global_process_inst(shader, path, accessqs->global); + +} + + +void shader_memory( shader_core_ctx_t *shader, unsigned int shader_number ) +{ + enum mem_stage_stall_type rc_fail = NO_RC_FAIL; // resource allocation + + //these should be local to the shader structure but can't because it is included in non c++ files. + //so provide static storage for it here + static std::vector shader_memory_queues; + if (shader_memory_queues.size() == 0) { + shader_memory_queues.resize(gpu_n_shader); + for (unsigned i = 0; i < gpu_n_shader; i++) { + shader_memory_queues[i].shared.reserve(pipe_simd_width); + shader_memory_queues[i].constant.reserve(pipe_simd_width); + shader_memory_queues[i].texture.reserve(pipe_simd_width); + shader_memory_queues[i].global.reserve(pipe_simd_width); + } + } + shader_queues_t *accessqs = &(shader_memory_queues[shader->sid]); + + if (shader->shader_memory_new_instruction_processed == 0) { + shader->shader_memory_new_instruction_processed = 1; //only do this once per pipeline occupant + shader_memory_queue(shader, accessqs); + } + + bool done = true; + mem_stage_access_type type; + + done &= shader_memory_shared_cycle(shader, accessqs->shared, rc_fail, type); + done &= shader_memory_constant_cycle(shader, accessqs->constant, rc_fail, type); + done &= shader_memory_texture_cycle(shader, accessqs->texture, rc_fail, type); + done &= shader_memory_global_cycle(shader, accessqs->global, rc_fail, type); + + //wb stalled? + int wb_stalled = 0; // check if next stage is stalled + for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) { + if (shader->pipeline_reg[MM_WB][i].hw_thread_id != -1 ) { + wb_stalled = 1; + break; + } + } + + if (!done) { + assert(rc_fail != NO_RC_FAIL); + //log stall types + gpu_stall_shd_mem++; + gpu_stall_shd_mem_breakdown[type][rc_fail]++; + } + + if (!done or wb_stalled) return; + + // this memory stage is done and not stalled by wb + // pipeline forward + + check_stage_pcs(shader,EX_MM); + // and pass instruction from EX_MM to MM_WB for cache hit + for (unsigned i=0; i< (unsigned) pipe_simd_width; i++) { + if (shader->pipeline_reg[EX_MM][i].hw_thread_id == -1 ) + continue; // bubble + shader->pipeline_reg[MM_WB][i] = shader->pipeline_reg[EX_MM][i]; + shader->pipeline_reg[MM_WB][i].mm_cycle = gpu_tot_sim_cycle + gpu_sim_cycle; + shader->pipeline_reg[EX_MM][i] = nop_inst; + } + + // reflect the change to EX|MM pipeline register to the pre_mem stage + if (gpgpu_pre_mem_stages) { + check_stage_pcs(shader,EX_MM); + for (unsigned i=0;i< (unsigned)pipe_simd_width ;i++ ) + shader->pre_mem_pipeline[gpgpu_pre_mem_stages][i] = shader->pipeline_reg[EX_MM][i]; + } +} + +int writeback_l1_miss =0 ; + + +void register_cta_thread_exit(shader_core_ctx_t *shader, int tid ) +{ + if (gpgpu_cuda_sim && gpgpu_spread_blocks_across_cores) { + unsigned padded_cta_size = ptx_sim_cta_size(); + if (padded_cta_size%warp_size) { + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + } + int cta_num = tid/padded_cta_size; + assert( shader->cta_status[cta_num] > 0 ); + shader->cta_status[cta_num]--; + if (!shader->cta_status[cta_num]) { + shader->n_active_cta--; + shader->deallocate_barrier(cta_num); + shader_CTA_count_unlog(shader->sid, 1); + printf("Shader %d finished CTA #%d (%lld,%lld)\n", shader->sid, cta_num, gpu_sim_cycle, gpu_tot_sim_cycle ); + } + } +} + +#if 0 +//this function is unecessary, cache is properly dirtied by fill with cache line function in gpu-sim.cc +void dirty_cache_lines(shader_core_ctx_t *shader, mshr_entry_t* mshr){ + shd_cache_line_t *hit_cacheline; + if (mshr->istexture) { + hit_cacheline = shd_cache_access(shader->L1texcache, + mshr->addr, WORD_SIZE, + mshr->iswrite, //should always be 0 + shader->gpu_cycle); + shd_cache_undo_stats( shader->L1texcache, !hit_cacheline ); + } else if (mshr->isconst) { + hit_cacheline = shd_cache_access(shader->L1constcache, + mshr->addr, WORD_SIZE, + mshr->iswrite, //should always be 0 + shader->gpu_cycle); + shd_cache_undo_stats( shader->L1constcache, !hit_cacheline ); + } else if (!gpgpu_no_dl1) { + hit_cacheline = shd_cache_access(shader->L1cache, + mshr->addr, WORD_SIZE, + mshr->iswrite, + shader->gpu_cycle); + shd_cache_undo_stats( shader->L1constcache, !hit_cacheline ); + if (!hit_cacheline) { + writeback_l1_miss++; + } + } +} +#endif + +typedef struct { + unsigned pc; + unsigned long latency; + void *ptx_thd_info; +} insn_latency_info; + +void obtain_insn_latency_info(insn_latency_info *latinfo, inst_t *insn) +{ + latinfo->pc = insn->pc; + latinfo->latency = gpu_tot_sim_cycle + gpu_sim_cycle - insn->ts_cycle; + latinfo->ptx_thd_info = insn->ptx_thd_info; +} + +unsigned gpu_n_max_mshr_writeback=1; +void shader_writeback( shader_core_ctx_t *shader, unsigned int shader_number, int grid_num ) +{ + static int *unlock_tid = NULL; + static int *freed_warp = NULL; + static int *mshr_tid = NULL; + static int *pl_tid = NULL; + static int *done_tid = NULL; + insn_latency_info *unlock_lat_info = NULL; + static insn_latency_info *mshr_lat_info = NULL; + static insn_latency_info *pl_lat_info = NULL; + + mshr_entry *mshr_head = NULL; + + int tid; + op_type op; + int o1, o2, o3, o4; + bool stalled_by_MSHR = false; + bool writeback_by_MSHR = false; + bool w2rf = false; + + if ( unlock_tid == NULL ) { + unlock_tid = (int*) malloc(sizeof(int)*pipe_simd_width); + mshr_tid = (int*) malloc(sizeof(int)*pipe_simd_width); + pl_tid = (int*) malloc(sizeof(int)*pipe_simd_width); + done_tid = (int*) malloc(sizeof(int)*pipe_simd_width); + freed_warp = (int *) malloc(sizeof(int)*pipe_simd_width); + mshr_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width); + pl_lat_info = (insn_latency_info*) malloc(sizeof(insn_latency_info) * pipe_simd_width); + } + memset(unlock_tid, -1, sizeof(int)*pipe_simd_width); + memset(mshr_tid, -1, sizeof(int)*pipe_simd_width); + memset(pl_tid, -1, sizeof(int)*pipe_simd_width); + memset(done_tid, -1, sizeof(int)*pipe_simd_width); + memset(freed_warp, 0, sizeof(int)*pipe_simd_width); + unlock_lat_info = NULL; + + check_stage_pcs(shader,MM_WB); + + /* Generate Condition for instruction writeback to register file. + A load miss *instruction* does not reach writeback until the data is fetched */ + for (int i=0; ipipeline_reg[MM_WB][i].hw_thread_id; + w2rf |= (tid >= 0); + pl_tid[i] = tid; + } + + //check mshrs for commit; + //in future do req bank checking here; + unsigned mshr_threads_unlocked = 0; + for (unsigned i = 0; i < gpu_n_max_mshr_writeback; i++) { + mshr_head = shader->mshr_unit->return_head(); + if (mshr_head) { + //bail if we can't unlock anymore threads, needs to be implemented better. + if (mshr_threads_unlocked + mshr_head->insts.size() > (unsigned) pipe_simd_width) break;//todo, do this right + assert(!gpgpu_strict_simd_wrbk);//implementation commented out below +/* + //for stalling in the middle of an mshr writeback due to mshr having threads from multiple warps, does this happen anymore? + static unsigned next_mshr_index = 0; + int mshr_warpid = -1; + bool mshr_not_blocked_by_samewarp = true; + //use below: for (j = next_mshr_index; ... +*/ + assert (mshr_head->insts.size()); + for (unsigned j = 0; j < mshr_head->insts.size(); j++) { + inst_t &insn = mshr_head->insts[j]; + time_vector_update(insn.uid,MR_WRITEBACK,gpu_sim_cycle+gpu_tot_sim_cycle,RD_REQ); + obtain_insn_latency_info(&mshr_lat_info[mshr_threads_unlocked], &(mshr_head->insts[j])); + inflight_memory_insn_sub(shader, &mshr_head->insts[j]); + assert (insn.hw_thread_id >= 0); + unlock_tid[mshr_threads_unlocked] = insn.hw_thread_id; + shader->pending_mem_access--; + // for ensuring that we don't unlock more than the code allows, needs to be fixed. + mshr_threads_unlocked++; +/* + next_mshr_index++; + if ((shader->model == POST_DOMINATOR || shader->model == NO_RECONVERGE) && gpgpu_strict_simd_wrbk) { + // restricting the threads from mshr to be in the same warp + if (mshr_warpid == -1) { + mshr_warpid = mshr_head->insts[j].hw_thread_id / warp_size; + } else if ((unsigned)mshr_warpid != (mshr_head->insts[j].hw_thread_id / warp_size)) { + warp_conflict_at_writeback++; + mshr_not_blocked_by_samewarp = false; + break; + } + } +*/ + } + //done with it since garanteed to wb +/* + if (mshr_not_blocked_by_samewarp) { +*/ + //this mshr is done this cycle, can pop it + shader->mshr_unit->pop_return_head(); +/* + //reset for next mshr + next_mshr_index = 0; + } +*/ + writeback_by_MSHR = true; + unlock_lat_info = mshr_lat_info; + if (w2rf) { + stalled_by_MSHR = true; + } + assert(mshr_threads_unlocked); + } + } + if (stalled_by_MSHR) { + gpu_stall_by_MSHRwb++; + } + + if (!writeback_by_MSHR) { //!writeback_by_MSHR + for (int i=0; ipipeline_reg[MM_WB][i].op; + tid = shader->pipeline_reg[MM_WB][i].hw_thread_id; + o1 = shader->pipeline_reg[MM_WB][i].out[0]; + o2 = shader->pipeline_reg[MM_WB][i].out[1]; + o3 = shader->pipeline_reg[MM_WB][i].out[2]; + o4 = shader->pipeline_reg[MM_WB][i].out[3]; + + unlock_tid[i] = pl_tid[i]; + obtain_insn_latency_info(&pl_lat_info[i], &shader->pipeline_reg[MM_WB][i]); + } + unlock_lat_info = pl_lat_info; + } + int thd_unlocked = 0; + for (int i=0; i= 0 ) { // not unlocking an invalid thread (ie. due to a bubble) + // thread completed if it is going to fetching beyond code boundry + if ( gpgpu_cuda_sim && ptx_thread_done(shader->thread[unlock_tid[i]].ptx_thd_info) ) { + + finished_trace += 1; + shader->not_completed -= 1; + gpu_completed_thread += 1; + + int warp_id = wid_from_hw_tid(unlock_tid[i],warp_size); + if (!(shader->warp[warp_id].n_completed < (unsigned)warp_size)) { + printf("shader[%d]->warp[%d].n_completed = %d; warp_size = %d\n", + shader->sid,warp_id, shader->warp[warp_id].n_completed, warp_size); + } + assert( shader->warp[warp_id].n_completed < (unsigned)warp_size ); + shader->warp[warp_id].n_completed++; + if ( shader->model == NO_RECONVERGE ) { + update_max_branch_priority(shader,warp_id,grid_num); + } + + if (gpgpu_no_divg_load) { + int amask = wpt_signal_complete(unlock_tid[i], shader); + freed_warp[i] = (amask != 0)? 1 : 0; + } else { + register_cta_thread_exit(shader, unlock_tid[i] ); + } + } else { //thread is not finished yet + // program is not finished yet, allow more fetch + if (gpgpu_no_divg_load) { + freed_warp[i] = wpt_signal_avail(unlock_tid[i], shader); + } else { + shader->thread[unlock_tid[i]].avail4fetch++; + assert(shader->thread[unlock_tid[i]].avail4fetch <= 1); + assert( shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch < (unsigned)warp_size ); + shader->warp[wid_from_hw_tid(unlock_tid[i],warp_size)].n_avail4fetch++; + thd_unlocked = 1; + } + } + + // At any rate, a real instruction is committed + // - don't count cache miss + if ( shader->pipeline_reg[MM_WB][i].inst_type != NO_OP_FLAG ) { + gpu_sim_insn++; + if ( !is_const(shader->pipeline_reg[MM_WB][i].space) ) + gpu_sim_insn_no_ld_const++; + gpu_sim_insn_last_update = gpu_sim_cycle; + shader->num_sim_insn++; + shader->thread[unlock_tid[i]].n_insn++; + shader->thread[unlock_tid[i]].n_insn_ac++; + } + + if (enable_ptx_file_line_stats) { + unsigned pc = unlock_lat_info[i].pc; + unsigned long latency = unlock_lat_info[i].latency; + ptx_file_line_stats_add_latency(unlock_lat_info[i].ptx_thd_info, pc, latency); + } + } + } + if (shader->using_commit_queue && thd_unlocked) { + int *tid_unlocked = alloc_commit_warp(); + memcpy(tid_unlocked, unlock_tid, sizeof(int)*pipe_simd_width); //NOTE: this maybe warp_size + dq_push(shader->thd_commit_queue,(void*)tid_unlocked); + } + + + /* The pipeline can be stalled by MSHR */ + if (!stalled_by_MSHR) { + for (int i=0; ipipeline_reg[WB_RT][i] = shader->pipeline_reg[MM_WB][i]; + shader->pipeline_reg[MM_WB][i] = nop_inst; + } + } +} + + +void shader_print_runtime_stat( FILE *fout ) { + unsigned i; + + fprintf(fout, "SHD_INSN: "); + for (i=0;inum_sim_insn); + } + fprintf(fout, "\n"); + fprintf(fout, "SHD_THDS: "); + for (i=0;inot_completed); + } + fprintf(fout, "\n"); + fprintf(fout, "SHD_DIVG: "); + for (i=0;in_diverge); + } + fprintf(fout, "\n"); + + fprintf(fout, "THD_INSN: "); + for (i=0; ithread[i].n_insn); + } + fprintf(fout, "\n"); +} + + +void shader_print_l1_miss_stat( FILE *fout ) { + unsigned i; + + fprintf(fout, "THD_INSN_AC: "); + for (i=0; ithread[i].n_insn_ac); + } + fprintf(fout, "\n"); + + fprintf(fout, "T_L1_Mss: "); //l1 miss rate per thread + for (i=0; ithread[i].n_l1_mis_ac); + } + fprintf(fout, "\n"); + + fprintf(fout, "T_L1_Mgs: "); //l1 merged miss rate per thread + for (i=0; ithread[i].n_l1_mis_ac - sc[0]->thread[i].n_l1_mrghit_ac); + } + fprintf(fout, "\n"); + + fprintf(fout, "T_L1_Acc: "); //l1 access per thread + for (i=0; ithread[i].n_l1_access_ac); + } + fprintf(fout, "\n"); + + //per warp + int temp =0; + fprintf(fout, "W_L1_Mss: "); //l1 miss rate per warp + for (i=0; ithread[i].n_l1_mis_ac; + if (i%warp_size == (unsigned)(warp_size-1)) { + fprintf(fout, "%d ", temp); + temp = 0; + } + } + fprintf(fout, "\n"); + temp=0; + fprintf(fout, "W_L1_Mgs: "); //l1 merged miss rate per warp + for (i=0; ithread[i].n_l1_mis_ac - sc[0]->thread[i].n_l1_mrghit_ac); + if (i%warp_size == (unsigned)(warp_size-1)) { + fprintf(fout, "%d ", temp); + temp = 0; + } + } + fprintf(fout, "\n"); + temp =0; + fprintf(fout, "W_L1_Acc: "); //l1 access per warp + for (i=0; ithread[i].n_l1_access_ac; + if (i%warp_size == (unsigned)(warp_size-1)) { + fprintf(fout, "%d ", temp); + temp = 0; + } + } + fprintf(fout, "\n"); + +} + +void shader_print_stage(shader_core_ctx_t *shader, unsigned int stage, + FILE *fout, int stage_width, int print_mem, int mask ) +{ + int i, j, warp_id = -1; + + for (i=0; ipipeline_reg[stage][i].hw_thread_id > -1) { + warp_id = shader->pipeline_reg[stage][i].hw_thread_id / warp_size; + break; + } + } + i = (i>=stage_width)? 0 : i; + + fprintf(fout,"0x%04x ", shader->pipeline_reg[stage][i].pc ); + + if( mask & 2 ) { + fprintf(fout, "(" ); + for (j=0; jpipeline_reg[stage][j].hw_thread_id); + fprintf(fout, "): "); + } else { + fprintf(fout, "w%02d[", warp_id); + for (j=0; jpipeline_reg[stage][j].hw_thread_id != -1)?'1':'0') ); + fprintf(fout, "]: "); + } + + if( warp_id != -1 && shader->model == POST_DOMINATOR ) { + pdom_warp_ctx_t *warp=&(shader->pdom_warp[warp_id]); + if( warp->m_recvg_pc[warp->m_stack_top] == (unsigned)-1 ) { + fprintf(fout," rp:--- "); + } else { + fprintf(fout," rp:0x%03x ", warp->m_recvg_pc[warp->m_stack_top] ); + } + } + + ptx_print_insn( shader->pipeline_reg[stage][i].pc, fout ); + + if( mask & 0x10 ) { + if ( (shader->pipeline_reg[stage][i].op == STORE_OP || + shader->pipeline_reg[stage][i].op == LOAD_OP) && print_mem ) + fprintf(fout, " mem: 0x%016llx", shader->pipeline_reg[stage][i].memreqaddr); + } + fprintf(fout, "\n"); +} + +void shader_print_pre_mem_stages(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask ) +{ + int i, j; + int warp_id; + + if (!gpgpu_pre_mem_stages) return; + + for (unsigned pms = 0; pms <= gpgpu_pre_mem_stages - 1; pms++) { + fprintf(fout, "PM[%01d] = ", pms); + + warp_id = -1; + + for (i=0; ipre_mem_pipeline[pms][i].hw_thread_id > -1) { + warp_id = shader->pre_mem_pipeline[pms][i].hw_thread_id / warp_size; + break; + } + } + i = (i>=pipe_simd_width)? 0 : i; + + fprintf(fout,"0x%04x ", shader->pre_mem_pipeline[pms][i].pc ); + + if( mask & 2 ) { + fprintf(fout, "(" ); + for (j=0; jpre_mem_pipeline[pms][j].hw_thread_id); + fprintf(fout, "): "); + } else { + fprintf(fout, "w%02d[", warp_id); + for (j=0; jpre_mem_pipeline[pms][j].hw_thread_id != -1)?'1':'0') ); + fprintf(fout, "]: "); + } + + if( warp_id != -1 && shader->model == POST_DOMINATOR ) { + pdom_warp_ctx_t *warp=&(shader->pdom_warp[warp_id]); + if( warp->m_recvg_pc[warp->m_stack_top] == (unsigned)-1 ) { + printf(" rp:--- "); + } else { + printf(" rp:0x%03x ", warp->m_recvg_pc[warp->m_stack_top] ); + } + } + + ptx_print_insn( shader->pre_mem_pipeline[pms][i].pc, fout ); + + if( mask & 0x10 ) { + if ( ( shader->pre_mem_pipeline[pms][i].op == LOAD_OP || + shader->pre_mem_pipeline[pms][i].op == STORE_OP ) && print_mem ) + fprintf(fout, " mem: 0x%016llx", shader->pre_mem_pipeline[pms][i].memreqaddr); + } + fprintf(fout, "\n"); + } +} + +const char * ptx_get_fname( unsigned PC ); + +void shader_display_pipeline(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask ) +{ + // call this function from within gdb to print out status of pipeline + // if you encounter a bug, or to visualize pipeline operation + // (this is a good way to "verify" your pipeline model makes sense!) + + fprintf(fout, "=================================================\n"); + fprintf(fout, "shader %u at cycle %Lu+%Lu (%u threads running)\n", shader->sid, + gpu_tot_sim_cycle, gpu_sim_cycle, shader->not_completed); + fprintf(fout, "=================================================\n"); + + if ( (mask & 4) && shader->model == POST_DOMINATOR ) { + fprintf(fout,"warp status:\n"); + unsigned n = shader->n_threads / warp_size; + for (unsigned i=0; i < n; i++) { + unsigned nactive = 0; + for (unsigned j=0; jthread[tid].ptx_thd_info ); + nactive += (ptx_thread_done( shader->thread[tid].ptx_thd_info )?0:1); + if ( done && (mask & 8) ) { + unsigned done_cycle = ptx_thread_donecycle( shader->thread[tid].ptx_thd_info ); + if ( done_cycle ) { + printf("\n w%02u:t%03u: done @ cycle %u", i, tid, done_cycle ); + } + } + } + if ( nactive == 0 ) { + continue; + } + pdom_warp_ctx_t *warp=&(shader->pdom_warp[i]); + for ( int k=0; k <= warp->m_stack_top; k++ ) { + if ( k==0 ) { + fprintf(fout, "w%02d (%2u thds active): %2u ", i, nactive, k ); + } else { + fprintf(fout, " %2u ", k ); + } + for (unsigned m=1,j=0; jm_active_mask[k] & m)?'1':'0') ); + fprintf(fout, " pc: %4u", warp->m_pc[k] ); + if ( warp->m_recvg_pc[k] == (unsigned)-1 ) { + fprintf(fout," rp: ---- cd: %2u ", warp->m_calldepth[k] ); + } else { + fprintf(fout," rp: %4u cd: %2u ", warp->m_recvg_pc[k], warp->m_calldepth[k] ); + } + if ( warp->m_branch_div_cycle[k] != 0 ) { + fprintf(fout," bd@%6u ", (unsigned) warp->m_branch_div_cycle[k] ); + } else { + fprintf(fout," " ); + } + //fprintf(fout," func=\'%s\' ", ptx_get_fname( warp->m_pc[k] ) ); + ptx_print_insn( warp->m_pc[k], fout ); + fprintf(fout,"\n"); + } + } + fprintf(fout,"\n"); + } + + if ( mask & 0x20 ) { + fprintf(fout, "TS/IF = "); + shader_print_stage(shader, TS_IF, fout, warp_size, print_mem, mask); + } + + fprintf(fout, "IF/ID = "); + shader_print_stage(shader, IF_ID, fout, pipe_simd_width, print_mem, mask ); + + if (shader->using_rrstage) { + fprintf(fout, "ID/RR = "); + shader_print_stage(shader, ID_RR, fout, pipe_simd_width, print_mem, mask); + } + + fprintf(fout, "ID/EX = "); + shader_print_stage(shader, ID_EX, fout, pipe_simd_width, print_mem, mask); + + shader_print_pre_mem_stages(shader, fout, print_mem, mask); + + if (!gpgpu_pre_mem_stages) + fprintf(fout, "EX/MEM= "); + else + fprintf(fout, "PM/MEM= "); + shader_print_stage(shader, EX_MM, fout, pipe_simd_width, print_mem, mask); + + fprintf(fout, "MEM/WB= "); + shader_print_stage(shader, MM_WB, fout, pipe_simd_width, print_mem, mask); + + fprintf(fout, "\n"); +} + +void shader_dump_thread_state(shader_core_ctx_t *shader, FILE *fout ) +{ + fprintf( fout, "\n"); + for ( unsigned w = 0; w < gpu_n_thread_per_shader/warp_size; w++ ) { + int tid = w*warp_size; + if ( shader->warp[w].n_completed < (unsigned)warp_size ) { + fprintf( fout, " %u:%3u fetch state = c:%u a4f:%u bw:%u (completed: ", shader->sid, tid, + shader->warp[w].n_completed, + shader->warp[w].n_avail4fetch, + shader->warp[w].n_waiting_at_barrier ); + + for ( unsigned i = tid; i < (w+1)*warp_size; i++ ) { + if ( gpgpu_cuda_sim && ptx_thread_done(shader->thread[i].ptx_thd_info) ) { + fprintf(fout,"1"); + } else { + fprintf(fout,"0"); + } + if ( (((i+1)%4) == 0) && (i+1) < (w+1)*warp_size ) { + fprintf(fout,","); + } + } + fprintf(fout,")\n"); + } + } +} + +void shader_dp(shader_core_ctx_t *shader, int print_mem) { + shader_display_pipeline(shader, stdout, print_mem, 7 ); +} + + +unsigned int max_cta_per_shader( shader_core_ctx_t *shader) +{ + unsigned int result; + unsigned int padded_cta_size; + + padded_cta_size = ptx_sim_cta_size(); + if (padded_cta_size%warp_size) { + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + //printf("padded_cta_size=%u\n", padded_cta_size); + } + + //Limit by n_threads/shader + unsigned int result_thread = shader->n_threads / padded_cta_size; + + const struct gpgpu_ptx_sim_kernel_info *kernel_info = ptx_sim_kernel_info(); + + //Limit by shmem/shader + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = shader->shmem_size / kernel_info->smem; + + //Limit by register count, rounded up to multiple of 4. + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = shader->n_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); + + //Limit by CTA + unsigned int result_cta = shader->n_cta; + + result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + static const struct gpgpu_ptx_sim_kernel_info* last_kinfo = NULL; + if (last_kinfo != kernel_info) { //Only print out stats if kernel_info struct changes + last_kinfo = kernel_info; + printf ("CTA/core = %u, limited by:", result); + if (result == result_thread) printf (" threads"); + if (result == result_shmem) printf (" shmem"); + if (result == result_regs) printf (" regs"); + if (result == result_cta) printf (" cta_limit"); + printf ("\n"); + } + + if (result < 1) { + printf ("Error: max_cta_per_shader(\"%s\") returning %d. Kernel requires more resources than shader has?\n", shader->name, result); + abort(); + } + return result; +} + +void shader_cycle( shader_core_ctx_t *shader, + unsigned int shader_number, + int grid_num ) +{ + + // last pipeline stage + shader_writeback(shader, shader_number, grid_num); + + // three parallel stages (only one does something on a given cycle) + //shader_const_memory (shader, shader_number); + shader_memory (shader, shader_number); + //shader_texture_memory (shader, shader_number); + + // empty stage + if (gpgpu_pre_mem_stages) + shader_pre_memory(shader, shader_number); + + shader_execute (shader, shader_number); + if (shader->using_rrstage) { + // model register bank conflicts + // (see Fung et al. MICRO'07 paper or ACM TACO paper) + shader_preexecute (shader, shader_number); + } + + shader_decode (shader, shader_number, grid_num); + + shader_fetch (shader, shader_number, grid_num); +} + +// performance counter that are not local to one shader +void shader_print_accstats( FILE* fout ) +{ + fprintf(fout, "gpgpu_n_load_insn = %d\n", gpgpu_n_load_insn); + fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn); + fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn); + fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn); + fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn); + fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn); + + fprintf(fout, "gpgpu_n_shmem_bkconflict = %d\n", gpgpu_n_shmem_bkconflict); + fprintf(fout, "gpgpu_n_cache_bkconflict = %d\n", gpgpu_n_cache_bkconflict); + + fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n", gpgpu_n_intrawarp_mshr_merge); + fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict); + + fprintf(fout, "gpgpu_n_writeback_l1_miss = %d\n", writeback_l1_miss); + + fprintf(fout, "gpgpu_n_partial_writes = %d\n", gpgpu_n_partial_writes); + + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]); + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n", gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem][coal_stall] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][COAL_STALL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); + fprintf(fout, "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n", gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]); + + fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n", gpu_reg_bank_conflict_stalls); + + if (warp_occ_detailed) { + int n_warp = gpu_n_thread_per_shader / warp_size; + + for (unsigned s = 0; sL1cache; + shd_cache_line_t *pline; + + for (i=0; inset*cp->assoc; i++) { + pline = &(cp->lines[i]); + set = i / cp->assoc; + if ((pline->status & (DIRTY|VALID)) == (DIRTY|VALID)) { + flush_addr = pline->addr; + + sc->fq_push(flush_addr, sc->L1cache->line_sz, 1, NO_PARTIAL_WRITE, sc->sid, 0, NULL, 0, GLOBAL_ACC_W, -1); + + pline->status &= ~VALID; + pline->status &= ~DIRTY; + } else if (pline->status & VALID) { + pline->status &= ~VALID; + } + } +} + +barrier_set_t::barrier_set_t( unsigned max_warps_per_core, unsigned max_cta_per_core ) +{ + m_max_warps_per_core = max_warps_per_core; + m_max_cta_per_core = max_cta_per_core; + if( max_warps_per_core > WARP_PER_CTA_MAX ) { + printf("ERROR ** increase WARP_PER_CTA_MAX in shader.h from %u to >= %u or warps per cta in gpgpusim.config\n", + WARP_PER_CTA_MAX, max_warps_per_core ); + exit(1); + } + m_warp_active.reset(); + m_warp_at_barrier.reset(); +} + +// during cta allocation +void barrier_set_t::allocate_barrier( unsigned cta_id, warp_set_t warps ) +{ + assert( cta_id < m_max_cta_per_core ); + cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); + assert( w == m_cta_to_warps.end() ); // cta should not already be active or allocated barrier resources + m_cta_to_warps[cta_id] = warps; + assert( m_cta_to_warps.size() <= m_max_cta_per_core ); // catch cta's that were not properly deallocated + + m_warp_active |= warps; + m_warp_at_barrier &= ~warps; +} + +// during cta deallocation +void barrier_set_t::deallocate_barrier( unsigned cta_id ) +{ + cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); + if( w == m_cta_to_warps.end() ) + return; + warp_set_t warps = w->second; + warp_set_t at_barrier = warps & m_warp_at_barrier; + assert( at_barrier.any() == false ); // no warps stuck at barrier + warp_set_t active = warps & m_warp_active; + assert( active.any() == false ); // no warps in CTA still running + m_warp_active &= ~warps; + m_warp_at_barrier &= ~warps; + m_cta_to_warps.erase(w); +} + +// individual warp hits barrier +void barrier_set_t::warp_reaches_barrier( unsigned cta_id, unsigned warp_id ) +{ + cta_to_warp_t::iterator w=m_cta_to_warps.find(cta_id); + + if( w == m_cta_to_warps.end() ) { // cta is active + printf("ERROR ** cta_id %u not found in barrier set on cycle %llu+%llu...\n", cta_id, gpu_tot_sim_cycle, gpu_sim_cycle ); + dump(); + abort(); + } + assert( w->second.test(warp_id) == true ); // warp is in cta + + m_warp_at_barrier.set(warp_id); + + warp_set_t warps_in_cta = w->second; + warp_set_t at_barrier = warps_in_cta & m_warp_at_barrier; + warp_set_t active = warps_in_cta & m_warp_active; + + if( at_barrier == active ) { + // all warps have reached barrier, so release waiting warps... + m_warp_at_barrier &= ~at_barrier; + } +} + +// fetching a warp +bool barrier_set_t::available_for_fetch( unsigned warp_id ) const +{ + return m_warp_active.test(warp_id) && m_warp_at_barrier.test(warp_id); +} + +// warp reaches exit +void barrier_set_t::warp_exit( unsigned warp_id ) +{ + // caller needs to verify all threads in warp are done, e.g., by checking PDOM stack to + // see it has only one entry during exit_impl() + m_warp_active.reset(warp_id); +} + +// assertions +bool barrier_set_t::warp_waiting_at_barrier( unsigned warp_id ) +{ + return m_warp_at_barrier.test(warp_id); +} + +void barrier_set_t::dump() const +{ + printf( "barrier set information\n"); + printf( " m_max_cta_per_core = %u\n", m_max_cta_per_core ); + printf( " m_max_warps_per_core = %u\n", m_max_warps_per_core ); + printf( " cta_to_warps:\n"); + + cta_to_warp_t::const_iterator i; + for( i=m_cta_to_warps.begin(); i!=m_cta_to_warps.end(); i++ ) { + unsigned cta_id = i->first; + warp_set_t warps = i->second; + printf(" cta_id %u : %s\n", cta_id, warps.to_string().c_str() ); + } + printf(" warp_active: %s\n", m_warp_active.to_string().c_str() ); + printf(" warp_at_barrier: %s\n", m_warp_at_barrier.to_string().c_str() ); + fflush(stdout); +} + +shader_core_ctx::shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core ) + : m_barriers( max_warps_per_cta, max_cta_per_core ) +{ +} + +void shader_core_ctx::set_at_barrier( unsigned cta_id, unsigned warp_id ) +{ + m_barriers.warp_reaches_barrier(cta_id,warp_id); +} + +void shader_core_ctx::warp_exit( unsigned warp_id ) +{ + m_barriers.warp_exit( warp_id ); +} + +bool shader_core_ctx::warp_waiting_at_barrier( unsigned warp_id ) +{ + return m_barriers.warp_waiting_at_barrier(warp_id); +} + +void shader_core_ctx::allocate_barrier( unsigned cta_id, warp_set_t warps ) +{ + m_barriers.allocate_barrier(cta_id,warps); +} + +void shader_core_ctx::deallocate_barrier( unsigned cta_id ) +{ + m_barriers.deallocate_barrier(cta_id); +} diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h new file mode 100644 index 0000000..8333202 --- /dev/null +++ b/src/gpgpu-sim/shader.h @@ -0,0 +1,524 @@ +/* + * shader.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan, Ivan Sham, Henry Wong, Dan O'Connor, Henry Tran and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include +#include +#include +#include +#include + +#include "../cuda-sim/ptx.tab.h" +#include "../cuda-sim/dram_callback.h" + +#include "gpu-cache.h" +#include "delayqueue.h" +#include "stack.h" +#include "dram.h" +#include "../abstract_hardware_model.h" + +#ifndef SHADER_H +#define SHADER_H + +#define NO_OP_FLAG 0xFF + +//READ_PACKET_SIZE: bytes: 6 address (flit can specify chanel so this gives up to ~2GB/channel, so good for now), 2 bytes [shaderid + mshrid](14 bits) + req_size(0-2 bits (if req_size variable) - so up to 2^14 = 16384 mshr total +#define READ_PACKET_SIZE 8 +//WRITE_PACKET_SIZE: bytes: 6 address, 2 miscelaneous. +#define WRITE_PACKET_SIZE 8 + +#include +const unsigned partial_write_mask_bits = 128; //must be at least size of largest memory access. +typedef std::bitset partial_write_mask_t; + +#define WRITE_MASK_SIZE 8 +#define NO_PARTIAL_WRITE (partial_write_mask_t()) + +//this is used a lot of places where it maybe should be more variable? +#define WORD_SIZE 4 + +//Set a hard limit of 32 CTAs per shader [cuda only has 8] +#define MAX_CTA_PER_SHADER 32 + +typedef unsigned op_type; + +enum { + NO_RECONVERGE = 0, + POST_DOMINATOR = 1, + MIMD = 2, + DWF = 3, + NUM_SIMD_MODEL +}; + +//Defines number of threads grouped together to be executed together + + +typedef struct { + + address_type pc; + + op_type op; + int space; + + unsigned long long int memreqaddr; + //Each instruction keeps track of which hardware thread it came from + short hw_thread_id; + short wlane; + + /* reg label of the instruction */ + unsigned out[4]; + unsigned in[4]; + unsigned char is_vectorin; + unsigned char is_vectorout; + int arch_reg[8]; // register number for bank conflict evaluation + unsigned data_size; // what is the size of the word being operated on? + + int reg_bank_access_pending; + int reg_bank_conflict_stall_checked; // flag to turn off register bank conflict checker to avoid double stalling + + unsigned char inst_type; + + unsigned priority; + + unsigned uid; + + void *ptx_thd_info; + dram_callback_t callback; + unsigned warp_active_mask; + unsigned long long ts_cycle; + unsigned long long if_cycle; + unsigned long long id_cycle; + unsigned long long ex_cycle; + unsigned long long mm_cycle; + +} inst_t; + +typedef struct { + + void *ptx_thd_info; // pointer to the functional state of the thread in cuda-sim + + int avail4fetch; // 1 if its instrucion can be fetch into the pipeline, 0 otherwise + int warp_priority; + + int id; + + //unsigned n_completed; // number of threads in warp completed -- set for first thread in each warp + //unsigned n_avail4fetch; // number of threads in warp available to fetch -- set for first thread in each warp + //int n_waiting_at_barrier; // number of threads in warp that have reached the barrier + unsigned in_scheduler; // used by dynamic warp formation for error check + + int m_waiting_at_barrier; + int m_reached_barrier; + + unsigned n_insn; + unsigned n_insn_ac; + unsigned n_l1_mis_ac, + n_l1_mrghit_ac, + n_l1_access_ac; //used to collect "per thread" l1 miss statistics + // ac stands for accumulative. + unsigned cta_id; // which hardware CTA does this thread belong to? +} thread_ctx_t; + +struct shd_warp_t +{ + shd_warp_t(unsigned warp_size){reset(warp_size); assert(warp_size <= bitset_size);} + void reset(unsigned warp_size){n_completed = warp_size; n_avail4fetch = n_waiting_at_barrier = 0; threads_completed.reset(); threads_functionally_executed.reset();} + + unsigned wid; + unsigned n_completed; // number of threads in warp completed + unsigned n_avail4fetch; // number of threads in warp available to fetch + int n_waiting_at_barrier; // number of threads in warp that have reached the barrier + + const static unsigned bitset_size = 32; + std::bitset threads_completed; + std::bitset threads_functionally_executed; +}; + +inline unsigned hw_tid_from_wid(unsigned wid, unsigned warp_size, unsigned i){return wid * warp_size + i;}; +inline unsigned wid_from_hw_tid(unsigned tid, unsigned warp_size){return tid/warp_size;}; + +typedef struct { + + int m_stack_top; + + address_type *m_pc; + unsigned int *m_active_mask; + address_type *m_recvg_pc; + unsigned int *m_calldepth; + + unsigned long long *m_branch_div_cycle; + +} pdom_warp_ctx_t; // bounded stack that implements pdom reconvergence (see MICRO'07 paper) + + +enum mshr_status { + INITIALIZED = 0, + IN_ICNT2MEM, + IN_CBTOL2QUEUE, + IN_L2TODRAMQUEUE, + IN_DRAM_REQ_QUEUE, + IN_DRAMRETURN_Q, + IN_DRAMTOL2QUEUE, + IN_L2TOCBQUEUE_HIT, + IN_L2TOCBQUEUE_MISS, + IN_ICNT2SHADER, + FETCHED, + NUM_MSHR_STATUS +}; + +//used to stages that time_vector will keep track of their timing +enum mem_req_stat { + MR_UNUSED, + MR_FQPUSHED, + MR_ICNT_PUSHED, + MR_ICNT_INJECTED, + MR_ICNT_AT_DEST, + MR_DRAMQ, //icnt_pop at dram side and mem_ctrl_push + MR_DRAM_PROCESSING_START, + MR_DRAM_PROCESSING_END, + MR_DRAM_OUTQ, + MR_2SH_ICNT_PUSHED, // icnt_push and mem_ctl_pop //STORES END HERE! + MR_2SH_ICNT_INJECTED, + MR_2SH_ICNT_AT_DEST, + MR_2SH_FQ_POP, //icnt_pop called inside fq_pop + MR_RETURN_Q, + MR_WRITEBACK, //done + NUM_MEM_REQ_STAT +}; +#include +typedef struct mshr_entry_t { +#ifdef _GLIBCXX_DEBUG + //satisfy cxx debug conditions on iterators, needs to be nonsingular to copy, which messes completely with structures containing them. + mshr_entry_t(){ + static std::vector dummy_vector; + this_mshr = dummy_vector.begin(); //initialize it to something nonsingular so it can be copied. + } +#endif +private: + friend class mshr_shader_unit; + std::vector::iterator this_mshr; //to ease tracking and update. +public: + unsigned request_uid; + + /* memory address of the data */ + unsigned long long int addr; + + // instructions are stored here. + std::vector insts; + + /* Current stage of the load: fetched or not? */ + bool fetched(){return status == FETCHED;}; + + bool iswrite; + + bool merged_on_other_reqest; //true if waiting for another mshr - this mshr doesn't send a memory request + struct mshr_entry_t *merged_requests; //mshrs waiting on this mshr + + enum mshr_status status; + + void *mf; // link to corresponding memory fetch structure + + //unsigned space; //does below. + bool istexture; //if it's a request from the texture cache + bool isconst; //if it's a request from the constant cache + bool islocal; //if it's a request to the local memory of a thread + + bool wt_no_w2cache; //in write_through, sometimes need to prevent writing back returning data into cache, because its been written in the meantime. +} mshr_entry; + +enum mem_access_type { + GLOBAL_ACC_R = 0, + LOCAL_ACC_R = 1, + CONST_ACC_R = 2, + TEXTURE_ACC_R = 3, + GLOBAL_ACC_W = 4, + LOCAL_ACC_W = 5, + L2_WRBK_ACC = 6, + NUM_MEM_ACCESS_TYPE = 7 +}; + + +/* A pointer to the function that glues the shader with the memory hiearchy */ +typedef unsigned char (*fq_push_t)(unsigned long long int addr, int bsize, unsigned char readwrite, + partial_write_mask_t, + int sid, int wid, mshr_entry* mshr, int cache_hits_waiting, + enum mem_access_type mem_acc, address_type pc); + +typedef unsigned char (*fq_has_buffer_t)(unsigned long long int addr, int bsize, bool write, int sid); + +const unsigned WARP_PER_CTA_MAX = 32; +typedef std::bitset warp_set_t; + +class barrier_set_t { +public: + barrier_set_t( unsigned max_warps_per_core, unsigned max_cta_per_core ); + + // during cta allocation + void allocate_barrier( unsigned cta_id, warp_set_t warps ); + + // during cta deallocation + void deallocate_barrier( unsigned cta_id ); + + typedef std::map cta_to_warp_t; + + // individual warp hits barrier + void warp_reaches_barrier( unsigned cta_id, unsigned warp_id ); + + // fetching a warp + bool available_for_fetch( unsigned warp_id ) const; + + // warp reaches exit + void warp_exit( unsigned warp_id ); + + // assertions + bool warp_waiting_at_barrier( unsigned warp_id ); + + // debug + void dump() const; + +private: + unsigned m_max_cta_per_core; + unsigned m_max_warps_per_core; + + cta_to_warp_t m_cta_to_warps; + warp_set_t m_warp_active; + warp_set_t m_warp_at_barrier; +}; + +class mshr_shader_unit; + +extern unsigned int warp_size; + +typedef struct shader_core_ctx : public core_t +{ + shader_core_ctx( unsigned max_warps_per_cta, unsigned max_cta_per_core ); + + virtual void set_at_barrier( unsigned cta_id, unsigned warp_id ); + virtual void warp_exit( unsigned warp_id ); + virtual bool warp_waiting_at_barrier( unsigned warp_id ); + void allocate_barrier( unsigned cta_id, warp_set_t warps ); + void deallocate_barrier( unsigned cta_id ); + +//// + + const char *name; + int sid; + + // array of the threads running on this shader core + thread_ctx_t *thread; + unsigned int n_threads; + unsigned int last_issued_thread; + + //per warp information array + std::vector warp; + + barrier_set_t m_barriers; + + //Keeps track of which warp of instructions to fetch/execute + int next_warp; + + // number of threads to be completed ( ==0 when all thread on this core completed) + int not_completed; + // number of Cuda Thread Arrays (blocks) currently running on this shader. + int n_active_cta; + //Keep track of multiple CTAs in shader + int cta_status[MAX_CTA_PER_SHADER]; + // registers holding the instruction between pipeline stages. + // see below for definition of pipeline stages + inst_t** pipeline_reg; + inst_t** pre_mem_pipeline; + int warp_part2issue; // which part of warp to issue to pipeline + int new_warp_TS; // new warp at TS pipeline register + + shd_cache_t *L1cache; + shd_cache_t *L1texcache; + shd_cache_t *L1constcache; + + // pointer to memory access wrapping function + fq_push_t fq_push; + fq_has_buffer_t fq_has_buffer; + + // simulation cycles happened to the shader, kept for cacheline replacement + unsigned int gpu_cycle; + // number of instructions committed by this shader core + unsigned int num_sim_insn; + + // reconvergence + unsigned int model; + + // Structure is used to keep track of the branching within the warp of instructions. + // As a group of instructions is grouped together from different threads to be executed, when + // a branch does occur, then the sub-set that does not get run will be given the value of warp_priority, + // and warp_priority will increase. Each time a sub-set branches further, a similar scheme is used. + // When a sub-set completes fully, then this table will determine which next sub-set to finish, which + // will be the next largest value in the table. + int branch_priority; + int* max_branch_priority; //Keeps track of the maximum priority of the threads running within a warp. need n_threads number of these + + // pdom reconvergence context for each warp + pdom_warp_ctx_t *pdom_warp; + + int waiting_at_barrier; // number of threads current waiting at a barrier in this shader. + int RR_k; //counter for register read pipeline + + int using_dwf; //is the scheduler using dynamic warp formation + int using_rrstage; //is the pipeline using an extra stage for register read + int using_commit_queue; //is the scheduler using commit_queue? + + delay_queue *thd_commit_queue; + + int pending_shmem_bkacc; // 0 = check conflict for new insn + int pending_cache_bkacc; // 0 = check conflict for new insn + + bool shader_memory_new_instruction_processed; + + int pending_mem_access; // number of memory access to be serviced (use for W0 classification) + + int pending_cmem_acc; //number of accesses to differrnt addresses in the constant memory cache + + unsigned int n_diverge; // number of divergence occurred in this shader + + //Shader core resources + unsigned int shmem_size; + unsigned int n_registers; //registers available in the shader core + unsigned int n_cta; //Limit on number of concurrent CTAs in shader core + + //void *req_hist; //not used anywhere + + mshr_shader_unit *mshr_unit; +} shader_core_ctx_t; + + +shader_core_ctx_t* shader_create( const char *name, int sid, unsigned int n_threads, + unsigned int n_mshr, fq_push_t fq_push, fq_has_buffer_t fq_has_buffer, unsigned int model); +unsigned shader_reinit(shader_core_ctx_t *sc, int start_thread, int end_thread); +void shader_init_CTA(shader_core_ctx_t *shader, int start_thread, int end_thread); + +void shader_fetch( shader_core_ctx_t *shader, + unsigned int shader_number, + int grid_num ); +void shader_decode( shader_core_ctx_t *shader, + unsigned int shader_number, + unsigned int grid_num ); +void shader_preexecute( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_execute( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_pre_memory( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_const_memory( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_texture_memory( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_memory( shader_core_ctx_t *shader, + unsigned int shader_number ); +void shader_writeback( shader_core_ctx_t *shader, + unsigned int shader_number, + int grid_num ); + +void shader_display_pipeline(shader_core_ctx_t *shader, FILE *fout, int print_mem, int mask3bit ); +void shader_dump_thread_state(shader_core_ctx_t *shader, FILE *fout ); +void shader_cycle( shader_core_ctx_t *shader, + unsigned int shader_number, + int grid_num ); + +void mshr_print(FILE *fp, shader_core_ctx_t *shader); + +void mshr_update_status(mshr_entry* mshr, enum mshr_status new_status); + +mshr_entry* fetchMSHR(delay_queue** mshr, shader_core_ctx_t* sc); +mshr_entry* shader_check_mshr4tag(shader_core_ctx_t* sc, unsigned long long int addr,int mem_type); +void shader_update_mshr(shader_core_ctx_t* sc, unsigned long long int fetched_addr, unsigned int mshr_idx, int mem_type ); +void shader_visualizer_dump(FILE *fp, shader_core_ctx_t* sc); + +void init_mshr_pool(); +mshr_entry* alloc_mshr_entry(); +void free_mshr_entry( mshr_entry * ); + +void shader_clean(shader_core_ctx_t *sc, unsigned int n_threads); +void shader_cache_flush(shader_core_ctx_t* sc); + +// print out the accumulative statistics for shaders (those that are not local to one shader) +void shader_print_accstats( FILE* fout ); +void shader_print_runtime_stat( FILE *fout ); +void shader_print_l1_miss_stat( FILE *fout ); + +//return the maximum CTAs that can be running at the same on shader +//based on on the current kernel's CTA size and is 1 if mutiple CTA per block is not supported +unsigned int max_cta_per_shader( shader_core_ctx_t *shader); + +#define N_PIPELINE_STAGES 7 +#define TS_IF 0 +#define IF_ID 1 +#define ID_RR 2 +#define ID_EX 3 +#define RR_EX 3 +#define EX_MM 4 +#define MM_WB 5 +#define WB_RT 6 + + +#endif /* SHADER_H */ diff --git a/src/gpgpu-sim/stack.cc b/src/gpgpu-sim/stack.cc new file mode 100644 index 0000000..7f7cc70 --- /dev/null +++ b/src/gpgpu-sim/stack.cc @@ -0,0 +1,127 @@ +/* + * stack.c + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * Ivan Sham and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. +* + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "stack.h" + +#include +#include + +void push_stack(Stack *S, address_type val) { + assert(S->top < S->max_size); + S->v[S->top] = val; + (S->top)++; + +} + +address_type pop_stack(Stack *S) { + (S->top)--; + return(S->v[S->top]); +} + +address_type top_stack(Stack *S) { + assert(S->top >= 1); + return(S->v[S->top - 1]); +} + +Stack* new_stack(int size) { + Stack* S; + S = (Stack*)malloc(sizeof(Stack)); + S->max_size = size; + S->top = 0; + S->v = (address_type*)calloc(size, sizeof(address_type)); + return S; +} + +void free_stack(Stack *S) { + free(S->v); + free(S); +} + +int size_stack(Stack *S) { + return S->top; +} + +int full_stack(Stack *S) { + return S->top >= S->max_size; +} + +int empty_stack(Stack *S) { + return S->top == 0; +} + +int element_exist_stack(Stack *S, address_type value) { + int i; + for (i = 0; i < S->top; ++i) { + if (value == S->v[i]) { + return 1; + } + } + return 0; +} + +void reset_stack(Stack *S) { + S->top = 0; +} diff --git a/src/gpgpu-sim/stack.h b/src/gpgpu-sim/stack.h new file mode 100644 index 0000000..b00e25b --- /dev/null +++ b/src/gpgpu-sim/stack.h @@ -0,0 +1,90 @@ +/* + * stack.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * Ivan Sham and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files + * src/gpgpusim_entrypoint.c and src/simplesim-3.0/ are derived from the + * SimpleScalar Toolset available from http://www.simplescalar.com/ + * (property of SimpleScalar LLC) and the files src/intersim/ are derived + * from Booksim (Simulator provided with the textbook "Principles and + * Practices of Interconnection Networks" available from + * http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef _MY_STACK_ +#define _MY_STACK_ + +#include "../util.h" + +typedef struct { + address_type *v; + int max_size; + int top; +} Stack; + +void push_stack(Stack *S, address_type val); +address_type pop_stack(Stack *S); +address_type top_stack(Stack *S); +Stack* new_stack(int size); +void free_stack(Stack *S); +int size_stack(Stack *S); +int full_stack(Stack *S); +int empty_stack(Stack *S); +int element_exist_stack(Stack *S, address_type value); +void reset_stack(Stack *S); +#endif // _MY_STACK_ diff --git a/src/gpgpu-sim/stat-tool.cc b/src/gpgpu-sim/stat-tool.cc new file mode 100644 index 0000000..2dc9c0b --- /dev/null +++ b/src/gpgpu-sim/stat-tool.cc @@ -0,0 +1,1081 @@ +/* + * stat-tool.cc + * + * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, + * George L. Yuan and the University of British Columbia, Vancouver, + * BC V6T 1Z4, All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include +#include +#include +#include +#include + +// detect gcc 4.3 and use unordered map (part of c++0x) +// unordered map doesn't play nice with _GLIBCXX_DEBUG, just use a map if its enabled. +#if defined( __GNUC__ ) and not defined( _GLIBCXX_DEBUG ) +#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 3 + #include + #define my_hash_map std::unordered_map +#else + #include + namespace std { + using namespace __gnu_cxx; + } + #define my_hash_map std::hash_map +#endif +#else + #include + #define my_hash_map std::map + #define USE_MAP +#endif + +#include "histogram.h" + +binned_histogram::binned_histogram (std::string name, int nbins, int* bins) + : m_name(name), m_nbins(nbins), m_bins(NULL), m_bin_cnts(new int[m_nbins]), m_maximum(0) +{ + if (bins) { + m_bins = new int[m_nbins]; + for (int i = 0; i < nbins; i++) { + m_bins[i] = bins[i]; + } + } + + reset_bins(); +} + +binned_histogram::binned_histogram (const binned_histogram& other) + : m_name(other.m_name), m_nbins(other.m_nbins), m_bins(NULL), + m_bin_cnts(new int[m_nbins]), m_maximum(0) +{ + for (int i = 0; i < m_nbins; i++) { + m_bin_cnts[i] = other.m_bin_cnts[i]; + } +} + +void binned_histogram::reset_bins () { + for (int i = 0; i < m_nbins; i++) { + m_bin_cnts[i] = 0; + } +} + +void binned_histogram::add2bin (int sample) { + assert(0); + m_maximum = (sample > m_maximum)? sample : m_maximum; +} + +void binned_histogram::fprint (FILE *fout) { + if (m_name.c_str() != NULL) fprintf(fout, "%s = ", m_name.c_str()); + for (int i = 0; i < m_nbins; i++) { + fprintf(fout, "%d ", m_bin_cnts[i]); + } + fprintf(fout, "max=%d ", m_maximum); +} + +binned_histogram::~binned_histogram () { + if (m_bins) delete[] m_bins; + delete[] m_bin_cnts; +} + +pow2_histogram::pow2_histogram (std::string name, int nbins, int* bins) + : binned_histogram (name, nbins, bins) {} + +void pow2_histogram::add2bin (int sample) { + assert(sample >= 0); + + int bin; + int v = sample; + register unsigned int shift; + + bin = (v > 0xFFFF) << 4; v >>= bin; + shift = (v > 0xFF ) << 3; v >>= shift; bin |= shift; + shift = (v > 0xF ) << 2; v >>= shift; bin |= shift; + shift = (v > 0x3 ) << 1; v >>= shift; bin |= shift; + bin |= (v >> 1); + bin += (sample > 0)? 1:0; + + m_bin_cnts[bin] += 1; + + m_maximum = (sample > m_maximum)? sample : m_maximum; +} + +linear_histogram::linear_histogram (int stride, const char *name, int nbins, int* bins) + : binned_histogram (name, nbins, bins), m_stride(stride) +{ +} + +void linear_histogram::add2bin (int sample) { + assert(sample >= 0); + + int bin = sample / m_stride; + + m_bin_cnts[bin] += 1; + + m_maximum = (sample > m_maximum)? sample : m_maximum; +} + + +#include +#include +#include +#include +#include +#include "../util.h" + +#include "cflogger.h" + +///////////////////////////////////////////////////////////////////////////////////// +// logger snapshot trigger: +// - automate the snap_shot part of loggers to avoid modifying simulation loop everytime +// a new time-dependent stat is added +///////////////////////////////////////////////////////////////////////////////////// + +class snap_shot_trigger { +protected: + unsigned long long m_snap_shot_interval; + +public: + snap_shot_trigger(unsigned long long interval) : m_snap_shot_interval(interval) {} + virtual ~snap_shot_trigger() {} + + const unsigned long long & get_interval() const { return m_snap_shot_interval;} + + void try_snap_shot(unsigned long long current_cycle) { + if ((current_cycle % m_snap_shot_interval == 0) && current_cycle != 0) { + snap_shot(current_cycle); + } + } + + virtual void snap_shot(unsigned long long current_cycle) = 0; +}; + +static unsigned long long min_snap_shot_interval = 0; +static unsigned long long next_snap_shot_cycle = 0; +static std::list list_ss_trigger; + +void add_snap_shot_trigger (snap_shot_trigger* ss_trigger) +{ + // quick optimization assuming that all snap shot intervals are perfect multiples of each other + if (min_snap_shot_interval == 0 || min_snap_shot_interval > ss_trigger->get_interval()) { + min_snap_shot_interval = ss_trigger->get_interval(); + next_snap_shot_cycle = min_snap_shot_interval; // assume that snap shots haven't started yet + } + list_ss_trigger.push_back(ss_trigger); +} + +void remove_snap_shot_trigger (snap_shot_trigger* ss_trigger) +{ + list_ss_trigger.remove(ss_trigger); +} + +void try_snap_shot (unsigned long long current_cycle) +{ + if (min_snap_shot_interval == 0) return; + if (current_cycle != next_snap_shot_cycle) return; + + std::list::iterator ss_trigger_iter = list_ss_trigger.begin(); + for(; ss_trigger_iter != list_ss_trigger.end(); ++ss_trigger_iter) { + (*ss_trigger_iter)->snap_shot(current_cycle); // WF: should be try_snap_shot + } + next_snap_shot_cycle = current_cycle + min_snap_shot_interval; // WF: stateful testing, maybe bad +} + +///////////////////////////////////////////////////////////////////////////////////// +// spill log interface: +// - unified interface to spill log to file to avoid infinite memory usage for logging +///////////////////////////////////////////////////////////////////////////////////// + +class spill_log_interface { + public: + spill_log_interface() {} + virtual ~spill_log_interface() {} + + virtual void spill(FILE *fout, bool final) = 0; +}; + +static unsigned long long spill_interval = 0; +static unsigned long long next_spill_cycle = 0; +static std::list list_spill_log; + +void add_spill_log (spill_log_interface* spill_log) +{ + list_spill_log.push_back(spill_log); +} + +void remove_spill_log (spill_log_interface* spill_log) +{ + list_spill_log.remove(spill_log); +} + +void set_spill_interval (unsigned long long interval) +{ + spill_interval = interval; + next_spill_cycle = spill_interval; +} + +void spill_log_to_file (FILE *fout, int final, unsigned long long current_cycle) +{ + if (!final && spill_interval == 0) return; + if (!final && current_cycle <= next_spill_cycle) return; + + fprintf(fout, "\n"); // ensure that the spill occurs at a new line + std::list::iterator i_spill_log = list_spill_log.begin(); + for(; i_spill_log != list_spill_log.end(); ++i_spill_log) { + (*i_spill_log)->spill(fout, final); + } + fflush(fout); + + next_spill_cycle = current_cycle + spill_interval; // WF: stateful testing, maybe bad +} + +///////////////////////////////////////////////////////////////////////////////////// +// thread control-flow locality logger +///////////////////////////////////////////////////////////////////////////////////// +unsigned translate_pc_to_ptxlineno(unsigned pc); +class thread_insn_span { +private: + + typedef my_hash_map span_count_map; + unsigned long long m_cycle; + int m_n_insn; + span_count_map m_insn_span_count; + +public: + + thread_insn_span(unsigned long long cycle, int n_insn) + : m_cycle(cycle), m_n_insn(n_insn), +#ifdef USE_MAP + m_insn_span_count() +#else + m_insn_span_count(n_insn * 2) +#endif + { } + + ~thread_insn_span() { } + + thread_insn_span(const thread_insn_span& other) + : m_cycle(other.m_cycle), m_n_insn(other.m_n_insn), + m_insn_span_count(other.m_insn_span_count) + { } + + thread_insn_span& operator=(const thread_insn_span& other) + { + printf("thread_insn_span& operator=\n"); + if (this != &other && m_n_insn != other.m_n_insn) { + m_n_insn = other.m_n_insn; + m_insn_span_count = other.m_insn_span_count; + m_cycle = other.m_cycle; + } + return *this; + } + + thread_insn_span& operator+=(const thread_insn_span& other) + { + assert(m_n_insn == other.m_n_insn); // no way to aggregate if they are different programs + span_count_map::const_iterator i_sc = other.m_insn_span_count.begin(); + for (; i_sc != other.m_insn_span_count.end(); ++i_sc) { + m_insn_span_count[i_sc->first] += i_sc->second; + } + return *this; + } + + void set_span( address_type pc ) { + if( ((int)pc) >= 0 ) + m_insn_span_count[pc] += 1; + } + + void reset(unsigned long long cycle) { + m_cycle = cycle; + m_insn_span_count.clear(); + } + + void print_span(FILE *fout) { + fprintf(fout, "%d: ", (int)m_cycle); + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + fprintf(fout, "%d ", i_sc->first); + } + fprintf(fout, "\n"); + } + + void print_histo(FILE *fout) { + fprintf(fout, "%d:", (int)m_cycle); + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + fprintf(fout, "%d ", i_sc->second); + } + fprintf(fout, "\n"); + } + + void print_sparse_histo(FILE *fout) { + int n_printed_entries = 0; + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first); + fprintf(fout, "%u %d ", ptx_lineno, i_sc->second); + n_printed_entries++; + } + if (n_printed_entries == 0) { + fprintf(fout, "0 0 "); + } + fprintf(fout, "\n"); + } + + void print_sparse_histo(gzFile fout) { + int n_printed_entries = 0; + span_count_map::const_iterator i_sc = m_insn_span_count.begin(); + for (; i_sc != m_insn_span_count.end(); ++i_sc) { + unsigned ptx_lineno = translate_pc_to_ptxlineno(i_sc->first); + gzprintf(fout, "%u %d ", ptx_lineno, i_sc->second); + n_printed_entries++; + } + if (n_printed_entries == 0) { + gzprintf(fout, "0 0 "); + } + gzprintf(fout, "\n"); + } +}; + +class thread_CFlocality : public snap_shot_trigger, public spill_log_interface { +private: + + std::string m_name; + + int m_nthreads; + std::vector m_thread_pc; + + unsigned long long m_cycle; + thread_insn_span m_thd_span; + std::list m_thd_span_archive; + +public: + + thread_CFlocality(std::string name, unsigned long long snap_shot_interval, + int nthreads, int n_insn, address_type start_pc, unsigned long long start_cycle = 0) + : snap_shot_trigger(snap_shot_interval), m_name(name), + m_nthreads(nthreads), m_thread_pc(nthreads, start_pc), m_cycle(start_cycle), + m_thd_span(start_cycle, n_insn) + { + std::fill(m_thread_pc.begin(), m_thread_pc.end(), -1); // so that hw thread with no work assigned will not clobber results + } + + ~thread_CFlocality() {} + + void update_thread_pc( int thread_id, address_type pc ) { + m_thread_pc[thread_id] = pc; + m_thd_span.set_span(pc); + } + + void snap_shot(unsigned long long current_cycle) { + m_thd_span_archive.push_back(m_thd_span); + m_thd_span.reset(current_cycle); + for (int i = 0; i < (int)m_thread_pc.size(); i++) { + m_thd_span.set_span(m_thread_pc[i]); + } + } + + void spill(FILE *fout, bool final) { + std::list::iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); lit = m_thd_span_archive.erase(lit) ) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_histo(fout); + } + assert( m_thd_span_archive.empty() ); + if (final) { + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_histo(fout); + } + } + + void print_visualizer(FILE *fout) { + fprintf(fout, "%s: ", m_name.c_str()); + if (m_thd_span_archive.empty()) { + + // visualizer do no require snap_shots + m_thd_span.print_sparse_histo(fout); + + // clean the thread span + m_thd_span.reset(0); + for (int i = 0; i < (int)m_thread_pc.size(); i++) { + m_thd_span.set_span(m_thread_pc[i]); + } + } else { + assert(0); // TODO: implement fall back so that visualizer can work with snap shots + } + } + + void print_visualizer(gzFile fout) { + gzprintf(fout, "%s: ", m_name.c_str()); + if (m_thd_span_archive.empty()) { + + // visualizer do no require snap_shots + m_thd_span.print_sparse_histo(fout); + + // clean the thread span + m_thd_span.reset(0); + for (int i = 0; i < (int)m_thread_pc.size(); i++) { + m_thd_span.set_span(m_thread_pc[i]); + } + } else { + assert(0); // TODO: implement fall back so that visualizer can work with snap shots + } + } + + void print_span(FILE *fout) { + std::list::iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); ++lit) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_span(fout); + } + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_span(fout); + } + + void print_histo(FILE *fout) { + std::list::iterator lit = m_thd_span_archive.begin(); + for (; lit != m_thd_span_archive.end(); ++lit) { + fprintf(fout, "%s-", m_name.c_str()); + lit->print_histo(fout); + } + fprintf(fout, "%s-", m_name.c_str()); + m_thd_span.print_histo(fout); + } +}; + +static int n_thread_CFloggers = 0; +static thread_CFlocality** thread_CFlogger = NULL; + +void create_thread_CFlogger( int n_loggers, int n_threads, int n_insn, address_type start_pc, unsigned long long logging_interval) +{ + destroy_thread_CFlogger(); + + n_thread_CFloggers = n_loggers; + thread_CFlogger = new thread_CFlocality*[n_loggers]; + + std::string name_tpl("CFLog"); + char buffer[32]; + for (int i = 0; i < n_thread_CFloggers; i++) { + snprintf(buffer, 32, "%02d", i); + thread_CFlogger[i] = new thread_CFlocality( name_tpl + buffer, logging_interval, n_threads, n_insn, start_pc); + if (logging_interval != 0) { + add_snap_shot_trigger(thread_CFlogger[i]); + add_spill_log(thread_CFlogger[i]); + } + } +} + +void destroy_thread_CFlogger( ) +{ + if (thread_CFlogger != NULL) { + for (int i = 0; i < n_thread_CFloggers; i++) { + remove_snap_shot_trigger(thread_CFlogger[i]); + remove_spill_log(thread_CFlogger[i]); + delete thread_CFlogger[i]; + } + delete thread_CFlogger; + thread_CFlogger = NULL; + } +} + +void cflog_update_thread_pc( int logger_id, int thread_id, address_type pc ) +{ + if (thread_id < 0) return; + thread_CFlogger[logger_id]->update_thread_pc(thread_id, pc); +} + +void cflog_snapshot( int logger_id, unsigned long long cycle ) +{ + thread_CFlogger[logger_id]->snap_shot(cycle); +} + +void cflog_print(FILE *fout) +{ + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_histo(fout); + } +} + +void cflog_visualizer_print(FILE *fout) +{ + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_visualizer(fout); + } +} + +void cflog_visualizer_gzprint(gzFile fout) +{ + for (int i = 0; i < n_thread_CFloggers; i++) { + thread_CFlogger[i]->print_visualizer(fout); + } +} + +///////////////////////////////////////////////////////////////////////////////////// +// per-insn active thread distribution (warp occ) logger +///////////////////////////////////////////////////////////////////////////////////// + +class insn_warp_occ_logger{ +private: + int m_simd_width; + std::vector m_insn_warp_occ; + int m_id; + static int s_ids; + +public: + insn_warp_occ_logger(int simd_width, int n_insn) + : m_simd_width(simd_width), + m_insn_warp_occ(n_insn, linear_histogram(1, "", m_simd_width)), + m_id(s_ids++) {} + + insn_warp_occ_logger(const insn_warp_occ_logger& other) + : m_simd_width(other.m_simd_width), + m_insn_warp_occ(other.m_insn_warp_occ.size(), linear_histogram(1, "", m_simd_width)), + m_id(s_ids++) {} + + insn_warp_occ_logger& operator=(const insn_warp_occ_logger& p) { + printf("insn_warp_occ_logger Operator= called: %02d \n", m_id); + assert(0); + return *this; + } + + ~insn_warp_occ_logger() {} + + void set_id(int id) { + m_id = id; + } + + void log(address_type pc, int warp_occ) { + m_insn_warp_occ[pc].add2bin(warp_occ - 1); + } + + void print(FILE *fout) { + for (unsigned i = 0; i < m_insn_warp_occ.size(); i++) { + fprintf(fout, "InsnWarpOcc%02d-%d", m_id, i); + m_insn_warp_occ[i].fprint(fout); + fprintf(fout, "\n"); + } + } +}; +int insn_warp_occ_logger::s_ids = 0; + +static std::vector iwo_logger; + +void insn_warp_occ_create( int n_loggers, int simd_width, int n_insn) +{ + iwo_logger.clear(); + iwo_logger.assign(n_loggers, insn_warp_occ_logger(simd_width, n_insn)); + for (unsigned i = 0; i < iwo_logger.size(); i++) { + iwo_logger[i].set_id(i); + } +} + +void insn_warp_occ_log( int logger_id, address_type pc, int warp_occ) +{ + if (warp_occ <= 0) return; + iwo_logger[logger_id].log(pc, warp_occ); +} + +void insn_warp_occ_print( FILE *fout ) +{ + for (unsigned i = 0; i < iwo_logger.size(); i++) { + iwo_logger[i].print(fout); + } +} + +///////////////////////////////////////////////////////////////////////////////////// +// generic linear histogram logger +///////////////////////////////////////////////////////////////////////////////////// + +class linear_histogram_snapshot { +private: + unsigned long long m_cycle; + std::vector m_linear_histogram; +public: + linear_histogram_snapshot(int n_bins, unsigned long long cycle) + : m_cycle(cycle), + m_linear_histogram(n_bins,0) + { } + + linear_histogram_snapshot(const linear_histogram_snapshot& other) + : m_cycle(other.m_cycle), + m_linear_histogram(other.m_linear_histogram) + { } + + ~linear_histogram_snapshot() { } + + void addsample(int pos) { + assert((size_t)pos < m_linear_histogram.size()); + m_linear_histogram[pos] += 1; + } + + void subsample(int pos) { + assert((size_t)pos < m_linear_histogram.size()); + m_linear_histogram[pos] -= 1; + } + + void reset(unsigned long long cycle) { + m_cycle = cycle; + m_linear_histogram.assign(m_linear_histogram.size(), 0); + } + + void set_cycle(unsigned long long cycle) { + m_cycle = cycle; + } + + void print(FILE *fout) { + fprintf(fout, "%d = ", (int)m_cycle); + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + fprintf(fout, "%d ", m_linear_histogram[i]); + } + } + + void print_visualizer(FILE *fout) { + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + fprintf(fout, "%d ", m_linear_histogram[i]); + } + } + + void print_visualizer(gzFile fout) { + for (unsigned int i = 0; i < m_linear_histogram.size(); i++) { + gzprintf(fout, "%d ", m_linear_histogram[i]); + } + } +}; + +class linear_histogram_logger : public snap_shot_trigger, public spill_log_interface { +private: + int m_n_bins; + linear_histogram_snapshot m_curr_lin_hist; + std::list m_lin_hist_archive; + unsigned long long m_cycle; + bool m_reset_at_snap_shot; + std::string m_name; + int m_id; + static int s_ids; + +public: + linear_histogram_logger(int n_bins, + unsigned long long snap_shot_interval, + const char *name, + bool reset_at_snap_shot = true, + unsigned long long start_cycle = 0) + : snap_shot_trigger(snap_shot_interval), + m_n_bins(n_bins), + m_curr_lin_hist(m_n_bins, start_cycle), + m_lin_hist_archive(), + m_cycle(start_cycle), + m_reset_at_snap_shot(reset_at_snap_shot), + m_name(name), + m_id(s_ids++) {} + + linear_histogram_logger(const linear_histogram_logger& other) // WF: Buggy - Not really copying data over + : snap_shot_trigger(other.get_interval()), + m_n_bins(other.m_n_bins), + m_curr_lin_hist(m_n_bins, other.m_cycle), + m_lin_hist_archive(), + m_cycle(other.m_cycle), + m_reset_at_snap_shot(other.m_reset_at_snap_shot), + m_name(other.m_name), + m_id(s_ids++) {} + + // using default assignment operator! + + ~linear_histogram_logger() { + // printf("Destroyer called: %s%02d \n", m_name.c_str(), m_id); + remove_snap_shot_trigger(this); + remove_spill_log(this); + } + + void set_id(int id) { + m_id = id; + } + + void log(int pos) { + m_curr_lin_hist.addsample(pos); + } + + void unlog(int pos) { + m_curr_lin_hist.subsample(pos); + } + + void snap_shot(unsigned long long current_cycle) { + m_lin_hist_archive.push_back(m_curr_lin_hist); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(current_cycle); + } else { + m_curr_lin_hist.set_cycle(current_cycle); + } + } + + void spill(FILE *fout, bool final) { + std::list::iterator iter = m_lin_hist_archive.begin(); + for (; iter != m_lin_hist_archive.end(); iter = m_lin_hist_archive.erase(iter) ) { + fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0); + iter->print(fout); + fprintf(fout, "\n"); + } + assert( m_lin_hist_archive.empty() ); + if (final) { + fprintf(fout, "%s%02d-", m_name.c_str(), (m_id >= 0)? m_id : 0); + m_curr_lin_hist.print(fout); + fprintf(fout, "\n"); + } + } + + void print(FILE *fout) { + std::list::iterator iter = m_lin_hist_archive.begin(); + for (; iter != m_lin_hist_archive.end(); ++iter) { + fprintf(fout, "%s%02d-", m_name.c_str(), m_id); + iter->print(fout); + fprintf(fout, "\n"); + } + fprintf(fout, "%s%02d-", m_name.c_str(), m_id); + m_curr_lin_hist.print(fout); + fprintf(fout, "\n"); + } + + void print_visualizer(FILE *fout) { + assert(m_lin_hist_archive.empty()); // don't support snapshot for now + fprintf(fout, "%s", m_name.c_str()); + if (m_id >= 0) { + fprintf(fout, "%02d: ", m_id); + } else { + fprintf(fout, ": "); + } + m_curr_lin_hist.print_visualizer(fout); + fprintf(fout, "\n"); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(0); + } + } + + void print_visualizer(gzFile fout) { + assert(m_lin_hist_archive.empty()); // don't support snapshot for now + gzprintf(fout, "%s", m_name.c_str()); + if (m_id >= 0) { + gzprintf(fout, "%02d: ", m_id); + } else { + gzprintf(fout, ": "); + } + m_curr_lin_hist.print_visualizer(fout); + gzprintf(fout, "\n"); + if (m_reset_at_snap_shot) { + m_curr_lin_hist.reset(0); + } + } +}; +int linear_histogram_logger::s_ids = 0; + +///////////////////////////////////////////////////////////////////////////////////// +// per-shadercore active thread distribution (warp occ) logger +///////////////////////////////////////////////////////////////////////////////////// + +static std::vector s_warp_occ_logger; + +void shader_warp_occ_create( int n_loggers, int simd_width, unsigned long long logging_interval) +{ + // simd_width + 1 to include the case with full warp + s_warp_occ_logger.assign(n_loggers, + linear_histogram_logger(simd_width + 1, logging_interval, "ShdrWarpOcc")); + for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { + s_warp_occ_logger[i].set_id(i); + add_snap_shot_trigger(&(s_warp_occ_logger[i])); + add_spill_log(&(s_warp_occ_logger[i])); + } +} + +void shader_warp_occ_log( int logger_id, int warp_occ) +{ + s_warp_occ_logger[logger_id].log(warp_occ); +} + +void shader_warp_occ_snapshot( int logger_id, unsigned long long current_cycle) +{ + s_warp_occ_logger[logger_id].snap_shot(current_cycle); +} + +void shader_warp_occ_print( FILE *fout ) +{ + for (unsigned i = 0; i < s_warp_occ_logger.size(); i++) { + s_warp_occ_logger[i].print(fout); + } +} + + +///////////////////////////////////////////////////////////////////////////////////// +// per-shadercore memory-access logger +///////////////////////////////////////////////////////////////////////////////////// + +static int s_mem_acc_logger_n_dram = 0; +static int s_mem_acc_logger_n_bank = 0; +static std::vector s_mem_acc_logger; + +void shader_mem_acc_create( int n_loggers, int n_dram, int n_bank, unsigned long long logging_interval) +{ + // (n_bank + 1) to space data out; 2x to separate read and write + s_mem_acc_logger.assign(n_loggers, + linear_histogram_logger(2 * n_dram * (n_bank + 1), logging_interval, "ShdrMemAcc")); + + s_mem_acc_logger_n_dram = n_dram; + s_mem_acc_logger_n_bank = n_bank; + for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { + s_mem_acc_logger[i].set_id(i); + add_snap_shot_trigger(&(s_mem_acc_logger[i])); + add_spill_log(&(s_mem_acc_logger[i])); + } +} + +void shader_mem_acc_log( int logger_id, int dram_id, int bank, char rw) +{ + if (s_mem_acc_logger_n_dram == 0) return; + int write_offset = 0; + switch(rw) { + case 'r': write_offset = 0; break; + case 'w': write_offset = (s_mem_acc_logger_n_bank + 1) * s_mem_acc_logger_n_dram; break; + default: assert(0); break; + } + s_mem_acc_logger[logger_id].log(dram_id * s_mem_acc_logger_n_bank + bank + write_offset); +} + +void shader_mem_acc_snapshot( int logger_id, unsigned long long current_cycle) +{ + s_mem_acc_logger[logger_id].snap_shot(current_cycle); +} + +void shader_mem_acc_print( FILE *fout ) +{ + for (unsigned i = 0; i < s_mem_acc_logger.size(); i++) { + s_mem_acc_logger[i].print(fout); + } +} + + +///////////////////////////////////////////////////////////////////////////////////// +// per-shadercore memory-latency logger +///////////////////////////////////////////////////////////////////////////////////// + +static bool s_mem_lat_logger_used = false; +static int s_mem_lat_logger_nbins = 48; // up to 2^24 = 16M +static std::vector s_mem_lat_logger; + +void shader_mem_lat_create( int n_loggers, unsigned long long logging_interval) +{ + s_mem_lat_logger.assign(n_loggers, + linear_histogram_logger(s_mem_lat_logger_nbins, logging_interval, "ShdrMemLat")); + + for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { + s_mem_lat_logger[i].set_id(i); + add_snap_shot_trigger(&(s_mem_lat_logger[i])); + add_spill_log(&(s_mem_lat_logger[i])); + } + + s_mem_lat_logger_used = true; +} + +void shader_mem_lat_log( int logger_id, int latency) +{ + if (s_mem_lat_logger_used == false) return; + if (latency > (1<<(s_mem_lat_logger_nbins/2))) assert(0); // guard for out of bound bin + assert(latency > 0); + + int latency_bin; + + int bin; // LOG_2(latency) + int v = latency; + register unsigned int shift; + + bin = (v > 0xFFFF) << 4; v >>= bin; + shift = (v > 0xFF ) << 3; v >>= shift; bin |= shift; + shift = (v > 0xF ) << 2; v >>= shift; bin |= shift; + shift = (v > 0x3 ) << 1; v >>= shift; bin |= shift; + bin |= (v >> 1); + latency_bin = 2 * bin; + if (bin > 0) { + latency_bin += ((latency & (1 << (bin - 1))) != 0)? 1 : 0; // approx. for LOG_sqrt2(latency) + } + + s_mem_lat_logger[logger_id].log(latency_bin); +} + +void shader_mem_lat_snapshot( int logger_id, unsigned long long current_cycle) +{ + s_mem_lat_logger[logger_id].snap_shot(current_cycle); +} + +void shader_mem_lat_print( FILE *fout ) +{ + for (unsigned i = 0; i < s_mem_lat_logger.size(); i++) { + s_mem_lat_logger[i].print(fout); + } +} + + +///////////////////////////////////////////////////////////////////////////////////// +// per-shadercore cache-miss logger +///////////////////////////////////////////////////////////////////////////////////// + +static int s_cache_access_logger_n_types = 0; +static std::vector s_cache_access_logger; + +enum cache_access_logger_types { + NORMAL, TEXTURE, CONSTANT +}; + +int get_shader_normal_cache_id() { return NORMAL; } +int get_shader_texture_cache_id() { return TEXTURE; } +int get_shader_constant_cache_id() { return CONSTANT; } + +void shader_cache_access_create( int n_loggers, int n_types, unsigned long long logging_interval) +{ + // There are different type of cache (x2 for recording accesses and misses) + s_cache_access_logger.assign(n_loggers, + linear_histogram_logger(n_types * 2, logging_interval, "ShdrCacheMiss")); + + s_cache_access_logger_n_types = n_types; + for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { + s_cache_access_logger[i].set_id(i); + add_snap_shot_trigger(&(s_cache_access_logger[i])); + add_spill_log(&(s_cache_access_logger[i])); + } +} + +void shader_cache_access_log( int logger_id, int type, int miss) +{ + if (s_cache_access_logger_n_types == 0) return; + if (logger_id < 0) return; + assert(type == NORMAL || type == TEXTURE || type == CONSTANT); + assert(miss == 0 || miss == 1); + + s_cache_access_logger[logger_id].log(2 * type + miss); +} + +void shader_cache_access_unlog( int logger_id, int type, int miss) +{ + if (s_cache_access_logger_n_types == 0) return; + if (logger_id < 0) return; + assert(type == NORMAL || type == TEXTURE || type == CONSTANT); + assert(miss == 0 || miss == 1); + + s_cache_access_logger[logger_id].unlog(2 * type + miss); +} + +void shader_cache_access_print( FILE *fout ) +{ + for (unsigned i = 0; i < s_cache_access_logger.size(); i++) { + s_cache_access_logger[i].print(fout); + } +} + + +///////////////////////////////////////////////////////////////////////////////////// +// per-shadercore CTA count logger (only make sense with gpgpu_spread_blocks_across_cores) +///////////////////////////////////////////////////////////////////////////////////// + +static linear_histogram_logger *s_CTA_count_logger = NULL; + +void shader_CTA_count_create( int n_shaders, unsigned long long logging_interval) +{ + // only need one logger to track all the shaders + if (s_CTA_count_logger != NULL) delete s_CTA_count_logger; + s_CTA_count_logger = new linear_histogram_logger(n_shaders, logging_interval, "ShdrCTACount", false); + + s_CTA_count_logger->set_id(-1); + if (logging_interval != 0) { + add_snap_shot_trigger(s_CTA_count_logger); + add_spill_log(s_CTA_count_logger); +} +} + +void shader_CTA_count_log( int shader_id, int nCTAadded ) +{ + if (s_CTA_count_logger == NULL) return; + + for (int i = 0; i < nCTAadded; i++) { + s_CTA_count_logger->log(shader_id); + } +} + +void shader_CTA_count_unlog( int shader_id, int nCTAdone ) +{ + if (s_CTA_count_logger == NULL) return; + + for (int i = 0; i < nCTAdone; i++) { + s_CTA_count_logger->unlog(shader_id); + } +} + +void shader_CTA_count_print( FILE *fout ) +{ + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print(fout); +} + +void shader_CTA_count_visualizer_print( FILE *fout ) +{ + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print_visualizer(fout); +} + +void shader_CTA_count_visualizer_gzprint( gzFile fout ) +{ + if (s_CTA_count_logger == NULL) return; + s_CTA_count_logger->print_visualizer(fout); +} + diff --git a/src/gpgpu-sim/visualizer.cc b/src/gpgpu-sim/visualizer.cc new file mode 100644 index 0000000..3772fc2 --- /dev/null +++ b/src/gpgpu-sim/visualizer.cc @@ -0,0 +1,602 @@ +/* + * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung and the University of + * British Columbia, Vancouver, BC V6T 1Z4, All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "gpu-sim.h" +#include "../option_parser.h" +#include +#include +#include + +extern unsigned int gpu_n_shader; +extern unsigned int gpu_n_mem; +extern unsigned int gpu_mem_n_bk; +extern shader_core_ctx_t **sc; +extern dram_t **dram; +extern unsigned int L1_read_miss; +extern unsigned int L1_write_miss; +extern unsigned int L1_texture_miss; +extern unsigned int L1_const_miss; +extern unsigned L2_write_miss; +extern unsigned L2_write_hit; +extern unsigned L2_read_hit; +extern unsigned L2_read_miss; +extern unsigned long long int mf_total_lat; +extern unsigned num_mfs; +extern unsigned long long gpu_sim_cycle; +extern unsigned long long gpu_sim_insn; +extern unsigned long long gpu_tot_sim_insn; +extern unsigned long long gpu_completed_thread; +extern unsigned int gpgpu_n_sent_writes; +extern unsigned int gpgpu_n_processed_writes; +extern unsigned int gpgpu_n_cache_bkconflict; +extern unsigned int gpgpu_n_shmem_bkconflict; +extern unsigned int gpu_stall_by_MSHRwb; +extern unsigned int *max_return_queue_length; +extern unsigned max_mrq_latency; +extern unsigned max_dq_latency; +extern unsigned max_mf_latency; +extern unsigned max_icnt2mem_latency; +extern unsigned max_icnt2sh_latency; +extern int gpgpu_warpdistro_shader; +extern unsigned ***mem_access_type_stats; + +extern unsigned int warp_size; +extern unsigned int *shader_cycle_distro; +void time_vector_print_interval2file(FILE *outfile); +void time_vector_print_interval2gzfile(gzFile outfile); +void cflog_visualizer_gzprint(gzFile fout); +void shader_CTA_count_visualizer_gzprint(gzFile fout); +float shd_cache_windowed_cache_miss_rate(shd_cache_t*, int); +void shd_cache_new_window(shd_cache_t*); + +int g_visualizer_enabled = 1; +char *g_visualizer_filename = NULL; +int g_visualizer_zlevel = 6; + +void visualizer_options(option_parser_t opp) +{ + option_parser_register(opp, "-visualizer_enabled", OPT_BOOL, + &g_visualizer_enabled, "Turn on visualizer output (1=On, 0=Off)", + "1"); + + option_parser_register(opp, "-visualizer_outputfile", OPT_CSTR, + &g_visualizer_filename, "Specifies the output log file for visualizer", + NULL); + + option_parser_register(opp, "-visualizer_zlevel", OPT_INT32, + &g_visualizer_zlevel, "Compression level of the visualizer output log (0=no comp, 9=highest)", + "6"); + +} + +void visualizer_printstat() +{ + static unsigned int *last_shader_cycle_distro = NULL; + gzFile visualizer_file = NULL; // gzFile is basically a pointer to a struct, so it is fine to initialize it as NULL + unsigned i; + if ( !g_visualizer_enabled ) + return; + if (!last_shader_cycle_distro) + last_shader_cycle_distro = (unsigned int*) calloc(warp_size + 3, sizeof(unsigned int)); + + if ( g_visualizer_filename == NULL ) { + time_t curr_time; + time(&curr_time); + char *date = ctime(&curr_time); + char *s = date; + while (*s) { + if (*s == ' ' || *s == '\t' || *s == ':') *s = '-'; + if (*s == '\n' || *s == '\r' ) *s = 0; + s++; + } + char buf[1024]; + snprintf(buf,1024,"gpgpusim_visualizer__%s.log.gz",date); + visualizer_file = gzopen(buf, "w"); + if (visualizer_file == NULL) { + printf("error - could not open visualizer trace file.\n"); + exit(1); + } + gzsetparams(visualizer_file, g_visualizer_zlevel, Z_DEFAULT_STRATEGY); + g_visualizer_filename = strdup(buf); + } else { + visualizer_file = gzopen(g_visualizer_filename,"a"); + if (visualizer_file == NULL) { + printf("error - could not open visualizer trace file.\n"); + exit(1); + } + gzsetparams(visualizer_file, g_visualizer_zlevel, Z_DEFAULT_STRATEGY); + } + + // instruction count per shader core + gzprintf(visualizer_file, "shaderinsncount: "); + for (i=0;inum_sim_insn); + } + gzprintf(visualizer_file, "\n"); + + // warp divergence per shader core + gzprintf(visualizer_file, "shaderwarpdiv: "); + for (i=0;in_diverge); + } + gzprintf(visualizer_file, "\n"); + + cflog_visualizer_gzprint(visualizer_file); + shader_CTA_count_visualizer_gzprint(visualizer_file); + + // per shader core cache miss rate + gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_All: "); + for (i=0;iL1cache, 0)); + } + gzprintf(visualizer_file, "\n"); + + gzprintf(visualizer_file, "CacheMissRate_TextureL1_All: "); + for (i=0;iL1texcache, 0)); + } + gzprintf(visualizer_file, "\n"); + + gzprintf(visualizer_file, "CacheMissRate_ConstL1_All: "); + for (i=0;iL1constcache, 0)); + } + gzprintf(visualizer_file, "\n"); + + gzprintf(visualizer_file, "CacheMissRate_GlobalLocalL1_noMgHt: "); + for (i=0;iL1cache, 1)); + } + gzprintf(visualizer_file, "\n"); + + gzprintf(visualizer_file, "CacheMissRate_TextureL1_noMgHt: "); + for (i=0;iL1texcache, 1)); + } + gzprintf(visualizer_file, "\n"); + + gzprintf(visualizer_file, "CacheMissRate_ConstL1_noMgHt: "); + for (i=0;iL1constcache, 1)); + } + gzprintf(visualizer_file, "\n"); + + // reset for next interval + for (i=0;iL1cache); + shd_cache_new_window(sc[i]->L1texcache); + shd_cache_new_window(sc[i]->L1constcache); + } + + // dram specific statistics + for (i=0;iid, dram[i]->n_cmd_partial); + gzprintf(visualizer_file, "dramnop: %u %u\n",dram[i]->id,dram[i]->n_nop_partial); + gzprintf(visualizer_file,"dramnact: %u %u\n",dram[i]->id,dram[i]->n_act_partial); + gzprintf(visualizer_file,"dramnpre: %u %u\n",dram[i]->id,dram[i]->n_pre_partial); + gzprintf(visualizer_file,"dramnreq: %u %u\n",dram[i]->id,dram[i]->n_req_partial); + gzprintf(visualizer_file,"dramavemrqs: %u %u\n",dram[i]->id, + dram[i]->n_cmd_partial?(dram[i]->ave_mrqs_partial/dram[i]->n_cmd_partial ):0); + + // utilization and efficiency + gzprintf(visualizer_file,"dramutil: %u %u\n", + dram[i]->id,dram[i]->n_cmd_partial?100*dram[i]->bwutil_partial/dram[i]->n_cmd_partial:0); + gzprintf(visualizer_file,"drameff: %u %u\n", + dram[i]->id,dram[i]->n_activity_partial?100*dram[i]->bwutil_partial/dram[i]->n_activity_partial:0); + + // reset for next interval + dram[i]->bwutil_partial = 0; + dram[i]->n_activity_partial = 0; + dram[i]->ave_mrqs_partial = 0; + dram[i]->n_cmd_partial = 0; + dram[i]->n_nop_partial = 0; + dram[i]->n_act_partial = 0; + dram[i]->n_pre_partial = 0; + dram[i]->n_req_partial = 0; + } + + // dram access type classification + for (i=0;iid, j, + mem_access_type_stats[GLOBAL_ACC_R][dram[i]->id][j]); + gzprintf(visualizer_file,"dramglobal_acc_w: %u %u %u\n", dram[i]->id, j, + mem_access_type_stats[GLOBAL_ACC_W][dram[i]->id][j]); + gzprintf(visualizer_file,"dramlocal_acc_r: %u %u %u\n", dram[i]->id, j, + mem_access_type_stats[LOCAL_ACC_R][dram[i]->id][j]); + gzprintf(visualizer_file,"dramlocal_acc_w: %u %u %u\n", dram[i]->id, j, + mem_access_type_stats[LOCAL_ACC_W][dram[i]->id][j]); + gzprintf(visualizer_file,"dramconst_acc_r: %u %u %u\n", dram[i]->id, j, + mem_access_type_stats[CONST_ACC_R][dram[i]->id][j]); + gzprintf(visualizer_file,"dramtexture_acc_r: %u %u %u\n", dram[i]->id, j, + mem_access_type_stats[TEXTURE_ACC_R][dram[i]->id][j]); + } + } + + // overall cache miss rates + gzprintf(visualizer_file, "Lonetexturemiss: %d\n", L1_texture_miss); + gzprintf(visualizer_file, "Loneconstmiss: %d\n", L1_const_miss); + gzprintf(visualizer_file, "Lonereadmiss: %d\n", L1_read_miss); + gzprintf(visualizer_file, "Lonewritemiss: %d\n", L1_write_miss); + gzprintf(visualizer_file, "Ltwowritemiss: %d\n", L2_write_miss); + gzprintf(visualizer_file, "Ltwowritehit: %d\n", L2_write_hit); + gzprintf(visualizer_file, "Ltworeadmiss: %d\n", L2_read_miss); + gzprintf(visualizer_file, "Ltworeadhit: %d\n", L2_read_hit); + + // latency stats + if (num_mfs) { + gzprintf(visualizer_file, "averagemflatency: %lld\n", mf_total_lat/num_mfs); + } + + // other parameters for graphing + gzprintf(visualizer_file, "globalcyclecount: %lld\n", gpu_sim_cycle); + gzprintf(visualizer_file, "globalinsncount: %lld\n", gpu_sim_insn); + gzprintf(visualizer_file, "globaltotinsncount: %lld\n", gpu_tot_sim_insn); + gzprintf(visualizer_file, "gpucompletedthreads: %lld\n", gpu_completed_thread); + gzprintf(visualizer_file, "gpgpunsentwrites: %d\n", gpgpu_n_sent_writes); + gzprintf(visualizer_file, "gpgpunprocessedwrites: %d\n", gpgpu_n_processed_writes); + gzprintf(visualizer_file, "gpgpu_n_cache_bkconflict: %d\n", gpgpu_n_cache_bkconflict); + gzprintf(visualizer_file, "gpgpu_n_shmem_bkconflict: %d\n", gpgpu_n_shmem_bkconflict); + gzprintf(visualizer_file, "gpu_stall_by_MSHRwb: %d\n", gpu_stall_by_MSHRwb); + + // warp divergence breakdown + time_vector_print_interval2gzfile(visualizer_file); + gzprintf(visualizer_file, "WarpDivergenceBreakdown:"); + unsigned int total=0; + unsigned int cf = (gpgpu_warpdistro_shader==-1)?gpu_n_shader:1; + gzprintf(visualizer_file, " %d", (shader_cycle_distro[0] - last_shader_cycle_distro[0]) / cf ); + gzprintf(visualizer_file, " %d", (shader_cycle_distro[2] - last_shader_cycle_distro[2]) / cf ); + for (i=0; i=3 ) { + total += (shader_cycle_distro[i] - last_shader_cycle_distro[i]); + if ( ((i-3) % (warp_size/8)) == ((warp_size/8)-1) ) { + gzprintf(visualizer_file, " %d", total / cf ); + total=0; + } + } + last_shader_cycle_distro[i] = shader_cycle_distro[i]; + } + gzprintf(visualizer_file,"\n"); + + gzclose(visualizer_file); +} + +#include +#include +#include +#include +#include"../gpgpu-sim/shader.h" +class my_time_vector { +private: + std::map< unsigned int, std::vector > ld_time_map; + std::map< unsigned int, std::vector > st_time_map; + unsigned ld_vector_size; + unsigned st_vector_size; + std::vector ld_time_dist; + std::vector st_time_dist; + + std::vector overal_ld_time_dist; + std::vector overal_st_time_dist; + int overal_ld_count; + int overal_st_count; + +public: + my_time_vector(int ld_size,int st_size){ + ld_vector_size = ld_size; + st_vector_size = st_size; + ld_time_dist.resize(ld_size); + st_time_dist.resize(st_size); + overal_ld_time_dist.resize(ld_size); + overal_st_time_dist.resize(st_size); + overal_ld_count = 0; + overal_st_count= 0; + } + void update_ld(unsigned int uid,unsigned int slot, long int time) { + if ( ld_time_map.find( uid )!=ld_time_map.end() ) { + ld_time_map[uid][slot]=time; + } else if (slot <= MR_2SH_FQ_POP ) { + std::vector time_vec; + time_vec.resize(ld_vector_size); + time_vec[slot] = time; + ld_time_map[uid] = time_vec; + } else { + //It's a merged mshr! forget it + } + } + void update_st(unsigned int uid,unsigned int slot, long int time) { + if ( st_time_map.find( uid )!=st_time_map.end() ) { + st_time_map[uid][slot]=time; + } else { + std::vector time_vec; + time_vec.resize(st_vector_size); + time_vec[slot] = time; + st_time_map[uid] = time_vec; + } + } + void check_ld_update(unsigned int uid,unsigned int slot, long int latency) { + if ( ld_time_map.find( uid )!=ld_time_map.end() ) { + int our_latency = ld_time_map[uid][slot] - ld_time_map[uid][MR_ICNT_PUSHED]; + assert( our_latency == latency); + } else if (slot <= MR_2SH_FQ_POP ) { + abort(); + } + } + void check_st_update(unsigned int uid,unsigned int slot, long int latency) { + if ( st_time_map.find( uid )!=st_time_map.end() ) { + int our_latency = st_time_map[uid][slot] - st_time_map[uid][MR_ICNT_PUSHED]; + assert( our_latency == latency); + } else { + abort(); + } + } +private: + void calculate_ld_dist(void) { + unsigned i,first; + long int last_update,diff; + int finished_count=0; + ld_time_dist.clear(); + ld_time_dist.resize(ld_vector_size); + std::map< unsigned int, std::vector >::iterator iter, iter_temp; + iter =ld_time_map.begin() ; + while (iter != ld_time_map.end()) { + last_update=0; + first=-1; + if (!iter->second[MR_WRITEBACK]) { + //this request is not done yet skip it! + ++iter; + continue; + } + while ( !last_update ) { + first++; + assert( first < iter->second.size() ); + last_update = iter->second[first]; + } + + for ( i=first;isecond[i] - last_update; + if ( diff>0 ) { + ld_time_dist[i]+=diff; + last_update = iter->second[i]; + } + } + iter_temp = iter; + iter++; + ld_time_map.erase(iter_temp); + finished_count++; + } + if ( finished_count ) { + for ( i=0;i >::iterator iter,iter_temp; + iter =st_time_map.begin() ; + while ( iter != st_time_map.end() ) { + last_update=0; + first=-1; + if (!iter->second[MR_2SH_ICNT_PUSHED]) { + //this request is not done yet skip it! + ++iter; + continue; + } + while ( !last_update ) { + first++; + assert( first < iter->second.size() ); + last_update = iter->second[first]; + } + + for ( i=first;isecond[i] - last_update; + if ( diff>0 ) { + st_time_dist[i]+=diff; + last_update = iter->second[i]; + } + } + iter_temp = iter; + iter++; + st_time_map.erase(iter_temp); + finished_count++; + } + if ( finished_count ) { + for ( i=0;i >::iterator iter; + for ( iter =ld_time_map.begin() ; iter != ld_time_map.end(); ++iter ) { + std::cout<<"ld_uid"<first; + for ( i=0;isecond[i]; + } + std::cout<< std::endl; + } + } + + void print_all_st(void) { + unsigned i; + std::map< unsigned int, std::vector >::iterator iter; + + for ( iter =st_time_map.begin() ; iter != st_time_map.end(); ++iter ) { + std::cout<<"st_uid"<first; + for ( i=0;isecond[i]; + } + std::cout<print_dist(); +} + + +void time_vector_print_interval2file(FILE *outfile) { + g_my_time_vector->print_to_file(outfile); +} + + +void time_vector_print_interval2gzfile(gzFile outfile) { + g_my_time_vector->print_to_gzfile(outfile); +} + +#include "../gpgpu-sim/mem_fetch.h" + +void time_vector_update(unsigned int uid,int slot ,long int cycle,int type) { + if ( (type == RD_REQ) || (type == REPLY_DATA) ) { + g_my_time_vector->update_ld( uid, slot,cycle); + } else if ( type == WT_REQ ) { + g_my_time_vector->update_st( uid, slot,cycle); + } else { + abort(); + } +} + +void check_time_vector_update(unsigned int uid,int slot ,long int latency,int type) +{ + if ( (type == RD_REQ) || (type == REPLY_DATA) ) { + g_my_time_vector->check_ld_update( uid, slot, latency ); + } else if ( type == WT_REQ ) { + g_my_time_vector->check_st_update( uid, slot, latency ); + } else { + abort(); + } +} diff --git a/src/gpgpu-sim/warp_tracker.cc b/src/gpgpu-sim/warp_tracker.cc new file mode 100644 index 0000000..19f0e12 --- /dev/null +++ b/src/gpgpu-sim/warp_tracker.cc @@ -0,0 +1,471 @@ +/* + * waro_tracker.cc + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#include "warp_tracker.h" + +using namespace std; + +extern unsigned int warp_size; +extern unsigned int gpu_n_shader; +extern unsigned int gpu_n_thread_per_shader; + +#include + +class warp_tracker { +public: + + int *tid; // the threads in this warp + int n_thd; // total number of threads in this warp + int n_notavail; // number of threads still not available + shader_core_ctx_t *shd; // reference to shader core + + warp_tracker () { + tid = new int[warp_size]; + memset(tid, -1, sizeof(int)*warp_size); + n_thd = 0; + n_notavail = 0; + shd = NULL; + } + warp_tracker( int *tid, shader_core_ctx_t * shd ) { + this->tid = new int[warp_size]; + memcpy(this->tid, tid, sizeof(int)*warp_size); + this->n_thd = 0; + this->n_notavail = 0; + for (unsigned i=0; itid[i] >= 0) { + this->n_thd++; + } + } + this->n_notavail = this->n_thd; + this->shd = shd; + } + ~warp_tracker () { + delete[] tid; + } + + // set the warp to be consist of the given threads + void set_warp ( int *tid, shader_core_ctx_t *shd) { + memcpy(this->tid, tid, sizeof(int)*warp_size); + this->n_thd = 0; + this->n_notavail = 0; + for (unsigned i=0; itid[i] >= 0) { + this->n_thd++; + } + } + this->n_notavail = this->n_thd; + this->shd = shd; + } + + // signal that this thread is available for fetch + // if all threads in the warp are available, change all their status + // and return true + bool avail_thd ( int tid_in ) { + n_notavail--; + if (n_notavail) { + return false; + } else { + + int thd_unlocked = 0; + if (shd->model == POST_DOMINATOR || shd->model == NO_RECONVERGE) { + thd_unlocked = 1; + } else { + // unlock the threads here if scheduler is not PDOM or NO-RECONV + for (unsigned i=0; itid[i] >= 0) { + shd->thread[tid[i]].avail4fetch++; + assert(shd->thread[tid[i]].avail4fetch <= 1); + assert( shd->warp[tid[i]/warp_size].n_avail4fetch < warp_size ); + shd->warp[tid[i]/warp_size].n_avail4fetch++; + thd_unlocked = 1; + } + } + } + if (shd->using_commit_queue && thd_unlocked) { + int *tid_unlocked = alloc_commit_warp(); + memcpy(tid_unlocked, this->tid, sizeof(int)*warp_size); + dq_push(shd->thd_commit_queue,(void*)tid_unlocked); + } + + return true; + } + } + + // a bookkeeping method to allow a warp to be deallocated + // when its threads have finished executing. + bool complete_thd ( int tid_in ) { + n_notavail--; + if (n_notavail) { + return false; + } else { + return true; + } + } +}; + +static warp_tracker ***warp_tracker_map; +static unsigned **g_warp_tracker_map_setl_cycle; +static warp_tracker *warp_tracker_pool = NULL; +static list free_wpt; + +warp_tracker* alloc_warp_tracker( int *tid_in, shader_core_ctx_t *shd ) +{ + assert(!free_wpt.empty()); + warp_tracker* wpt = free_wpt.front(); + free_wpt.pop_front(); + + wpt->set_warp(tid_in, shd); + + return wpt; +} + +void free_warp_tracker(warp_tracker* wpt) +{ + free_wpt.push_back(wpt); +} + +void init_warp_tracker( ) +{ + unsigned int i; + + warp_tracker_map = (warp_tracker ***)calloc(gpu_n_shader, sizeof(warp_tracker **)); + g_warp_tracker_map_setl_cycle = (unsigned**)calloc(gpu_n_shader, sizeof(unsigned*)); + for (i=0; isid; + unsigned i; + int n_thd = 0; + for (i=0; i= 0) n_thd++; + } + + if (!n_thd) return; + + warp_tracker *wpt = alloc_warp_tracker(tid_in, shd); + + // assign the new warp_tracker to warp_tracker_map + for (i=0; i= 0) { + assert( warp_tracker_map[sid][tid_in[i]] == NULL ); + warp_tracker_map[sid][tid_in[i]] = wpt; + g_warp_tracker_map_setl_cycle[sid][tid_in[i]] = gpu_tot_sim_cycle + gpu_sim_cycle; + } + } +} + +int wpt_signal_avail( int tid, shader_core_ctx_t *shd ) +{ + int sid = shd->sid; + warp_tracker *wpt = warp_tracker_map[sid][tid]; + assert(wpt != NULL); + + + // signal the warp tracker + if (wpt->avail_thd(tid)) { + // if the warp is ready to be fetched again, remove this warp_tracker + for (unsigned i=0; itid[i] >= 0) { + warp_tracker_map[sid][wpt->tid[i]] = NULL; + g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle; + } + } + + free_warp_tracker( wpt ); + + return 1; + } else { + return 0; + } +} + +void register_cta_thread_exit(shader_core_ctx_t *shader, int cta_num ); + +int wpt_signal_complete( int tid, shader_core_ctx_t *shd ) +{ + int sid = shd->sid; + warp_tracker *wpt = warp_tracker_map[sid][tid]; + assert(wpt != NULL); + + // signal the warp tracker + if (wpt->complete_thd(tid)) { + // if the warp has completed execution, remove this warp_tracker + int warp_mask = 0; + for (unsigned i=0; itid[i] >= 0) { + register_cta_thread_exit(shd, wpt->tid[i] ); + warp_tracker_map[sid][wpt->tid[i]] = NULL; + g_warp_tracker_map_setl_cycle[sid][wpt->tid[i]] = gpu_tot_sim_cycle + gpu_sim_cycle; + warp_mask |= (1 << i); + } + } + + free_warp_tracker( wpt ); + + return warp_mask; + } else { + return 0; + } +} + +//------------------------------------------------------------------------------------ + +class thread_pc_tracker_class { +public: + address_type *thd_pc; // tracks the pc of each thread + map pc_count; + unsigned acc_pc_count; + int simd_width; + static map histogram; + + thread_pc_tracker_class( ) { + this->acc_pc_count = 0; + this->simd_width = 0; + this->thd_pc = NULL; + } + + thread_pc_tracker_class(int simd_width, int thread_count) { + this->acc_pc_count = 0; + this->simd_width = simd_width; + this->thd_pc = new address_type[thread_count]; + memset( this->thd_pc, 0, sizeof(address_type)*thread_count); + } + + void add_threads( int *tid, address_type pc ) { + for (int i=0; i= 0); + if (pc_count[pc] == 0) pc_count.erase(pc); // manually erasing entries with 0 count + } + } + } + + void update_acc_count( ) { + acc_pc_count += pc_count.size(); + histogram[pc_count.size()] += 1; + } + + void set_threads_pc ( int *tid, address_type pc ) { + sub_threads(tid); + add_threads(tid, pc); + update_acc_count( ); + } + + unsigned get_acc_pc_count( ) { return acc_pc_count;} + + unsigned count( ) { return pc_count.size();} + + static void histo_print( FILE* fout ) { + if (histogram.empty()) return; // do not output anything if the histogram is empty + map::iterator i; + fprintf(fout, "Thread PC Histogram: "); + for (i = histogram.begin(); i != histogram.end(); i++) { + fprintf(fout, "%d:%d ", i->first, i->second); + } + fprintf(fout, "\n"); + } +}; + +map thread_pc_tracker_class::histogram; + +thread_pc_tracker_class *thread_pc_tracker = NULL; + +void print_thread_pc_histogram( FILE *fout ) +{ + thread_pc_tracker_class::histo_print(fout); +} + +void print_thread_pc( FILE *fout ) +{ + fprintf(fout, "SHD_PC_C: "); + for (unsigned i=0; i free_commit_warp_q; + +void init_commit_warp( ) +{ + unsigned int num_warp = warp_size * gpu_n_shader * gpu_n_thread_per_shader; + commit_warp_pool = new int[num_warp]; + for (unsigned int i=0; imodel == MIMD ) + return; + + std::set tids; + + for ( int i = 0; i < pipe_simd_width; i++) { + if (shader->pipeline_reg[i][stage].hw_thread_id == -1 ) + continue; + if ( inst_pc == (address_type)-1 ) + inst_pc = shader->pipeline_reg[i][stage].pc; + tid = shader->pipeline_reg[i][stage].hw_thread_id; + assert( tids.find(tid) == tids.end() ); + tids.insert(tid); + assert( inst_pc == shader->pipeline_reg[i][stage].pc ); + } +#endif +} + +void check_pm_stage_pcs( shader_core_ctx_t *shader, unsigned stage ) +{ +#ifdef CHECK_WARP_CONSISTENCY + address_type inst_pc = (address_type)-1; + unsigned tid; + if( shader->model == MIMD ) + return; + + std::set tids; + + for (int i = 0; i < pipe_simd_width; i++) { + if (shader->pre_mem_pipeline[i][stage].hw_thread_id == -1 ) + continue; + if ( inst_pc == (address_type)-1 ) + inst_pc = shader->pre_mem_pipeline[i][stage].pc; + tid = shader->pre_mem_pipeline[i][stage].hw_thread_id; + assert( tids.find(tid) == tids.end() ); + tids.insert(tid); + assert( inst_pc == shader->pre_mem_pipeline[i][stage].pc ); + } +#endif +} diff --git a/src/gpgpu-sim/warp_tracker.h b/src/gpgpu-sim/warp_tracker.h new file mode 100644 index 0000000..a4faa09 --- /dev/null +++ b/src/gpgpu-sim/warp_tracker.h @@ -0,0 +1,100 @@ +/* + * waro_tracker.h + * + * Copyright (c) 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda and the + * University of British Columbia + * Vancouver, BC V6T 1Z4 + * All Rights Reserved. + * + * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE + * TERMS AND CONDITIONS. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h + * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda + * (property of NVIDIA). The files benchmarks/BlackScholes/ and + * benchmarks/template/ are derived from the CUDA SDK available from + * http://www.nvidia.com/cuda (also property of NVIDIA). The files from + * src/intersim/ are derived from Booksim (a simulator provided with the + * textbook "Principles and Practices of Interconnection Networks" available + * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by + * the corresponding legal terms and conditions set forth separately (original + * copyright notices are left in files from these sources and where we have + * modified a file our copyright notice appears before the original copyright + * notice). + * + * Using this version of GPGPU-Sim requires a complete installation of CUDA + * which is distributed seperately by NVIDIA under separate terms and + * conditions. To use this version of GPGPU-Sim with OpenCL requires a + * recent version of NVIDIA's drivers which support OpenCL. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the University of British Columbia nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only. + * + * 5. No nonprofit user may place any restrictions on the use of this software, + * including as modified by the user, by any other authorized user. + * + * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, + * Ali Bakhoda, George L. Yuan, at the University of British Columbia, + * Vancouver, BC V6T 1Z4 + */ + +#ifndef warp_tracker_h_INCLUDED +#define warp_tracker_h_INCLUDED + +#ifdef __cplusplus + + #include + #include + #include + #include + #include + #include + #include + #include + +#endif + +#include "../util.h" +#include "shader.h" + +void init_warp_tracker( ); + +void wpt_register_warp( int *tid_in, shader_core_ctx_t *shd ); + +int wpt_signal_avail( int tid, shader_core_ctx_t *shd ); + +int wpt_signal_complete( int tid, shader_core_ctx_t *shd ); + +void print_thread_pc_histogram( FILE *fout ); +void print_thread_pc( FILE *fout ); +void track_thread_pc( int shader_id, int *tid, address_type pc ); + +int* alloc_commit_warp( ); +void free_commit_warp( int *commit_warp ); + +#endif -- cgit v1.3