diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/gpgpu-sim/Makefile | 7 | ||||
| -rw-r--r-- | src/gpgpu-sim/addrdec.cc | 39 | ||||
| -rw-r--r-- | src/gpgpu-sim/addrdec.h | 7 | ||||
| -rw-r--r-- | src/gpgpu-sim/dram.cc | 7 | ||||
| -rw-r--r-- | src/gpgpu-sim/dram.h | 3 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.cc | 47 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.h | 26 | ||||
| -rw-r--r-- | src/gpgpu-sim/l2cache.cc | 340 | ||||
| -rw-r--r-- | src/gpgpu-sim/l2cache.h | 131 | ||||
| -rw-r--r-- | src/gpgpu-sim/l2cache_trace.h | 55 | ||||
| -rw-r--r-- | src/gpgpu-sim/mem_fetch.h | 1 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 2 | ||||
| -rw-r--r-- | src/trace.cc | 1 | ||||
| -rw-r--r-- | src/trace.h | 1 | ||||
| -rw-r--r-- | src/trace_streams.tup | 1 |
15 files changed, 516 insertions, 152 deletions
diff --git a/src/gpgpu-sim/Makefile b/src/gpgpu-sim/Makefile index e355b59..bead38a 100644 --- a/src/gpgpu-sim/Makefile +++ b/src/gpgpu-sim/Makefile @@ -33,10 +33,8 @@ TRACE?=0 ifeq ($(DEBUG),1) CXXFLAGS = -Wall -DDEBUG - CXXFLAGS_L2CACHE = -Wall -DDEBUG else CXXFLAGS = -Wall - CXXFLAGS_L2CACHE = -Wall endif ifeq ($(TRACE),1) @@ -47,13 +45,11 @@ include ../../version_detection.mk ifeq ($(GNUC_CPP0X), 1) CXXFLAGS += -std=c++0x - CXXFLAGS_L2CACHE += -std=c++0x endif ifneq ($(DEBUG),1) OPTFLAGS += -O3 else - CXXFLAGS_L2CACHE += CXXFLAGS += endif @@ -90,9 +86,6 @@ depend: touch $(OUTPUT_DIR)/Makefile.makedepend makedepend -f$(OUTPUT_DIR)/Makefile.makedepend -p$(OUTPUT_DIR)/ $(CSRCS) 2> /dev/null -$(OUTPUT_DIR)/l2cache.$(OEXT): l2cache.cc - $(CPP) $(OPTFLAGS) $(CXXFLAGS_L2CACHE) -o $*.$(OEXT) -c l2cache.cc - $(OUTPUT_DIR)/%.$(OEXT): %.cc $(CPP) $(OPTFLAGS) $(CXXFLAGS) $(POWER_FLAGS) -o $(OUTPUT_DIR)/$*.$(OEXT) -c $*.cc diff --git a/src/gpgpu-sim/addrdec.cc b/src/gpgpu-sim/addrdec.cc index 9b2f339..c1fa6e5 100644 --- a/src/gpgpu-sim/addrdec.cc +++ b/src/gpgpu-sim/addrdec.cc @@ -71,7 +71,7 @@ new_addr_type linear_to_raw_address_translation::partition_address( new_addr_typ } else { // see addrdec_tlx for explanation unsigned long long int partition_addr; - partition_addr = ( (addr>>ADDR_CHIP_S) / Nchips) << ADDR_CHIP_S; + partition_addr = ( (addr>>ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; partition_addr |= addr & ((1 << ADDR_CHIP_S) - 1); return partition_addr; } @@ -90,8 +90,8 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_ // Split the given address at ADDR_CHIP_S into (MSBs,LSBs) // - extract chip address using modulus of MSBs // - recreate the rest of the address by stitching the quotient of MSBs and the LSBs - addr_for_chip = (addr>>ADDR_CHIP_S) % Nchips; - rest_of_addr = ( (addr>>ADDR_CHIP_S) / Nchips) << ADDR_CHIP_S; + addr_for_chip = (addr>>ADDR_CHIP_S) % m_n_channel; + rest_of_addr = ( (addr>>ADDR_CHIP_S) / m_n_channel) << ADDR_CHIP_S; rest_of_addr |= addr & ((1 << ADDR_CHIP_S) - 1); tlx->chip = addr_for_chip; @@ -100,6 +100,11 @@ void linear_to_raw_address_translation::addrdec_tlx(new_addr_type addr, addrdec_ tlx->col = addrdec_packbits(addrdec_mask[COL], rest_of_addr, addrdec_mkhigh[COL], addrdec_mklow[COL]); tlx->burst= addrdec_packbits(addrdec_mask[BURST], rest_of_addr, addrdec_mkhigh[BURST], addrdec_mklow[BURST]); } + + // combine the chip address and the lower bits of DRAM bank address to form the subpartition ID + unsigned sub_partition_addr_mask = m_n_sub_partition_in_channel - 1; + tlx->sub_partition = tlx->chip * m_n_sub_partition_in_channel + + (tlx->bk & sub_partition_addr_mask); } void linear_to_raw_address_translation::addrdec_parseoption(const char *option) @@ -152,14 +157,15 @@ void linear_to_raw_address_translation::addrdec_parseoption(const char *option) } } -void linear_to_raw_address_translation::init(unsigned int nchips) +void linear_to_raw_address_translation::init(unsigned int n_channel, unsigned int n_sub_partition_in_channel) { unsigned i; unsigned long long int mask; - unsigned int nchipbits = ::LOGB2_32(nchips); - Nchips = nchips; + unsigned int nchipbits = ::LOGB2_32(n_channel); + m_n_channel = n_channel; + m_n_sub_partition_in_channel = n_sub_partition_in_channel; - gap = (nchips - ::powli(2,nchipbits)); + gap = (n_channel - ::powli(2,nchipbits)); if (gap) { nchipbits++; } @@ -280,9 +286,11 @@ void linear_to_raw_address_translation::init(unsigned int nchips) } } // otherwise, no need to change the masks } else { - // make sure nchips is power of two when explicit dram id mask is used - assert((nchips & (nchips - 1)) == 0); + // make sure n_channel is power of two when explicit dram id mask is used + assert((n_channel & (n_channel - 1)) == 0); } + // make sure m_n_sub_partition_in_channel is power of two + assert((m_n_sub_partition_in_channel & (m_n_sub_partition_in_channel - 1)) == 0); addrdec_getmasklimit(addrdec_mask[CHIP], &addrdec_mkhigh[CHIP], &addrdec_mklow[CHIP] ); addrdec_getmasklimit(addrdec_mask[BK], &addrdec_mkhigh[BK], &addrdec_mklow[BK] ); @@ -348,7 +356,7 @@ void linear_to_raw_address_translation::sweep_test() const printf("[AddrDec] ** Error: address decoding mapping aliases two addresses to same partition with same intra-partition address: %llx %llx\n", h->second, raw_addr); abort(); } else { - assert((int)tlx.chip < Nchips); + assert((int)tlx.chip < m_n_channel); // ensure that partition_address() returns the concatenated address if ((ADDR_CHIP_S != -1 and raw_addr >= (1ULL << ADDR_CHIP_S)) or (ADDR_CHIP_S == -1 and raw_addr >= (1ULL << addrdec_mklow[CHIP]))) { @@ -363,11 +371,12 @@ void linear_to_raw_address_translation::sweep_test() const void addrdec_t::print( FILE *fp ) const { - if (chip) fprintf(fp,"\tchip:%x ", chip); - if (row) fprintf(fp,"\trow:%x ", row); - if (col) fprintf(fp,"\tcol:%x ", col); - if (bk) fprintf(fp,"\tbk:%x ", bk); - if (burst) fprintf(fp,"\tburst:%x ", burst); + fprintf(fp,"\tchip:%x ", chip); + fprintf(fp,"\trow:%x ", row); + fprintf(fp,"\tcol:%x ", col); + fprintf(fp,"\tbk:%x ", bk); + fprintf(fp,"\tburst:%x ", burst); + fprintf(fp,"\tsub_partition:%x ", sub_partition); } diff --git a/src/gpgpu-sim/addrdec.h b/src/gpgpu-sim/addrdec.h index 94b113a..f16c9b1 100644 --- a/src/gpgpu-sim/addrdec.h +++ b/src/gpgpu-sim/addrdec.h @@ -43,13 +43,15 @@ struct addrdec_t { unsigned row; unsigned col; unsigned burst; + + unsigned sub_partition; }; class linear_to_raw_address_translation { public: linear_to_raw_address_translation(); void addrdec_setoption(option_parser_t opp); - void init(unsigned int nchips); + void init(unsigned int n_channel, unsigned int n_sub_partition_in_channel); // accessors void addrdec_tlx(new_addr_type addr, addrdec_t *tlx) const; @@ -78,7 +80,8 @@ private: new_addr_type addrdec_mask[N_ADDRDEC]; unsigned int gap; - int Nchips; + int m_n_channel; + int m_n_sub_partition_in_channel; }; #endif diff --git a/src/gpgpu-sim/dram.cc b/src/gpgpu-sim/dram.cc index f28db2f..92c9727 100644 --- a/src/gpgpu-sim/dram.cc +++ b/src/gpgpu-sim/dram.cc @@ -421,11 +421,16 @@ void dram_t::cycle() } //if mrq is being serviced by dram, gets popped after CL latency fulfilled -class mem_fetch* dram_t::pop() +class mem_fetch* dram_t::return_queue_pop() { return returnq->pop(); } +class mem_fetch* dram_t::return_queue_top() +{ + return returnq->top(); +} + void dram_t::print( FILE* simFile) const { unsigned i; diff --git a/src/gpgpu-sim/dram.h b/src/gpgpu-sim/dram.h index 8c0a2b8..a8bff14 100644 --- a/src/gpgpu-sim/dram.h +++ b/src/gpgpu-sim/dram.h @@ -104,7 +104,8 @@ public: unsigned int queue_limit() const; void visualizer_print( gzFile visualizer_file ); - class mem_fetch* pop(); + class mem_fetch* return_queue_pop(); + class mem_fetch* return_queue_top(); void push( class mem_fetch *data ); void cycle(); void dram_log (int task); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 5fa487a..475833f 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -159,6 +159,9 @@ void memory_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_n_mem", OPT_UINT32, &m_n_mem, "number of memory modules (e.g. memory controllers) in gpu", "8"); + option_parser_register(opp, "-gpgpu_n_sub_partition_per_mchannel", OPT_UINT32, &m_n_sub_partition_per_memory_channel, + "number of memory subpartition in each memory module", + "1"); option_parser_register(opp, "-gpgpu_n_mem_per_ctrlr", OPT_UINT32, &gpu_n_mem_per_ctrlr, "number of memory chips per memory controller", "1"); @@ -429,6 +432,9 @@ void gpgpu_sim_config::reg_options(option_parser_t opp) option_parser_register(opp, "-trace_sampling_core", OPT_INT32, &Trace::sampling_core, "The core which is printed using CORE_DPRINTF. Default 0", "0"); + option_parser_register(opp, "-trace_sampling_memory_partition", OPT_INT32, + &Trace::sampling_memory_partition, "The memory partition which is printed using MEMPART_DPRINTF. Default -1 (i.e. all)", + "-1"); ptx_file_line_stats_options(opp); } @@ -564,10 +570,16 @@ gpgpu_sim::gpgpu_sim( const gpgpu_sim_config &config ) m_cluster[i] = new simt_core_cluster(this,i,m_shader_config,m_memory_config,m_shader_stats,m_memory_stats); m_memory_partition_unit = new memory_partition_unit*[m_memory_config->m_n_mem]; - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) + m_memory_sub_partition = new memory_sub_partition*[m_memory_config->m_n_mem_sub_partition]; + for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { m_memory_partition_unit[i] = new memory_partition_unit(i, m_memory_config, m_memory_stats); + for (unsigned p = 0; p < m_memory_config->m_n_sub_partition_per_memory_channel; p++) { + unsigned submpid = i * m_memory_config->m_n_sub_partition_per_memory_channel + p; + m_memory_sub_partition[submpid] = m_memory_partition_unit[i]->get_sub_partition(p); + } + } - icnt_init(m_shader_config->n_simt_clusters,m_memory_config->m_n_mem); + icnt_init(m_shader_config->n_simt_clusters,m_memory_config->m_n_mem_sub_partition); time_vector_create(NUM_MEM_REQ_STAT); fprintf(stdout, "GPGPU-Sim uArch: performance model initialization complete.\n"); @@ -926,9 +938,9 @@ void gpgpu_sim::gpu_print_stat() total_l2_css.clear(); printf("\n========= L2 cache stats =========\n"); - for (unsigned i=0;i<m_memory_config->m_n_mem;i++){ - m_memory_partition_unit[i]->accumulate_L2cache_stats(l2_stats); - m_memory_partition_unit[i]->get_L2cache_sub_stats(l2_css); + for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++){ + m_memory_sub_partition[i]->accumulate_L2cache_stats(l2_stats); + m_memory_sub_partition[i]->get_L2cache_sub_stats(l2_css); fprintf( stdout, "L2_cache_bank[%d]: Access = %u, Miss = %u, Miss_rate = %.3lf, Pending_hits = %u, Reservation_fails = %u\n", i, l2_css.accesses, l2_css.misses, (double)l2_css.misses / (double)l2_css.accesses, l2_css.pending_hits, l2_css.res_fails); @@ -1150,8 +1162,8 @@ void gpgpu_sim::cycle() } if (clock_mask & ICNT) { // pop from memory controller to interconnect - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - mem_fetch* mf = m_memory_partition_unit[i]->top(); + for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) { + mem_fetch* mf = m_memory_sub_partition[i]->top(); if (mf) { unsigned response_size = mf->get_is_write()?mf->get_ctrl_size():mf->size(); if ( ::icnt_has_buffer( m_shader_config->mem2device(i), response_size ) ) { @@ -1159,12 +1171,12 @@ void gpgpu_sim::cycle() mf->set_return_timestamp(gpu_sim_cycle+gpu_tot_sim_cycle); mf->set_status(IN_ICNT_TO_SHADER,gpu_sim_cycle+gpu_tot_sim_cycle); ::icnt_push( m_shader_config->mem2device(i), mf->get_tpc(), mf, response_size ); - m_memory_partition_unit[i]->pop(); + m_memory_sub_partition[i]->pop(); } else { gpu_stall_icnt2sh++; } } else { - m_memory_partition_unit[i]->pop(); + m_memory_sub_partition[i]->pop(); } } } @@ -1182,17 +1194,17 @@ void gpgpu_sim::cycle() // L2 operations follow L2 clock domain if (clock_mask & L2) { m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX].clear(); - for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { + for (unsigned i=0;i<m_memory_config->m_n_mem_sub_partition;i++) { //move memory request from interconnect into memory partition (if not backed up) //Note:This needs to be called in DRAM clock domain if there is no L2 cache in the system - if ( m_memory_partition_unit[i]->full() ) { + if ( m_memory_sub_partition[i]->full() ) { gpu_stall_dramfull++; } else { mem_fetch* mf = (mem_fetch*) icnt_pop( m_shader_config->mem2device(i) ); - m_memory_partition_unit[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle ); + m_memory_sub_partition[i]->push( mf, gpu_sim_cycle + gpu_tot_sim_cycle ); } - m_memory_partition_unit[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle); - m_memory_partition_unit[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); + m_memory_sub_partition[i]->cache_cycle(gpu_sim_cycle+gpu_tot_sim_cycle); + m_memory_sub_partition[i]->accumulate_L2cache_stats(m_power_stats->pwr_mem_stat->l2_cache_stats[CURRENT_STAT_IDX]); } } @@ -1263,7 +1275,7 @@ void gpgpu_sim::cycle() if (m_memory_config->m_L2_config.get_num_lines()) { int dlc = 0; for (unsigned i=0;i<m_memory_config->m_n_mem;i++) { - dlc = m_memory_partition_unit[i]->flushL2(); + dlc = m_memory_sub_partition[i]->flushL2(); assert (dlc == 0); // need to model actual writes to DRAM here printf("Dirty lines flushed from L2 %d is %d\n", i, dlc ); } @@ -1386,8 +1398,3 @@ simt_core_cluster * gpgpu_sim::getSIMTCluster() return *m_cluster; } -void memory_partition_unit::visualizer_print( gzFile visualizer_file ) -{ - m_dram->visualizer_print(visualizer_file); -} - diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 03f9bf2..4e6b7a5 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -190,20 +190,28 @@ struct memory_config { option_parser_destroy(dram_opp); } - int nbkt = nbk/nbkgrp; - unsigned i; - for (i=0; nbkt>0; i++) { - nbkt = nbkt>>1; - } - bk_tag_length = i; + int nbkt = nbk/nbkgrp; + unsigned i; + for (i=0; nbkt>0; i++) { + nbkt = nbkt>>1; + } + bk_tag_length = i; assert(nbkgrp>0 && "Number of bank groups cannot be zero"); tRCDWR = tRCD-(WL+1); tRTW = (CL+(BL/data_command_freq_ratio)+2-WL); tWTR = (WL+(BL/data_command_freq_ratio)+tCDLR); tWTP = (WL+(BL/data_command_freq_ratio)+tWR); dram_atom_size = BL * busW * gpu_n_mem_per_ctrlr; // burst length x bus width x # chips per partition - m_address_mapping.init(m_n_mem); + + assert( m_n_sub_partition_per_memory_channel > 0 ); + assert( (nbk % m_n_sub_partition_per_memory_channel == 0) + && "Number of DRAM banks must be a perfect multiple of memory sub partition"); + m_n_mem_sub_partition = m_n_mem * m_n_sub_partition_per_memory_channel; + fprintf(stdout, "Total number of memory sub partition = %u\n", m_n_mem_sub_partition); + + m_address_mapping.init(m_n_mem, m_n_sub_partition_per_memory_channel); m_L2_config.init(&m_address_mapping); + m_valid = true; icnt_flit_size = 32; // Default 32 } @@ -221,6 +229,8 @@ struct memory_config { enum dram_ctrl_t scheduler_type; bool gpgpu_memlatency_stat; unsigned m_n_mem; + unsigned m_n_sub_partition_per_memory_channel; + unsigned m_n_mem_sub_partition; unsigned gpu_n_mem_per_ctrlr; unsigned rop_latency; @@ -416,7 +426,6 @@ private: int next_clock_domain(void); void issue_block2core(); void print_dram_stats(FILE *fout) const; - void L2c_print_cache_stat() const; void shader_print_runtime_stat( FILE *fout ); void shader_print_l1_miss_stat( FILE *fout ) const; void shader_print_cache_stats( FILE *fout ) const; @@ -430,6 +439,7 @@ private: class simt_core_cluster **m_cluster; class memory_partition_unit **m_memory_partition_unit; + class memory_sub_partition **m_memory_sub_partition; std::vector<kernel_info_t*> m_running_kernels; unsigned m_last_issued_kernel; diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc index a661b4b..d667fd7 100644 --- a/src/gpgpu-sim/l2cache.cc +++ b/src/gpgpu-sim/l2cache.cc @@ -43,6 +43,7 @@ #include "gpu-sim.h" #include "shader.h" #include "mem_latency_stat.h" +#include "l2cache_trace.h" mem_fetch * partition_mf_allocator::alloc(new_addr_type addr, mem_access_type type, unsigned size, bool wr ) const @@ -62,11 +63,234 @@ mem_fetch * partition_mf_allocator::alloc(new_addr_type addr, mem_access_type ty memory_partition_unit::memory_partition_unit( unsigned partition_id, const struct memory_config *config, class memory_stats_t *stats ) +: m_id(partition_id), m_config(config), m_stats(stats), m_arbitration_metadata(config) { - m_id = partition_id; + m_dram = new dram_t(m_id,m_config,m_stats,this); + + m_sub_partition = new memory_sub_partition*[m_config->m_n_sub_partition_per_memory_channel]; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + unsigned sub_partition_id = m_id * m_config->m_n_sub_partition_per_memory_channel + p; + m_sub_partition[p] = new memory_sub_partition(sub_partition_id, m_config, stats); + } +} + +memory_partition_unit::~memory_partition_unit() +{ + delete m_dram; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + delete m_sub_partition[p]; + } + delete[] m_sub_partition; +} + +memory_partition_unit::arbitration_metadata::arbitration_metadata(const struct memory_config *config) +: m_last_borrower(config->m_n_sub_partition_per_memory_channel - 1), + m_private_credit(config->m_n_sub_partition_per_memory_channel, 0), + m_shared_credit(0) +{ + // each sub partition get at least 1 credit for forward progress + // the rest is shared among with other partitions + m_private_credit_limit = 1; + m_shared_credit_limit = config->gpgpu_frfcfs_dram_sched_queue_size + + config->gpgpu_dram_return_queue_size + - (config->m_n_sub_partition_per_memory_channel - 1); + if (config->gpgpu_frfcfs_dram_sched_queue_size == 0 + or config->gpgpu_dram_return_queue_size == 0) + { + m_shared_credit_limit = 0; // no limit if either of the queue has no limit in size + } + assert(m_shared_credit_limit >= 0); +} + +bool memory_partition_unit::arbitration_metadata::has_credits(int inner_sub_partition_id) const +{ + int spid = inner_sub_partition_id; + if (m_private_credit[spid] < m_private_credit_limit) { + return true; + } else if (m_shared_credit_limit == 0 || m_shared_credit < m_shared_credit_limit) { + return true; + } else { + return false; + } +} + +void memory_partition_unit::arbitration_metadata::borrow_credit(int inner_sub_partition_id) +{ + int spid = inner_sub_partition_id; + if (m_private_credit[spid] < m_private_credit_limit) { + m_private_credit[spid] += 1; + } else if (m_shared_credit_limit == 0 || m_shared_credit < m_shared_credit_limit) { + m_shared_credit += 1; + } else { + assert(0 && "DRAM arbitration error: Borrowing from depleted credit!"); + } + m_last_borrower = spid; +} + +void memory_partition_unit::arbitration_metadata::return_credit(int inner_sub_partition_id) +{ + int spid = inner_sub_partition_id; + if (m_private_credit[spid] > 0) { + m_private_credit[spid] -= 1; + } else { + m_shared_credit -= 1; + } + assert((m_shared_credit >= 0) && "DRAM arbitration error: Returning more than available credits!"); +} + +void memory_partition_unit::arbitration_metadata::print( FILE *fp ) const +{ + fprintf(fp, "private_credit = "); + for (unsigned p = 0; p < m_private_credit.size(); p++) { + fprintf(fp, "%d ", m_private_credit[p]); + } + fprintf(fp, "(limit = %d)\n", m_private_credit_limit); + fprintf(fp, "shared_credit = %d (limit = %d)\n", m_shared_credit, m_shared_credit_limit); +} + +bool memory_partition_unit::busy() const +{ + bool busy = false; + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + if (m_sub_partition[p]->busy()) { + busy = true; + } + } + return busy; +} + +void memory_partition_unit::cache_cycle(unsigned cycle) +{ + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + m_sub_partition[p]->cache_cycle(cycle); + } +} + +void memory_partition_unit::visualizer_print( gzFile visualizer_file ) const +{ + m_dram->visualizer_print(visualizer_file); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + m_sub_partition[p]->visualizer_print(visualizer_file); + } +} + +// determine whether a given subpartition can issue to DRAM +bool memory_partition_unit::can_issue_to_dram(int inner_sub_partition_id) +{ + int spid = inner_sub_partition_id; + bool sub_partition_contention = m_sub_partition[spid]->dram_L2_queue_full(); + bool has_dram_resource = m_arbitration_metadata.has_credits(spid); + + MEMPART_DPRINTF("sub partition %d sub_partition_contention=%c has_dram_resource=%c\n", + spid, (sub_partition_contention)? 'T':'F', (has_dram_resource)? 'T':'F'); + + return (has_dram_resource && !sub_partition_contention); +} + +int memory_partition_unit::global_sub_partition_id_to_local_id(int global_sub_partition_id) const +{ + return (global_sub_partition_id - m_id * m_config->m_n_sub_partition_per_memory_channel); +} + +void memory_partition_unit::dram_cycle() +{ + // pop completed memory request from dram and push it to dram-to-L2 queue + // of the original sub partition + mem_fetch* mf_return = m_dram->return_queue_top(); + if (mf_return) { + unsigned dest_global_spid = mf_return->get_sub_partition_id(); + int dest_spid = global_sub_partition_id_to_local_id(dest_global_spid); + assert(m_sub_partition[dest_spid]->get_id() == dest_global_spid); + if (!m_sub_partition[dest_spid]->dram_L2_queue_full()) { + if( mf_return->get_access_type() == L1_WRBK_ACC ) { + m_sub_partition[dest_spid]->set_done(mf_return); + delete mf_return; + } else { + m_sub_partition[dest_spid]->dram_L2_queue_push(mf_return); + mf_return->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_arbitration_metadata.return_credit(dest_spid); + MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf_return, dest_spid); + } + m_dram->return_queue_pop(); + } + } else { + m_dram->return_queue_pop(); + } + + m_dram->cycle(); + m_dram->dram_log(SAMPLELOG); + + if( !m_dram->full() ) { + // L2->DRAM queue to DRAM latency queue + // Arbitrate among multiple L2 subpartitions + int last_issued_partition = m_arbitration_metadata.last_borrower(); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + int spid = (p + last_issued_partition + 1) % m_config->m_n_sub_partition_per_memory_channel; + if (!m_sub_partition[spid]->L2_dram_queue_empty() && can_issue_to_dram(spid)) { + mem_fetch *mf = m_sub_partition[spid]->L2_dram_queue_top(); + m_sub_partition[spid]->L2_dram_queue_pop(); + MEMPART_DPRINTF("Issue mem_fetch request %p from sub partition %d to dram\n", mf, spid); + dram_delay_t d; + d.req = mf; + d.ready_cycle = gpu_sim_cycle+gpu_tot_sim_cycle + m_config->dram_latency; + m_dram_latency_queue.push(d); + mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); + m_arbitration_metadata.borrow_credit(spid); + break; // the DRAM should only accept one request per cycle + } + } + } + + // DRAM latency queue + if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) { + mem_fetch* mf = m_dram_latency_queue.front().req; + m_dram_latency_queue.pop(); + m_dram->push(mf); + } +} + +void memory_partition_unit::set_done( mem_fetch *mf ) +{ + unsigned global_spid = mf->get_sub_partition_id(); + int spid = global_sub_partition_id_to_local_id(global_spid); + assert(m_sub_partition[spid]->get_id() == global_spid); + if (mf->get_access_type() == L1_WRBK_ACC || mf->get_access_type() == L2_WRBK_ACC) { + m_arbitration_metadata.return_credit(spid); + MEMPART_DPRINTF("mem_fetch request %p return from dram to sub partition %d\n", mf, spid); + } + m_sub_partition[spid]->set_done(mf); +} + +void memory_partition_unit::set_dram_power_stats(unsigned &n_cmd, + unsigned &n_activity, + unsigned &n_nop, + unsigned &n_act, + unsigned &n_pre, + unsigned &n_rd, + unsigned &n_wr, + unsigned &n_req) const +{ + m_dram->set_dram_power_stats(n_cmd, n_activity, n_nop, n_act, n_pre, n_rd, n_wr, n_req); +} + +void memory_partition_unit::print( FILE *fp ) const +{ + fprintf(fp, "Memory Partition %u: \n", m_id); + for (unsigned p = 0; p < m_config->m_n_sub_partition_per_memory_channel; p++) { + m_sub_partition[p]->print(fp); + } + m_dram->print(fp); +} + +memory_sub_partition::memory_sub_partition( unsigned sub_partition_id, + const struct memory_config *config, + class memory_stats_t *stats ) +{ + m_id = sub_partition_id; m_config=config; m_stats=stats; - m_dram = new dram_t(m_id,m_config,m_stats,this); + + assert(m_id < m_config->m_n_mem_sub_partition); char L2c_name[32]; snprintf(L2c_name, 32, "L2_bank_%03d", m_id); @@ -88,7 +312,7 @@ memory_partition_unit::memory_partition_unit( unsigned partition_id, wb_addr=-1; } -memory_partition_unit::~memory_partition_unit() +memory_sub_partition::~memory_sub_partition() { delete m_icnt_L2_queue; delete m_L2_dram_queue; @@ -98,7 +322,7 @@ memory_partition_unit::~memory_partition_unit() delete m_L2interface; } -void memory_partition_unit::cache_cycle( unsigned cycle ) +void memory_sub_partition::cache_cycle( unsigned cycle ) { // L2 fill responses if( !m_config->m_L2_config.disabled()) { @@ -189,22 +413,47 @@ void memory_partition_unit::cache_cycle( unsigned cycle ) } } -bool memory_partition_unit::full() const +bool memory_sub_partition::full() const { return m_icnt_L2_queue->full(); } -void memory_partition_unit::print_cache_stat(unsigned &accesses, unsigned &misses) const +bool memory_sub_partition::L2_dram_queue_empty() const +{ + return m_L2_dram_queue->empty(); +} + +class mem_fetch* memory_sub_partition::L2_dram_queue_top() const +{ + return m_L2_dram_queue->top(); +} + +void memory_sub_partition::L2_dram_queue_pop() +{ + m_L2_dram_queue->pop(); +} + +bool memory_sub_partition::dram_L2_queue_full() const +{ + return m_dram_L2_queue->full(); +} + +void memory_sub_partition::dram_L2_queue_push( class mem_fetch* mf ) +{ + m_dram_L2_queue->push(mf); +} + +void memory_sub_partition::print_cache_stat(unsigned &accesses, unsigned &misses) const { FILE *fp = stdout; if( !m_config->m_L2_config.disabled() ) m_L2cache->print(fp,accesses,misses); } -void memory_partition_unit::print( FILE *fp ) const +void memory_sub_partition::print( FILE *fp ) const { if ( !m_request_tracker.empty() ) { - fprintf(fp,"Memory Parition %u: pending memory requests:\n", m_id); + fprintf(fp,"Memory Sub Parition %u: pending memory requests:\n", m_id); for ( std::set<mem_fetch*>::const_iterator r=m_request_tracker.begin(); r != m_request_tracker.end(); ++r ) { mem_fetch *mf = *r; if ( mf ) @@ -215,7 +464,6 @@ void memory_partition_unit::print( FILE *fp ) const } if( !m_config->m_L2_config.disabled() ) m_L2cache->display_state(fp); - m_dram->print(fp); } void memory_stats_t::visualizer_print( gzFile visualizer_file ) @@ -264,26 +512,19 @@ void gpgpu_sim::print_dram_stats(FILE *fout) const fprintf(fout,"gpgpu_n_dram_precharges = %d\n",tot_pre ); fprintf(fout,"gpgpu_n_dram_requests = %d\n",tot_req ); } -void gpgpu_sim::L2c_print_cache_stat() const -{ - unsigned i, j, k; - for (i=0,j=0,k=0;i<m_memory_config->m_n_mem;i++) - m_memory_partition_unit[i]->print_cache_stat(k,j); - printf("L2 Cache Total Miss Rate = %0.3f\n", (float)j/k); -} -unsigned memory_partition_unit::flushL2() +unsigned memory_sub_partition::flushL2() { m_L2cache->flush(); return 0; // L2 is read only in this version } -bool memory_partition_unit::busy() const +bool memory_sub_partition::busy() const { return !m_request_tracker.empty(); } -void memory_partition_unit::push( mem_fetch* req, unsigned long long cycle ) +void memory_sub_partition::push( mem_fetch* req, unsigned long long cycle ) { if (req) { m_request_tracker.insert(req); @@ -301,7 +542,7 @@ void memory_partition_unit::push( mem_fetch* req, unsigned long long cycle ) } } -mem_fetch* memory_partition_unit::pop() +mem_fetch* memory_sub_partition::pop() { mem_fetch* mf = m_L2_icnt_queue->pop(); m_request_tracker.erase(mf); @@ -314,7 +555,7 @@ mem_fetch* memory_partition_unit::pop() return mf; } -mem_fetch* memory_partition_unit::top() +mem_fetch* memory_sub_partition::top() { mem_fetch *mf = m_L2_icnt_queue->top(); if( mf && (mf->get_access_type() == L2_WRBK_ACC || mf->get_access_type() == L1_WRBK_ACC) ) { @@ -326,62 +567,21 @@ mem_fetch* memory_partition_unit::top() return mf; } -void memory_partition_unit::set_done( mem_fetch *mf ) +void memory_sub_partition::set_done( mem_fetch *mf ) { m_request_tracker.erase(mf); } -void memory_partition_unit::dram_cycle() -{ - // pop completed memory request from dram and push it to dram-to-L2 queue - if ( !m_dram_L2_queue->full() ) { - mem_fetch* mf = m_dram->pop(); - if (mf) { - if( mf->get_access_type() == L1_WRBK_ACC ) { - m_request_tracker.erase(mf); - delete mf; - } else { - m_dram_L2_queue->push(mf); - mf->set_status(IN_PARTITION_DRAM_TO_L2_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - } - } - } - m_dram->cycle(); - m_dram->dram_log(SAMPLELOG); - - if( !m_dram->full() && !m_L2_dram_queue->empty() ) { - // L2->DRAM queue to DRAM latency queue - mem_fetch *mf = m_L2_dram_queue->pop(); - dram_delay_t d; - d.req = mf; - d.ready_cycle = gpu_sim_cycle+gpu_tot_sim_cycle + m_config->dram_latency; - m_dram_latency_queue.push(d); - mf->set_status(IN_PARTITION_DRAM_LATENCY_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle); - } - - // DRAM latency queue - if( !m_dram_latency_queue.empty() && ( (gpu_sim_cycle+gpu_tot_sim_cycle) >= m_dram_latency_queue.front().ready_cycle ) && !m_dram->full() ) { - mem_fetch* mf = m_dram_latency_queue.front().req; - m_dram_latency_queue.pop(); - m_dram->push(mf); - } +void memory_sub_partition::accumulate_L2cache_stats(class cache_stats &l2_stats) const { + l2_stats += m_L2cache->get_stats(); } -void memory_partition_unit::set_dram_power_stats(unsigned &n_cmd, - unsigned &n_activity, - unsigned &n_nop, - unsigned &n_act, - unsigned &n_pre, - unsigned &n_rd, - unsigned &n_wr, - unsigned &n_req) const{ - m_dram->set_dram_power_stats(n_cmd, n_activity, n_nop, n_act, n_pre, n_rd, n_wr, n_req); +void memory_sub_partition::get_L2cache_sub_stats(struct cache_sub_stats &css) const{ + m_L2cache->get_sub_stats(css); } -void memory_partition_unit::accumulate_L2cache_stats(class cache_stats &l2_stats) const { - l2_stats += m_L2cache->get_stats(); +void memory_sub_partition::visualizer_print( gzFile visualizer_file ) +{ + // TODO: Add visualizer stats for L2 cache } -void memory_partition_unit::get_L2cache_sub_stats(struct cache_sub_stats &css) const{ - m_L2cache->get_sub_stats(css); -} diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h index 6ef00a7..07e37ca 100644 --- a/src/gpgpu-sim/l2cache.h +++ b/src/gpgpu-sim/l2cache.h @@ -52,9 +52,12 @@ private: const memory_config *m_memory_config; }; -class memory_partition_unit +// Memory partition unit contains all the units assolcated with a single DRAM channel. +// - It arbitrates the DRAM channel among multiple sub partitions. +// - It does not connect directly with the interconnection network. +class memory_partition_unit { -public: +public: memory_partition_unit( unsigned partition_id, const struct memory_config *config, class memory_stats_t *stats ); ~memory_partition_unit(); @@ -63,6 +66,93 @@ public: void cache_cycle( unsigned cycle ); void dram_cycle(); + void set_done( mem_fetch *mf ); + + void visualizer_print( gzFile visualizer_file ) const; + void print_stat( FILE *fp ) { m_dram->print_stat(fp); } + void visualize() const { m_dram->visualize(); } + void print( FILE *fp ) const; + + class memory_sub_partition * get_sub_partition(int sub_partition_id) + { + return m_sub_partition[sub_partition_id]; + } + + // Power model + void set_dram_power_stats(unsigned &n_cmd, + unsigned &n_activity, + unsigned &n_nop, + unsigned &n_act, + unsigned &n_pre, + unsigned &n_rd, + unsigned &n_wr, + unsigned &n_req) const; + + int global_sub_partition_id_to_local_id(int global_sub_partition_id) const; + + unsigned get_mpid() const { return m_id; } + +private: + + unsigned m_id; + const struct memory_config *m_config; + class memory_stats_t *m_stats; + class memory_sub_partition **m_sub_partition; + class dram_t *m_dram; + + class arbitration_metadata + { + public: + arbitration_metadata(const struct memory_config *config); + + // check if a subpartition still has credit + bool has_credits(int inner_sub_partition_id) const; + // borrow a credit for a subpartition + void borrow_credit(int inner_sub_partition_id); + // return a credit from a subpartition + void return_credit(int inner_sub_partition_id); + + // return the last subpartition that borrowed credit + int last_borrower() const { return m_last_borrower; } + + void print( FILE *fp ) const; + private: + // id of the last subpartition that borrowed credit + int m_last_borrower; + + int m_shared_credit_limit; + int m_private_credit_limit; + + // credits borrowed by the subpartitions + std::vector<int> m_private_credit; + int m_shared_credit; + }; + arbitration_metadata m_arbitration_metadata; + + // determine wheither a given subpartition can issue to DRAM + bool can_issue_to_dram(int inner_sub_partition_id); + + // model DRAM access scheduler latency (fixed latency between L2 and DRAM) + struct dram_delay_t + { + unsigned long long ready_cycle; + class mem_fetch* req; + }; + std::queue<dram_delay_t> m_dram_latency_queue; +}; + +class memory_sub_partition +{ +public: + memory_sub_partition( unsigned sub_partition_id, const struct memory_config *config, class memory_stats_t *stats ); + ~memory_sub_partition(); + + unsigned get_id() const { return m_id; } + + bool busy() const; + + void cache_cycle( unsigned cycle ); + bool full() const; void push( class mem_fetch* mf, unsigned long long clock_cycle ); class mem_fetch* pop(); @@ -71,31 +161,26 @@ public: unsigned flushL2(); + // interface to L2_dram_queue + bool L2_dram_queue_empty() const; + class mem_fetch* L2_dram_queue_top() const; + void L2_dram_queue_pop(); + + // interface to dram_L2_queue + bool dram_L2_queue_full() const; + void dram_L2_queue_push( class mem_fetch* mf ); + void visualizer_print( gzFile visualizer_file ); void print_cache_stat(unsigned &accesses, unsigned &misses) const; - void print_stat( FILE *fp ) { m_dram->print_stat(fp); } - void visualize() const { m_dram->visualize(); } void print( FILE *fp ) const; - - // Power model - void set_dram_power_stats(unsigned &n_cmd, - unsigned &n_activity, - unsigned &n_nop, - unsigned &n_act, - unsigned &n_pre, - unsigned &n_rd, - unsigned &n_wr, - unsigned &n_req) const; - void accumulate_L2cache_stats(class cache_stats &l2_stats) const; void get_L2cache_sub_stats(struct cache_sub_stats &css) const; private: // data - unsigned m_id; + unsigned m_id; //< the global sub partition ID const struct memory_config *m_config; - class dram_t *m_dram; class l2_cache *m_L2cache; class L2interface *m_L2interface; partition_mf_allocator *m_mf_allocator; @@ -108,14 +193,6 @@ private: }; std::queue<rop_delay_t> m_rop; - // model DRAM access scheduler latency (fixed latency between L2 and DRAM) - struct dram_delay_t - { - unsigned long long ready_cycle; - class mem_fetch* req; - }; - std::queue<dram_delay_t> m_dram_latency_queue; - // these are various FIFOs between units within a memory partition fifo_pipeline<mem_fetch> *m_icnt_L2_queue; fifo_pipeline<mem_fetch> *m_L2_dram_queue; @@ -134,7 +211,7 @@ private: class L2interface : public mem_fetch_interface { public: - L2interface( memory_partition_unit *unit ) { m_unit=unit; } + L2interface( memory_sub_partition *unit ) { m_unit=unit; } virtual ~L2interface() {} virtual bool full( unsigned size, bool write) const { @@ -147,7 +224,7 @@ public: m_unit->m_L2_dram_queue->push(mf); } private: - memory_partition_unit *m_unit; + memory_sub_partition *m_unit; }; #endif diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h new file mode 100644 index 0000000..3dac87d --- /dev/null +++ b/src/gpgpu-sim/l2cache_trace.h @@ -0,0 +1,55 @@ +// Copyright (c) 2009-2011, Tor M. Aamodt, Tim Rogers, Wilson W. L. Fung +// The University of British Columbia +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright notice, this +// list of conditions and the following disclaimer in the documentation and/or +// other materials provided with the distribution. +// Neither the name of The University of British Columbia nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "../trace.h" + +#if TRACING_ON + +#define MEMPART_PRINT_STR SIM_PRINT_STR " %d - " +#define MEMPART_DTRACE(x) ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) ) + +// Intended to be called from inside components of a memory partition +// Depends on a get_mpid() function +#define MEMPART_DPRINTF(...) do {\ + if (MEMPART_DTRACE(MEMORY_PARTITION_UNIT)) {\ + printf( MEMPART_PRINT_STR,\ + gpu_sim_cycle + gpu_tot_sim_cycle,\ + Trace::trace_streams_str[Trace::MEMORY_PARTITION_UNIT],\ + get_mpid() );\ + printf(__VA_ARGS__);\ + }\ +} while (0) + +#else + +#define MEMPART_DTRACE(x) (false) +#define MEMPART_DPRINTF(x, ...) do {} while (0) + +#endif + diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index c4ba0dc..f788c2b 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -83,6 +83,7 @@ public: void set_addr(new_addr_type addr) { m_access.set_addr(addr); } new_addr_type get_addr() const { return m_access.get_addr(); } new_addr_type get_partition_addr() const { return m_partition_addr; } + unsigned get_sub_partition_id() const { return m_raw_addr.sub_partition; } bool get_is_write() const { return m_access.is_write(); } unsigned get_request_uid() const { return m_request_uid; } unsigned get_sid() const { return m_sid; } diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 87e59ed..73a8a62 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -3184,7 +3184,7 @@ void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf) packet_size = mf->get_ctrl_size(); } m_stats->m_outgoing_traffic_stats->record_traffic(mf, packet_size); - unsigned destination = mf->get_tlx_addr().chip; + unsigned destination = mf->get_sub_partition_id(); mf->set_status(IN_ICNT_TO_MEM,gpu_sim_cycle+gpu_tot_sim_cycle); if (!mf->get_is_write() && !mf->isatomic()) ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void*)mf, mf->get_ctrl_size() ); diff --git a/src/trace.cc b/src/trace.cc index 05b9d6b..5171e46 100644 --- a/src/trace.cc +++ b/src/trace.cc @@ -41,6 +41,7 @@ namespace Trace { bool enabled = false; int sampling_core = 0; + int sampling_memory_partition = -1; bool trace_streams_enabled[NUM_TRACE_STREAMS] = {false}; const char* config_str; diff --git a/src/trace.h b/src/trace.h index c7b389e..a79b4a0 100644 --- a/src/trace.h +++ b/src/trace.h @@ -46,6 +46,7 @@ namespace Trace { extern bool enabled; extern int sampling_core; + extern int sampling_memory_partition; extern const char* trace_streams_str[]; extern bool trace_streams_enabled[NUM_TRACE_STREAMS]; extern const char* config_str; diff --git a/src/trace_streams.tup b/src/trace_streams.tup index fa6fd1e..c41690e 100644 --- a/src/trace_streams.tup +++ b/src/trace_streams.tup @@ -28,5 +28,6 @@ TS_TUP_BEGIN( trace_streams_type ) TS_TUP( WARP_SCHEDULER ), TS_TUP( SCOREBOARD ), + TS_TUP( MEMORY_PARTITION_UNIT ), TS_TUP( NUM_TRACE_STREAMS ) TS_TUP_END( trace_streams_type ) |