diff options
Diffstat (limited to 'src/gpgpu-sim/shader.cc')
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 349 |
1 files changed, 278 insertions, 71 deletions
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 191beaf..3b2b17c 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -56,6 +56,11 @@ #define MAX(a, b) (((a) > (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b)) +// MEMCO v3: diagnostic globals aggregating inter-set merge activity across +// all ldst_units. Printed at gpgpu termination. +unsigned long long g_memcov3_n_merges_global = 0; +unsigned long long g_memcov3_n_merged_sources_global = 0; + mem_fetch *shader_core_mem_fetch_allocator::alloc( new_addr_type addr, mem_access_type type, unsigned size, bool wr, unsigned long long cycle, unsigned long long streamID) const { @@ -3515,40 +3520,53 @@ void ldst_unit::L1_latency_queue_cycle() { // Bug #2 fix: per-access dispatch via mf's own write bit, not // composite primary op. if (!mf_next->get_is_write()) { - // MEM co-issue: the access belongs to mf_next's stamped source - // warp. Resolve which instruction's out[] registers should be - // decremented (primary's or a co-issued set's). - unsigned src_split_mf = mf_next->get_access_source_split_id(); - mem_src_t src = resolve_source( - mf_next->get_inst(), mf_next->get_wid(), src_split_mf); - // "intra" = coissuer came from a secondary ibuffer slot (secondary - // scoreboard map). Not tied to warp equality with primary. - bool src_is_intra = (src_split_mf != (unsigned)-1); + // MEMCO v3: iterate mf's access source_list — a merged access has + // multiple contributing sources, each with its own out[] regs to + // decrement. Falls back to single-entry [(stamp_wid, stamp_split)] + // when the access wasn't normalized (legacy / non-coissue path). + std::vector<std::pair<unsigned, unsigned> > l1_srcs; + const std::vector<std::pair<unsigned, unsigned> > &mf_srcs = + mf_next->get_access_source_list(); + if (!mf_srcs.empty()) { + l1_srcs = mf_srcs; + } else { + l1_srcs.push_back(std::make_pair( + mf_next->get_wid(), mf_next->get_access_source_split_id())); + } bool any_completed = false; - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (src.out_inst->out[r] > 0) { - unsigned reg = src.out_inst->out[r]; - if (src_is_intra) { - assert(m_pending_writes_secondary[src.wid][src_split_mf] - [reg] > 0); - unsigned still_pending = --m_pending_writes_secondary - [src.wid][src_split_mf][reg]; - if (!still_pending) { - m_pending_writes_secondary[src.wid][src_split_mf].erase( - reg); - m_scoreboard->releaseRegisterSecondary(src.wid, reg); - any_completed = true; - } - } else { - assert(m_pending_writes[src.wid][reg] > 0); - unsigned still_pending = --m_pending_writes[src.wid][reg]; - if (!still_pending) { - m_pending_writes[src.wid].erase(reg); - m_scoreboard->releaseRegister(src.wid, reg); - any_completed = true; + for (unsigned si = 0; si < l1_srcs.size(); si++) { + unsigned src_wid_i = l1_srcs[si].first; + unsigned src_split_mf = l1_srcs[si].second; + mem_src_t src = resolve_source( + mf_next->get_inst(), src_wid_i, src_split_mf); + // "intra" = coissuer came from a secondary ibuffer slot (secondary + // scoreboard map). Not tied to warp equality with primary. + bool src_is_intra = (src_split_mf != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) + if (src.out_inst->out[r] > 0) { + unsigned reg = src.out_inst->out[r]; + if (src_is_intra) { + assert(m_pending_writes_secondary[src.wid][src_split_mf] + [reg] > 0); + unsigned still_pending = --m_pending_writes_secondary + [src.wid][src_split_mf][reg]; + if (!still_pending) { + m_pending_writes_secondary[src.wid][src_split_mf].erase( + reg); + m_scoreboard->releaseRegisterSecondary(src.wid, reg); + any_completed = true; + } + } else { + assert(m_pending_writes[src.wid][reg] > 0); + unsigned still_pending = --m_pending_writes[src.wid][reg]; + if (!still_pending) { + m_pending_writes[src.wid].erase(reg); + m_scoreboard->releaseRegister(src.wid, reg); + any_completed = true; + } } } - } + } if (any_completed) m_core->warp_inst_complete(mf_next->get_inst()); // release LDGSTS (primary-only; LDGSTS excluded from MEM co-issue) @@ -4050,6 +4068,9 @@ void ldst_unit::init(mem_fetch_interface *icnt, m_next_global = NULL; m_next_wb_src_wid = (unsigned)-1; m_next_wb_src_split_id = (unsigned)-1; + m_next_wb_src_list.clear(); + m_n_interset_merges = 0; + m_n_interset_merged_sources = 0; m_last_inst_gpu_sim_cycle = 0; m_last_inst_gpu_tot_sim_cycle = 0; } @@ -4123,6 +4144,144 @@ ldst_unit::mem_src_t ldst_unit::resolve_source( return r; } +// MEMCO v3: inter-set coalescing pass. Single post-splice walk of the +// composite's accessq that: +// (1) Normalizes each eligible access's source_list to contain the +// access's originating (wid, split_id) (from its single stamp). +// (2) Merges segment-equal accesses (same access_type + block_address + +// is_write) by unioning their warp/byte/sector masks into the first +// occurrence and appending the later source to the first's source_list, +// then erasing the merged duplicate. +// +// Provenance: each merged access carries the list of contributing sources. +// Writeback (ldst_unit::writeback) and L1 HIT path (L1_latency_queue_cycle) +// iterate this list to decrement pending_writes for each source. +// +// Exclusions: atomics (per-thread semantic), LDGSTS (excluded from co-issue), +// access types other than GLOBAL/LOCAL (shared doesn't populate accessq; +// const/texture take other pipe stages). +// +// Env vars: +// MEMCOV3_DISABLE_INTERSET_COALESCE=1 → skip pass entirely (Stage A behavior) +// MEMCOV3_DISABLE_MERGE=1 → normalize but skip merge (Stage B) +// MEMCOV3_TRACE_MERGES=1 → print one line per merge event +void ldst_unit::coalesce_accessq_across_sets(warp_inst_t &inst) { + static const bool disable = (getenv("MEMCOV3_DISABLE_INTERSET_COALESCE") != + NULL); + static const bool disable_merge = (getenv("MEMCOV3_DISABLE_MERGE") != NULL); + static const bool trace = (getenv("MEMCOV3_TRACE_MERGES") != NULL); + if (disable) return; + if (inst.isatomic()) return; + if (inst.m_is_ldgsts) return; + unsigned primary_wid = inst.warp_id(); + + // Pass 1: normalize source_list on every eligible access. + for (std::list<mem_access_t>::iterator it = inst.accessq_mut_begin(); + it != inst.accessq_mut_end(); ++it) { + mem_access_type at = it->get_type(); + bool eligible = (at == GLOBAL_ACC_R || at == GLOBAL_ACC_W || + at == LOCAL_ACC_R || at == LOCAL_ACC_W); + if (!eligible) continue; + if (it->has_source_list()) continue; + unsigned w = it->get_source_wid(); + unsigned sp = it->get_source_split_id(); + if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } + it->add_source(w, sp); + } + + if (disable_merge) return; + + // Pass 2: merge segment-equal accesses. Hash key = (type, addr, size, write). + // Include size to avoid merging accesses of different sector sizes (e.g. + // 32B atomic sector vs 128B line). Merging is safe only when every field + // the downstream cache/coalescer inspects is identical. + struct merge_key { + mem_access_type type; + new_addr_type addr; + unsigned size; + bool write; + bool operator<(const merge_key &o) const { + if (type != o.type) return type < o.type; + if (addr != o.addr) return addr < o.addr; + if (size != o.size) return size < o.size; + return write < o.write; + } + }; + std::map<merge_key, std::list<mem_access_t>::iterator> first_occ; + + unsigned merges_fired = 0; + std::list<mem_access_t>::iterator it = inst.accessq_mut_begin(); + while (it != inst.accessq_mut_end()) { + mem_access_type at = it->get_type(); + bool eligible = (at == GLOBAL_ACC_R || at == GLOBAL_ACC_W || + at == LOCAL_ACC_R || at == LOCAL_ACC_W); + if (!eligible) { ++it; continue; } + merge_key k; + k.type = at; + k.addr = it->get_addr(); + k.size = it->get_size(); + k.write = it->is_write(); + std::map<merge_key, std::list<mem_access_t>::iterator>::iterator + fit = first_occ.find(k); + if (fit == first_occ.end()) { + first_occ[k] = it; + ++it; + } else { + mem_access_t &existing = *(fit->second); + const mem_access_t &dup = *it; + // Union masks. + active_mask_t merged_active = existing.get_warp_mask() | + dup.get_warp_mask(); + mem_access_byte_mask_t merged_byte = existing.get_byte_mask() | + dup.get_byte_mask(); + mem_access_sector_mask_t merged_sector = + existing.get_sector_mask() | dup.get_sector_mask(); + // Mutate existing via a rebuilt mem_access_t (no in-place setters for + // masks). Preserve source_list from existing + append dup's sources. + std::vector<std::pair<unsigned, unsigned> > merged_srcs = + existing.get_source_list(); + const std::vector<std::pair<unsigned, unsigned> > &dup_srcs = + dup.get_source_list(); + for (unsigned i = 0; i < dup_srcs.size(); i++) { + bool already = false; + for (unsigned j = 0; j < merged_srcs.size(); j++) { + if (merged_srcs[j] == dup_srcs[i]) { already = true; break; } + } + if (!already) merged_srcs.push_back(dup_srcs[i]); + } + mem_access_t rebuilt(k.type, k.addr, k.size, k.write, merged_active, + merged_byte, merged_sector, + m_core->get_gpu()->gpgpu_ctx); + // Preserve single-stamp from existing (first-arrival source), so + // downstream code that still reads the single stamp sees a consistent + // first source; source_list carries the full truth for iteration. + rebuilt.set_source_wid(existing.get_source_wid()); + rebuilt.set_source_split_id(existing.get_source_split_id()); + for (unsigned i = 0; i < merged_srcs.size(); i++) { + rebuilt.add_source(merged_srcs[i].first, merged_srcs[i].second); + } + *(fit->second) = rebuilt; + it = inst.accessq_mut_erase(it); + merges_fired++; + m_n_interset_merges++; + m_n_interset_merged_sources += dup_srcs.size(); + extern unsigned long long g_memcov3_n_merges_global; + extern unsigned long long g_memcov3_n_merged_sources_global; + g_memcov3_n_merges_global++; + g_memcov3_n_merged_sources_global += dup_srcs.size(); + if (trace) { + fprintf(stdout, + "[MEMCOV3 merge] sid=%u pc=0x%llx type=%d addr=0x%llx " + "size=%u wr=%d sources_now=%u\n", + m_sid, (unsigned long long)inst.pc, (int)k.type, + (unsigned long long)k.addr, k.size, k.write ? 1 : 0, + (unsigned)merged_srcs.size()); + } + } + } + (void)merges_fired; +} + void ldst_unit::issue(register_set ®_set) { warp_inst_t *inst = *(reg_set.get_ready()); @@ -4138,21 +4297,40 @@ void ldst_unit::issue(register_set ®_set) { bool primary_is_tracked_load = inst->is_load() && inst->space.get_type() != shared_space; - if (inst->has_simd_sets() && - m_config->gpgpu_simd_partitioning && - m_config->gpgpu_mem_coissue) { + // MEMCO v3: for co-issue composites, run the inter-set coalescing / + // provenance-normalization pass before pending_writes accounting. Stage B: + // populate each access's source_list with [(wid, split)] derived from the + // access's single-stamp — no merging yet, same access count as before. + bool is_coissue_composite = inst->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue; + if (is_coissue_composite && !inst->accessq_empty()) { + coalesce_accessq_across_sets(*inst); + } + + if (is_coissue_composite) { // MEM co-issue: accesses in the composite's queue may originate from // different source warps (inter-warp) or different splits of the same // warp (intra-warp). Group them by (source_wid, source_split_id) to // charge pending_writes to the correct source's destination registers. - // Unstamped accesses (source_wid == -1) belong to the primary. + // Post-MEMCO-v3: each access has a source_list (one entry for unmerged, + // multiple for merged). Iterate the list so merged accesses count + // toward EACH contributing source. std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src; for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin(); it != inst->accessq_end(); ++it) { - unsigned w = it->get_source_wid(); - unsigned sp = it->get_source_split_id(); - if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } - n_acc_per_src[std::make_pair(w, sp)] += 1; + if (it->has_source_list()) { + const std::vector<std::pair<unsigned, unsigned> > &srcs = + it->get_source_list(); + for (unsigned i = 0; i < srcs.size(); i++) { + n_acc_per_src[srcs[i]] += 1; + } + } else { + unsigned w = it->get_source_wid(); + unsigned sp = it->get_source_split_id(); + if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } + n_acc_per_src[std::make_pair(w, sp)] += 1; + } } // Primary's own registers — gated on primary being a tracked load @@ -4277,46 +4455,69 @@ void ldst_unit::writeback() { unsigned src_split = m_next_wb_src_split_id; // is_intra = secondary-slot coissuer (not warp equality with primary). bool is_intra_coissued = (src_split != (unsigned)-1); - mem_src_t src = resolve_source(m_next_wb, src_wid, src_split); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (src.out_inst->out[r] > 0) { - if (!is_shared) { - unsigned reg = src.out_inst->out[r]; - if (is_intra_coissued) { - // Intra-warp co-issued: pending count lives in the secondary - // map (keyed by (wid, split_id, reg)). Decrement there; when - // zero, release from secondary scoreboard. - assert(m_pending_writes_secondary[src.wid][src_split][reg] > 0); - unsigned still_pending = - --m_pending_writes_secondary[src.wid][src_split][reg]; - if (!still_pending) { - m_pending_writes_secondary[src.wid][src_split].erase(reg); - m_scoreboard->releaseRegisterSecondary(src.wid, reg); - insn_completed = true; + if (!is_shared) { + // MEMCO v3: iterate m_next_wb_src_list — each contributing source + // gets its out[] regs decremented independently. Falls back to + // single-entry [(src_wid, src_split)] when the access wasn't + // normalized (legacy / non-coissue path). + std::vector<std::pair<unsigned, unsigned> > wb_srcs; + if (!m_next_wb_src_list.empty()) { + wb_srcs = m_next_wb_src_list; + } else { + wb_srcs.push_back(std::make_pair(src_wid, src_split)); + } + for (unsigned si = 0; si < wb_srcs.size(); si++) { + unsigned src_wid_i = wb_srcs[si].first; + unsigned src_split_i = wb_srcs[si].second; + bool is_intra_i = (src_split_i != (unsigned)-1); + mem_src_t src_i = resolve_source(m_next_wb, src_wid_i, src_split_i); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (src_i.out_inst->out[r] > 0) { + unsigned reg = src_i.out_inst->out[r]; + if (is_intra_i) { + assert(m_pending_writes_secondary[src_i.wid][src_split_i] + [reg] > 0); + unsigned still_pending = + --m_pending_writes_secondary[src_i.wid][src_split_i][reg]; + if (!still_pending) { + m_pending_writes_secondary[src_i.wid][src_split_i] + .erase(reg); + m_scoreboard->releaseRegisterSecondary(src_i.wid, reg); + insn_completed = true; + } + } else { + assert(m_pending_writes[src_i.wid][reg] > 0); + unsigned still_pending = + --m_pending_writes[src_i.wid][reg]; + if (!still_pending) { + m_pending_writes[src_i.wid].erase(reg); + m_scoreboard->releaseRegister(src_i.wid, reg); + insn_completed = true; + } } - } else { - assert(m_pending_writes[src.wid][reg] > 0); - unsigned still_pending = --m_pending_writes[src.wid][reg]; - if (!still_pending) { - m_pending_writes[src.wid].erase(reg); - m_scoreboard->releaseRegister(src.wid, reg); + } else if (m_next_wb.m_is_ldgsts) { + // LDGSTS excluded from co-issue; source_list always size<=1 + m_pending_ldgsts[primary_wid][m_next_wb.pc] + [m_next_wb.get_addr(0)]--; + if (m_pending_ldgsts[primary_wid][m_next_wb.pc] + [m_next_wb.get_addr(0)] == 0) { insn_completed = true; } + break; } - } else { // shared — release primary scoreboard only here; - // composite per-set releases happen below. - m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]); - insn_completed = true; } - } else if (m_next_wb.m_is_ldgsts) { // LDGSTS excluded from co-issue - m_pending_ldgsts[primary_wid][m_next_wb.pc] - [m_next_wb.get_addr(0)]--; - if (m_pending_ldgsts[primary_wid][m_next_wb.pc] - [m_next_wb.get_addr(0)] == 0) { + } + } else { + // shared — release primary scoreboard only here; composite per-set + // releases happen below. + mem_src_t src = resolve_source(m_next_wb, src_wid, src_split); + (void)is_intra_coissued; // unused on shared path + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (src.out_inst->out[r] > 0) { + m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]); insn_completed = true; } - break; } } @@ -4359,6 +4560,7 @@ void ldst_unit::writeback() { m_next_wb.clear(); m_next_wb_src_wid = (unsigned)-1; m_next_wb_src_split_id = (unsigned)-1; + m_next_wb_src_list.clear(); m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle; m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle; } // close inner block opened at line ~4176 @@ -4381,6 +4583,7 @@ void ldst_unit::writeback() { // handles the primary portion. m_next_wb_src_wid = (unsigned)-1; m_next_wb_src_split_id = (unsigned)-1; + m_next_wb_src_list.clear(); if (m_next_wb.isatomic()) { m_next_wb.do_atomic(); m_core->decrement_atomic_count(m_next_wb.warp_id(), @@ -4419,6 +4622,7 @@ void ldst_unit::writeback() { m_next_wb = mf->get_inst(); m_next_wb_src_wid = mf->get_wid(); m_next_wb_src_split_id = mf->get_access_source_split_id(); + m_next_wb_src_list = mf->get_access_source_list(); delete mf; serviced_client = next_client; } @@ -4429,6 +4633,7 @@ void ldst_unit::writeback() { m_next_wb = mf->get_inst(); m_next_wb_src_wid = mf->get_wid(); m_next_wb_src_split_id = mf->get_access_source_split_id(); + m_next_wb_src_list = mf->get_access_source_list(); delete mf; serviced_client = next_client; } @@ -4439,6 +4644,7 @@ void ldst_unit::writeback() { m_next_wb_src_wid = m_next_global->get_wid(); m_next_wb_src_split_id = m_next_global->get_access_source_split_id(); + m_next_wb_src_list = m_next_global->get_access_source_list(); if (m_next_global->isatomic()) { m_core->decrement_atomic_count( m_next_global->get_wid(), @@ -4455,6 +4661,7 @@ void ldst_unit::writeback() { m_next_wb = mf->get_inst(); m_next_wb_src_wid = mf->get_wid(); m_next_wb_src_split_id = mf->get_access_source_split_id(); + m_next_wb_src_list = mf->get_access_source_list(); delete mf; serviced_client = next_client; } |