diff options
| -rw-r--r-- | src/abstract_hardware_model.cc | 10 | ||||
| -rw-r--r-- | src/abstract_hardware_model.h | 23 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.cc | 9 | ||||
| -rw-r--r-- | src/gpgpu-sim/mem_fetch.cc | 7 | ||||
| -rw-r--r-- | src/gpgpu-sim/mem_fetch.h | 6 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 576 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.h | 62 |
7 files changed, 633 insertions, 60 deletions
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 86427ad..ed1be65 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -48,6 +48,8 @@ void mem_access_t::init(gpgpu_context *ctx) { m_uid = ++(gpgpu_ctx->sm_next_access_uid); m_addr = 0; m_req_size = 0; + m_source_wid = (unsigned)-1; + m_source_split_id = (unsigned)-1; } void warp_inst_t::issue(const active_mask_t &mask, unsigned warp_id, @@ -191,6 +193,14 @@ void warp_inst_t::set_source_inst_on_sets(const inst_t *src) { } } +void warp_inst_t::set_split_id_on_sets(unsigned split_id) { + for (unsigned s = 0; s < m_simd_sets.size(); s++) { + if (m_simd_sets[s].valid) { + m_simd_sets[s].split_id = split_id; + } + } +} + void warp_inst_t::merge_simd_sets( const std::vector<simd_set_info> &other_sets) { assert(m_simd_sets.size() == other_sets.size()); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 698c896..1e3c94e 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -859,6 +859,14 @@ class mem_access_t { mem_access_byte_mask_t get_byte_mask() const { return m_byte_mask; } mem_access_sector_mask_t get_sector_mask() const { return m_sector_mask; } + // Source-warp attribution for MEM co-issue composites. Unset by default; + // when set, overrides the composite's primary warp_id for mem_fetch + // allocation + pending_writes tracking + writeback scoreboard release. + unsigned get_source_wid() const { return m_source_wid; } + unsigned get_source_split_id() const { return m_source_split_id; } + void set_source_wid(unsigned w) { m_source_wid = w; } + void set_source_split_id(unsigned s) { m_source_split_id = s; } + void print(FILE *fp) const { fprintf(fp, "addr=0x%llx, %s, size=%u, ", m_addr, m_write ? "store" : "load ", m_req_size); @@ -909,6 +917,8 @@ class mem_access_t { active_mask_t m_warp_mask; mem_access_byte_mask_t m_byte_mask; mem_access_sector_mask_t m_sector_mask; + unsigned m_source_wid; // MEM co-issue: originating warp (-1 = unset) + unsigned m_source_split_id; // MEM co-issue: originating split (-1 = unset) }; class mem_fetch; @@ -1113,6 +1123,7 @@ const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; struct simd_set_info { unsigned set_id; // Set index (0 to num_sets-1) unsigned warp_id; // Owning warp + unsigned split_id; // Owning split (-1 = inter-warp / primary) simt_mask_t set_active_mask; // Active lanes within this set (lower set_width bits) active_mask_t active_mask_in_warp; // Active mask in original 32-bit warp thread domain unsigned thread_ids[MAX_WARP_SIZE]; // Original thread IDs for each lane (max set_width used) @@ -1121,8 +1132,8 @@ struct simd_set_info { const inst_t *source_inst; // Pointer to static decoded instruction (for MIMD) simd_set_info() - : set_id(0), warp_id(0), num_active_threads(0), valid(false), - source_inst(NULL) { + : set_id(0), warp_id(0), split_id((unsigned)-1), + num_active_threads(0), valid(false), source_inst(NULL) { set_active_mask.reset(); active_mask_in_warp.reset(); for (unsigned i = 0; i < MAX_WARP_SIZE; i++) thread_ids[i] = 0; @@ -1198,6 +1209,7 @@ class warp_inst_t : public inst_t { bool has_simd_sets() const { return !m_simd_sets.empty(); } void merge_simd_sets(const std::vector<simd_set_info> &other_sets); void set_source_inst_on_sets(const inst_t *src); + void set_split_id_on_sets(unsigned split_id); static bool simd_sets_overlap(const std::vector<simd_set_info> &a, const std::vector<simd_set_info> &b); @@ -1323,6 +1335,13 @@ class warp_inst_t : public inst_t { unsigned accessq_count() const { return m_accessq.size(); } const mem_access_t &accessq_back() { return m_accessq.back(); } void accessq_pop_back() { m_accessq.pop_back(); } + // Const iteration over the access queue for per-source grouping (MEM co-issue). + std::list<mem_access_t>::const_iterator accessq_begin() const { + return m_accessq.begin(); + } + std::list<mem_access_t>::const_iterator accessq_end() const { + return m_accessq.end(); + } bool dispatch_delay() { if (cycles > 0) cycles--; diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index cd38857..ca51c50 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -711,6 +711,15 @@ void shader_core_config::reg_options(class OptionParser *opp) { "the `gpu_tot_inter_warp_coissue` / `gpu_tot_intra_warp_coissue` " "stats; only turn this on for quick_tests that grep the raw lines.", "0"); + option_parser_register( + opp, "-gpgpu_mem_coissue", OPT_BOOL, &gpgpu_mem_coissue, + "Enable MEM/LDST co-issue under SIMD lane partitioning. When 0 " + "(default), MEM primaries do not participate in co-issue (bit-parity " + "with pre-MEM-coissue tree). When 1, global/local-space loads and " + "stores from different warps or different splits of the same warp " + "may share the LDST unit in one pipe slot; atomics, shared, tex, " + "const, barriers, LDGSTS, and BRU spill/fill are still excluded.", + "0"); for (unsigned j = 0; j < SPECIALIZED_UNIT_NUM; ++j) { std::stringstream ss; diff --git a/src/gpgpu-sim/mem_fetch.cc b/src/gpgpu-sim/mem_fetch.cc index 809c920..e171d9e 100644 --- a/src/gpgpu-sim/mem_fetch.cc +++ b/src/gpgpu-sim/mem_fetch.cc @@ -46,7 +46,12 @@ mem_fetch::mem_fetch(const mem_access_t &access, const warp_inst_t *inst, m_access = access; if (inst) { m_inst = *inst; - assert(wid == m_inst.warp_id()); + // Sanity: the stamped wid matches the inst's warp_id, UNLESS this is a + // MEM co-issue transaction where the access carries an override source + // warp_id. In that case m_inst is a copy of the composite (whose + // warp_id is the primary) but wid is the originating warp. + assert(wid == m_inst.warp_id() || + access.get_source_wid() != (unsigned)-1); } m_streamID = streamID; m_data_size = access.get_size(); diff --git a/src/gpgpu-sim/mem_fetch.h b/src/gpgpu-sim/mem_fetch.h index 7704218..add4792 100644 --- a/src/gpgpu-sim/mem_fetch.h +++ b/src/gpgpu-sim/mem_fetch.h @@ -96,6 +96,12 @@ class mem_fetch { unsigned get_sid() const { return m_sid; } unsigned get_tpc() const { return m_tpc; } unsigned get_wid() const { return m_wid; } + // MEM co-issue source attribution carried via the access. get_wid() still + // returns the composite's stamped wid (source-wid-overridden via allocator). + // This extra getter exposes the split id for intra-warp disambiguation. + unsigned get_access_source_split_id() const { + return m_access.get_source_split_id(); + } bool istexture() const; bool isconst() const; enum mf_type get_type() const { return m_type; } diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index c874733..f96e939 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -830,6 +830,19 @@ void shader_core_stats::print(FILE *fout) const { fprintf(fout, "gpu_tot_coissue_skipped_no_inst = %llu\n", tot_noinst); fprintf(fout, "gpu_tot_coissue_skipped_pc_mismatch = %llu\n", tot_pcm); fprintf(fout, "gpu_tot_coissue_skipped_samepc_pass0 = %llu\n", tot_spc0); + + // MEM co-issue-specific subtotals + unsigned long long tot_mem_inter = 0, tot_mem_intra = 0, + tot_mem_denied = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + tot_mem_inter += mem_coissue_inter_events[i]; + tot_mem_intra += mem_coissue_intra_events[i]; + tot_mem_denied += mem_coissue_denied_filter[i]; + } + fprintf(fout, "gpu_tot_mem_coissue_inter_events = %llu\n", tot_mem_inter); + fprintf(fout, "gpu_tot_mem_coissue_intra_events = %llu\n", tot_mem_intra); + fprintf(fout, "gpu_tot_mem_coissue_denied_filter = %llu\n", + tot_mem_denied); } } @@ -1457,8 +1470,29 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, // Set next PC for the co-issued warp m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize); - // Tag the co-issued sets with their source instruction for writeback + // Tag the co-issued sets with their source instruction + split_id for + // writeback routing (intra-warp composites need split_id to disambiguate + // between primary and secondary-split sets that share warp_id). temp_inst.set_source_inst_on_sets(next_inst); + temp_inst.set_split_id_on_sets(split_id); + + // For MEM co-issue: splice temp_inst's already-coalesced mem accesses + // into the composite, stamping each with source (warp_id, split_id) so + // the LDST unit routes pending_writes + scoreboard release back to the + // originating warp/split. This must happen BEFORE merge_simd_sets + // because we also need the temp_inst's own simd_sets to still carry + // valid flags here; but the accessq itself is independent. + if (temp_inst.is_load() || temp_inst.is_store()) { + unsigned src_wid = warp_id; + unsigned src_split = split_id; // -1 for inter-warp + while (!temp_inst.accessq_empty()) { + mem_access_t acc = temp_inst.accessq_back(); + acc.set_source_wid(src_wid); + acc.set_source_split_id(src_split); + composite->inject_mem_acccesses(acc); + temp_inst.accessq_pop_back(); + } + } // Merge the co-issued instruction's valid sets into the composite composite->merge_simd_sets(temp_inst.get_simd_sets()); @@ -1604,6 +1638,31 @@ void scheduler_unit::order_by_priority( // Change 4: co-issue scheduler helpers. // --------------------------------------------------------------------------- +bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const { + if (!inst) return true; + // Exclude TENSOR_CORE loads/stores (quirky pipeline), memory barriers, + // LDGSTS, and BRU spill/fill. These remain MEM-bucketed at classify time + // but are rejected from co-issue. + if (inst->op == TENSOR_CORE_LOAD_OP || inst->op == TENSOR_CORE_STORE_OP) + return true; + if (inst->op == MEMORY_BARRIER_OP) return true; + if (inst->m_is_ldgsts) return true; + if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() || + inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request()) + return true; + // Exclude atomics (decrement_atomic_count ties to a single warp) and + // non-global memory spaces. Global/local go through the coalescer + + // mem_fetch path that the source_wid stamp supports. Shared/tex/const + // use per-subwarp / per-line paths not yet per-set-aware. + if (inst->isatomic()) return true; + memory_space_t space = inst->space; + if (space.get_type() != global_space && + space.get_type() != local_space && + space.get_type() != param_space_local) + return true; + return false; +} + exec_unit_type_t scheduler_unit::classify_fu_type( const warp_inst_t *inst) const { if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) || @@ -1797,6 +1856,14 @@ void scheduler_unit::try_inter_warp_coissue( continue; } + // MEM co-issue safety filter. + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any. if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) { m_stats->coissue_skipped_samepc_pass0[get_sid()]++; @@ -1832,6 +1899,8 @@ void scheduler_unit::try_inter_warp_coissue( warp(cand_warp_id).ibuffer_step(); picked_warps.insert(cand_warp_id); m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; } } } @@ -1881,6 +1950,14 @@ void scheduler_unit::try_intra_warp_coissue( continue; } + // MEM co-issue safety filter. + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; @@ -1909,6 +1986,8 @@ void scheduler_unit::try_intra_warp_coissue( available_sets -= sec_sets_needed; next_free_set += sec_sets_needed; m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; warp(primary_warp_id).ibuffer_free_slot(sec_slot); @@ -1976,6 +2055,12 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; continue; } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } const active_mask_t &cand_mask = m_shader->get_active_mask(cand_warp_id, cand_inst); @@ -2033,6 +2118,12 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; continue; } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; @@ -2106,6 +2197,8 @@ void scheduler_unit::try_utilization_max_coissue( available_sets -= needed; next_free_set += needed; m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; warp(c.warp_id).ibuffer_free_slot(c.sec_slot); if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { warp(c.warp_id).ibuffer_flush_half(1); @@ -2136,6 +2229,8 @@ void scheduler_unit::try_utilization_max_coissue( available_sets -= needed; next_free_set += needed; m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; warp(c.warp_id).ibuffer_step(); } } @@ -2251,8 +2346,19 @@ void scheduler_unit::cycle() { m_id) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { - m_shader->issue_warp(*m_mem_out, pI, active_mask, warp_id, - m_id); + warp_inst_t *issued_pipe_reg = m_shader->issue_warp( + *m_mem_out, pI, active_mask, warp_id, m_id); + // When MEM co-issue is enabled, capture the composite pointer + // so the dispatch loop below can attempt to merge co-issuers + // into this MEM pipeline slot. Otherwise leave co_issue_* + // variables unset for bit-parity with the pre-MEM-coissue tree. + if (m_shader->m_config->gpgpu_simd_partitioning && + m_shader->m_config->gpgpu_mem_coissue) { + co_issue_composite = issued_pipe_reg; + co_issue_fu_type = exec_unit_type_t::MEM; + co_issue_reg_set = m_mem_out; + co_issue_primary_warp_id = warp_id; + } issued++; issued_inst = true; warp_inst_issued = true; @@ -2484,10 +2590,13 @@ void scheduler_unit::cycle() { // SIMD set co-issue dispatch: after issuing the primary instruction, // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority. - // Only for non-MEM FUs (LDST excluded from partitioning). + // MEM co-issue is opt-in via -gpgpu_mem_coissue (the primary-issue branch + // above leaves co_issue_composite=NULL when the flag is off, so this + // block is skipped naturally in that case). if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL && - co_issue_fu_type != exec_unit_type_t::MEM && - co_issue_fu_type != exec_unit_type_t::NONE) { + co_issue_fu_type != exec_unit_type_t::NONE && + !(co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(co_issue_composite))) { available_sets = 0; next_free_set = 0; const std::vector<simd_set_info> &primary_sets = @@ -3114,13 +3223,21 @@ mem_stage_stall_type ldst_unit::process_cache_access( mem_stage_stall_type result = NO_RC_FAIL; bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); + // Attribute store acks / load HIT register decrements to the access's + // source warp + its source instruction (for MEM co-issue composites). + unsigned src_wid_access = mf ? mf->get_wid() : inst.warp_id(); + unsigned src_split_access = + mf ? mf->get_access_source_split_id() : (unsigned)-1; + mem_src_t src = resolve_source(inst, src_wid_access, src_split_access); + bool src_is_intra_coissued = + (src.wid == inst.warp_id() && src_split_access != (unsigned)-1); if (write_sent) { unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) ? (mf->get_data_size() / SECTOR_SIZE) : 1; for (unsigned i = 0; i < inc_ack; ++i) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(src.wid); } if (status == HIT) { assert(!read_sent); @@ -3129,9 +3246,15 @@ mem_stage_stall_type ldst_unit::process_cache_access( release_virtual_entries(inst); } else if (inst.is_load()) { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--; + if (src.out_inst->out[r] > 0) { + if (src_is_intra_coissued) + m_pending_writes_secondary[src.wid][src_split_access] + [src.out_inst->out[r]]--; + else + m_pending_writes[src.wid][src.out_inst->out[r]]--; + } - // release LDGSTS + // release LDGSTS (primary-only: LDGSTS is excluded from MEM co-issue) if (inst.m_is_ldgsts) { m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)]--; if (m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)] == 0) { @@ -3203,7 +3326,7 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( : 1; for (unsigned i = 0; i < inc_ack; ++i) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(mf->get_wid()); } inst.accessq_pop_back(); @@ -3251,23 +3374,43 @@ void ldst_unit::L1_latency_queue_cycle() { assert(!read_sent); l1_latency_queue[j][0] = NULL; if (mf_next->get_inst().is_load()) { + // MEM co-issue: the access belongs to mf_next's stamped source + // warp. Resolve which instruction's out[] registers should be + // decremented (primary's or a co-issued set's). + unsigned src_split_mf = mf_next->get_access_source_split_id(); + mem_src_t src = resolve_source( + mf_next->get_inst(), mf_next->get_wid(), src_split_mf); + bool src_is_intra = + (src.wid == mf_next->get_inst().warp_id() && + src_split_mf != (unsigned)-1); + bool any_completed = false; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (mf_next->get_inst().out[r] > 0) { - assert(m_pending_writes[mf_next->get_inst().warp_id()] - [mf_next->get_inst().out[r]] > 0); - unsigned still_pending = - --m_pending_writes[mf_next->get_inst().warp_id()] - [mf_next->get_inst().out[r]]; - if (!still_pending) { - m_pending_writes[mf_next->get_inst().warp_id()].erase( - mf_next->get_inst().out[r]); - m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(), - mf_next->get_inst().out[r]); - m_core->warp_inst_complete(mf_next->get_inst()); + if (src.out_inst->out[r] > 0) { + unsigned reg = src.out_inst->out[r]; + if (src_is_intra) { + assert(m_pending_writes_secondary[src.wid][src_split_mf] + [reg] > 0); + unsigned still_pending = --m_pending_writes_secondary + [src.wid][src_split_mf][reg]; + if (!still_pending) { + m_pending_writes_secondary[src.wid][src_split_mf].erase( + reg); + m_scoreboard->releaseRegisterSecondary(src.wid, reg); + any_completed = true; + } + } else { + assert(m_pending_writes[src.wid][reg] > 0); + unsigned still_pending = --m_pending_writes[src.wid][reg]; + if (!still_pending) { + m_pending_writes[src.wid].erase(reg); + m_scoreboard->releaseRegister(src.wid, reg); + any_completed = true; + } } } + if (any_completed) m_core->warp_inst_complete(mf_next->get_inst()); - // release LDGSTS + // release LDGSTS (primary-only; LDGSTS excluded from MEM co-issue) if (mf_next->get_inst().m_is_ldgsts) { m_pending_ldgsts[mf_next->get_inst().warp_id()] [mf_next->get_inst().pc] @@ -3417,6 +3560,9 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, stall_cond = ICNT_RC_FAIL; break; } else { + // Resolve source attribution for MEM co-issued accesses. + unsigned src_wid = access.get_source_wid(); + if (src_wid == (unsigned)-1) src_wid = inst.warp_id(); mem_fetch *mf = m_mf_allocator->alloc(inst, access, m_core->get_gpu()->gpu_sim_cycle + @@ -3425,11 +3571,24 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, inst.accessq_pop_back(); // inst.clear_active( access.get_warp_mask() ); if (inst.is_load()) { + // For MEM-co-issue composites, sanity-check against the source + // warp's pending_writes on its source out[] regs (if identifiable). + const inst_t *src_out_inst = &inst; + if (src_wid != inst.warp_id() && inst.has_simd_sets()) { + const std::vector<simd_set_info> &sets = inst.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (sets[s].valid && sets[s].source_inst != NULL && + sets[s].warp_id == src_wid) { + src_out_inst = sets[s].source_inst; + break; + } + } + } for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (inst.out[r] > 0) - assert(m_pending_writes[inst.warp_id()][inst.out[r]] > 0); + if (src_out_inst->out[r] > 0) + assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0); } else if (inst.is_store()) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(src_wid); } } } else { @@ -3739,6 +3898,8 @@ void ldst_unit::init(mem_fetch_interface *icnt, 5; // = shared memory, global/local (uncached), L1D, L1T, L1C m_writeback_arb = 0; m_next_global = NULL; + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; m_last_inst_gpu_sim_cycle = 0; m_last_inst_gpu_tot_sim_cycle = 0; } @@ -3785,6 +3946,33 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt, mem_config, stats, sid, tpc); } +ldst_unit::mem_src_t ldst_unit::resolve_source( + const warp_inst_t &inst, unsigned access_src_wid, + unsigned access_src_split_id) const { + mem_src_t r; + r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid; + r.out_inst = &inst; + if (!inst.has_simd_sets()) return r; + // Walk simd_sets for a co-issued set matching BOTH (wid, split_id). For + // inter-warp accesses split_id is -1 and wid differs from primary; for + // intra-warp accesses split_id is non-neg and wid equals primary. Primary + // accesses (source_wid == -1 or split_id == -1 with wid==primary) keep + // out_inst = &inst (composite's out[], which is primary's). + bool is_intra = + (r.wid == inst.warp_id() && access_src_split_id != (unsigned)-1); + bool is_inter = (r.wid != inst.warp_id()); + if (!is_intra && !is_inter) return r; + const std::vector<simd_set_info> &sets = inst.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != r.wid) continue; + if (is_intra && sets[s].split_id != access_src_split_id) continue; + r.out_inst = sets[s].source_inst; + break; + } + return r; +} + void ldst_unit::issue(register_set ®_set) { warp_inst_t *inst = *(reg_set.get_ready()); @@ -3792,16 +3980,75 @@ void ldst_unit::issue(register_set ®_set) { // instruction assert(inst->empty() == false); if (inst->is_load() and inst->space.get_type() != shared_space) { - unsigned warp_id = inst->warp_id(); - unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = inst->out[r]; - if (reg_id > 0) { - m_pending_writes[warp_id][reg_id] += n_accesses; + unsigned primary_wid = inst->warp_id(); + + // MEM co-issue: accesses in the composite's queue may originate from + // different source warps (inter-warp) or different splits of the same + // warp (intra-warp). Group them by (source_wid, source_split_id) to + // charge pending_writes to the correct source's destination registers. + // Unstamped accesses (source_wid == -1) belong to the primary. + if (inst->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src; + for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin(); + it != inst->accessq_end(); ++it) { + unsigned w = it->get_source_wid(); + unsigned sp = it->get_source_split_id(); + if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } + n_acc_per_src[std::make_pair(w, sp)] += 1; + } + + // Primary's own registers + unsigned n_primary = + n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)]; + if (n_primary > 0) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary; + } + } + + // Per co-issued set's registers (use set's source_inst->out[]). + // Inter-warp sets: different warp_id → primary pending_writes map is + // fine (no key collision with primary). Intra-warp sets: same warp_id + // but different split_id → use the secondary pending_writes map to + // avoid conflating with primary's pending_writes[primary_wid][reg]. + const std::vector<simd_set_info> &sets = inst->get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> accounted; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id); + if (accounted.count(key)) continue; + accounted.insert(key); + unsigned n_src = n_acc_per_src[key]; + if (n_src == 0) continue; + bool is_intra = (sets[s].warp_id == primary_wid && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id > 0) { + if (is_intra) { + m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id] + [reg_id] += n_src; + } else { + m_pending_writes[sets[s].warp_id][reg_id] += n_src; + } + } + } + } + } else { + // Legacy (non-co-issue) path: single-warp pending_writes. + unsigned n_accesses = inst->accessq_count(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses; } } + if (inst->m_is_ldgsts) { - m_pending_ldgsts[warp_id][inst->pc][inst->get_addr(0)] += n_accesses; + m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] += + inst->accessq_count(); } } @@ -3847,35 +4094,92 @@ void ldst_unit::writeback() { m_next_wb.is_bru_rt_fill_request()) { release_virtual_entries(m_next_wb); } else if (m_operand_collector->writeback(m_next_wb)) { + { bool insn_completed = false; + // Resolve (src_wid, src_out_inst) for this writeback. For non-shared + // spaces the source is stamped on the feeding mem_fetch. For shared + // the source is always the composite's primary here; co-issued sets' + // releases are handled in a dedicated pass below. + unsigned primary_wid = m_next_wb.warp_id(); + bool is_shared = (m_next_wb.space.get_type() == shared_space); + unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1) + ? m_next_wb_src_wid + : primary_wid; + unsigned src_split = m_next_wb_src_split_id; + bool is_intra_coissued = + (src_wid == primary_wid && src_split != (unsigned)-1); + mem_src_t src = resolve_source(m_next_wb, src_wid, src_split); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (m_next_wb.out[r] > 0) { - if (m_next_wb.space.get_type() != shared_space) { - assert(m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0); - unsigned still_pending = - --m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]]; - if (!still_pending) { - m_pending_writes[m_next_wb.warp_id()].erase(m_next_wb.out[r]); - m_scoreboard->releaseRegister(m_next_wb.warp_id(), - m_next_wb.out[r]); - insn_completed = true; + if (src.out_inst->out[r] > 0) { + if (!is_shared) { + unsigned reg = src.out_inst->out[r]; + if (is_intra_coissued) { + // Intra-warp co-issued: pending count lives in the secondary + // map (keyed by (wid, split_id, reg)). Decrement there; when + // zero, release from secondary scoreboard. + assert(m_pending_writes_secondary[src.wid][src_split][reg] > 0); + unsigned still_pending = + --m_pending_writes_secondary[src.wid][src_split][reg]; + if (!still_pending) { + m_pending_writes_secondary[src.wid][src_split].erase(reg); + m_scoreboard->releaseRegisterSecondary(src.wid, reg); + insn_completed = true; + } + } else { + assert(m_pending_writes[src.wid][reg] > 0); + unsigned still_pending = --m_pending_writes[src.wid][reg]; + if (!still_pending) { + m_pending_writes[src.wid].erase(reg); + m_scoreboard->releaseRegister(src.wid, reg); + insn_completed = true; + } } - } else { // shared - m_scoreboard->releaseRegister(m_next_wb.warp_id(), - m_next_wb.out[r]); + } else { // shared — release primary scoreboard only here; + // composite per-set releases happen below. + m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]); insn_completed = true; } - } else if (m_next_wb.m_is_ldgsts) { // for LDGSTS instructions where no - // output register is used - m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc] + } else if (m_next_wb.m_is_ldgsts) { // LDGSTS excluded from co-issue + m_pending_ldgsts[primary_wid][m_next_wb.pc] [m_next_wb.get_addr(0)]--; - if (m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc] + if (m_pending_ldgsts[primary_wid][m_next_wb.pc] [m_next_wb.get_addr(0)] == 0) { insn_completed = true; } break; } } + + // Shared-memory composite: release each co-issued set's scoreboard + // regs in one pass (shared writeback is single-shot, not per-mf). + if (is_shared && m_next_wb.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = m_next_wb.get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool coissued_intra = (sets[s].warp_id == primary_wid && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (sets[s].source_inst->out[r] > 0) { + if (coissued_intra) + m_scoreboard->releaseRegisterSecondary( + sets[s].warp_id, sets[s].source_inst->out[r]); + else + m_scoreboard->releaseRegister( + sets[s].warp_id, sets[s].source_inst->out[r]); + insn_completed = true; + } + } + } + } + if (insn_completed) { m_core->warp_inst_complete(m_next_wb); if (m_next_wb.m_is_ldgsts) { @@ -3884,10 +4188,13 @@ void ldst_unit::writeback() { } m_next_wb.clear(); + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle; m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle; - } - } + } // close inner block opened at line ~4176 + } // close else + } // close if (!m_next_wb.empty()) unsigned serviced_client = -1; for (unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients); @@ -3897,12 +4204,42 @@ void ldst_unit::writeback() { case 0: // shared memory if (!m_pipeline_reg[0]->empty()) { m_next_wb = *m_pipeline_reg[0]; + // Shared-mem writeback bypasses mem_fetch: there is no + // per-transaction source. Treat this as primary-only. For a MEM + // co-issue composite, enumerate the composite's simd_sets and + // release each source's scoreboard/pipeline in a dedicated pass + // below; here we just stamp to "primary" so the default path + // handles the primary portion. + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; if (m_next_wb.isatomic()) { m_next_wb.do_atomic(); m_core->decrement_atomic_count(m_next_wb.warp_id(), m_next_wb.active_count()); } - m_core->dec_inst_in_pipeline(m_pipeline_reg[0]->warp_id()); + // dec_inst_in_pipeline for each unique (source wid, split_id) + // participating in the composite (mirrors SP writeback at + // shader.cc~3019-3044). Non-composite MEM inst still dec's once. + unsigned primary_wid = m_pipeline_reg[0]->warp_id(); + m_core->dec_inst_in_pipeline(primary_wid); + if (m_next_wb.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + std::set<unsigned> inter_dec; + bool intra_dec = false; + const std::vector<simd_set_info> &sets = + m_next_wb.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != primary_wid) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(primary_wid); + intra_dec = true; + } + } + } m_pipeline_reg[0]->clear(); serviced_client = next_client; } @@ -3911,6 +4248,8 @@ void ldst_unit::writeback() { if (m_L1T->access_ready()) { mem_fetch *mf = m_L1T->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -3919,6 +4258,8 @@ void ldst_unit::writeback() { if (m_L1C->access_ready()) { mem_fetch *mf = m_L1C->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -3926,6 +4267,9 @@ void ldst_unit::writeback() { case 3: // global/local if (m_next_global) { m_next_wb = m_next_global->get_inst(); + m_next_wb_src_wid = m_next_global->get_wid(); + m_next_wb_src_split_id = + m_next_global->get_access_source_split_id(); if (m_next_global->isatomic()) { m_core->decrement_atomic_count( m_next_global->get_wid(), @@ -3940,6 +4284,8 @@ void ldst_unit::writeback() { if (m_L1D && m_L1D->access_ready()) { mem_fetch *mf = m_L1D->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -4104,10 +4450,77 @@ void ldst_unit::cycle() { } } } + // MEM co-issue: also check per-source pending for co-issued sets. + // Inter-warp: m_pending_writes[coissue_wid][source_inst.out[r]]. + // Intra-warp: m_pending_writes_secondary[wid][split][source_inst.out[r]]. + if (!pending_requests && pipe_reg.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> checked; + for (unsigned s = 0; s < sets.size() && !pending_requests; s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (checked.count(key)) continue; + checked.insert(key); + bool is_intra = (sets[s].warp_id == warp_id && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) { + auto &m = m_pending_writes_secondary[sets[s].warp_id] + [sets[s].split_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_requests = true; break; + } + } else { + auto &m = m_pending_writes[sets[s].warp_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_requests = true; break; + } + } + } + } + } if (!pending_requests) { m_core->warp_inst_complete(*m_dispatch_reg); m_scoreboard->releaseRegisters(m_dispatch_reg); + // MEM co-issue retirement: also release each co-issued set's + // scoreboard + call warp_inst_complete for each unique source + // warp/split (inter-warp gets its own warp_inst_complete bump; + // intra-warp keeps sharing the primary's warp_id but gets a + // separate stats increment). + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool is_intra = (sets[s].warp_id == warp_id && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) + m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, + reg_id); + else + m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + } + } + } + // release LDGSTS if (m_dispatch_reg->m_is_ldgsts) { // m_pending_ldgsts[m_dispatch_reg->warp_id()][m_dispatch_reg->pc][m_dispatch_reg->get_addr(0)]--; @@ -4118,12 +4531,52 @@ void ldst_unit::cycle() { } } m_core->dec_inst_in_pipeline(warp_id); + // MEM co-issue: each unique co-issued (warp, split) was incremented + // at decode time, so decrement once per unique here. Mirrors the + // SP/SFU/INT retirement at shader.cc~3019-3044. + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<unsigned> inter_dec; + bool intra_dec = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(warp_id); + intra_dec = true; + } + } + } m_dispatch_reg->clear(); } } else { // stores exit pipeline here m_core->dec_inst_in_pipeline(warp_id); m_core->warp_inst_complete(*m_dispatch_reg); + // MEM co-issue: stores retiring — dec pipe + complete per source too. + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<unsigned> inter_dec; + bool intra_dec = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(warp_id); + intra_dec = true; + } + } + } m_dispatch_reg->clear(); } } @@ -5503,7 +5956,17 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) { } } - // Phase 2: Co-issued sets' destination registers (different instructions) + // Phase 2: Co-issued sets' destination registers (different instructions). + // Phase 1 has already stamped dst=-1 on entries it wrote. If Phase 2 hits + // a busy bank (can happen when primary.reg and coissued.reg hash to the + // same bank under sub-core_model — e.g., (reg+wid_A)%banks_per_sched == + // (reg+wid_B)%banks_per_sched), SKIP that op instead of returning false. + // Returning false would be swallowed by shader_core_ctx::writeback for + // EX_WB composites (see comment at ~shader.cc:3096), but ldst_unit's + // writeback uses the return value to decide whether to clear m_next_wb, + // so a persistent false = deadlock. Bank-busy bookkeeping doesn't affect + // functional correctness — thread register values are committed during + // func_exec_inst, not by this bank model. if (inst.has_simd_sets()) { const std::vector<simd_set_info> &sets = inst.get_simd_sets(); for (unsigned s = 0; s < sets.size(); s++) { @@ -5520,9 +5983,10 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) { m_arbiter.allocate_bank_for_write( bank, op_t(sets[s].warp_id, reg_num, m_num_banks, sub_core_model, m_num_banks_per_sched, sched_id)); - } else { - return false; } + // else: bank busy — skip the bookkeeping for this op. The + // composite still completes; we just don't model this particular + // bank-write's energy for this cycle. } } } diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 2800b81..cd03aec 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -556,6 +556,13 @@ class scheduler_unit { // this can be copied freely, so can be used in std // and intra-warp pass). exec_unit_type_t classify_fu_type(const warp_inst_t *inst) const; + // MEM co-issue safety filter. Returns true if the given inst must NOT + // participate (as primary or co-issuer) in a MEM composite. v1 excludes + // atomics, shared-space ops (deferred per-set bank-conflict rework), + // texture/const caches (line-based coalesce model not per-set-aware), + // MEMORY_BARRIER_OP, LDGSTS, and BRU spill/fill requests. + bool mem_coissue_disallowed(const warp_inst_t *inst) const; + // Check whether `cand_inst` can co-issue alongside `composite` given // `available_sets`. Writes the candidate's sets_needed to *sets_needed // on success. Uses the current compaction_mode to decide between the @@ -1648,6 +1655,17 @@ class ldst_unit : public pipelined_simd_unit { warp_inst_t &inst); mem_stage_stall_type process_memory_access_queue_l1cache(l1_cache *cache, warp_inst_t &inst); + + // MEM co-issue: resolve which source warp + instruction an access/mem_fetch + // belongs to (for routing pending_writes / scoreboard release back to the + // originating warp under composites). Returns {src_wid, src_out_inst}. + // For non-composite or primary-source accesses, returns {primary_wid, &inst}. + // split_id differentiates intra-warp co-issued sets (same wid as primary) + // from the primary set itself; use (unsigned)-1 when not applicable. + struct mem_src_t { unsigned wid; const inst_t *out_inst; }; + mem_src_t resolve_source(const warp_inst_t &inst, + unsigned access_src_wid, + unsigned access_src_split_id) const; gpgpu_sim *m_gpu; const memory_config *m_memory_config; @@ -1669,6 +1687,22 @@ class ldst_unit : public pipelined_simd_unit { mem_fetch *m_next_global; warp_inst_t m_next_wb; + // MEM co-issue: source warp + split_id for the mem_fetch that fed m_next_wb. + // Used by writeback() to decrement the correct source's pending_writes / + // release the correct source's scoreboard registers (primary vs secondary). + // Set at each "m_next_wb = ..." assignment; defaults to composite primary. + unsigned m_next_wb_src_wid; + unsigned m_next_wb_src_split_id; + // MEM co-issue intra-warp pending writes: primary and intra-warp-coissued + // may write the same register number (R5) for disjoint thread subsets. + // The existing m_pending_writes[wid][reg] is single-counter per (wid,reg) + // and would conflate the two — leading to premature / delayed scoreboard + // release (deadlock). Use a separate map keyed by (wid, split_id, reg) + // for intra-warp-coissued accesses so counts stay distinct. + std::map<unsigned /*wid*/, + std::map<unsigned /*split_id*/, + std::map<unsigned /*reg*/, unsigned /*count*/>>> + m_pending_writes_secondary; unsigned m_writeback_arb; // round-robin arbiter for writeback contention // between L1T, L1C, shared unsigned m_num_writeback_clients; @@ -1961,6 +1995,7 @@ class shader_core_config : public core_config { unsigned gpgpu_opndcoll_read_latency; // extra cycles to collector unsigned gpgpu_co_issue_priority; // 0=greedy..4=same-PC bool gpgpu_simd_partitioning_debug; // verbose SIMD_SETS printf gate + bool gpgpu_mem_coissue; // enable MEM/LDST co-issue path unsigned n_simt_cores_per_cluster; unsigned n_simt_clusters; @@ -2112,6 +2147,14 @@ struct shader_core_stats_pod { unsigned long long *coissue_skipped_no_inst; unsigned long long *coissue_skipped_pc_mismatch; unsigned long long *coissue_skipped_samepc_pass0; + + // MEM co-issue specific counters (the subset of coissue_* events where + // the composite's FU type is MEM). Useful for diagnosing whether the + // -gpgpu_mem_coissue flag is actually firing and how much of the total + // co-issue activity is memory-backed on a given workload. + unsigned long long *mem_coissue_inter_events; + unsigned long long *mem_coissue_intra_events; + unsigned long long *mem_coissue_denied_filter; // safety filter rejects }; class shader_core_stats : public shader_core_stats_pod { @@ -2269,6 +2312,14 @@ class shader_core_stats : public shader_core_stats_pod { config->num_shader(), sizeof(unsigned long long)); coissue_skipped_samepc_pass0 = (unsigned long long *)calloc( config->num_shader(), sizeof(unsigned long long)); + + // MEM co-issue-specific counters + mem_coissue_inter_events = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + mem_coissue_intra_events = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + mem_coissue_denied_filter = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); } ~shader_core_stats() { @@ -2341,6 +2392,9 @@ class shader_core_stats : public shader_core_stats_pod { free(coissue_skipped_no_inst); free(coissue_skipped_pc_mismatch); free(coissue_skipped_samepc_pass0); + free(mem_coissue_inter_events); + free(mem_coissue_intra_events); + free(mem_coissue_denied_filter); } void new_grid() {} @@ -2404,10 +2458,16 @@ class shader_core_mem_fetch_allocator : public mem_fetch_allocator { mem_fetch *alloc(const warp_inst_t &inst, const mem_access_t &access, unsigned long long cycle) const { warp_inst_t inst_copy = inst; + // If the access was tagged with a source warp_id (MEM co-issue), + // stamp the mem_fetch with that wid so writeback routes back to the + // correct warp's scoreboard / pending_writes entry. Otherwise fall + // back to the composite's primary warp_id (legacy behavior). + unsigned wid = access.get_source_wid(); + if (wid == (unsigned)-1) wid = inst.warp_id(); mem_fetch *mf = new mem_fetch( access, &inst_copy, inst.get_streamID(), access.is_write() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, - inst.warp_id(), m_core_id, m_cluster_id, m_memory_config, cycle); + wid, m_core_id, m_cluster_id, m_memory_config, cycle); return mf; } |