diff options
Diffstat (limited to 'src/gpgpu-sim/shader.cc')
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 576 |
1 files changed, 520 insertions, 56 deletions
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index c874733..f96e939 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -830,6 +830,19 @@ void shader_core_stats::print(FILE *fout) const { fprintf(fout, "gpu_tot_coissue_skipped_no_inst = %llu\n", tot_noinst); fprintf(fout, "gpu_tot_coissue_skipped_pc_mismatch = %llu\n", tot_pcm); fprintf(fout, "gpu_tot_coissue_skipped_samepc_pass0 = %llu\n", tot_spc0); + + // MEM co-issue-specific subtotals + unsigned long long tot_mem_inter = 0, tot_mem_intra = 0, + tot_mem_denied = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + tot_mem_inter += mem_coissue_inter_events[i]; + tot_mem_intra += mem_coissue_intra_events[i]; + tot_mem_denied += mem_coissue_denied_filter[i]; + } + fprintf(fout, "gpu_tot_mem_coissue_inter_events = %llu\n", tot_mem_inter); + fprintf(fout, "gpu_tot_mem_coissue_intra_events = %llu\n", tot_mem_intra); + fprintf(fout, "gpu_tot_mem_coissue_denied_filter = %llu\n", + tot_mem_denied); } } @@ -1457,8 +1470,29 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, // Set next PC for the co-issued warp m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize); - // Tag the co-issued sets with their source instruction for writeback + // Tag the co-issued sets with their source instruction + split_id for + // writeback routing (intra-warp composites need split_id to disambiguate + // between primary and secondary-split sets that share warp_id). temp_inst.set_source_inst_on_sets(next_inst); + temp_inst.set_split_id_on_sets(split_id); + + // For MEM co-issue: splice temp_inst's already-coalesced mem accesses + // into the composite, stamping each with source (warp_id, split_id) so + // the LDST unit routes pending_writes + scoreboard release back to the + // originating warp/split. This must happen BEFORE merge_simd_sets + // because we also need the temp_inst's own simd_sets to still carry + // valid flags here; but the accessq itself is independent. + if (temp_inst.is_load() || temp_inst.is_store()) { + unsigned src_wid = warp_id; + unsigned src_split = split_id; // -1 for inter-warp + while (!temp_inst.accessq_empty()) { + mem_access_t acc = temp_inst.accessq_back(); + acc.set_source_wid(src_wid); + acc.set_source_split_id(src_split); + composite->inject_mem_acccesses(acc); + temp_inst.accessq_pop_back(); + } + } // Merge the co-issued instruction's valid sets into the composite composite->merge_simd_sets(temp_inst.get_simd_sets()); @@ -1604,6 +1638,31 @@ void scheduler_unit::order_by_priority( // Change 4: co-issue scheduler helpers. // --------------------------------------------------------------------------- +bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const { + if (!inst) return true; + // Exclude TENSOR_CORE loads/stores (quirky pipeline), memory barriers, + // LDGSTS, and BRU spill/fill. These remain MEM-bucketed at classify time + // but are rejected from co-issue. + if (inst->op == TENSOR_CORE_LOAD_OP || inst->op == TENSOR_CORE_STORE_OP) + return true; + if (inst->op == MEMORY_BARRIER_OP) return true; + if (inst->m_is_ldgsts) return true; + if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() || + inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request()) + return true; + // Exclude atomics (decrement_atomic_count ties to a single warp) and + // non-global memory spaces. Global/local go through the coalescer + + // mem_fetch path that the source_wid stamp supports. Shared/tex/const + // use per-subwarp / per-line paths not yet per-set-aware. + if (inst->isatomic()) return true; + memory_space_t space = inst->space; + if (space.get_type() != global_space && + space.get_type() != local_space && + space.get_type() != param_space_local) + return true; + return false; +} + exec_unit_type_t scheduler_unit::classify_fu_type( const warp_inst_t *inst) const { if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) || @@ -1797,6 +1856,14 @@ void scheduler_unit::try_inter_warp_coissue( continue; } + // MEM co-issue safety filter. + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any. if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) { m_stats->coissue_skipped_samepc_pass0[get_sid()]++; @@ -1832,6 +1899,8 @@ void scheduler_unit::try_inter_warp_coissue( warp(cand_warp_id).ibuffer_step(); picked_warps.insert(cand_warp_id); m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; } } } @@ -1881,6 +1950,14 @@ void scheduler_unit::try_intra_warp_coissue( continue; } + // MEM co-issue safety filter. + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; @@ -1909,6 +1986,8 @@ void scheduler_unit::try_intra_warp_coissue( available_sets -= sec_sets_needed; next_free_set += sec_sets_needed; m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; warp(primary_warp_id).ibuffer_free_slot(sec_slot); @@ -1976,6 +2055,12 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; continue; } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } const active_mask_t &cand_mask = m_shader->get_active_mask(cand_warp_id, cand_inst); @@ -2033,6 +2118,12 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; continue; } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; @@ -2106,6 +2197,8 @@ void scheduler_unit::try_utilization_max_coissue( available_sets -= needed; next_free_set += needed; m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; warp(c.warp_id).ibuffer_free_slot(c.sec_slot); if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { warp(c.warp_id).ibuffer_flush_half(1); @@ -2136,6 +2229,8 @@ void scheduler_unit::try_utilization_max_coissue( available_sets -= needed; next_free_set += needed; m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; warp(c.warp_id).ibuffer_step(); } } @@ -2251,8 +2346,19 @@ void scheduler_unit::cycle() { m_id) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { - m_shader->issue_warp(*m_mem_out, pI, active_mask, warp_id, - m_id); + warp_inst_t *issued_pipe_reg = m_shader->issue_warp( + *m_mem_out, pI, active_mask, warp_id, m_id); + // When MEM co-issue is enabled, capture the composite pointer + // so the dispatch loop below can attempt to merge co-issuers + // into this MEM pipeline slot. Otherwise leave co_issue_* + // variables unset for bit-parity with the pre-MEM-coissue tree. + if (m_shader->m_config->gpgpu_simd_partitioning && + m_shader->m_config->gpgpu_mem_coissue) { + co_issue_composite = issued_pipe_reg; + co_issue_fu_type = exec_unit_type_t::MEM; + co_issue_reg_set = m_mem_out; + co_issue_primary_warp_id = warp_id; + } issued++; issued_inst = true; warp_inst_issued = true; @@ -2484,10 +2590,13 @@ void scheduler_unit::cycle() { // SIMD set co-issue dispatch: after issuing the primary instruction, // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority. - // Only for non-MEM FUs (LDST excluded from partitioning). + // MEM co-issue is opt-in via -gpgpu_mem_coissue (the primary-issue branch + // above leaves co_issue_composite=NULL when the flag is off, so this + // block is skipped naturally in that case). if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL && - co_issue_fu_type != exec_unit_type_t::MEM && - co_issue_fu_type != exec_unit_type_t::NONE) { + co_issue_fu_type != exec_unit_type_t::NONE && + !(co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(co_issue_composite))) { available_sets = 0; next_free_set = 0; const std::vector<simd_set_info> &primary_sets = @@ -3114,13 +3223,21 @@ mem_stage_stall_type ldst_unit::process_cache_access( mem_stage_stall_type result = NO_RC_FAIL; bool write_sent = was_write_sent(events); bool read_sent = was_read_sent(events); + // Attribute store acks / load HIT register decrements to the access's + // source warp + its source instruction (for MEM co-issue composites). + unsigned src_wid_access = mf ? mf->get_wid() : inst.warp_id(); + unsigned src_split_access = + mf ? mf->get_access_source_split_id() : (unsigned)-1; + mem_src_t src = resolve_source(inst, src_wid_access, src_split_access); + bool src_is_intra_coissued = + (src.wid == inst.warp_id() && src_split_access != (unsigned)-1); if (write_sent) { unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) ? (mf->get_data_size() / SECTOR_SIZE) : 1; for (unsigned i = 0; i < inc_ack; ++i) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(src.wid); } if (status == HIT) { assert(!read_sent); @@ -3129,9 +3246,15 @@ mem_stage_stall_type ldst_unit::process_cache_access( release_virtual_entries(inst); } else if (inst.is_load()) { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--; + if (src.out_inst->out[r] > 0) { + if (src_is_intra_coissued) + m_pending_writes_secondary[src.wid][src_split_access] + [src.out_inst->out[r]]--; + else + m_pending_writes[src.wid][src.out_inst->out[r]]--; + } - // release LDGSTS + // release LDGSTS (primary-only: LDGSTS is excluded from MEM co-issue) if (inst.m_is_ldgsts) { m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)]--; if (m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)] == 0) { @@ -3203,7 +3326,7 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( : 1; for (unsigned i = 0; i < inc_ack; ++i) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(mf->get_wid()); } inst.accessq_pop_back(); @@ -3251,23 +3374,43 @@ void ldst_unit::L1_latency_queue_cycle() { assert(!read_sent); l1_latency_queue[j][0] = NULL; if (mf_next->get_inst().is_load()) { + // MEM co-issue: the access belongs to mf_next's stamped source + // warp. Resolve which instruction's out[] registers should be + // decremented (primary's or a co-issued set's). + unsigned src_split_mf = mf_next->get_access_source_split_id(); + mem_src_t src = resolve_source( + mf_next->get_inst(), mf_next->get_wid(), src_split_mf); + bool src_is_intra = + (src.wid == mf_next->get_inst().warp_id() && + src_split_mf != (unsigned)-1); + bool any_completed = false; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (mf_next->get_inst().out[r] > 0) { - assert(m_pending_writes[mf_next->get_inst().warp_id()] - [mf_next->get_inst().out[r]] > 0); - unsigned still_pending = - --m_pending_writes[mf_next->get_inst().warp_id()] - [mf_next->get_inst().out[r]]; - if (!still_pending) { - m_pending_writes[mf_next->get_inst().warp_id()].erase( - mf_next->get_inst().out[r]); - m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(), - mf_next->get_inst().out[r]); - m_core->warp_inst_complete(mf_next->get_inst()); + if (src.out_inst->out[r] > 0) { + unsigned reg = src.out_inst->out[r]; + if (src_is_intra) { + assert(m_pending_writes_secondary[src.wid][src_split_mf] + [reg] > 0); + unsigned still_pending = --m_pending_writes_secondary + [src.wid][src_split_mf][reg]; + if (!still_pending) { + m_pending_writes_secondary[src.wid][src_split_mf].erase( + reg); + m_scoreboard->releaseRegisterSecondary(src.wid, reg); + any_completed = true; + } + } else { + assert(m_pending_writes[src.wid][reg] > 0); + unsigned still_pending = --m_pending_writes[src.wid][reg]; + if (!still_pending) { + m_pending_writes[src.wid].erase(reg); + m_scoreboard->releaseRegister(src.wid, reg); + any_completed = true; + } } } + if (any_completed) m_core->warp_inst_complete(mf_next->get_inst()); - // release LDGSTS + // release LDGSTS (primary-only; LDGSTS excluded from MEM co-issue) if (mf_next->get_inst().m_is_ldgsts) { m_pending_ldgsts[mf_next->get_inst().warp_id()] [mf_next->get_inst().pc] @@ -3417,6 +3560,9 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, stall_cond = ICNT_RC_FAIL; break; } else { + // Resolve source attribution for MEM co-issued accesses. + unsigned src_wid = access.get_source_wid(); + if (src_wid == (unsigned)-1) src_wid = inst.warp_id(); mem_fetch *mf = m_mf_allocator->alloc(inst, access, m_core->get_gpu()->gpu_sim_cycle + @@ -3425,11 +3571,24 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, inst.accessq_pop_back(); // inst.clear_active( access.get_warp_mask() ); if (inst.is_load()) { + // For MEM-co-issue composites, sanity-check against the source + // warp's pending_writes on its source out[] regs (if identifiable). + const inst_t *src_out_inst = &inst; + if (src_wid != inst.warp_id() && inst.has_simd_sets()) { + const std::vector<simd_set_info> &sets = inst.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (sets[s].valid && sets[s].source_inst != NULL && + sets[s].warp_id == src_wid) { + src_out_inst = sets[s].source_inst; + break; + } + } + } for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) - if (inst.out[r] > 0) - assert(m_pending_writes[inst.warp_id()][inst.out[r]] > 0); + if (src_out_inst->out[r] > 0) + assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0); } else if (inst.is_store()) - m_core->inc_store_req(inst.warp_id()); + m_core->inc_store_req(src_wid); } } } else { @@ -3739,6 +3898,8 @@ void ldst_unit::init(mem_fetch_interface *icnt, 5; // = shared memory, global/local (uncached), L1D, L1T, L1C m_writeback_arb = 0; m_next_global = NULL; + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; m_last_inst_gpu_sim_cycle = 0; m_last_inst_gpu_tot_sim_cycle = 0; } @@ -3785,6 +3946,33 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt, mem_config, stats, sid, tpc); } +ldst_unit::mem_src_t ldst_unit::resolve_source( + const warp_inst_t &inst, unsigned access_src_wid, + unsigned access_src_split_id) const { + mem_src_t r; + r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid; + r.out_inst = &inst; + if (!inst.has_simd_sets()) return r; + // Walk simd_sets for a co-issued set matching BOTH (wid, split_id). For + // inter-warp accesses split_id is -1 and wid differs from primary; for + // intra-warp accesses split_id is non-neg and wid equals primary. Primary + // accesses (source_wid == -1 or split_id == -1 with wid==primary) keep + // out_inst = &inst (composite's out[], which is primary's). + bool is_intra = + (r.wid == inst.warp_id() && access_src_split_id != (unsigned)-1); + bool is_inter = (r.wid != inst.warp_id()); + if (!is_intra && !is_inter) return r; + const std::vector<simd_set_info> &sets = inst.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != r.wid) continue; + if (is_intra && sets[s].split_id != access_src_split_id) continue; + r.out_inst = sets[s].source_inst; + break; + } + return r; +} + void ldst_unit::issue(register_set ®_set) { warp_inst_t *inst = *(reg_set.get_ready()); @@ -3792,16 +3980,75 @@ void ldst_unit::issue(register_set ®_set) { // instruction assert(inst->empty() == false); if (inst->is_load() and inst->space.get_type() != shared_space) { - unsigned warp_id = inst->warp_id(); - unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = inst->out[r]; - if (reg_id > 0) { - m_pending_writes[warp_id][reg_id] += n_accesses; + unsigned primary_wid = inst->warp_id(); + + // MEM co-issue: accesses in the composite's queue may originate from + // different source warps (inter-warp) or different splits of the same + // warp (intra-warp). Group them by (source_wid, source_split_id) to + // charge pending_writes to the correct source's destination registers. + // Unstamped accesses (source_wid == -1) belong to the primary. + if (inst->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src; + for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin(); + it != inst->accessq_end(); ++it) { + unsigned w = it->get_source_wid(); + unsigned sp = it->get_source_split_id(); + if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } + n_acc_per_src[std::make_pair(w, sp)] += 1; + } + + // Primary's own registers + unsigned n_primary = + n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)]; + if (n_primary > 0) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary; + } + } + + // Per co-issued set's registers (use set's source_inst->out[]). + // Inter-warp sets: different warp_id → primary pending_writes map is + // fine (no key collision with primary). Intra-warp sets: same warp_id + // but different split_id → use the secondary pending_writes map to + // avoid conflating with primary's pending_writes[primary_wid][reg]. + const std::vector<simd_set_info> &sets = inst->get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> accounted; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id); + if (accounted.count(key)) continue; + accounted.insert(key); + unsigned n_src = n_acc_per_src[key]; + if (n_src == 0) continue; + bool is_intra = (sets[s].warp_id == primary_wid && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id > 0) { + if (is_intra) { + m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id] + [reg_id] += n_src; + } else { + m_pending_writes[sets[s].warp_id][reg_id] += n_src; + } + } + } + } + } else { + // Legacy (non-co-issue) path: single-warp pending_writes. + unsigned n_accesses = inst->accessq_count(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses; } } + if (inst->m_is_ldgsts) { - m_pending_ldgsts[warp_id][inst->pc][inst->get_addr(0)] += n_accesses; + m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] += + inst->accessq_count(); } } @@ -3847,35 +4094,92 @@ void ldst_unit::writeback() { m_next_wb.is_bru_rt_fill_request()) { release_virtual_entries(m_next_wb); } else if (m_operand_collector->writeback(m_next_wb)) { + { bool insn_completed = false; + // Resolve (src_wid, src_out_inst) for this writeback. For non-shared + // spaces the source is stamped on the feeding mem_fetch. For shared + // the source is always the composite's primary here; co-issued sets' + // releases are handled in a dedicated pass below. + unsigned primary_wid = m_next_wb.warp_id(); + bool is_shared = (m_next_wb.space.get_type() == shared_space); + unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1) + ? m_next_wb_src_wid + : primary_wid; + unsigned src_split = m_next_wb_src_split_id; + bool is_intra_coissued = + (src_wid == primary_wid && src_split != (unsigned)-1); + mem_src_t src = resolve_source(m_next_wb, src_wid, src_split); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (m_next_wb.out[r] > 0) { - if (m_next_wb.space.get_type() != shared_space) { - assert(m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0); - unsigned still_pending = - --m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]]; - if (!still_pending) { - m_pending_writes[m_next_wb.warp_id()].erase(m_next_wb.out[r]); - m_scoreboard->releaseRegister(m_next_wb.warp_id(), - m_next_wb.out[r]); - insn_completed = true; + if (src.out_inst->out[r] > 0) { + if (!is_shared) { + unsigned reg = src.out_inst->out[r]; + if (is_intra_coissued) { + // Intra-warp co-issued: pending count lives in the secondary + // map (keyed by (wid, split_id, reg)). Decrement there; when + // zero, release from secondary scoreboard. + assert(m_pending_writes_secondary[src.wid][src_split][reg] > 0); + unsigned still_pending = + --m_pending_writes_secondary[src.wid][src_split][reg]; + if (!still_pending) { + m_pending_writes_secondary[src.wid][src_split].erase(reg); + m_scoreboard->releaseRegisterSecondary(src.wid, reg); + insn_completed = true; + } + } else { + assert(m_pending_writes[src.wid][reg] > 0); + unsigned still_pending = --m_pending_writes[src.wid][reg]; + if (!still_pending) { + m_pending_writes[src.wid].erase(reg); + m_scoreboard->releaseRegister(src.wid, reg); + insn_completed = true; + } } - } else { // shared - m_scoreboard->releaseRegister(m_next_wb.warp_id(), - m_next_wb.out[r]); + } else { // shared — release primary scoreboard only here; + // composite per-set releases happen below. + m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]); insn_completed = true; } - } else if (m_next_wb.m_is_ldgsts) { // for LDGSTS instructions where no - // output register is used - m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc] + } else if (m_next_wb.m_is_ldgsts) { // LDGSTS excluded from co-issue + m_pending_ldgsts[primary_wid][m_next_wb.pc] [m_next_wb.get_addr(0)]--; - if (m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc] + if (m_pending_ldgsts[primary_wid][m_next_wb.pc] [m_next_wb.get_addr(0)] == 0) { insn_completed = true; } break; } } + + // Shared-memory composite: release each co-issued set's scoreboard + // regs in one pass (shared writeback is single-shot, not per-mf). + if (is_shared && m_next_wb.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = m_next_wb.get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool coissued_intra = (sets[s].warp_id == primary_wid && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (sets[s].source_inst->out[r] > 0) { + if (coissued_intra) + m_scoreboard->releaseRegisterSecondary( + sets[s].warp_id, sets[s].source_inst->out[r]); + else + m_scoreboard->releaseRegister( + sets[s].warp_id, sets[s].source_inst->out[r]); + insn_completed = true; + } + } + } + } + if (insn_completed) { m_core->warp_inst_complete(m_next_wb); if (m_next_wb.m_is_ldgsts) { @@ -3884,10 +4188,13 @@ void ldst_unit::writeback() { } m_next_wb.clear(); + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle; m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle; - } - } + } // close inner block opened at line ~4176 + } // close else + } // close if (!m_next_wb.empty()) unsigned serviced_client = -1; for (unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients); @@ -3897,12 +4204,42 @@ void ldst_unit::writeback() { case 0: // shared memory if (!m_pipeline_reg[0]->empty()) { m_next_wb = *m_pipeline_reg[0]; + // Shared-mem writeback bypasses mem_fetch: there is no + // per-transaction source. Treat this as primary-only. For a MEM + // co-issue composite, enumerate the composite's simd_sets and + // release each source's scoreboard/pipeline in a dedicated pass + // below; here we just stamp to "primary" so the default path + // handles the primary portion. + m_next_wb_src_wid = (unsigned)-1; + m_next_wb_src_split_id = (unsigned)-1; if (m_next_wb.isatomic()) { m_next_wb.do_atomic(); m_core->decrement_atomic_count(m_next_wb.warp_id(), m_next_wb.active_count()); } - m_core->dec_inst_in_pipeline(m_pipeline_reg[0]->warp_id()); + // dec_inst_in_pipeline for each unique (source wid, split_id) + // participating in the composite (mirrors SP writeback at + // shader.cc~3019-3044). Non-composite MEM inst still dec's once. + unsigned primary_wid = m_pipeline_reg[0]->warp_id(); + m_core->dec_inst_in_pipeline(primary_wid); + if (m_next_wb.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + std::set<unsigned> inter_dec; + bool intra_dec = false; + const std::vector<simd_set_info> &sets = + m_next_wb.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != primary_wid) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(primary_wid); + intra_dec = true; + } + } + } m_pipeline_reg[0]->clear(); serviced_client = next_client; } @@ -3911,6 +4248,8 @@ void ldst_unit::writeback() { if (m_L1T->access_ready()) { mem_fetch *mf = m_L1T->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -3919,6 +4258,8 @@ void ldst_unit::writeback() { if (m_L1C->access_ready()) { mem_fetch *mf = m_L1C->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -3926,6 +4267,9 @@ void ldst_unit::writeback() { case 3: // global/local if (m_next_global) { m_next_wb = m_next_global->get_inst(); + m_next_wb_src_wid = m_next_global->get_wid(); + m_next_wb_src_split_id = + m_next_global->get_access_source_split_id(); if (m_next_global->isatomic()) { m_core->decrement_atomic_count( m_next_global->get_wid(), @@ -3940,6 +4284,8 @@ void ldst_unit::writeback() { if (m_L1D && m_L1D->access_ready()) { mem_fetch *mf = m_L1D->next_access(); m_next_wb = mf->get_inst(); + m_next_wb_src_wid = mf->get_wid(); + m_next_wb_src_split_id = mf->get_access_source_split_id(); delete mf; serviced_client = next_client; } @@ -4104,10 +4450,77 @@ void ldst_unit::cycle() { } } } + // MEM co-issue: also check per-source pending for co-issued sets. + // Inter-warp: m_pending_writes[coissue_wid][source_inst.out[r]]. + // Intra-warp: m_pending_writes_secondary[wid][split][source_inst.out[r]]. + if (!pending_requests && pipe_reg.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> checked; + for (unsigned s = 0; s < sets.size() && !pending_requests; s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (checked.count(key)) continue; + checked.insert(key); + bool is_intra = (sets[s].warp_id == warp_id && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) { + auto &m = m_pending_writes_secondary[sets[s].warp_id] + [sets[s].split_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_requests = true; break; + } + } else { + auto &m = m_pending_writes[sets[s].warp_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_requests = true; break; + } + } + } + } + } if (!pending_requests) { m_core->warp_inst_complete(*m_dispatch_reg); m_scoreboard->releaseRegisters(m_dispatch_reg); + // MEM co-issue retirement: also release each co-issued set's + // scoreboard + call warp_inst_complete for each unique source + // warp/split (inter-warp gets its own warp_inst_complete bump; + // intra-warp keeps sharing the primary's warp_id but gets a + // separate stats increment). + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool is_intra = (sets[s].warp_id == warp_id && + sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) + m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, + reg_id); + else + m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + } + } + } + // release LDGSTS if (m_dispatch_reg->m_is_ldgsts) { // m_pending_ldgsts[m_dispatch_reg->warp_id()][m_dispatch_reg->pc][m_dispatch_reg->get_addr(0)]--; @@ -4118,12 +4531,52 @@ void ldst_unit::cycle() { } } m_core->dec_inst_in_pipeline(warp_id); + // MEM co-issue: each unique co-issued (warp, split) was incremented + // at decode time, so decrement once per unique here. Mirrors the + // SP/SFU/INT retirement at shader.cc~3019-3044. + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<unsigned> inter_dec; + bool intra_dec = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(warp_id); + intra_dec = true; + } + } + } m_dispatch_reg->clear(); } } else { // stores exit pipeline here m_core->dec_inst_in_pipeline(warp_id); m_core->warp_inst_complete(*m_dispatch_reg); + // MEM co-issue: stores retiring — dec pipe + complete per source too. + if (m_dispatch_reg->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { + const std::vector<simd_set_info> &sets = + m_dispatch_reg->get_simd_sets(); + std::set<unsigned> inter_dec; + bool intra_dec = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(warp_id); + intra_dec = true; + } + } + } m_dispatch_reg->clear(); } } @@ -5503,7 +5956,17 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) { } } - // Phase 2: Co-issued sets' destination registers (different instructions) + // Phase 2: Co-issued sets' destination registers (different instructions). + // Phase 1 has already stamped dst=-1 on entries it wrote. If Phase 2 hits + // a busy bank (can happen when primary.reg and coissued.reg hash to the + // same bank under sub-core_model — e.g., (reg+wid_A)%banks_per_sched == + // (reg+wid_B)%banks_per_sched), SKIP that op instead of returning false. + // Returning false would be swallowed by shader_core_ctx::writeback for + // EX_WB composites (see comment at ~shader.cc:3096), but ldst_unit's + // writeback uses the return value to decide whether to clear m_next_wb, + // so a persistent false = deadlock. Bank-busy bookkeeping doesn't affect + // functional correctness — thread register values are committed during + // func_exec_inst, not by this bank model. if (inst.has_simd_sets()) { const std::vector<simd_set_info> &sets = inst.get_simd_sets(); for (unsigned s = 0; s < sets.size(); s++) { @@ -5520,9 +5983,10 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) { m_arbiter.allocate_bank_for_write( bank, op_t(sets[s].warp_id, reg_num, m_num_banks, sub_core_model, m_num_banks_per_sched, sched_id)); - } else { - return false; } + // else: bank busy — skip the bookkeeping for this op. The + // composite still completes; we just don't model this particular + // bank-write's energy for this cycle. } } } |