diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/abstract_hardware_model.h | 105 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 705 |
2 files changed, 706 insertions, 104 deletions
diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 1e3c94e..1a42657 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -1119,6 +1119,20 @@ enum divergence_support_t { const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8; +// Mixed-space MEM co-issue (v2): per-lane address storage for a shared- +// memory coissuer set. Captured at co_issue_warp time BEFORE temp_inst +// dies, so later per-set bank conflict recomputes can run against each +// set's own per-lane addresses without colliding on the composite's +// m_per_scalar_thread (which is keyed by thread-in-warp and would be +// overwritten across inter-warp coissuers). +struct simd_set_addr_entry { + new_addr_type memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + simd_set_addr_entry() { + for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) + memreqaddr[i] = 0; + } +}; + // SIMD lane partitioning: per-set execution data struct simd_set_info { unsigned set_id; // Set index (0 to num_sets-1) @@ -1131,13 +1145,29 @@ struct simd_set_info { bool valid; // Whether this set has any active threads const inst_t *source_inst; // Pointer to static decoded instruction (for MIMD) + // Mixed-space MEM co-issue (v2): per-set shared state. + // Populated only when source_inst's space is shared/sstarr; otherwise + // unused. `cycles` is the set's bank-conflict depth (modelled per-set + // in shared_cycle). `per_thread_addrs` is the captured per-lane + // memreqaddr for this set. + unsigned cycles; + std::vector<simd_set_addr_entry> per_thread_addrs; + simd_set_info() : set_id(0), warp_id(0), split_id((unsigned)-1), - num_active_threads(0), valid(false), source_inst(NULL) { + num_active_threads(0), valid(false), source_inst(NULL), cycles(0) { set_active_mask.reset(); active_mask_in_warp.reset(); for (unsigned i = 0; i < MAX_WARP_SIZE; i++) thread_ids[i] = 0; } + + // Mirror of warp_inst_t::has_dispatch_delay / dispatch_delay, but + // per-set. Used by the per-set shared_cycle path in ldst_unit. + bool has_dispatch_delay() const { return cycles > 0; } + bool dispatch_delay() { + if (cycles > 0) cycles--; + return cycles > 0; + } }; class warp_inst_t : public inst_t { @@ -1205,8 +1235,68 @@ class warp_inst_t : public inst_t { const std::vector<simd_set_info> &get_simd_sets() const { return m_simd_sets; } + // Mixed-space MEM co-issue (v2): mutable accessor for capture path. + std::vector<simd_set_info> &get_simd_sets_mutable() { return m_simd_sets; } unsigned num_active_simd_sets() const; + + // Mixed-space MEM co-issue (v2): space-bucket classification. "shared + // bucket" = shared_space/sstarr_space; "global bucket" = + // global/local/param_local. + static bool is_shared_bucket_space(enum _memory_space_t t) { + return t == shared_space || t == sstarr_space; + } + static bool is_global_bucket_space(enum _memory_space_t t) { + return t == global_space || t == local_space || t == param_space_local; + } + // A composite has a shared/global "subcomposite" if any valid simd_set + // (including the primary set when source_inst==NULL) has a source space + // in that bucket. Primary set's space is the composite's own `space`. + bool has_shared_subcomposite() const { + for (unsigned s = 0; s < m_simd_sets.size(); s++) { + if (!m_simd_sets[s].valid) continue; + enum _memory_space_t sp = (m_simd_sets[s].source_inst == NULL) + ? space.get_type() + : m_simd_sets[s].source_inst->space.get_type(); + if (is_shared_bucket_space(sp)) return true; + } + return false; + } + bool has_global_subcomposite() const { + for (unsigned s = 0; s < m_simd_sets.size(); s++) { + if (!m_simd_sets[s].valid) continue; + enum _memory_space_t sp = (m_simd_sets[s].source_inst == NULL) + ? space.get_type() + : m_simd_sets[s].source_inst->space.get_type(); + if (is_global_bucket_space(sp)) return true; + } + return false; + } + bool is_mixed_space_composite() const { + return has_shared_subcomposite() && has_global_subcomposite(); + } bool has_simd_sets() const { return !m_simd_sets.empty(); } + + // Mixed-space MEM co-issue (v2): copy this instruction's per-lane + // memreqaddr entries into `set.per_thread_addrs`, and copy this + // instruction's bank-conflict `cycles` into `set.cycles`. Used by + // co_issue_warp for SHARED coissuer sets BEFORE temp_inst dies, so + // later per-set bank-conflict re-checks can read each set's own + // addresses without colliding on the composite's m_per_scalar_thread. + void capture_per_set_shared_state(simd_set_info &set, + unsigned set_width) const { + set.cycles = cycles; + if (!m_per_scalar_thread_valid) return; + set.per_thread_addrs.assign(set_width, simd_set_addr_entry()); + for (unsigned lane = 0; lane < set_width; lane++) { + if (!set.set_active_mask.test(lane)) continue; + unsigned t = set.thread_ids[lane]; + if (t >= m_per_scalar_thread.size()) continue; + for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) { + set.per_thread_addrs[lane].memreqaddr[i] = + m_per_scalar_thread[t].memreqaddr[i]; + } + } + } void merge_simd_sets(const std::vector<simd_set_info> &other_sets); void set_source_inst_on_sets(const inst_t *src); void set_split_id_on_sets(unsigned split_id); @@ -1408,6 +1498,19 @@ class warp_inst_t : public inst_t { bool m_is_depbar; unsigned int m_depbar_group_no; + + // Mixed-space MEM co-issue (Option A / Stage 6): tracks whether the + // shared-source scoreboards of this MIXED composite have already been + // released via the early-release path in ldst_unit::cycle. Set once; + // checked by both the early-release path (to avoid re-release as + // shared_cycle keeps returning true) and by the final mixed-retire + // (to avoid double-releasing shared sources at dispatch_reg clear). + // Default false. Only meaningful for mixed composites. + public: + bool shared_side_released() const { return m_shared_side_released; } + void set_shared_side_released(bool v) { m_shared_side_released = v; } + private: + bool m_shared_side_released = false; }; void move_warp(warp_inst_t *&dst, warp_inst_t *&src); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index f96e939..191beaf 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -1476,6 +1476,24 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, temp_inst.set_source_inst_on_sets(next_inst); temp_inst.set_split_id_on_sets(split_id); + // Mixed-space MEM co-issue (v2): for SHARED coissuers, capture per-lane + // memreqaddr + temp_inst.cycles into each valid set BEFORE merge. The + // captured state survives temp_inst's destruction at function exit and + // is used by per-set shared_cycle + any per-set bank-conflict recompute + // at retirement. No-op for global coissuers (shared_cycle and the + // retirement walk gate on source_inst->space). Guarded on is_load || + // is_store to skip non-memory ops. + if ((temp_inst.is_load() || temp_inst.is_store()) && + (next_inst->space.get_type() == shared_space || + next_inst->space.get_type() == sstarr_space)) { + unsigned set_width = m_config->simd_set_width; + std::vector<simd_set_info> &sets = temp_inst.get_simd_sets_mutable(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid) continue; + temp_inst.capture_per_set_shared_state(sets[s], set_width); + } + } + // For MEM co-issue: splice temp_inst's already-coalesced mem accesses // into the composite, stamping each with source (warp_id, split_id) so // the LDST unit routes pending_writes + scoreboard release back to the @@ -1650,15 +1668,20 @@ bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const { if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() || inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request()) return true; - // Exclude atomics (decrement_atomic_count ties to a single warp) and - // non-global memory spaces. Global/local go through the coalescer + - // mem_fetch path that the source_wid stamp supports. Shared/tex/const - // use per-subwarp / per-line paths not yet per-set-aware. + // Exclude atomics (decrement_atomic_count ties to a single warp). + // Mixed-space MEM co-issue (v2): allow shared_space + sstarr_space in + // addition to the original global/local/param_local. Cross-bucket + // composites are rejected at the candidate-feasibility check in + // try_utilization_max_coissue (same_space only). MEMCOV2_DISABLE_MIXED=1 + // tightens to v1 (global/local only) for A/B testing. if (inst->isatomic()) return true; + static const bool disable_shared = + (getenv("MEMCOV2_DISABLE_MIXED") != NULL); memory_space_t space = inst->space; - if (space.get_type() != global_space && - space.get_type() != local_space && - space.get_type() != param_space_local) + enum _memory_space_t t = space.get_type(); + bool in_global_bucket = warp_inst_t::is_global_bucket_space(t); + bool in_shared_bucket = warp_inst_t::is_shared_bucket_space(t); + if (!in_global_bucket && (disable_shared || !in_shared_bucket)) return true; return false; } @@ -1863,6 +1886,21 @@ void scheduler_unit::try_inter_warp_coissue( m_stats->mem_coissue_denied_filter[get_sid()]++; continue; } + // Same-bucket (Stage 4): primary and candidate must share bucket. + // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging. + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = cand_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any. if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) { @@ -1957,6 +1995,21 @@ void scheduler_unit::try_intra_warp_coissue( m_stats->mem_coissue_denied_filter[get_sid()]++; continue; } + // Same-bucket (Stage 4): primary and candidate must share bucket. + // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging. + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = sec_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; @@ -2061,6 +2114,23 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->mem_coissue_denied_filter[get_sid()]++; continue; } + // Mixed-space MEM co-issue (v2, same-bucket only): primary and + // candidate must be in the same space bucket (shared or global). + // Cross-bucket composites are disallowed here — Stage 5 will add a + // mixed-retire path. MEMCOV2_ALLOW_MIXED=1 bypasses for debugging. + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = cand_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } const active_mask_t &cand_mask = m_shader->get_active_mask(cand_warp_id, cand_inst); @@ -2124,6 +2194,21 @@ void scheduler_unit::try_utilization_max_coissue( m_stats->mem_coissue_denied_filter[get_sid()]++; continue; } + // Same-bucket check: reject cross-bucket intra candidates. + // MEMCOV2_ALLOW_MIXED=1 bypasses for Stage 5 debugging. + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = sec_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; @@ -3198,22 +3283,62 @@ void shader_core_ctx::writeback() { bool ldst_unit::shared_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, mem_stage_access_type &fail_type) { - if (inst.space.get_type() != shared_space) return true; + // Mixed-space MEM co-issue (v2): relax the primary-space gate. Shared + // work can live in (a) the primary's own cycles counter if primary is + // shared, or (b) per-set `cycles` on any valid simd_set whose source + // is a shared coissuer. Return immediately if the instruction has no + // shared work at all. + bool primary_is_shared = (inst.space.get_type() == shared_space || + inst.space.get_type() == sstarr_space); + bool has_shared_coissuer_set = false; + if (inst.has_simd_sets()) { + const std::vector<simd_set_info> &sets = inst.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + enum _memory_space_t src_sp = sets[s].source_inst->space.get_type(); + if (src_sp == shared_space || src_sp == sstarr_space) { + has_shared_coissuer_set = true; + break; + } + } + } + if (!primary_is_shared && !has_shared_coissuer_set) return true; if (inst.active_count() == 0) return true; - if (inst.has_dispatch_delay()) { - m_stats->gpgpu_n_shmem_bank_access[m_sid]++; + bool any_stall = false; + + // Primary's own cycles (only counts if primary is shared). + if (primary_is_shared) { + if (inst.has_dispatch_delay()) { + m_stats->gpgpu_n_shmem_bank_access[m_sid]++; + } + if (inst.dispatch_delay()) any_stall = true; } - bool stall = inst.dispatch_delay(); - if (stall) { + // Per-set cycles for shared coissuer sets. Each set's cycles counts + // down independently; we stall as long as ANY still has cycles > 0. + if (inst.has_simd_sets()) { + std::vector<simd_set_info> &sets = inst.get_simd_sets_mutable(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + enum _memory_space_t src_sp = sets[s].source_inst->space.get_type(); + if (src_sp != shared_space && src_sp != sstarr_space) continue; + if (sets[s].has_dispatch_delay()) { + m_stats->gpgpu_n_shmem_bank_access[m_sid]++; + } + if (sets[s].dispatch_delay()) any_stall = true; + } + } + + if (any_stall) { fail_type = S_MEM; rc_fail = BK_CONF; m_stats->gpgpu_n_shmem_bkconflict++; - } else + } else { rc_fail = NO_RC_FAIL; - return !stall; + } + return !any_stall; } mem_stage_stall_type ldst_unit::process_cache_access( @@ -3229,8 +3354,12 @@ mem_stage_stall_type ldst_unit::process_cache_access( unsigned src_split_access = mf ? mf->get_access_source_split_id() : (unsigned)-1; mem_src_t src = resolve_source(inst, src_wid_access, src_split_access); - bool src_is_intra_coissued = - (src.wid == inst.warp_id() && src_split_access != (unsigned)-1); + // "intra" here means the coissuer came from a secondary ibuffer slot + // (scoreboard was reserved on the secondary map). It does NOT mean + // same-warp-as-primary — try_utilization_max_coissue scans every warp's + // secondary slots, so a coissuer from warp Y's secondary slot can pair + // with warp X as primary. The correct discriminator is split_id != -1. + bool src_is_intra_coissued = (src_split_access != (unsigned)-1); if (write_sent) { unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) ? (mf->get_data_size() / SECTOR_SIZE) @@ -3244,7 +3373,11 @@ mem_stage_stall_type ldst_unit::process_cache_access( inst.accessq_pop_back(); if (inst.is_bru_st_fill_request() || inst.is_bru_rt_fill_request()) { release_virtual_entries(inst); - } else if (inst.is_load()) { + } else if (mf && !mf->get_is_write()) { + // Bug #2 fix: use per-access is_write (NOT composite primary op). + // A STORE-primary composite can carry LOAD coissuer accesses; each + // access's register-decrement or store-ack bookkeeping is selected + // by the access's own type. for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) if (src.out_inst->out[r] > 0) { if (src_is_intra_coissued) @@ -3319,7 +3452,13 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache( NULL) { l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1] = mf; - if (mf->get_inst().is_store()) { + // Bug #2 fix: per-access write bit, NOT composite primary op. + // A LOAD-primary composite can carry STORE coissuer accesses (and + // vice versa). Each mf's own write bit determines whether it + // needs an inc_store_req — the mf will be store_ack'd later when + // WRITE_ACK returns, and that dec_store_req requires a matching + // inc here. + if (mf->get_is_write()) { unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) ? (mf->get_data_size() / SECTOR_SIZE) @@ -3373,16 +3512,18 @@ void ldst_unit::L1_latency_queue_cycle() { if (status == HIT) { assert(!read_sent); l1_latency_queue[j][0] = NULL; - if (mf_next->get_inst().is_load()) { + // Bug #2 fix: per-access dispatch via mf's own write bit, not + // composite primary op. + if (!mf_next->get_is_write()) { // MEM co-issue: the access belongs to mf_next's stamped source // warp. Resolve which instruction's out[] registers should be // decremented (primary's or a co-issued set's). unsigned src_split_mf = mf_next->get_access_source_split_id(); mem_src_t src = resolve_source( mf_next->get_inst(), mf_next->get_wid(), src_split_mf); - bool src_is_intra = - (src.wid == mf_next->get_inst().warp_id() && - src_split_mf != (unsigned)-1); + // "intra" = coissuer came from a secondary ibuffer slot (secondary + // scoreboard map). Not tied to warp equality with primary. + bool src_is_intra = (src_split_mf != (unsigned)-1); bool any_completed = false; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) if (src.out_inst->out[r] > 0) { @@ -3423,8 +3564,8 @@ void ldst_unit::L1_latency_queue_cycle() { } } - // For write hit in WB policy - if (mf_next->get_inst().is_store() && !write_sent) { + // For write hit in WB policy (Bug #2 fix: per-access write bit) + if (mf_next->get_is_write() && !write_sent) { unsigned dec_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC) ? (mf_next->get_data_size() / SECTOR_SIZE) @@ -3443,8 +3584,9 @@ void ldst_unit::L1_latency_queue_cycle() { } else { assert(status == MISS || status == HIT_RESERVED); l1_latency_queue[j][0] = NULL; + // Bug #2 fix: per-access write bit, not composite primary op. if (m_config->m_L1D_config.get_write_policy() != WRITE_THROUGH && - mf_next->get_inst().is_store() && + mf_next->get_is_write() && (m_config->m_L1D_config.get_write_allocate_policy() == FETCH_ON_WRITE || m_config->m_L1D_config.get_write_allocate_policy() == @@ -3518,10 +3660,14 @@ bool ldst_unit::texture_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail, bool ldst_unit::memory_cycle(warp_inst_t &inst, mem_stage_stall_type &stall_reason, mem_stage_access_type &access_type) { - if (inst.empty() || ((inst.space.get_type() != global_space) && - (inst.space.get_type() != local_space) && - (inst.space.get_type() != param_space_local))) - return true; + // Mixed-space MEM co-issue (v2): accessq is the source of truth for + // global/local/param_local work. A shared-primary composite with a + // global coissuer will have global accesses in its accessq (stamped + // with source_wid per access), so we must enter this function even if + // primary's space is shared. Accesses themselves carry their own + // attribution; primary space only matters for the `else` stat-only + // branch at the bottom of this function (access_type encoding). + if (inst.empty()) return true; if (inst.active_count() == 0) return true; if (inst.accessq_empty()) return true; @@ -3570,7 +3716,10 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, m_icnt->push(mf); inst.accessq_pop_back(); // inst.clear_active( access.get_warp_mask() ); - if (inst.is_load()) { + // Bug #2 fix: use per-access is_write, NOT composite primary op. + // A STORE-primary composite can carry LOAD coissuer accesses, and + // vice versa. Each access's type determines its own bookkeeping. + if (!access.is_write()) { // For MEM-co-issue composites, sanity-check against the source // warp's pending_writes on its source out[] regs (if identifiable). const inst_t *src_out_inst = &inst; @@ -3587,8 +3736,9 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst, for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) if (src_out_inst->out[r] > 0) assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0); - } else if (inst.is_store()) + } else { m_core->inc_store_req(src_wid); + } } } } else { @@ -3952,23 +4102,23 @@ ldst_unit::mem_src_t ldst_unit::resolve_source( mem_src_t r; r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid; r.out_inst = &inst; + // Primary access: (wid == primary AND split_id == -1). Return early with + // out_inst = composite (primary's out[]). Note: split_id is the + // discriminator — a coissuer can have the same warp_id as primary when + // it came from the primary warp's own secondary ibuffer slot. + if (r.wid == inst.warp_id() && access_src_split_id == (unsigned)-1) { + return r; + } if (!inst.has_simd_sets()) return r; - // Walk simd_sets for a co-issued set matching BOTH (wid, split_id). For - // inter-warp accesses split_id is -1 and wid differs from primary; for - // intra-warp accesses split_id is non-neg and wid equals primary. Primary - // accesses (source_wid == -1 or split_id == -1 with wid==primary) keep - // out_inst = &inst (composite's out[], which is primary's). - bool is_intra = - (r.wid == inst.warp_id() && access_src_split_id != (unsigned)-1); - bool is_inter = (r.wid != inst.warp_id()); - if (!is_intra && !is_inter) return r; + // Coissued access: search sets for an exact (wid, split_id) match. const std::vector<simd_set_info> &sets = inst.get_simd_sets(); for (unsigned s = 0; s < sets.size(); s++) { if (!sets[s].valid || sets[s].source_inst == NULL) continue; - if (sets[s].warp_id != r.wid) continue; - if (is_intra && sets[s].split_id != access_src_split_id) continue; - r.out_inst = sets[s].source_inst; - break; + if (sets[s].warp_id == r.wid && + sets[s].split_id == access_src_split_id) { + r.out_inst = sets[s].source_inst; + break; + } } return r; } @@ -3979,27 +4129,35 @@ void ldst_unit::issue(register_set ®_set) { // record how many pending register writes/memory accesses there are for this // instruction assert(inst->empty() == false); - if (inst->is_load() and inst->space.get_type() != shared_space) { - unsigned primary_wid = inst->warp_id(); + // Bug #1 fix: a composite's PRIMARY op may be STORE or shared while one + // or more coissuer SETS carry LOAD accesses in non-shared spaces. Don't + // gate the whole pending_writes bookkeeping block on primary. Gate the + // primary portion on primary's own is_load+non-shared; gate per-set + // accumulation on EACH set's source_inst is_load+non-shared. + unsigned primary_wid = inst->warp_id(); + bool primary_is_tracked_load = inst->is_load() && + inst->space.get_type() != shared_space; + if (inst->has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue) { // MEM co-issue: accesses in the composite's queue may originate from // different source warps (inter-warp) or different splits of the same // warp (intra-warp). Group them by (source_wid, source_split_id) to // charge pending_writes to the correct source's destination registers. // Unstamped accesses (source_wid == -1) belong to the primary. - if (inst->has_simd_sets() && - m_config->gpgpu_simd_partitioning && - m_config->gpgpu_mem_coissue) { - std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src; - for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin(); - it != inst->accessq_end(); ++it) { - unsigned w = it->get_source_wid(); - unsigned sp = it->get_source_split_id(); - if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } - n_acc_per_src[std::make_pair(w, sp)] += 1; - } + std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src; + for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin(); + it != inst->accessq_end(); ++it) { + unsigned w = it->get_source_wid(); + unsigned sp = it->get_source_split_id(); + if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; } + n_acc_per_src[std::make_pair(w, sp)] += 1; + } - // Primary's own registers + // Primary's own registers — gated on primary being a tracked load + // (stores have no out[]; shared loads don't use pending_writes). + if (primary_is_tracked_load) { unsigned n_primary = n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)]; if (n_primary > 0) { @@ -4008,50 +4166,53 @@ void ldst_unit::issue(register_set ®_set) { if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary; } } + } - // Per co-issued set's registers (use set's source_inst->out[]). - // Inter-warp sets: different warp_id → primary pending_writes map is - // fine (no key collision with primary). Intra-warp sets: same warp_id - // but different split_id → use the secondary pending_writes map to - // avoid conflating with primary's pending_writes[primary_wid][reg]. - const std::vector<simd_set_info> &sets = inst->get_simd_sets(); - std::set<std::pair<unsigned, unsigned>> accounted; - for (unsigned s = 0; s < sets.size(); s++) { - if (!sets[s].valid || sets[s].source_inst == NULL) continue; - std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id); - if (accounted.count(key)) continue; - accounted.insert(key); - unsigned n_src = n_acc_per_src[key]; - if (n_src == 0) continue; - bool is_intra = (sets[s].warp_id == primary_wid && - sets[s].split_id != (unsigned)-1); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = sets[s].source_inst->out[r]; - if (reg_id > 0) { - if (is_intra) { - m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id] - [reg_id] += n_src; - } else { - m_pending_writes[sets[s].warp_id][reg_id] += n_src; - } + // Per co-issued set's registers (use set's source_inst->out[]). + // Gate per-set on the SET's source being a tracked load, independent + // of primary. A STORE-primary composite can carry LOAD coissuers; + // their pending_writes must still be tracked so their mf returns + // decrement correctly. + const std::vector<simd_set_info> &sets = inst->get_simd_sets(); + std::set<std::pair<unsigned, unsigned>> accounted; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + if (sets[s].source_inst->space.get_type() == shared_space) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id); + if (accounted.count(key)) continue; + accounted.insert(key); + unsigned n_src = n_acc_per_src[key]; + if (n_src == 0) continue; + // is_intra = "uses secondary scoreboard map" = split came from a + // secondary ibuffer slot. Not tied to warp_id == primary_wid. + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id > 0) { + if (is_intra) { + m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id] + [reg_id] += n_src; + } else { + m_pending_writes[sets[s].warp_id][reg_id] += n_src; } } } - } else { - // Legacy (non-co-issue) path: single-warp pending_writes. - unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - unsigned reg_id = inst->out[r]; - if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses; - } } - - if (inst->m_is_ldgsts) { - m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] += - inst->accessq_count(); + } else if (primary_is_tracked_load) { + // Legacy (non-co-issue) path: single-warp pending_writes. + unsigned n_accesses = inst->accessq_count(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = inst->out[r]; + if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses; } } + if (primary_is_tracked_load && inst->m_is_ldgsts) { + m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] += + inst->accessq_count(); + } + inst->op_pipe = MEM__OP; // stat collection m_core->mem_instruction_stats(*inst); @@ -4101,13 +4262,21 @@ void ldst_unit::writeback() { // the source is always the composite's primary here; co-issued sets' // releases are handled in a dedicated pass below. unsigned primary_wid = m_next_wb.warp_id(); - bool is_shared = (m_next_wb.space.get_type() == shared_space); + // Mixed-space MEM co-issue (v2): for MIXED composites with shared + // primary, a global coissuer's mf return arrives here via case 3/4 + // with m_next_wb_src_wid stamped to coissuer_wid. The mf's origin + // is GLOBAL — it should take the non-shared pending_writes path, + // not the shared-primary release path. Use the mf stamp to + // disambiguate: stamped = mf-driven (non-shared routing), unstamped + // = case 0 shared retire (shared routing). + bool is_shared = (m_next_wb.space.get_type() == shared_space && + m_next_wb_src_wid == (unsigned)-1); unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1) ? m_next_wb_src_wid : primary_wid; unsigned src_split = m_next_wb_src_split_id; - bool is_intra_coissued = - (src_wid == primary_wid && src_split != (unsigned)-1); + // is_intra = secondary-slot coissuer (not warp equality with primary). + bool is_intra_coissued = (src_split != (unsigned)-1); mem_src_t src = resolve_source(m_next_wb, src_wid, src_split); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { @@ -4164,8 +4333,8 @@ void ldst_unit::writeback() { sets[s].split_id); if (released.count(key)) continue; released.insert(key); - bool coissued_intra = (sets[s].warp_id == primary_wid && - sets[s].split_id != (unsigned)-1); + // Secondary slot coissuer → secondary scoreboard map. + bool coissued_intra = (sets[s].split_id != (unsigned)-1); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { if (sets[s].source_inst->out[r] > 0) { if (coissued_intra) @@ -4334,6 +4503,28 @@ inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id(); void ldst_unit::cycle() { writeback(); + // MEMCOV2_STATE_DUMP: periodic state-counter dump to detect leaks. + // Prints total pending_writes map sizes + sum of their values every + // 4096 cycles. Used for Stage 5 mixed-retire leak investigation. + static const bool mc_state_dump = (getenv("MEMCOV2_STATE_DUMP") != NULL); + if (mc_state_dump) { + unsigned long long cyc = + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle; + if ((cyc & 0xFFF) == 0) { + unsigned pw_keys = 0, pw_sum = 0; + for (auto &kv : m_pending_writes) + for (auto &rv : kv.second) { pw_keys++; pw_sum += rv.second; } + unsigned sw_keys = 0, sw_sum = 0; + for (auto &kv : m_pending_writes_secondary) + for (auto &sp : kv.second) + for (auto &rv : sp.second) { sw_keys++; sw_sum += rv.second; } + fprintf(stderr, + "[MC2 state@%llu] sid=%u pw_keys=%u pw_sum=%u " + "sw_keys=%u sw_sum=%u\n", + cyc, m_sid, pw_keys, pw_sum, sw_keys, sw_sum); + } + } + for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++) if (m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage + 1]->empty()) move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]); @@ -4412,6 +4603,80 @@ void ldst_unit::cycle() { done &= memory_cycle(pipe_reg, rc_fail, type); m_mem_rc = rc_fail; + // Option A (hardware-faithful per-source scoreboard release) for MIXED + // composites: as soon as the SHARED side's bank-conflict cycles drain, + // release shared-source scoreboards (primary if shared + shared + // coissuer sets) regardless of whether global mfs are still in flight. + // Global-source scoreboards continue to release via the pending_writes + // path in writeback() on their own timeline. This matches hardware + // where the shared-mem bank arbiter signals completion to the + // scoreboard independently of global-memory transactions. One-shot + // guarded by warp_inst_t::shared_side_released() to avoid re-release + // as shared_cycle keeps returning true on subsequent cycles. + if (!pipe_reg.empty() && pipe_reg.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue && + pipe_reg.is_mixed_space_composite() && + !pipe_reg.shared_side_released()) { + bool shared_done = true; + // Primary's own cycles count only if primary itself is shared. + if (warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type()) && + pipe_reg.has_dispatch_delay()) { + shared_done = false; + } + if (shared_done) { + const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + enum _memory_space_t sp = sets[s].source_inst->space.get_type(); + if (warp_inst_t::is_shared_bucket_space(sp) && + sets[s].has_dispatch_delay()) { + shared_done = false; + break; + } + } + } + if (shared_done) { + // Release shared-source scoreboards. + const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets(); + // Primary scoreboard if primary is a shared LOAD. + if (pipe_reg.is_load() && + warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type())) { + m_scoreboard->releaseRegisters(m_dispatch_reg); + } + // Shared coissuer LOAD sets. + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + enum _memory_space_t sp = sets[s].source_inst->space.get_type(); + if (!warp_inst_t::is_shared_bucket_space(sp)) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg = sets[s].source_inst->out[r]; + if (reg == 0) continue; + if (is_intra) + m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg); + else + m_scoreboard->releaseRegister(sets[s].warp_id, reg); + } + } + pipe_reg.set_shared_side_released(true); + if (getenv("MEMCOV2_TRACE")) { + unsigned long long cyc = m_core->get_gpu()->gpu_sim_cycle + + m_core->get_gpu()->gpu_tot_sim_cycle; + fprintf(stderr, + "[MC2 early-rel@%llu] wid=%u primary_space=%d " + "primary_is_load=%d sets=%u\n", + cyc, pipe_reg.warp_id(), (int)pipe_reg.space.get_type(), + (int)pipe_reg.is_load(), (unsigned)sets.size()); + } + } + } + if (!done) { // log stall types and return assert(rc_fail != NO_RC_FAIL); m_stats->gpgpu_n_stall_shd_mem++; @@ -4419,6 +4684,210 @@ void ldst_unit::cycle() { return; } + // Mixed-space MEM co-issue (v2): new inline retirement path for + // composites with BOTH shared and global valid sets. Fires only when + // has_simd_sets + partitioning + mem_coissue enabled + actually mixed. + // Pure-shared (only shared sets) and pure-global (only global sets) + // composites fall through to the v1 load/store branches below + // unchanged. We retire at dispatch_reg (skipping pipeline_reg smem + // chain + case 0 writeback) because mixed composites need to wait for + // BOTH shared-cycle completion and global pending_writes completion; + // routing them through pipeline_reg[smem_latency-1] first would cause + // premature primary-scoreboard release at case 0 while global mfs are + // still in flight (see re-plan doc). + if (!pipe_reg.empty() && pipe_reg.has_simd_sets() && + m_config->gpgpu_simd_partitioning && + m_config->gpgpu_mem_coissue && + pipe_reg.is_mixed_space_composite()) { + unsigned warp_id = pipe_reg.warp_id(); + const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets(); + + // mixed_retire_ready: shared_cycle already returned true (done), and + // memory_cycle returned accessq_empty. We still need to wait for + // outstanding global-source mfs to drain via the pending_writes + // path. A global LOAD coissuer's scoreboard is released by the + // pending_writes decrement in writeback() when its last mf returns; + // our inline retire waits for that BEFORE releasing primary or + // shared coissuer scoreboards. + bool pending_writes_outstanding = false; + // Check primary's own pending_writes (only if primary is a global + // load — shared loads don't use pending_writes). + if (pipe_reg.is_load() && + warp_inst_t::is_global_bucket_space(pipe_reg.space.get_type())) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = pipe_reg.out[r]; + if (reg_id == 0) continue; + auto it = m_pending_writes[warp_id].find(reg_id); + if (it != m_pending_writes[warp_id].end()) { + if (it->second > 0) { + pending_writes_outstanding = true; + break; + } else { + m_pending_writes[warp_id].erase(reg_id); + } + } + } + } + // Per-coissuer check for global LOAD sources. + if (!pending_writes_outstanding) { + std::set<std::pair<unsigned, unsigned>> checked; + for (unsigned s = 0; s < sets.size() && !pending_writes_outstanding; + s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + enum _memory_space_t src_sp = + sets[s].source_inst->space.get_type(); + if (!warp_inst_t::is_global_bucket_space(src_sp)) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (checked.count(key)) continue; + checked.insert(key); + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) { + auto &m = m_pending_writes_secondary[sets[s].warp_id] + [sets[s].split_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_writes_outstanding = true; break; + } + } else { + auto &m = m_pending_writes[sets[s].warp_id]; + auto it = m.find(reg_id); + if (it != m.end() && it->second > 0) { + pending_writes_outstanding = true; break; + } + } + } + } + } + static const bool mc_trace = (getenv("MEMCOV2_TRACE") != NULL); + if (pending_writes_outstanding) { + if (mc_trace) { + unsigned long long cyc = + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle; + fprintf(stderr, + "[MC2 stall@%llu] wid=%u pending_writes_outstanding\n", + cyc, warp_id); + } + return; + } + + // RF write-port fidelity: stall if operand collector not ready. + if (!m_operand_collector->writeback(*m_dispatch_reg)) { + if (mc_trace) { + unsigned long long cyc = + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle; + if ((cyc & 0x3FFF) == 0) + fprintf(stderr, "[MC2 stall@%llu] wid=%u opnd_coll not ready\n", + cyc, warp_id); + } + return; + } + if (mc_trace) { + unsigned long long cyc = + m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle; + fprintf(stderr, + "[MC2 retire@%llu] wid=%u is_load=%d primary_space=%d sets=%u\n", + cyc, warp_id, (int)pipe_reg.is_load(), + (int)pipe_reg.space.get_type(), (unsigned)sets.size()); + } + + // Primary retirement: release primary scoreboard if LOAD; always + // call warp_inst_complete and dec_inst_in_pipeline for primary. + // Stage 6 (Option A): if primary is shared, its scoreboard was + // already released in the early-release block above — skip here to + // avoid a spurious release that could race with a subsequent + // reservation on the same reg. + bool primary_is_shared = + warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type()); + if (pipe_reg.is_load() && + !(primary_is_shared && pipe_reg.shared_side_released())) { + m_scoreboard->releaseRegisters(m_dispatch_reg); + } + m_core->warp_inst_complete(*m_dispatch_reg); + + // Release ONLY shared-source coissuer scoreboards that haven't been + // released yet. Global-source coissuer scoreboards were already + // released via the pending_writes decrement in writeback() when + // their last mf returned; releasing again here would race with the + // coissuer's next-instruction reservation. Shared-source coissuer + // scoreboards are released here OR in the early-release block above + // (Stage 6 Option A) — guarded by shared_side_released() to avoid + // double-release. + if (!pipe_reg.shared_side_released()) { + std::set<std::pair<unsigned, unsigned>> released_shared; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + enum _memory_space_t src_sp = + sets[s].source_inst->space.get_type(); + if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released_shared.count(key)) continue; + released_shared.insert(key); + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) + m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg_id); + else + m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + } + } + } + + // Defensive: global-source coissuer pending_writes should be drained + // by now (we waited on them above). Assert to catch bookkeeping drift. + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + enum _memory_space_t src_sp = + sets[s].source_inst->space.get_type(); + if (!warp_inst_t::is_global_bucket_space(src_sp)) continue; + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) { + auto &m = m_pending_writes_secondary[sets[s].warp_id] + [sets[s].split_id]; + auto it = m.find(reg_id); + assert(it == m.end() || it->second == 0); + } else { + auto &m = m_pending_writes[sets[s].warp_id]; + auto it = m.find(reg_id); + assert(it == m.end() || it->second == 0); + } + } + } + + // dec_inst_in_pipeline for primary + each unique coissuer. Mirrors + // case 0 shared writeback dedup: inter-warp coissuer decs once per + // warp; intra (secondary slot of primary warp) decs primary warp + // a second time. + m_core->dec_inst_in_pipeline(warp_id); + std::set<unsigned> inter_dec; + bool intra_dec = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + if (inter_dec.insert(sets[s].warp_id).second) + m_core->dec_inst_in_pipeline(sets[s].warp_id); + } else if (!intra_dec) { + m_core->dec_inst_in_pipeline(warp_id); + intra_dec = true; + } + } + + m_dispatch_reg->clear(); + return; + } + if (!pipe_reg.empty()) { unsigned warp_id = pipe_reg.warp_id(); if (pipe_reg.is_load()) { @@ -4464,8 +4933,8 @@ void ldst_unit::cycle() { sets[s].split_id); if (checked.count(key)) continue; checked.insert(key); - bool is_intra = (sets[s].warp_id == warp_id && - sets[s].split_id != (unsigned)-1); + // Secondary slot → secondary pending_writes map. + bool is_intra = (sets[s].split_id != (unsigned)-1); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id == 0) continue; @@ -4507,8 +4976,8 @@ void ldst_unit::cycle() { sets[s].split_id); if (released.count(key)) continue; released.insert(key); - bool is_intra = (sets[s].warp_id == warp_id && - sets[s].split_id != (unsigned)-1); + // Secondary slot → secondary scoreboard map. + bool is_intra = (sets[s].split_id != (unsigned)-1); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id == 0) continue; @@ -4576,6 +5045,36 @@ void ldst_unit::cycle() { intra_dec = true; } } + // Mixed-space MEM co-issue (v2): for a STORE-primary composite, + // SHARED LOAD coissuers bypass all the normal scoreboard-release + // machinery (no mf/pending_writes for shared, no case 0 writeback + // since composite never enters pipeline_reg via is_load branch). + // Release their scoreboards here explicitly, mirroring case 0 + // writeback's shared-coissue release. Global LOAD coissuers are + // handled by the pending_writes decrement path in writeback() and + // MUST NOT be released here (would double-release). + std::set<std::pair<unsigned, unsigned>> released; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (!sets[s].source_inst->is_load()) continue; + enum _memory_space_t src_sp = + sets[s].source_inst->space.get_type(); + if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue; + std::pair<unsigned, unsigned> key(sets[s].warp_id, + sets[s].split_id); + if (released.count(key)) continue; + released.insert(key); + bool is_intra = (sets[s].split_id != (unsigned)-1); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + unsigned reg_id = sets[s].source_inst->out[r]; + if (reg_id == 0) continue; + if (is_intra) + m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, + reg_id); + else + m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + } + } } m_dispatch_reg->clear(); } |