co-issue global mem requests

1 files changed, 520 insertions, 56 deletions

diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index c874733..f96e939 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -830,6 +830,19 @@ void shader_core_stats::print(FILE *fout) const {
     fprintf(fout, "gpu_tot_coissue_skipped_no_inst = %llu\n", tot_noinst);
     fprintf(fout, "gpu_tot_coissue_skipped_pc_mismatch = %llu\n", tot_pcm);
     fprintf(fout, "gpu_tot_coissue_skipped_samepc_pass0 = %llu\n", tot_spc0);
+
+    // MEM co-issue-specific subtotals
+    unsigned long long tot_mem_inter = 0, tot_mem_intra = 0,
+                       tot_mem_denied = 0;
+    for (unsigned i = 0; i < m_config->num_shader(); i++) {
+      tot_mem_inter += mem_coissue_inter_events[i];
+      tot_mem_intra += mem_coissue_intra_events[i];
+      tot_mem_denied += mem_coissue_denied_filter[i];
+    }
+    fprintf(fout, "gpu_tot_mem_coissue_inter_events = %llu\n", tot_mem_inter);
+    fprintf(fout, "gpu_tot_mem_coissue_intra_events = %llu\n", tot_mem_intra);
+    fprintf(fout, "gpu_tot_mem_coissue_denied_filter = %llu\n",
+            tot_mem_denied);
   }
 }
 
@@ -1457,8 +1470,29 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite,
   // Set next PC for the co-issued warp
   m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize);
 
-  // Tag the co-issued sets with their source instruction for writeback
+  // Tag the co-issued sets with their source instruction + split_id for
+  // writeback routing (intra-warp composites need split_id to disambiguate
+  // between primary and secondary-split sets that share warp_id).
   temp_inst.set_source_inst_on_sets(next_inst);
+  temp_inst.set_split_id_on_sets(split_id);
+
+  // For MEM co-issue: splice temp_inst's already-coalesced mem accesses
+  // into the composite, stamping each with source (warp_id, split_id) so
+  // the LDST unit routes pending_writes + scoreboard release back to the
+  // originating warp/split. This must happen BEFORE merge_simd_sets
+  // because we also need the temp_inst's own simd_sets to still carry
+  // valid flags here; but the accessq itself is independent.
+  if (temp_inst.is_load() || temp_inst.is_store()) {
+    unsigned src_wid = warp_id;
+    unsigned src_split = split_id;  // -1 for inter-warp
+    while (!temp_inst.accessq_empty()) {
+      mem_access_t acc = temp_inst.accessq_back();
+      acc.set_source_wid(src_wid);
+      acc.set_source_split_id(src_split);
+      composite->inject_mem_acccesses(acc);
+      temp_inst.accessq_pop_back();
+    }
+  }
 
   // Merge the co-issued instruction's valid sets into the composite
   composite->merge_simd_sets(temp_inst.get_simd_sets());
@@ -1604,6 +1638,31 @@ void scheduler_unit::order_by_priority(
 // Change 4: co-issue scheduler helpers.
 // ---------------------------------------------------------------------------
 
+bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const {
+  if (!inst) return true;
+  // Exclude TENSOR_CORE loads/stores (quirky pipeline), memory barriers,
+  // LDGSTS, and BRU spill/fill. These remain MEM-bucketed at classify time
+  // but are rejected from co-issue.
+  if (inst->op == TENSOR_CORE_LOAD_OP || inst->op == TENSOR_CORE_STORE_OP)
+    return true;
+  if (inst->op == MEMORY_BARRIER_OP) return true;
+  if (inst->m_is_ldgsts) return true;
+  if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() ||
+      inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request())
+    return true;
+  // Exclude atomics (decrement_atomic_count ties to a single warp) and
+  // non-global memory spaces. Global/local go through the coalescer +
+  // mem_fetch path that the source_wid stamp supports. Shared/tex/const
+  // use per-subwarp / per-line paths not yet per-set-aware.
+  if (inst->isatomic()) return true;
+  memory_space_t space = inst->space;
+  if (space.get_type() != global_space &&
+      space.get_type() != local_space &&
+      space.get_type() != param_space_local)
+    return true;
+  return false;
+}
+
 exec_unit_type_t scheduler_unit::classify_fu_type(
     const warp_inst_t *inst) const {
   if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) ||
@@ -1797,6 +1856,14 @@ void scheduler_unit::try_inter_warp_coissue(
         continue;
       }
 
+      // MEM co-issue safety filter.
+      if (co_issue_fu_type == exec_unit_type_t::MEM &&
+          mem_coissue_disallowed(cand_inst)) {
+        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+        m_stats->mem_coissue_denied_filter[get_sid()]++;
+        continue;
+      }
+
       // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any.
       if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) {
         m_stats->coissue_skipped_samepc_pass0[get_sid()]++;
@@ -1832,6 +1899,8 @@ void scheduler_unit::try_inter_warp_coissue(
       warp(cand_warp_id).ibuffer_step();
       picked_warps.insert(cand_warp_id);
       m_stats->inter_warp_coissue_events[get_sid()]++;
+      if (co_issue_fu_type == exec_unit_type_t::MEM)
+        m_stats->mem_coissue_inter_events[get_sid()]++;
     }
   }
 }
@@ -1881,6 +1950,14 @@ void scheduler_unit::try_intra_warp_coissue(
       continue;
     }
 
+    // MEM co-issue safety filter.
+    if (co_issue_fu_type == exec_unit_type_t::MEM &&
+        mem_coissue_disallowed(sec_inst)) {
+      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+      m_stats->mem_coissue_denied_filter[get_sid()]++;
+      continue;
+    }
+
     if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) {
       m_stats->coissue_denied_by_scoreboard[get_sid()]++;
       continue;
@@ -1909,6 +1986,8 @@ void scheduler_unit::try_intra_warp_coissue(
     available_sets -= sec_sets_needed;
     next_free_set += sec_sets_needed;
     m_stats->intra_warp_coissue_events[get_sid()]++;
+    if (co_issue_fu_type == exec_unit_type_t::MEM)
+      m_stats->mem_coissue_intra_events[get_sid()]++;
 
     warp(primary_warp_id).ibuffer_free_slot(sec_slot);
 
@@ -1976,6 +2055,12 @@ void scheduler_unit::try_utilization_max_coissue(
       m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
       continue;
     }
+    if (co_issue_fu_type == exec_unit_type_t::MEM &&
+        mem_coissue_disallowed(cand_inst)) {
+      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+      m_stats->mem_coissue_denied_filter[get_sid()]++;
+      continue;
+    }
 
     const active_mask_t &cand_mask =
         m_shader->get_active_mask(cand_warp_id, cand_inst);
@@ -2033,6 +2118,12 @@ void scheduler_unit::try_utilization_max_coissue(
           m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
           continue;
         }
+        if (co_issue_fu_type == exec_unit_type_t::MEM &&
+            mem_coissue_disallowed(sec_inst)) {
+          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+          m_stats->mem_coissue_denied_filter[get_sid()]++;
+          continue;
+        }
         if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) {
           m_stats->coissue_denied_by_scoreboard[get_sid()]++;
           continue;
@@ -2106,6 +2197,8 @@ void scheduler_unit::try_utilization_max_coissue(
       available_sets -= needed;
       next_free_set += needed;
       m_stats->intra_warp_coissue_events[get_sid()]++;
+      if (co_issue_fu_type == exec_unit_type_t::MEM)
+        m_stats->mem_coissue_intra_events[get_sid()]++;
       warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
       if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
         warp(c.warp_id).ibuffer_flush_half(1);
@@ -2136,6 +2229,8 @@ void scheduler_unit::try_utilization_max_coissue(
       available_sets -= needed;
       next_free_set += needed;
       m_stats->inter_warp_coissue_events[get_sid()]++;
+      if (co_issue_fu_type == exec_unit_type_t::MEM)
+        m_stats->mem_coissue_inter_events[get_sid()]++;
       warp(c.warp_id).ibuffer_step();
     }
   }
@@ -2251,8 +2346,19 @@ void scheduler_unit::cycle() {
                                       m_id) &&
                   (!diff_exec_units ||
                    previous_issued_inst_exec_type != exec_unit_type_t::MEM)) {
-                m_shader->issue_warp(*m_mem_out, pI, active_mask, warp_id,
-                                     m_id);
+                warp_inst_t *issued_pipe_reg = m_shader->issue_warp(
+                    *m_mem_out, pI, active_mask, warp_id, m_id);
+                // When MEM co-issue is enabled, capture the composite pointer
+                // so the dispatch loop below can attempt to merge co-issuers
+                // into this MEM pipeline slot. Otherwise leave co_issue_*
+                // variables unset for bit-parity with the pre-MEM-coissue tree.
+                if (m_shader->m_config->gpgpu_simd_partitioning &&
+                    m_shader->m_config->gpgpu_mem_coissue) {
+                  co_issue_composite = issued_pipe_reg;
+                  co_issue_fu_type = exec_unit_type_t::MEM;
+                  co_issue_reg_set = m_mem_out;
+                  co_issue_primary_warp_id = warp_id;
+                }
                 issued++;
                 issued_inst = true;
                 warp_inst_issued = true;
@@ -2484,10 +2590,13 @@ void scheduler_unit::cycle() {
 
   // SIMD set co-issue dispatch: after issuing the primary instruction,
   // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority.
-  // Only for non-MEM FUs (LDST excluded from partitioning).
+  // MEM co-issue is opt-in via -gpgpu_mem_coissue (the primary-issue branch
+  // above leaves co_issue_composite=NULL when the flag is off, so this
+  // block is skipped naturally in that case).
   if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL &&
-      co_issue_fu_type != exec_unit_type_t::MEM &&
-      co_issue_fu_type != exec_unit_type_t::NONE) {
+      co_issue_fu_type != exec_unit_type_t::NONE &&
+      !(co_issue_fu_type == exec_unit_type_t::MEM &&
+        mem_coissue_disallowed(co_issue_composite))) {
     available_sets = 0;
     next_free_set = 0;
     const std::vector<simd_set_info> &primary_sets =
@@ -3114,13 +3223,21 @@ mem_stage_stall_type ldst_unit::process_cache_access(
   mem_stage_stall_type result = NO_RC_FAIL;
   bool write_sent = was_write_sent(events);
   bool read_sent = was_read_sent(events);
+  // Attribute store acks / load HIT register decrements to the access's
+  // source warp + its source instruction (for MEM co-issue composites).
+  unsigned src_wid_access = mf ? mf->get_wid() : inst.warp_id();
+  unsigned src_split_access =
+      mf ? mf->get_access_source_split_id() : (unsigned)-1;
+  mem_src_t src = resolve_source(inst, src_wid_access, src_split_access);
+  bool src_is_intra_coissued =
+      (src.wid == inst.warp_id() && src_split_access != (unsigned)-1);
   if (write_sent) {
     unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                            ? (mf->get_data_size() / SECTOR_SIZE)
                            : 1;
 
     for (unsigned i = 0; i < inc_ack; ++i)
-      m_core->inc_store_req(inst.warp_id());
+      m_core->inc_store_req(src.wid);
   }
   if (status == HIT) {
     assert(!read_sent);
@@ -3129,9 +3246,15 @@ mem_stage_stall_type ldst_unit::process_cache_access(
       release_virtual_entries(inst);
     } else if (inst.is_load()) {
       for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
-        if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--;
+        if (src.out_inst->out[r] > 0) {
+          if (src_is_intra_coissued)
+            m_pending_writes_secondary[src.wid][src_split_access]
+                                      [src.out_inst->out[r]]--;
+          else
+            m_pending_writes[src.wid][src.out_inst->out[r]]--;
+        }
 
-      // release LDGSTS
+      // release LDGSTS (primary-only: LDGSTS is excluded from MEM co-issue)
       if (inst.m_is_ldgsts) {
         m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)]--;
         if (m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)] == 0) {
@@ -3203,7 +3326,7 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache(
                   : 1;
 
           for (unsigned i = 0; i < inc_ack; ++i)
-            m_core->inc_store_req(inst.warp_id());
+            m_core->inc_store_req(mf->get_wid());
         }
 
         inst.accessq_pop_back();
@@ -3251,23 +3374,43 @@ void ldst_unit::L1_latency_queue_cycle() {
         assert(!read_sent);
         l1_latency_queue[j][0] = NULL;
         if (mf_next->get_inst().is_load()) {
+          // MEM co-issue: the access belongs to mf_next's stamped source
+          // warp. Resolve which instruction's out[] registers should be
+          // decremented (primary's or a co-issued set's).
+          unsigned src_split_mf = mf_next->get_access_source_split_id();
+          mem_src_t src = resolve_source(
+              mf_next->get_inst(), mf_next->get_wid(), src_split_mf);
+          bool src_is_intra =
+              (src.wid == mf_next->get_inst().warp_id() &&
+               src_split_mf != (unsigned)-1);
+          bool any_completed = false;
           for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
-            if (mf_next->get_inst().out[r] > 0) {
-              assert(m_pending_writes[mf_next->get_inst().warp_id()]
-                                     [mf_next->get_inst().out[r]] > 0);
-              unsigned still_pending =
-                  --m_pending_writes[mf_next->get_inst().warp_id()]
-                                    [mf_next->get_inst().out[r]];
-              if (!still_pending) {
-                m_pending_writes[mf_next->get_inst().warp_id()].erase(
-                    mf_next->get_inst().out[r]);
-                m_scoreboard->releaseRegister(mf_next->get_inst().warp_id(),
-                                              mf_next->get_inst().out[r]);
-                m_core->warp_inst_complete(mf_next->get_inst());
+            if (src.out_inst->out[r] > 0) {
+              unsigned reg = src.out_inst->out[r];
+              if (src_is_intra) {
+                assert(m_pending_writes_secondary[src.wid][src_split_mf]
+                                                 [reg] > 0);
+                unsigned still_pending = --m_pending_writes_secondary
+                    [src.wid][src_split_mf][reg];
+                if (!still_pending) {
+                  m_pending_writes_secondary[src.wid][src_split_mf].erase(
+                      reg);
+                  m_scoreboard->releaseRegisterSecondary(src.wid, reg);
+                  any_completed = true;
+                }
+              } else {
+                assert(m_pending_writes[src.wid][reg] > 0);
+                unsigned still_pending = --m_pending_writes[src.wid][reg];
+                if (!still_pending) {
+                  m_pending_writes[src.wid].erase(reg);
+                  m_scoreboard->releaseRegister(src.wid, reg);
+                  any_completed = true;
+                }
               }
             }
+          if (any_completed) m_core->warp_inst_complete(mf_next->get_inst());
 
-          // release LDGSTS
+          // release LDGSTS (primary-only; LDGSTS excluded from MEM co-issue)
           if (mf_next->get_inst().m_is_ldgsts) {
             m_pending_ldgsts[mf_next->get_inst().warp_id()]
                             [mf_next->get_inst().pc]
@@ -3417,6 +3560,9 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst,
         stall_cond = ICNT_RC_FAIL;
         break;
       } else {
+        // Resolve source attribution for MEM co-issued accesses.
+        unsigned src_wid = access.get_source_wid();
+        if (src_wid == (unsigned)-1) src_wid = inst.warp_id();
         mem_fetch *mf =
             m_mf_allocator->alloc(inst, access,
                                   m_core->get_gpu()->gpu_sim_cycle +
@@ -3425,11 +3571,24 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst,
         inst.accessq_pop_back();
         // inst.clear_active( access.get_warp_mask() );
         if (inst.is_load()) {
+          // For MEM-co-issue composites, sanity-check against the source
+          // warp's pending_writes on its source out[] regs (if identifiable).
+          const inst_t *src_out_inst = &inst;
+          if (src_wid != inst.warp_id() && inst.has_simd_sets()) {
+            const std::vector<simd_set_info> &sets = inst.get_simd_sets();
+            for (unsigned s = 0; s < sets.size(); s++) {
+              if (sets[s].valid && sets[s].source_inst != NULL &&
+                  sets[s].warp_id == src_wid) {
+                src_out_inst = sets[s].source_inst;
+                break;
+              }
+            }
+          }
           for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
-            if (inst.out[r] > 0)
-              assert(m_pending_writes[inst.warp_id()][inst.out[r]] > 0);
+            if (src_out_inst->out[r] > 0)
+              assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0);
         } else if (inst.is_store())
-          m_core->inc_store_req(inst.warp_id());
+          m_core->inc_store_req(src_wid);
       }
     }
   } else {
@@ -3739,6 +3898,8 @@ void ldst_unit::init(mem_fetch_interface *icnt,
       5;  // = shared memory, global/local (uncached), L1D, L1T, L1C
   m_writeback_arb = 0;
   m_next_global = NULL;
+  m_next_wb_src_wid = (unsigned)-1;
+  m_next_wb_src_split_id = (unsigned)-1;
   m_last_inst_gpu_sim_cycle = 0;
   m_last_inst_gpu_tot_sim_cycle = 0;
 }
@@ -3785,6 +3946,33 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt,
        mem_config, stats, sid, tpc);
 }
 
+ldst_unit::mem_src_t ldst_unit::resolve_source(
+    const warp_inst_t &inst, unsigned access_src_wid,
+    unsigned access_src_split_id) const {
+  mem_src_t r;
+  r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid;
+  r.out_inst = &inst;
+  if (!inst.has_simd_sets()) return r;
+  // Walk simd_sets for a co-issued set matching BOTH (wid, split_id). For
+  // inter-warp accesses split_id is -1 and wid differs from primary; for
+  // intra-warp accesses split_id is non-neg and wid equals primary. Primary
+  // accesses (source_wid == -1 or split_id == -1 with wid==primary) keep
+  // out_inst = &inst (composite's out[], which is primary's).
+  bool is_intra =
+      (r.wid == inst.warp_id() && access_src_split_id != (unsigned)-1);
+  bool is_inter = (r.wid != inst.warp_id());
+  if (!is_intra && !is_inter) return r;
+  const std::vector<simd_set_info> &sets = inst.get_simd_sets();
+  for (unsigned s = 0; s < sets.size(); s++) {
+    if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+    if (sets[s].warp_id != r.wid) continue;
+    if (is_intra && sets[s].split_id != access_src_split_id) continue;
+    r.out_inst = sets[s].source_inst;
+    break;
+  }
+  return r;
+}
+
 void ldst_unit::issue(register_set &reg_set) {
   warp_inst_t *inst = *(reg_set.get_ready());
 
@@ -3792,16 +3980,75 @@ void ldst_unit::issue(register_set &reg_set) {
   // instruction
   assert(inst->empty() == false);
   if (inst->is_load() and inst->space.get_type() != shared_space) {
-    unsigned warp_id = inst->warp_id();
-    unsigned n_accesses = inst->accessq_count();
-    for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
-      unsigned reg_id = inst->out[r];
-      if (reg_id > 0) {
-        m_pending_writes[warp_id][reg_id] += n_accesses;
+    unsigned primary_wid = inst->warp_id();
+
+    // MEM co-issue: accesses in the composite's queue may originate from
+    // different source warps (inter-warp) or different splits of the same
+    // warp (intra-warp). Group them by (source_wid, source_split_id) to
+    // charge pending_writes to the correct source's destination registers.
+    // Unstamped accesses (source_wid == -1) belong to the primary.
+    if (inst->has_simd_sets() &&
+        m_config->gpgpu_simd_partitioning &&
+        m_config->gpgpu_mem_coissue) {
+      std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src;
+      for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin();
+           it != inst->accessq_end(); ++it) {
+        unsigned w = it->get_source_wid();
+        unsigned sp = it->get_source_split_id();
+        if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; }
+        n_acc_per_src[std::make_pair(w, sp)] += 1;
+      }
+
+      // Primary's own registers
+      unsigned n_primary =
+          n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)];
+      if (n_primary > 0) {
+        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+          unsigned reg_id = inst->out[r];
+          if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary;
+        }
+      }
+
+      // Per co-issued set's registers (use set's source_inst->out[]).
+      // Inter-warp sets: different warp_id → primary pending_writes map is
+      // fine (no key collision with primary). Intra-warp sets: same warp_id
+      // but different split_id → use the secondary pending_writes map to
+      // avoid conflating with primary's pending_writes[primary_wid][reg].
+      const std::vector<simd_set_info> &sets = inst->get_simd_sets();
+      std::set<std::pair<unsigned, unsigned>> accounted;
+      for (unsigned s = 0; s < sets.size(); s++) {
+        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+        std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
+        if (accounted.count(key)) continue;
+        accounted.insert(key);
+        unsigned n_src = n_acc_per_src[key];
+        if (n_src == 0) continue;
+        bool is_intra = (sets[s].warp_id == primary_wid &&
+                         sets[s].split_id != (unsigned)-1);
+        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+          unsigned reg_id = sets[s].source_inst->out[r];
+          if (reg_id > 0) {
+            if (is_intra) {
+              m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id]
+                                        [reg_id] += n_src;
+            } else {
+              m_pending_writes[sets[s].warp_id][reg_id] += n_src;
+            }
+          }
+        }
+      }
+    } else {
+      // Legacy (non-co-issue) path: single-warp pending_writes.
+      unsigned n_accesses = inst->accessq_count();
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+        unsigned reg_id = inst->out[r];
+        if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses;
       }
     }
+
     if (inst->m_is_ldgsts) {
-      m_pending_ldgsts[warp_id][inst->pc][inst->get_addr(0)] += n_accesses;
+      m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] +=
+          inst->accessq_count();
     }
   }
 
@@ -3847,35 +4094,92 @@ void ldst_unit::writeback() {
         m_next_wb.is_bru_rt_fill_request()) {
       release_virtual_entries(m_next_wb);
     } else if (m_operand_collector->writeback(m_next_wb)) {
+      {
       bool insn_completed = false;
+      // Resolve (src_wid, src_out_inst) for this writeback.  For non-shared
+      // spaces the source is stamped on the feeding mem_fetch.  For shared
+      // the source is always the composite's primary here; co-issued sets'
+      // releases are handled in a dedicated pass below.
+      unsigned primary_wid = m_next_wb.warp_id();
+      bool is_shared = (m_next_wb.space.get_type() == shared_space);
+      unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1)
+                             ? m_next_wb_src_wid
+                             : primary_wid;
+      unsigned src_split = m_next_wb_src_split_id;
+      bool is_intra_coissued =
+          (src_wid == primary_wid && src_split != (unsigned)-1);
+      mem_src_t src = resolve_source(m_next_wb, src_wid, src_split);
+
       for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
-        if (m_next_wb.out[r] > 0) {
-          if (m_next_wb.space.get_type() != shared_space) {
-            assert(m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0);
-            unsigned still_pending =
-                --m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]];
-            if (!still_pending) {
-              m_pending_writes[m_next_wb.warp_id()].erase(m_next_wb.out[r]);
-              m_scoreboard->releaseRegister(m_next_wb.warp_id(),
-                                            m_next_wb.out[r]);
-              insn_completed = true;
+        if (src.out_inst->out[r] > 0) {
+          if (!is_shared) {
+            unsigned reg = src.out_inst->out[r];
+            if (is_intra_coissued) {
+              // Intra-warp co-issued: pending count lives in the secondary
+              // map (keyed by (wid, split_id, reg)). Decrement there; when
+              // zero, release from secondary scoreboard.
+              assert(m_pending_writes_secondary[src.wid][src_split][reg] > 0);
+              unsigned still_pending =
+                  --m_pending_writes_secondary[src.wid][src_split][reg];
+              if (!still_pending) {
+                m_pending_writes_secondary[src.wid][src_split].erase(reg);
+                m_scoreboard->releaseRegisterSecondary(src.wid, reg);
+                insn_completed = true;
+              }
+            } else {
+              assert(m_pending_writes[src.wid][reg] > 0);
+              unsigned still_pending = --m_pending_writes[src.wid][reg];
+              if (!still_pending) {
+                m_pending_writes[src.wid].erase(reg);
+                m_scoreboard->releaseRegister(src.wid, reg);
+                insn_completed = true;
+              }
             }
-          } else {  // shared
-            m_scoreboard->releaseRegister(m_next_wb.warp_id(),
-                                          m_next_wb.out[r]);
+          } else {  // shared — release primary scoreboard only here;
+                   // composite per-set releases happen below.
+            m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]);
             insn_completed = true;
           }
-        } else if (m_next_wb.m_is_ldgsts) {  // for LDGSTS instructions where no
-                                             // output register is used
-          m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc]
+        } else if (m_next_wb.m_is_ldgsts) {  // LDGSTS excluded from co-issue
+          m_pending_ldgsts[primary_wid][m_next_wb.pc]
                           [m_next_wb.get_addr(0)]--;
-          if (m_pending_ldgsts[m_next_wb.warp_id()][m_next_wb.pc]
+          if (m_pending_ldgsts[primary_wid][m_next_wb.pc]
                               [m_next_wb.get_addr(0)] == 0) {
             insn_completed = true;
           }
           break;
         }
       }
+
+      // Shared-memory composite: release each co-issued set's scoreboard
+      // regs in one pass (shared writeback is single-shot, not per-mf).
+      if (is_shared && m_next_wb.has_simd_sets() &&
+          m_config->gpgpu_simd_partitioning &&
+          m_config->gpgpu_mem_coissue) {
+        const std::vector<simd_set_info> &sets = m_next_wb.get_simd_sets();
+        std::set<std::pair<unsigned, unsigned>> released;
+        for (unsigned s = 0; s < sets.size(); s++) {
+          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+          std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                             sets[s].split_id);
+          if (released.count(key)) continue;
+          released.insert(key);
+          bool coissued_intra = (sets[s].warp_id == primary_wid &&
+                                  sets[s].split_id != (unsigned)-1);
+          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+            if (sets[s].source_inst->out[r] > 0) {
+              if (coissued_intra)
+                m_scoreboard->releaseRegisterSecondary(
+                    sets[s].warp_id, sets[s].source_inst->out[r]);
+              else
+                m_scoreboard->releaseRegister(
+                    sets[s].warp_id, sets[s].source_inst->out[r]);
+              insn_completed = true;
+            }
+          }
+        }
+      }
+
       if (insn_completed) {
         m_core->warp_inst_complete(m_next_wb);
         if (m_next_wb.m_is_ldgsts) {
@@ -3884,10 +4188,13 @@ void ldst_unit::writeback() {
       }
 
       m_next_wb.clear();
+      m_next_wb_src_wid = (unsigned)-1;
+      m_next_wb_src_split_id = (unsigned)-1;
       m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle;
       m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle;
-    }
-  }
+      }  // close inner block opened at line ~4176
+    }  // close else
+  }  // close if (!m_next_wb.empty())
 
   unsigned serviced_client = -1;
   for (unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients);
@@ -3897,12 +4204,42 @@ void ldst_unit::writeback() {
       case 0:  // shared memory
         if (!m_pipeline_reg[0]->empty()) {
           m_next_wb = *m_pipeline_reg[0];
+          // Shared-mem writeback bypasses mem_fetch: there is no
+          // per-transaction source. Treat this as primary-only. For a MEM
+          // co-issue composite, enumerate the composite's simd_sets and
+          // release each source's scoreboard/pipeline in a dedicated pass
+          // below; here we just stamp to "primary" so the default path
+          // handles the primary portion.
+          m_next_wb_src_wid = (unsigned)-1;
+          m_next_wb_src_split_id = (unsigned)-1;
           if (m_next_wb.isatomic()) {
             m_next_wb.do_atomic();
             m_core->decrement_atomic_count(m_next_wb.warp_id(),
                                            m_next_wb.active_count());
           }
-          m_core->dec_inst_in_pipeline(m_pipeline_reg[0]->warp_id());
+          // dec_inst_in_pipeline for each unique (source wid, split_id)
+          // participating in the composite (mirrors SP writeback at
+          // shader.cc~3019-3044). Non-composite MEM inst still dec's once.
+          unsigned primary_wid = m_pipeline_reg[0]->warp_id();
+          m_core->dec_inst_in_pipeline(primary_wid);
+          if (m_next_wb.has_simd_sets() &&
+              m_config->gpgpu_simd_partitioning &&
+              m_config->gpgpu_mem_coissue) {
+            std::set<unsigned> inter_dec;
+            bool intra_dec = false;
+            const std::vector<simd_set_info> &sets =
+                m_next_wb.get_simd_sets();
+            for (unsigned s = 0; s < sets.size(); s++) {
+              if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+              if (sets[s].warp_id != primary_wid) {
+                if (inter_dec.insert(sets[s].warp_id).second)
+                  m_core->dec_inst_in_pipeline(sets[s].warp_id);
+              } else if (!intra_dec) {
+                m_core->dec_inst_in_pipeline(primary_wid);
+                intra_dec = true;
+              }
+            }
+          }
           m_pipeline_reg[0]->clear();
           serviced_client = next_client;
         }
@@ -3911,6 +4248,8 @@ void ldst_unit::writeback() {
         if (m_L1T->access_ready()) {
           mem_fetch *mf = m_L1T->next_access();
           m_next_wb = mf->get_inst();
+          m_next_wb_src_wid = mf->get_wid();
+          m_next_wb_src_split_id = mf->get_access_source_split_id();
           delete mf;
           serviced_client = next_client;
         }
@@ -3919,6 +4258,8 @@ void ldst_unit::writeback() {
         if (m_L1C->access_ready()) {
           mem_fetch *mf = m_L1C->next_access();
           m_next_wb = mf->get_inst();
+          m_next_wb_src_wid = mf->get_wid();
+          m_next_wb_src_split_id = mf->get_access_source_split_id();
           delete mf;
           serviced_client = next_client;
         }
@@ -3926,6 +4267,9 @@ void ldst_unit::writeback() {
       case 3:  // global/local
         if (m_next_global) {
           m_next_wb = m_next_global->get_inst();
+          m_next_wb_src_wid = m_next_global->get_wid();
+          m_next_wb_src_split_id =
+              m_next_global->get_access_source_split_id();
           if (m_next_global->isatomic()) {
             m_core->decrement_atomic_count(
                 m_next_global->get_wid(),
@@ -3940,6 +4284,8 @@ void ldst_unit::writeback() {
         if (m_L1D && m_L1D->access_ready()) {
           mem_fetch *mf = m_L1D->next_access();
           m_next_wb = mf->get_inst();
+          m_next_wb_src_wid = mf->get_wid();
+          m_next_wb_src_split_id = mf->get_access_source_split_id();
           delete mf;
           serviced_client = next_client;
         }
@@ -4104,10 +4450,77 @@ void ldst_unit::cycle() {
             }
           }
         }
+        // MEM co-issue: also check per-source pending for co-issued sets.
+        // Inter-warp: m_pending_writes[coissue_wid][source_inst.out[r]].
+        // Intra-warp: m_pending_writes_secondary[wid][split][source_inst.out[r]].
+        if (!pending_requests && pipe_reg.has_simd_sets() &&
+            m_config->gpgpu_simd_partitioning &&
+            m_config->gpgpu_mem_coissue) {
+          const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
+          std::set<std::pair<unsigned, unsigned>> checked;
+          for (unsigned s = 0; s < sets.size() && !pending_requests; s++) {
+            if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+            std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                               sets[s].split_id);
+            if (checked.count(key)) continue;
+            checked.insert(key);
+            bool is_intra = (sets[s].warp_id == warp_id &&
+                             sets[s].split_id != (unsigned)-1);
+            for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+              unsigned reg_id = sets[s].source_inst->out[r];
+              if (reg_id == 0) continue;
+              if (is_intra) {
+                auto &m = m_pending_writes_secondary[sets[s].warp_id]
+                                                    [sets[s].split_id];
+                auto it = m.find(reg_id);
+                if (it != m.end() && it->second > 0) {
+                  pending_requests = true; break;
+                }
+              } else {
+                auto &m = m_pending_writes[sets[s].warp_id];
+                auto it = m.find(reg_id);
+                if (it != m.end() && it->second > 0) {
+                  pending_requests = true; break;
+                }
+              }
+            }
+          }
+        }
         if (!pending_requests) {
           m_core->warp_inst_complete(*m_dispatch_reg);
           m_scoreboard->releaseRegisters(m_dispatch_reg);
 
+          // MEM co-issue retirement: also release each co-issued set's
+          // scoreboard + call warp_inst_complete for each unique source
+          // warp/split (inter-warp gets its own warp_inst_complete bump;
+          // intra-warp keeps sharing the primary's warp_id but gets a
+          // separate stats increment).
+          if (m_dispatch_reg->has_simd_sets() &&
+              m_config->gpgpu_simd_partitioning &&
+              m_config->gpgpu_mem_coissue) {
+            const std::vector<simd_set_info> &sets =
+                m_dispatch_reg->get_simd_sets();
+            std::set<std::pair<unsigned, unsigned>> released;
+            for (unsigned s = 0; s < sets.size(); s++) {
+              if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+              std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                                 sets[s].split_id);
+              if (released.count(key)) continue;
+              released.insert(key);
+              bool is_intra = (sets[s].warp_id == warp_id &&
+                               sets[s].split_id != (unsigned)-1);
+              for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+                unsigned reg_id = sets[s].source_inst->out[r];
+                if (reg_id == 0) continue;
+                if (is_intra)
+                  m_scoreboard->releaseRegisterSecondary(sets[s].warp_id,
+                                                         reg_id);
+                else
+                  m_scoreboard->releaseRegister(sets[s].warp_id, reg_id);
+              }
+            }
+          }
+
           // release LDGSTS
           if (m_dispatch_reg->m_is_ldgsts) {
             // m_pending_ldgsts[m_dispatch_reg->warp_id()][m_dispatch_reg->pc][m_dispatch_reg->get_addr(0)]--;
@@ -4118,12 +4531,52 @@ void ldst_unit::cycle() {
           }
         }
         m_core->dec_inst_in_pipeline(warp_id);
+        // MEM co-issue: each unique co-issued (warp, split) was incremented
+        // at decode time, so decrement once per unique here. Mirrors the
+        // SP/SFU/INT retirement at shader.cc~3019-3044.
+        if (m_dispatch_reg->has_simd_sets() &&
+            m_config->gpgpu_simd_partitioning &&
+            m_config->gpgpu_mem_coissue) {
+          const std::vector<simd_set_info> &sets =
+              m_dispatch_reg->get_simd_sets();
+          std::set<unsigned> inter_dec;
+          bool intra_dec = false;
+          for (unsigned s = 0; s < sets.size(); s++) {
+            if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+            if (sets[s].warp_id != warp_id) {
+              if (inter_dec.insert(sets[s].warp_id).second)
+                m_core->dec_inst_in_pipeline(sets[s].warp_id);
+            } else if (!intra_dec) {
+              m_core->dec_inst_in_pipeline(warp_id);
+              intra_dec = true;
+            }
+          }
+        }
         m_dispatch_reg->clear();
       }
     } else {
       // stores exit pipeline here
       m_core->dec_inst_in_pipeline(warp_id);
       m_core->warp_inst_complete(*m_dispatch_reg);
+      // MEM co-issue: stores retiring — dec pipe + complete per source too.
+      if (m_dispatch_reg->has_simd_sets() &&
+          m_config->gpgpu_simd_partitioning &&
+          m_config->gpgpu_mem_coissue) {
+        const std::vector<simd_set_info> &sets =
+            m_dispatch_reg->get_simd_sets();
+        std::set<unsigned> inter_dec;
+        bool intra_dec = false;
+        for (unsigned s = 0; s < sets.size(); s++) {
+          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+          if (sets[s].warp_id != warp_id) {
+            if (inter_dec.insert(sets[s].warp_id).second)
+              m_core->dec_inst_in_pipeline(sets[s].warp_id);
+          } else if (!intra_dec) {
+            m_core->dec_inst_in_pipeline(warp_id);
+            intra_dec = true;
+          }
+        }
+      }
       m_dispatch_reg->clear();
     }
   }
@@ -5503,7 +5956,17 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) {
     }
   }
 
-  // Phase 2: Co-issued sets' destination registers (different instructions)
+  // Phase 2: Co-issued sets' destination registers (different instructions).
+  // Phase 1 has already stamped dst=-1 on entries it wrote. If Phase 2 hits
+  // a busy bank (can happen when primary.reg and coissued.reg hash to the
+  // same bank under sub-core_model — e.g., (reg+wid_A)%banks_per_sched ==
+  // (reg+wid_B)%banks_per_sched), SKIP that op instead of returning false.
+  // Returning false would be swallowed by shader_core_ctx::writeback for
+  // EX_WB composites (see comment at ~shader.cc:3096), but ldst_unit's
+  // writeback uses the return value to decide whether to clear m_next_wb,
+  // so a persistent false = deadlock.  Bank-busy bookkeeping doesn't affect
+  // functional correctness — thread register values are committed during
+  // func_exec_inst, not by this bank model.
   if (inst.has_simd_sets()) {
     const std::vector<simd_set_info> &sets = inst.get_simd_sets();
     for (unsigned s = 0; s < sets.size(); s++) {
@@ -5520,9 +5983,10 @@ bool opndcoll_rfu_t::writeback(warp_inst_t &inst) {
             m_arbiter.allocate_bank_for_write(
                 bank, op_t(sets[s].warp_id, reg_num, m_num_banks,
                            sub_core_model, m_num_banks_per_sched, sched_id));
-          } else {
-            return false;
           }
+          // else: bank busy — skip the bookkeeping for this op. The
+          // composite still completes; we just don't model this particular
+          // bank-write's energy for this cycle.
         }
       }
     }