impl mixed-space mem instruction co-issuing

2 files changed, 706 insertions, 104 deletions

diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h
index 1e3c94e..1a42657 100644
--- a/src/abstract_hardware_model.h
+++ b/src/abstract_hardware_model.h
@@ -1119,6 +1119,20 @@ enum divergence_support_t {
 
 const unsigned MAX_ACCESSES_PER_INSN_PER_THREAD = 8;
 
+// Mixed-space MEM co-issue (v2): per-lane address storage for a shared-
+// memory coissuer set. Captured at co_issue_warp time BEFORE temp_inst
+// dies, so later per-set bank conflict recomputes can run against each
+// set's own per-lane addresses without colliding on the composite's
+// m_per_scalar_thread (which is keyed by thread-in-warp and would be
+// overwritten across inter-warp coissuers).
+struct simd_set_addr_entry {
+  new_addr_type memreqaddr[MAX_ACCESSES_PER_INSN_PER_THREAD];
+  simd_set_addr_entry() {
+    for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++)
+      memreqaddr[i] = 0;
+  }
+};
+
 // SIMD lane partitioning: per-set execution data
 struct simd_set_info {
   unsigned set_id;                      // Set index (0 to num_sets-1)
@@ -1131,13 +1145,29 @@ struct simd_set_info {
   bool valid;                           // Whether this set has any active threads
   const inst_t *source_inst;            // Pointer to static decoded instruction (for MIMD)
 
+  // Mixed-space MEM co-issue (v2): per-set shared state.
+  // Populated only when source_inst's space is shared/sstarr; otherwise
+  // unused. `cycles` is the set's bank-conflict depth (modelled per-set
+  // in shared_cycle). `per_thread_addrs` is the captured per-lane
+  // memreqaddr for this set.
+  unsigned cycles;
+  std::vector<simd_set_addr_entry> per_thread_addrs;
+
   simd_set_info()
       : set_id(0), warp_id(0), split_id((unsigned)-1),
-        num_active_threads(0), valid(false), source_inst(NULL) {
+        num_active_threads(0), valid(false), source_inst(NULL), cycles(0) {
     set_active_mask.reset();
     active_mask_in_warp.reset();
     for (unsigned i = 0; i < MAX_WARP_SIZE; i++) thread_ids[i] = 0;
   }
+
+  // Mirror of warp_inst_t::has_dispatch_delay / dispatch_delay, but
+  // per-set. Used by the per-set shared_cycle path in ldst_unit.
+  bool has_dispatch_delay() const { return cycles > 0; }
+  bool dispatch_delay() {
+    if (cycles > 0) cycles--;
+    return cycles > 0;
+  }
 };
 
 class warp_inst_t : public inst_t {
@@ -1205,8 +1235,68 @@ class warp_inst_t : public inst_t {
   const std::vector<simd_set_info> &get_simd_sets() const {
     return m_simd_sets;
   }
+  // Mixed-space MEM co-issue (v2): mutable accessor for capture path.
+  std::vector<simd_set_info> &get_simd_sets_mutable() { return m_simd_sets; }
   unsigned num_active_simd_sets() const;
+
+  // Mixed-space MEM co-issue (v2): space-bucket classification. "shared
+  // bucket" = shared_space/sstarr_space; "global bucket" =
+  // global/local/param_local.
+  static bool is_shared_bucket_space(enum _memory_space_t t) {
+    return t == shared_space || t == sstarr_space;
+  }
+  static bool is_global_bucket_space(enum _memory_space_t t) {
+    return t == global_space || t == local_space || t == param_space_local;
+  }
+  // A composite has a shared/global "subcomposite" if any valid simd_set
+  // (including the primary set when source_inst==NULL) has a source space
+  // in that bucket. Primary set's space is the composite's own `space`.
+  bool has_shared_subcomposite() const {
+    for (unsigned s = 0; s < m_simd_sets.size(); s++) {
+      if (!m_simd_sets[s].valid) continue;
+      enum _memory_space_t sp = (m_simd_sets[s].source_inst == NULL)
+                                    ? space.get_type()
+                                    : m_simd_sets[s].source_inst->space.get_type();
+      if (is_shared_bucket_space(sp)) return true;
+    }
+    return false;
+  }
+  bool has_global_subcomposite() const {
+    for (unsigned s = 0; s < m_simd_sets.size(); s++) {
+      if (!m_simd_sets[s].valid) continue;
+      enum _memory_space_t sp = (m_simd_sets[s].source_inst == NULL)
+                                    ? space.get_type()
+                                    : m_simd_sets[s].source_inst->space.get_type();
+      if (is_global_bucket_space(sp)) return true;
+    }
+    return false;
+  }
+  bool is_mixed_space_composite() const {
+    return has_shared_subcomposite() && has_global_subcomposite();
+  }
   bool has_simd_sets() const { return !m_simd_sets.empty(); }
+
+  // Mixed-space MEM co-issue (v2): copy this instruction's per-lane
+  // memreqaddr entries into `set.per_thread_addrs`, and copy this
+  // instruction's bank-conflict `cycles` into `set.cycles`. Used by
+  // co_issue_warp for SHARED coissuer sets BEFORE temp_inst dies, so
+  // later per-set bank-conflict re-checks can read each set's own
+  // addresses without colliding on the composite's m_per_scalar_thread.
+  void capture_per_set_shared_state(simd_set_info &set,
+                                    unsigned set_width) const {
+    set.cycles = cycles;
+    if (!m_per_scalar_thread_valid) return;
+    set.per_thread_addrs.assign(set_width, simd_set_addr_entry());
+    for (unsigned lane = 0; lane < set_width; lane++) {
+      if (!set.set_active_mask.test(lane)) continue;
+      unsigned t = set.thread_ids[lane];
+      if (t >= m_per_scalar_thread.size()) continue;
+      for (unsigned i = 0; i < MAX_ACCESSES_PER_INSN_PER_THREAD; i++) {
+        set.per_thread_addrs[lane].memreqaddr[i] =
+            m_per_scalar_thread[t].memreqaddr[i];
+      }
+    }
+  }
   void merge_simd_sets(const std::vector<simd_set_info> &other_sets);
   void set_source_inst_on_sets(const inst_t *src);
   void set_split_id_on_sets(unsigned split_id);
@@ -1408,6 +1498,19 @@ class warp_inst_t : public inst_t {
   bool m_is_depbar;
 
   unsigned int m_depbar_group_no;
+
+  // Mixed-space MEM co-issue (Option A / Stage 6): tracks whether the
+  // shared-source scoreboards of this MIXED composite have already been
+  // released via the early-release path in ldst_unit::cycle. Set once;
+  // checked by both the early-release path (to avoid re-release as
+  // shared_cycle keeps returning true) and by the final mixed-retire
+  // (to avoid double-releasing shared sources at dispatch_reg clear).
+  // Default false. Only meaningful for mixed composites.
+ public:
+  bool shared_side_released() const { return m_shared_side_released; }
+  void set_shared_side_released(bool v) { m_shared_side_released = v; }
+ private:
+  bool m_shared_side_released = false;
 };
 
 void move_warp(warp_inst_t *&dst, warp_inst_t *&src);
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index f96e939..191beaf 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1476,6 +1476,24 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite,
   temp_inst.set_source_inst_on_sets(next_inst);
   temp_inst.set_split_id_on_sets(split_id);
 
+  // Mixed-space MEM co-issue (v2): for SHARED coissuers, capture per-lane
+  // memreqaddr + temp_inst.cycles into each valid set BEFORE merge. The
+  // captured state survives temp_inst's destruction at function exit and
+  // is used by per-set shared_cycle + any per-set bank-conflict recompute
+  // at retirement. No-op for global coissuers (shared_cycle and the
+  // retirement walk gate on source_inst->space). Guarded on is_load ||
+  // is_store to skip non-memory ops.
+  if ((temp_inst.is_load() || temp_inst.is_store()) &&
+      (next_inst->space.get_type() == shared_space ||
+       next_inst->space.get_type() == sstarr_space)) {
+    unsigned set_width = m_config->simd_set_width;
+    std::vector<simd_set_info> &sets = temp_inst.get_simd_sets_mutable();
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid) continue;
+      temp_inst.capture_per_set_shared_state(sets[s], set_width);
+    }
+  }
+
   // For MEM co-issue: splice temp_inst's already-coalesced mem accesses
   // into the composite, stamping each with source (warp_id, split_id) so
   // the LDST unit routes pending_writes + scoreboard release back to the
@@ -1650,15 +1668,20 @@ bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const {
   if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() ||
       inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request())
     return true;
-  // Exclude atomics (decrement_atomic_count ties to a single warp) and
-  // non-global memory spaces. Global/local go through the coalescer +
-  // mem_fetch path that the source_wid stamp supports. Shared/tex/const
-  // use per-subwarp / per-line paths not yet per-set-aware.
+  // Exclude atomics (decrement_atomic_count ties to a single warp).
+  // Mixed-space MEM co-issue (v2): allow shared_space + sstarr_space in
+  // addition to the original global/local/param_local. Cross-bucket
+  // composites are rejected at the candidate-feasibility check in
+  // try_utilization_max_coissue (same_space only). MEMCOV2_DISABLE_MIXED=1
+  // tightens to v1 (global/local only) for A/B testing.
   if (inst->isatomic()) return true;
+  static const bool disable_shared =
+      (getenv("MEMCOV2_DISABLE_MIXED") != NULL);
   memory_space_t space = inst->space;
-  if (space.get_type() != global_space &&
-      space.get_type() != local_space &&
-      space.get_type() != param_space_local)
+  enum _memory_space_t t = space.get_type();
+  bool in_global_bucket = warp_inst_t::is_global_bucket_space(t);
+  bool in_shared_bucket = warp_inst_t::is_shared_bucket_space(t);
+  if (!in_global_bucket && (disable_shared || !in_shared_bucket))
     return true;
   return false;
 }
@@ -1863,6 +1886,21 @@ void scheduler_unit::try_inter_warp_coissue(
         m_stats->mem_coissue_denied_filter[get_sid()]++;
         continue;
       }
+      // Same-bucket (Stage 4): primary and candidate must share bucket.
+      // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging.
+      if (co_issue_fu_type == exec_unit_type_t::MEM) {
+        static const bool allow_mixed =
+            (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
+        enum _memory_space_t pspace = co_issue_composite->space.get_type();
+        enum _memory_space_t cspace = cand_inst->space.get_type();
+        bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
+        bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
+        if (!allow_mixed && p_shared != c_shared) {
+          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+          m_stats->mem_coissue_denied_filter[get_sid()]++;
+          continue;
+        }
+      }
 
       // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any.
       if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) {
@@ -1957,6 +1995,21 @@ void scheduler_unit::try_intra_warp_coissue(
       m_stats->mem_coissue_denied_filter[get_sid()]++;
       continue;
     }
+    // Same-bucket (Stage 4): primary and candidate must share bucket.
+    // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging.
+    if (co_issue_fu_type == exec_unit_type_t::MEM) {
+      static const bool allow_mixed =
+          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
+      enum _memory_space_t pspace = co_issue_composite->space.get_type();
+      enum _memory_space_t cspace = sec_inst->space.get_type();
+      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
+      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
+      if (!allow_mixed && p_shared != c_shared) {
+        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+        m_stats->mem_coissue_denied_filter[get_sid()]++;
+        continue;
+      }
+    }
 
     if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) {
       m_stats->coissue_denied_by_scoreboard[get_sid()]++;
@@ -2061,6 +2114,23 @@ void scheduler_unit::try_utilization_max_coissue(
       m_stats->mem_coissue_denied_filter[get_sid()]++;
       continue;
     }
+    // Mixed-space MEM co-issue (v2, same-bucket only): primary and
+    // candidate must be in the same space bucket (shared or global).
+    // Cross-bucket composites are disallowed here — Stage 5 will add a
+    // mixed-retire path. MEMCOV2_ALLOW_MIXED=1 bypasses for debugging.
+    if (co_issue_fu_type == exec_unit_type_t::MEM) {
+      static const bool allow_mixed =
+          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
+      enum _memory_space_t pspace = co_issue_composite->space.get_type();
+      enum _memory_space_t cspace = cand_inst->space.get_type();
+      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
+      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
+      if (!allow_mixed && p_shared != c_shared) {
+        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+        m_stats->mem_coissue_denied_filter[get_sid()]++;
+        continue;
+      }
+    }
 
     const active_mask_t &cand_mask =
         m_shader->get_active_mask(cand_warp_id, cand_inst);
@@ -2124,6 +2194,21 @@ void scheduler_unit::try_utilization_max_coissue(
           m_stats->mem_coissue_denied_filter[get_sid()]++;
           continue;
         }
+        // Same-bucket check: reject cross-bucket intra candidates.
+        // MEMCOV2_ALLOW_MIXED=1 bypasses for Stage 5 debugging.
+        if (co_issue_fu_type == exec_unit_type_t::MEM) {
+          static const bool allow_mixed =
+              (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
+          enum _memory_space_t pspace = co_issue_composite->space.get_type();
+          enum _memory_space_t cspace = sec_inst->space.get_type();
+          bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
+          bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
+          if (!allow_mixed && p_shared != c_shared) {
+            m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+            m_stats->mem_coissue_denied_filter[get_sid()]++;
+            continue;
+          }
+        }
         if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) {
           m_stats->coissue_denied_by_scoreboard[get_sid()]++;
           continue;
@@ -3198,22 +3283,62 @@ void shader_core_ctx::writeback() {
 
 bool ldst_unit::shared_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
                              mem_stage_access_type &fail_type) {
-  if (inst.space.get_type() != shared_space) return true;
+  // Mixed-space MEM co-issue (v2): relax the primary-space gate. Shared
+  // work can live in (a) the primary's own cycles counter if primary is
+  // shared, or (b) per-set `cycles` on any valid simd_set whose source
+  // is a shared coissuer. Return immediately if the instruction has no
+  // shared work at all.
+  bool primary_is_shared = (inst.space.get_type() == shared_space ||
+                            inst.space.get_type() == sstarr_space);
+  bool has_shared_coissuer_set = false;
+  if (inst.has_simd_sets()) {
+    const std::vector<simd_set_info> &sets = inst.get_simd_sets();
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+      enum _memory_space_t src_sp = sets[s].source_inst->space.get_type();
+      if (src_sp == shared_space || src_sp == sstarr_space) {
+        has_shared_coissuer_set = true;
+        break;
+      }
+    }
+  }
+  if (!primary_is_shared && !has_shared_coissuer_set) return true;
 
   if (inst.active_count() == 0) return true;
 
-  if (inst.has_dispatch_delay()) {
-    m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
+  bool any_stall = false;
+
+  // Primary's own cycles (only counts if primary is shared).
+  if (primary_is_shared) {
+    if (inst.has_dispatch_delay()) {
+      m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
+    }
+    if (inst.dispatch_delay()) any_stall = true;
   }
 
-  bool stall = inst.dispatch_delay();
-  if (stall) {
+  // Per-set cycles for shared coissuer sets. Each set's cycles counts
+  // down independently; we stall as long as ANY still has cycles > 0.
+  if (inst.has_simd_sets()) {
+    std::vector<simd_set_info> &sets = inst.get_simd_sets_mutable();
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+      enum _memory_space_t src_sp = sets[s].source_inst->space.get_type();
+      if (src_sp != shared_space && src_sp != sstarr_space) continue;
+      if (sets[s].has_dispatch_delay()) {
+        m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
+      }
+      if (sets[s].dispatch_delay()) any_stall = true;
+    }
+  }
+
+  if (any_stall) {
     fail_type = S_MEM;
     rc_fail = BK_CONF;
     m_stats->gpgpu_n_shmem_bkconflict++;
-  } else
+  } else {
     rc_fail = NO_RC_FAIL;
-  return !stall;
+  }
+  return !any_stall;
 }
 
 mem_stage_stall_type ldst_unit::process_cache_access(
@@ -3229,8 +3354,12 @@ mem_stage_stall_type ldst_unit::process_cache_access(
   unsigned src_split_access =
       mf ? mf->get_access_source_split_id() : (unsigned)-1;
   mem_src_t src = resolve_source(inst, src_wid_access, src_split_access);
-  bool src_is_intra_coissued =
-      (src.wid == inst.warp_id() && src_split_access != (unsigned)-1);
+  // "intra" here means the coissuer came from a secondary ibuffer slot
+  // (scoreboard was reserved on the secondary map). It does NOT mean
+  // same-warp-as-primary — try_utilization_max_coissue scans every warp's
+  // secondary slots, so a coissuer from warp Y's secondary slot can pair
+  // with warp X as primary. The correct discriminator is split_id != -1.
+  bool src_is_intra_coissued = (src_split_access != (unsigned)-1);
   if (write_sent) {
     unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                            ? (mf->get_data_size() / SECTOR_SIZE)
@@ -3244,7 +3373,11 @@ mem_stage_stall_type ldst_unit::process_cache_access(
     inst.accessq_pop_back();
     if (inst.is_bru_st_fill_request() || inst.is_bru_rt_fill_request()) {
       release_virtual_entries(inst);
-    } else if (inst.is_load()) {
+    } else if (mf && !mf->get_is_write()) {
+      // Bug #2 fix: use per-access is_write (NOT composite primary op).
+      // A STORE-primary composite can carry LOAD coissuer accesses; each
+      // access's register-decrement or store-ack bookkeeping is selected
+      // by the access's own type.
       for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
         if (src.out_inst->out[r] > 0) {
           if (src_is_intra_coissued)
@@ -3319,7 +3452,13 @@ mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache(
           NULL) {
         l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1] = mf;
 
-        if (mf->get_inst().is_store()) {
+        // Bug #2 fix: per-access write bit, NOT composite primary op.
+        // A LOAD-primary composite can carry STORE coissuer accesses (and
+        // vice versa). Each mf's own write bit determines whether it
+        // needs an inc_store_req — the mf will be store_ack'd later when
+        // WRITE_ACK returns, and that dec_store_req requires a matching
+        // inc here.
+        if (mf->get_is_write()) {
           unsigned inc_ack =
               (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                   ? (mf->get_data_size() / SECTOR_SIZE)
@@ -3373,16 +3512,18 @@ void ldst_unit::L1_latency_queue_cycle() {
       if (status == HIT) {
         assert(!read_sent);
         l1_latency_queue[j][0] = NULL;
-        if (mf_next->get_inst().is_load()) {
+        // Bug #2 fix: per-access dispatch via mf's own write bit, not
+        // composite primary op.
+        if (!mf_next->get_is_write()) {
           // MEM co-issue: the access belongs to mf_next's stamped source
           // warp. Resolve which instruction's out[] registers should be
           // decremented (primary's or a co-issued set's).
           unsigned src_split_mf = mf_next->get_access_source_split_id();
           mem_src_t src = resolve_source(
               mf_next->get_inst(), mf_next->get_wid(), src_split_mf);
-          bool src_is_intra =
-              (src.wid == mf_next->get_inst().warp_id() &&
-               src_split_mf != (unsigned)-1);
+          // "intra" = coissuer came from a secondary ibuffer slot (secondary
+          // scoreboard map). Not tied to warp equality with primary.
+          bool src_is_intra = (src_split_mf != (unsigned)-1);
           bool any_completed = false;
           for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
             if (src.out_inst->out[r] > 0) {
@@ -3423,8 +3564,8 @@ void ldst_unit::L1_latency_queue_cycle() {
           }
         }
 
-        // For write hit in WB policy
-        if (mf_next->get_inst().is_store() && !write_sent) {
+        // For write hit in WB policy (Bug #2 fix: per-access write bit)
+        if (mf_next->get_is_write() && !write_sent) {
           unsigned dec_ack =
               (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                   ? (mf_next->get_data_size() / SECTOR_SIZE)
@@ -3443,8 +3584,9 @@ void ldst_unit::L1_latency_queue_cycle() {
       } else {
         assert(status == MISS || status == HIT_RESERVED);
         l1_latency_queue[j][0] = NULL;
+        // Bug #2 fix: per-access write bit, not composite primary op.
         if (m_config->m_L1D_config.get_write_policy() != WRITE_THROUGH &&
-            mf_next->get_inst().is_store() &&
+            mf_next->get_is_write() &&
             (m_config->m_L1D_config.get_write_allocate_policy() ==
                  FETCH_ON_WRITE ||
              m_config->m_L1D_config.get_write_allocate_policy() ==
@@ -3518,10 +3660,14 @@ bool ldst_unit::texture_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
 bool ldst_unit::memory_cycle(warp_inst_t &inst,
                              mem_stage_stall_type &stall_reason,
                              mem_stage_access_type &access_type) {
-  if (inst.empty() || ((inst.space.get_type() != global_space) &&
-                       (inst.space.get_type() != local_space) &&
-                       (inst.space.get_type() != param_space_local)))
-    return true;
+  // Mixed-space MEM co-issue (v2): accessq is the source of truth for
+  // global/local/param_local work. A shared-primary composite with a
+  // global coissuer will have global accesses in its accessq (stamped
+  // with source_wid per access), so we must enter this function even if
+  // primary's space is shared. Accesses themselves carry their own
+  // attribution; primary space only matters for the `else` stat-only
+  // branch at the bottom of this function (access_type encoding).
+  if (inst.empty()) return true;
   if (inst.active_count() == 0) return true;
   if (inst.accessq_empty()) return true;
 
@@ -3570,7 +3716,10 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst,
         m_icnt->push(mf);
         inst.accessq_pop_back();
         // inst.clear_active( access.get_warp_mask() );
-        if (inst.is_load()) {
+        // Bug #2 fix: use per-access is_write, NOT composite primary op.
+        // A STORE-primary composite can carry LOAD coissuer accesses, and
+        // vice versa. Each access's type determines its own bookkeeping.
+        if (!access.is_write()) {
           // For MEM-co-issue composites, sanity-check against the source
           // warp's pending_writes on its source out[] regs (if identifiable).
           const inst_t *src_out_inst = &inst;
@@ -3587,8 +3736,9 @@ bool ldst_unit::memory_cycle(warp_inst_t &inst,
           for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
             if (src_out_inst->out[r] > 0)
               assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0);
-        } else if (inst.is_store())
+        } else {
           m_core->inc_store_req(src_wid);
+        }
       }
     }
   } else {
@@ -3952,23 +4102,23 @@ ldst_unit::mem_src_t ldst_unit::resolve_source(
   mem_src_t r;
   r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid;
   r.out_inst = &inst;
+  // Primary access: (wid == primary AND split_id == -1). Return early with
+  // out_inst = composite (primary's out[]). Note: split_id is the
+  // discriminator — a coissuer can have the same warp_id as primary when
+  // it came from the primary warp's own secondary ibuffer slot.
+  if (r.wid == inst.warp_id() && access_src_split_id == (unsigned)-1) {
+    return r;
+  }
   if (!inst.has_simd_sets()) return r;
-  // Walk simd_sets for a co-issued set matching BOTH (wid, split_id). For
-  // inter-warp accesses split_id is -1 and wid differs from primary; for
-  // intra-warp accesses split_id is non-neg and wid equals primary. Primary
-  // accesses (source_wid == -1 or split_id == -1 with wid==primary) keep
-  // out_inst = &inst (composite's out[], which is primary's).
-  bool is_intra =
-      (r.wid == inst.warp_id() && access_src_split_id != (unsigned)-1);
-  bool is_inter = (r.wid != inst.warp_id());
-  if (!is_intra && !is_inter) return r;
+  // Coissued access: search sets for an exact (wid, split_id) match.
   const std::vector<simd_set_info> &sets = inst.get_simd_sets();
   for (unsigned s = 0; s < sets.size(); s++) {
     if (!sets[s].valid || sets[s].source_inst == NULL) continue;
-    if (sets[s].warp_id != r.wid) continue;
-    if (is_intra && sets[s].split_id != access_src_split_id) continue;
-    r.out_inst = sets[s].source_inst;
-    break;
+    if (sets[s].warp_id == r.wid &&
+        sets[s].split_id == access_src_split_id) {
+      r.out_inst = sets[s].source_inst;
+      break;
+    }
   }
   return r;
 }
@@ -3979,27 +4129,35 @@ void ldst_unit::issue(register_set &reg_set) {
   // record how many pending register writes/memory accesses there are for this
   // instruction
   assert(inst->empty() == false);
-  if (inst->is_load() and inst->space.get_type() != shared_space) {
-    unsigned primary_wid = inst->warp_id();
+  // Bug #1 fix: a composite's PRIMARY op may be STORE or shared while one
+  // or more coissuer SETS carry LOAD accesses in non-shared spaces. Don't
+  // gate the whole pending_writes bookkeeping block on primary. Gate the
+  // primary portion on primary's own is_load+non-shared; gate per-set
+  // accumulation on EACH set's source_inst is_load+non-shared.
+  unsigned primary_wid = inst->warp_id();
+  bool primary_is_tracked_load = inst->is_load() &&
+                                 inst->space.get_type() != shared_space;
 
+  if (inst->has_simd_sets() &&
+      m_config->gpgpu_simd_partitioning &&
+      m_config->gpgpu_mem_coissue) {
     // MEM co-issue: accesses in the composite's queue may originate from
     // different source warps (inter-warp) or different splits of the same
     // warp (intra-warp). Group them by (source_wid, source_split_id) to
     // charge pending_writes to the correct source's destination registers.
     // Unstamped accesses (source_wid == -1) belong to the primary.
-    if (inst->has_simd_sets() &&
-        m_config->gpgpu_simd_partitioning &&
-        m_config->gpgpu_mem_coissue) {
-      std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src;
-      for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin();
-           it != inst->accessq_end(); ++it) {
-        unsigned w = it->get_source_wid();
-        unsigned sp = it->get_source_split_id();
-        if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; }
-        n_acc_per_src[std::make_pair(w, sp)] += 1;
-      }
+    std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src;
+    for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin();
+         it != inst->accessq_end(); ++it) {
+      unsigned w = it->get_source_wid();
+      unsigned sp = it->get_source_split_id();
+      if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; }
+      n_acc_per_src[std::make_pair(w, sp)] += 1;
+    }
 
-      // Primary's own registers
+    // Primary's own registers — gated on primary being a tracked load
+    // (stores have no out[]; shared loads don't use pending_writes).
+    if (primary_is_tracked_load) {
       unsigned n_primary =
           n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)];
       if (n_primary > 0) {
@@ -4008,50 +4166,53 @@ void ldst_unit::issue(register_set &reg_set) {
           if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary;
         }
       }
+    }
 
-      // Per co-issued set's registers (use set's source_inst->out[]).
-      // Inter-warp sets: different warp_id → primary pending_writes map is
-      // fine (no key collision with primary). Intra-warp sets: same warp_id
-      // but different split_id → use the secondary pending_writes map to
-      // avoid conflating with primary's pending_writes[primary_wid][reg].
-      const std::vector<simd_set_info> &sets = inst->get_simd_sets();
-      std::set<std::pair<unsigned, unsigned>> accounted;
-      for (unsigned s = 0; s < sets.size(); s++) {
-        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
-        std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
-        if (accounted.count(key)) continue;
-        accounted.insert(key);
-        unsigned n_src = n_acc_per_src[key];
-        if (n_src == 0) continue;
-        bool is_intra = (sets[s].warp_id == primary_wid &&
-                         sets[s].split_id != (unsigned)-1);
-        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
-          unsigned reg_id = sets[s].source_inst->out[r];
-          if (reg_id > 0) {
-            if (is_intra) {
-              m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id]
-                                        [reg_id] += n_src;
-            } else {
-              m_pending_writes[sets[s].warp_id][reg_id] += n_src;
-            }
+    // Per co-issued set's registers (use set's source_inst->out[]).
+    // Gate per-set on the SET's source being a tracked load, independent
+    // of primary. A STORE-primary composite can carry LOAD coissuers;
+    // their pending_writes must still be tracked so their mf returns
+    // decrement correctly.
+    const std::vector<simd_set_info> &sets = inst->get_simd_sets();
+    std::set<std::pair<unsigned, unsigned>> accounted;
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+      if (!sets[s].source_inst->is_load()) continue;
+      if (sets[s].source_inst->space.get_type() == shared_space) continue;
+      std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
+      if (accounted.count(key)) continue;
+      accounted.insert(key);
+      unsigned n_src = n_acc_per_src[key];
+      if (n_src == 0) continue;
+      // is_intra = "uses secondary scoreboard map" = split came from a
+      // secondary ibuffer slot. Not tied to warp_id == primary_wid.
+      bool is_intra = (sets[s].split_id != (unsigned)-1);
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+        unsigned reg_id = sets[s].source_inst->out[r];
+        if (reg_id > 0) {
+          if (is_intra) {
+            m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id]
+                                      [reg_id] += n_src;
+          } else {
+            m_pending_writes[sets[s].warp_id][reg_id] += n_src;
           }
         }
       }
-    } else {
-      // Legacy (non-co-issue) path: single-warp pending_writes.
-      unsigned n_accesses = inst->accessq_count();
-      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
-        unsigned reg_id = inst->out[r];
-        if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses;
-      }
     }
-
-    if (inst->m_is_ldgsts) {
-      m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] +=
-          inst->accessq_count();
+  } else if (primary_is_tracked_load) {
+    // Legacy (non-co-issue) path: single-warp pending_writes.
+    unsigned n_accesses = inst->accessq_count();
+    for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+      unsigned reg_id = inst->out[r];
+      if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses;
     }
   }
 
+  if (primary_is_tracked_load && inst->m_is_ldgsts) {
+    m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] +=
+        inst->accessq_count();
+  }
+
   inst->op_pipe = MEM__OP;
   // stat collection
   m_core->mem_instruction_stats(*inst);
@@ -4101,13 +4262,21 @@ void ldst_unit::writeback() {
       // the source is always the composite's primary here; co-issued sets'
       // releases are handled in a dedicated pass below.
       unsigned primary_wid = m_next_wb.warp_id();
-      bool is_shared = (m_next_wb.space.get_type() == shared_space);
+      // Mixed-space MEM co-issue (v2): for MIXED composites with shared
+      // primary, a global coissuer's mf return arrives here via case 3/4
+      // with m_next_wb_src_wid stamped to coissuer_wid. The mf's origin
+      // is GLOBAL — it should take the non-shared pending_writes path,
+      // not the shared-primary release path. Use the mf stamp to
+      // disambiguate: stamped = mf-driven (non-shared routing), unstamped
+      // = case 0 shared retire (shared routing).
+      bool is_shared = (m_next_wb.space.get_type() == shared_space &&
+                        m_next_wb_src_wid == (unsigned)-1);
       unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1)
                              ? m_next_wb_src_wid
                              : primary_wid;
       unsigned src_split = m_next_wb_src_split_id;
-      bool is_intra_coissued =
-          (src_wid == primary_wid && src_split != (unsigned)-1);
+      // is_intra = secondary-slot coissuer (not warp equality with primary).
+      bool is_intra_coissued = (src_split != (unsigned)-1);
       mem_src_t src = resolve_source(m_next_wb, src_wid, src_split);
 
       for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
@@ -4164,8 +4333,8 @@ void ldst_unit::writeback() {
                                              sets[s].split_id);
           if (released.count(key)) continue;
           released.insert(key);
-          bool coissued_intra = (sets[s].warp_id == primary_wid &&
-                                  sets[s].split_id != (unsigned)-1);
+          // Secondary slot coissuer → secondary scoreboard map.
+          bool coissued_intra = (sets[s].split_id != (unsigned)-1);
           for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
             if (sets[s].source_inst->out[r] > 0) {
               if (coissued_intra)
@@ -4334,6 +4503,28 @@ inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id();
 void ldst_unit::cycle() {
   writeback();
 
+  // MEMCOV2_STATE_DUMP: periodic state-counter dump to detect leaks.
+  // Prints total pending_writes map sizes + sum of their values every
+  // 4096 cycles. Used for Stage 5 mixed-retire leak investigation.
+  static const bool mc_state_dump = (getenv("MEMCOV2_STATE_DUMP") != NULL);
+  if (mc_state_dump) {
+    unsigned long long cyc =
+        m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
+    if ((cyc & 0xFFF) == 0) {
+      unsigned pw_keys = 0, pw_sum = 0;
+      for (auto &kv : m_pending_writes)
+        for (auto &rv : kv.second) { pw_keys++; pw_sum += rv.second; }
+      unsigned sw_keys = 0, sw_sum = 0;
+      for (auto &kv : m_pending_writes_secondary)
+        for (auto &sp : kv.second)
+          for (auto &rv : sp.second) { sw_keys++; sw_sum += rv.second; }
+      fprintf(stderr,
+              "[MC2 state@%llu] sid=%u pw_keys=%u pw_sum=%u "
+              "sw_keys=%u sw_sum=%u\n",
+              cyc, m_sid, pw_keys, pw_sum, sw_keys, sw_sum);
+    }
+  }
+
   for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++)
     if (m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage + 1]->empty())
       move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]);
@@ -4412,6 +4603,80 @@ void ldst_unit::cycle() {
   done &= memory_cycle(pipe_reg, rc_fail, type);
   m_mem_rc = rc_fail;
 
+  // Option A (hardware-faithful per-source scoreboard release) for MIXED
+  // composites: as soon as the SHARED side's bank-conflict cycles drain,
+  // release shared-source scoreboards (primary if shared + shared
+  // coissuer sets) regardless of whether global mfs are still in flight.
+  // Global-source scoreboards continue to release via the pending_writes
+  // path in writeback() on their own timeline. This matches hardware
+  // where the shared-mem bank arbiter signals completion to the
+  // scoreboard independently of global-memory transactions. One-shot
+  // guarded by warp_inst_t::shared_side_released() to avoid re-release
+  // as shared_cycle keeps returning true on subsequent cycles.
+  if (!pipe_reg.empty() && pipe_reg.has_simd_sets() &&
+      m_config->gpgpu_simd_partitioning &&
+      m_config->gpgpu_mem_coissue &&
+      pipe_reg.is_mixed_space_composite() &&
+      !pipe_reg.shared_side_released()) {
+    bool shared_done = true;
+    // Primary's own cycles count only if primary itself is shared.
+    if (warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type()) &&
+        pipe_reg.has_dispatch_delay()) {
+      shared_done = false;
+    }
+    if (shared_done) {
+      const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
+      for (unsigned s = 0; s < sets.size(); s++) {
+        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+        enum _memory_space_t sp = sets[s].source_inst->space.get_type();
+        if (warp_inst_t::is_shared_bucket_space(sp) &&
+            sets[s].has_dispatch_delay()) {
+          shared_done = false;
+          break;
+        }
+      }
+    }
+    if (shared_done) {
+      // Release shared-source scoreboards.
+      const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
+      // Primary scoreboard if primary is a shared LOAD.
+      if (pipe_reg.is_load() &&
+          warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type())) {
+        m_scoreboard->releaseRegisters(m_dispatch_reg);
+      }
+      // Shared coissuer LOAD sets.
+      std::set<std::pair<unsigned, unsigned>> released;
+      for (unsigned s = 0; s < sets.size(); s++) {
+        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+        if (!sets[s].source_inst->is_load()) continue;
+        enum _memory_space_t sp = sets[s].source_inst->space.get_type();
+        if (!warp_inst_t::is_shared_bucket_space(sp)) continue;
+        std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
+        if (released.count(key)) continue;
+        released.insert(key);
+        bool is_intra = (sets[s].split_id != (unsigned)-1);
+        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+          unsigned reg = sets[s].source_inst->out[r];
+          if (reg == 0) continue;
+          if (is_intra)
+            m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg);
+          else
+            m_scoreboard->releaseRegister(sets[s].warp_id, reg);
+        }
+      }
+      pipe_reg.set_shared_side_released(true);
+      if (getenv("MEMCOV2_TRACE")) {
+        unsigned long long cyc = m_core->get_gpu()->gpu_sim_cycle +
+                                 m_core->get_gpu()->gpu_tot_sim_cycle;
+        fprintf(stderr,
+                "[MC2 early-rel@%llu] wid=%u primary_space=%d "
+                "primary_is_load=%d sets=%u\n",
+                cyc, pipe_reg.warp_id(), (int)pipe_reg.space.get_type(),
+                (int)pipe_reg.is_load(), (unsigned)sets.size());
+      }
+    }
+  }
+
   if (!done) {  // log stall types and return
     assert(rc_fail != NO_RC_FAIL);
     m_stats->gpgpu_n_stall_shd_mem++;
@@ -4419,6 +4684,210 @@ void ldst_unit::cycle() {
     return;
   }
 
+  // Mixed-space MEM co-issue (v2): new inline retirement path for
+  // composites with BOTH shared and global valid sets. Fires only when
+  // has_simd_sets + partitioning + mem_coissue enabled + actually mixed.
+  // Pure-shared (only shared sets) and pure-global (only global sets)
+  // composites fall through to the v1 load/store branches below
+  // unchanged. We retire at dispatch_reg (skipping pipeline_reg smem
+  // chain + case 0 writeback) because mixed composites need to wait for
+  // BOTH shared-cycle completion and global pending_writes completion;
+  // routing them through pipeline_reg[smem_latency-1] first would cause
+  // premature primary-scoreboard release at case 0 while global mfs are
+  // still in flight (see re-plan doc).
+  if (!pipe_reg.empty() && pipe_reg.has_simd_sets() &&
+      m_config->gpgpu_simd_partitioning &&
+      m_config->gpgpu_mem_coissue &&
+      pipe_reg.is_mixed_space_composite()) {
+    unsigned warp_id = pipe_reg.warp_id();
+    const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
+
+    // mixed_retire_ready: shared_cycle already returned true (done), and
+    // memory_cycle returned accessq_empty. We still need to wait for
+    // outstanding global-source mfs to drain via the pending_writes
+    // path. A global LOAD coissuer's scoreboard is released by the
+    // pending_writes decrement in writeback() when its last mf returns;
+    // our inline retire waits for that BEFORE releasing primary or
+    // shared coissuer scoreboards.
+    bool pending_writes_outstanding = false;
+    // Check primary's own pending_writes (only if primary is a global
+    // load — shared loads don't use pending_writes).
+    if (pipe_reg.is_load() &&
+        warp_inst_t::is_global_bucket_space(pipe_reg.space.get_type())) {
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+        unsigned reg_id = pipe_reg.out[r];
+        if (reg_id == 0) continue;
+        auto it = m_pending_writes[warp_id].find(reg_id);
+        if (it != m_pending_writes[warp_id].end()) {
+          if (it->second > 0) {
+            pending_writes_outstanding = true;
+            break;
+          } else {
+            m_pending_writes[warp_id].erase(reg_id);
+          }
+        }
+      }
+    }
+    // Per-coissuer check for global LOAD sources.
+    if (!pending_writes_outstanding) {
+      std::set<std::pair<unsigned, unsigned>> checked;
+      for (unsigned s = 0; s < sets.size() && !pending_writes_outstanding;
+           s++) {
+        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+        if (!sets[s].source_inst->is_load()) continue;
+        enum _memory_space_t src_sp =
+            sets[s].source_inst->space.get_type();
+        if (!warp_inst_t::is_global_bucket_space(src_sp)) continue;
+        std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                           sets[s].split_id);
+        if (checked.count(key)) continue;
+        checked.insert(key);
+        bool is_intra = (sets[s].split_id != (unsigned)-1);
+        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+          unsigned reg_id = sets[s].source_inst->out[r];
+          if (reg_id == 0) continue;
+          if (is_intra) {
+            auto &m = m_pending_writes_secondary[sets[s].warp_id]
+                                                [sets[s].split_id];
+            auto it = m.find(reg_id);
+            if (it != m.end() && it->second > 0) {
+              pending_writes_outstanding = true; break;
+            }
+          } else {
+            auto &m = m_pending_writes[sets[s].warp_id];
+            auto it = m.find(reg_id);
+            if (it != m.end() && it->second > 0) {
+              pending_writes_outstanding = true; break;
+            }
+          }
+        }
+      }
+    }
+    static const bool mc_trace = (getenv("MEMCOV2_TRACE") != NULL);
+    if (pending_writes_outstanding) {
+      if (mc_trace) {
+        unsigned long long cyc =
+            m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
+        fprintf(stderr,
+                "[MC2 stall@%llu] wid=%u pending_writes_outstanding\n",
+                cyc, warp_id);
+      }
+      return;
+    }
+
+    // RF write-port fidelity: stall if operand collector not ready.
+    if (!m_operand_collector->writeback(*m_dispatch_reg)) {
+      if (mc_trace) {
+        unsigned long long cyc =
+            m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
+        if ((cyc & 0x3FFF) == 0)
+          fprintf(stderr, "[MC2 stall@%llu] wid=%u opnd_coll not ready\n",
+                  cyc, warp_id);
+      }
+      return;
+    }
+    if (mc_trace) {
+      unsigned long long cyc =
+          m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
+      fprintf(stderr,
+              "[MC2 retire@%llu] wid=%u is_load=%d primary_space=%d sets=%u\n",
+              cyc, warp_id, (int)pipe_reg.is_load(),
+              (int)pipe_reg.space.get_type(), (unsigned)sets.size());
+    }
+
+    // Primary retirement: release primary scoreboard if LOAD; always
+    // call warp_inst_complete and dec_inst_in_pipeline for primary.
+    // Stage 6 (Option A): if primary is shared, its scoreboard was
+    // already released in the early-release block above — skip here to
+    // avoid a spurious release that could race with a subsequent
+    // reservation on the same reg.
+    bool primary_is_shared =
+        warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type());
+    if (pipe_reg.is_load() &&
+        !(primary_is_shared && pipe_reg.shared_side_released())) {
+      m_scoreboard->releaseRegisters(m_dispatch_reg);
+    }
+    m_core->warp_inst_complete(*m_dispatch_reg);
+
+    // Release ONLY shared-source coissuer scoreboards that haven't been
+    // released yet. Global-source coissuer scoreboards were already
+    // released via the pending_writes decrement in writeback() when
+    // their last mf returned; releasing again here would race with the
+    // coissuer's next-instruction reservation. Shared-source coissuer
+    // scoreboards are released here OR in the early-release block above
+    // (Stage 6 Option A) — guarded by shared_side_released() to avoid
+    // double-release.
+    if (!pipe_reg.shared_side_released()) {
+      std::set<std::pair<unsigned, unsigned>> released_shared;
+      for (unsigned s = 0; s < sets.size(); s++) {
+        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+        if (!sets[s].source_inst->is_load()) continue;
+        enum _memory_space_t src_sp =
+            sets[s].source_inst->space.get_type();
+        if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue;
+        std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                           sets[s].split_id);
+        if (released_shared.count(key)) continue;
+        released_shared.insert(key);
+        bool is_intra = (sets[s].split_id != (unsigned)-1);
+        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+          unsigned reg_id = sets[s].source_inst->out[r];
+          if (reg_id == 0) continue;
+          if (is_intra)
+            m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg_id);
+          else
+            m_scoreboard->releaseRegister(sets[s].warp_id, reg_id);
+        }
+      }
+    }
+
+    // Defensive: global-source coissuer pending_writes should be drained
+    // by now (we waited on them above). Assert to catch bookkeeping drift.
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+      if (!sets[s].source_inst->is_load()) continue;
+      enum _memory_space_t src_sp =
+          sets[s].source_inst->space.get_type();
+      if (!warp_inst_t::is_global_bucket_space(src_sp)) continue;
+      bool is_intra = (sets[s].split_id != (unsigned)-1);
+      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+        unsigned reg_id = sets[s].source_inst->out[r];
+        if (reg_id == 0) continue;
+        if (is_intra) {
+          auto &m = m_pending_writes_secondary[sets[s].warp_id]
+                                              [sets[s].split_id];
+          auto it = m.find(reg_id);
+          assert(it == m.end() || it->second == 0);
+        } else {
+          auto &m = m_pending_writes[sets[s].warp_id];
+          auto it = m.find(reg_id);
+          assert(it == m.end() || it->second == 0);
+        }
+      }
+    }
+
+    // dec_inst_in_pipeline for primary + each unique coissuer. Mirrors
+    // case 0 shared writeback dedup: inter-warp coissuer decs once per
+    // warp; intra (secondary slot of primary warp) decs primary warp
+    // a second time.
+    m_core->dec_inst_in_pipeline(warp_id);
+    std::set<unsigned> inter_dec;
+    bool intra_dec = false;
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+      if (sets[s].warp_id != warp_id) {
+        if (inter_dec.insert(sets[s].warp_id).second)
+          m_core->dec_inst_in_pipeline(sets[s].warp_id);
+      } else if (!intra_dec) {
+        m_core->dec_inst_in_pipeline(warp_id);
+        intra_dec = true;
+      }
+    }
+
+    m_dispatch_reg->clear();
+    return;
+  }
+
   if (!pipe_reg.empty()) {
     unsigned warp_id = pipe_reg.warp_id();
     if (pipe_reg.is_load()) {
@@ -4464,8 +4933,8 @@ void ldst_unit::cycle() {
                                                sets[s].split_id);
             if (checked.count(key)) continue;
             checked.insert(key);
-            bool is_intra = (sets[s].warp_id == warp_id &&
-                             sets[s].split_id != (unsigned)-1);
+            // Secondary slot → secondary pending_writes map.
+            bool is_intra = (sets[s].split_id != (unsigned)-1);
             for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
               unsigned reg_id = sets[s].source_inst->out[r];
               if (reg_id == 0) continue;
@@ -4507,8 +4976,8 @@ void ldst_unit::cycle() {
                                                  sets[s].split_id);
               if (released.count(key)) continue;
               released.insert(key);
-              bool is_intra = (sets[s].warp_id == warp_id &&
-                               sets[s].split_id != (unsigned)-1);
+              // Secondary slot → secondary scoreboard map.
+              bool is_intra = (sets[s].split_id != (unsigned)-1);
               for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
                 unsigned reg_id = sets[s].source_inst->out[r];
                 if (reg_id == 0) continue;
@@ -4576,6 +5045,36 @@ void ldst_unit::cycle() {
             intra_dec = true;
           }
         }
+        // Mixed-space MEM co-issue (v2): for a STORE-primary composite,
+        // SHARED LOAD coissuers bypass all the normal scoreboard-release
+        // machinery (no mf/pending_writes for shared, no case 0 writeback
+        // since composite never enters pipeline_reg via is_load branch).
+        // Release their scoreboards here explicitly, mirroring case 0
+        // writeback's shared-coissue release. Global LOAD coissuers are
+        // handled by the pending_writes decrement path in writeback() and
+        // MUST NOT be released here (would double-release).
+        std::set<std::pair<unsigned, unsigned>> released;
+        for (unsigned s = 0; s < sets.size(); s++) {
+          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
+          if (!sets[s].source_inst->is_load()) continue;
+          enum _memory_space_t src_sp =
+              sets[s].source_inst->space.get_type();
+          if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue;
+          std::pair<unsigned, unsigned> key(sets[s].warp_id,
+                                             sets[s].split_id);
+          if (released.count(key)) continue;
+          released.insert(key);
+          bool is_intra = (sets[s].split_id != (unsigned)-1);
+          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
+            unsigned reg_id = sets[s].source_inst->out[r];
+            if (reg_id == 0) continue;
+            if (is_intra)
+              m_scoreboard->releaseRegisterSecondary(sets[s].warp_id,
+                                                     reg_id);
+            else
+              m_scoreboard->releaseRegister(sets[s].warp_id, reg_id);
+          }
+        }
       }
       m_dispatch_reg->clear();
     }