add more scheduling policies; update stats counters

3 files changed, 676 insertions, 238 deletions

diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index dd3bba7..7b947fb 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -700,8 +700,9 @@ void shader_core_config::reg_options(class OptionParser *opp) {
       opp, "-gpgpu_co_issue_priority", OPT_UINT32,
       &gpgpu_co_issue_priority,
       "Co-issue scheduler: 0=greedy (utilization-max), 1=intra-first, "
-      "2=inter-first, 3=fewest-lanes, 4=same-PC (default 0).",
-      "0");
+      "2=inter-first (matches pre-Change-4 behavior), 3=fewest-lanes, "
+      "4=same-PC (default 2).",
+      "2");
 
   for (unsigned j = 0; j < SPECIALIZED_UNIT_NUM; ++j) {
     std::stringstream ss;
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index ffbb0c6..921fa60 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -756,6 +756,47 @@ void shader_core_stats::print(FILE *fout) const {
 
   m_outgoing_traffic_stats->print(fout);
   m_incoming_traffic_stats->print(fout);
+
+  // Change 5: SIMD partitioning co-issue stats (aggregated across cores)
+  {
+    unsigned long long tot_inter = 0, tot_intra = 0;
+    unsigned long long tot_denied_fu = 0, tot_denied_sb = 0, tot_denied_ns = 0;
+    unsigned long long tot_active_lanes = 0, tot_composite_cycles = 0;
+    for (unsigned i = 0; i < m_config->num_shader(); i++) {
+      tot_inter += inter_warp_coissue_events[i];
+      tot_intra += intra_warp_coissue_events[i];
+      tot_denied_fu += coissue_denied_by_fu_mismatch[i];
+      tot_denied_sb += coissue_denied_by_scoreboard[i];
+      tot_denied_ns += coissue_denied_by_no_sets[i];
+      tot_active_lanes += coissue_total_active_lanes[i];
+      tot_composite_cycles += coissue_composite_cycles[i];
+    }
+    fprintf(fout, "SIMD Partitioning Co-issue Stats:\n");
+    fprintf(fout, "gpu_tot_inter_warp_coissue = %llu\n", tot_inter);
+    fprintf(fout, "gpu_tot_intra_warp_coissue = %llu\n", tot_intra);
+    fprintf(fout, "gpu_tot_coissue_composite_cycles = %llu\n",
+            tot_composite_cycles);
+    fprintf(fout, "gpu_tot_coissue_denied_fu_mismatch = %llu\n", tot_denied_fu);
+    fprintf(fout, "gpu_tot_coissue_denied_scoreboard = %llu\n", tot_denied_sb);
+    fprintf(fout, "gpu_tot_coissue_denied_no_sets = %llu\n", tot_denied_ns);
+    double avg_lanes = tot_composite_cycles > 0
+                           ? (double)tot_active_lanes /
+                                 (double)tot_composite_cycles
+                           : 0.0;
+    fprintf(fout, "gpu_tot_avg_active_lanes_per_coissue_cycle = %.3f\n",
+            avg_lanes);
+
+    unsigned nbins = m_config->gpgpu_num_simd_sets + 1;
+    fprintf(fout, "gpu_tot_simd_sets_used_histogram = [");
+    for (unsigned b = 0; b < nbins; b++) {
+      unsigned long long tot_bin = 0;
+      for (unsigned c = 0; c < m_config->num_shader(); c++) {
+        tot_bin += simd_sets_used_histogram[c * nbins + b];
+      }
+      fprintf(fout, "%s%llu", b == 0 ? "" : ", ", tot_bin);
+    }
+    fprintf(fout, "]\n");
+  }
 }
 
 void shader_core_stats::event_warp_issued(unsigned s_id, unsigned warp_id,
@@ -1525,6 +1566,493 @@ void scheduler_unit::order_by_priority(
   }
 }
 
+// ---------------------------------------------------------------------------
+// Change 4: co-issue scheduler helpers.
+// ---------------------------------------------------------------------------
+
+exec_unit_type_t scheduler_unit::classify_fu_type(
+    const warp_inst_t *inst) const {
+  if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) ||
+      (inst->op == MEMORY_BARRIER_OP) ||
+      (inst->op == TENSOR_CORE_LOAD_OP) ||
+      (inst->op == TENSOR_CORE_STORE_OP)) {
+    return exec_unit_type_t::MEM;
+  } else if (inst->op == SP_OP ||
+             (inst->op != DP_OP && inst->op != SFU_OP &&
+              inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP &&
+              inst->op < SPEC_UNIT_START_ID &&
+              m_shader->m_config->gpgpu_num_int_units == 0)) {
+    return exec_unit_type_t::SP;
+  } else if (inst->op != SP_OP && inst->op != DP_OP && inst->op != SFU_OP &&
+             inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP &&
+             inst->op < SPEC_UNIT_START_ID &&
+             m_shader->m_config->gpgpu_num_int_units > 0) {
+    return exec_unit_type_t::INT;
+  } else if (inst->op == DP_OP) {
+    return exec_unit_type_t::DP;
+  } else if (inst->op == SFU_OP || inst->op == ALU_SFU_OP) {
+    return exec_unit_type_t::SFU;
+  } else if (inst->op == TENSOR_CORE_OP) {
+    return exec_unit_type_t::TENSOR;
+  } else if (inst->op >= SPEC_UNIT_START_ID) {
+    return exec_unit_type_t::SPECIALIZED;
+  }
+  return exec_unit_type_t::NONE;
+}
+
+bool scheduler_unit::check_coissue_feasibility(
+    const warp_inst_t *composite, const warp_inst_t *cand_inst,
+    const active_mask_t &cand_mask, unsigned cand_warp_id,
+    unsigned available_sets, unsigned *sets_needed) {
+  const unsigned num_sets = m_shader->m_config->gpgpu_num_simd_sets;
+  const unsigned set_width = m_shader->m_config->simd_set_width;
+
+  if (m_shader->m_config->gpgpu_compaction_mode == 2) {
+    // Full compaction: just count sets needed; placement starts at
+    // next_free_set so no overlap is possible.
+    unsigned active = cand_mask.count();
+    unsigned needed = (active + set_width - 1) / set_width;
+    if (needed > available_sets) return false;
+    *sets_needed = needed;
+    return true;
+  }
+
+  // No compaction or XOR-static: build temp sets with the selected
+  // mapping and check set-level overlap vs. the composite.
+  warp_inst_t cand_temp(m_shader->m_config);
+  cand_temp = *cand_inst;
+  cand_temp.issue(cand_mask, cand_warp_id, 0, 0, 0, 0);
+  if (m_shader->m_config->gpgpu_compaction_mode == 1) {
+    cand_temp.compute_simd_sets_xor_static(num_sets, set_width);
+  } else {
+    cand_temp.compute_simd_sets(num_sets, set_width);
+  }
+  if (warp_inst_t::simd_sets_overlap(composite->get_simd_sets(),
+                                      cand_temp.get_simd_sets()))
+    return false;
+  unsigned needed = cand_temp.num_active_simd_sets();
+  if (needed > available_sets) return false;
+  *sets_needed = needed;
+  return true;
+}
+
+void scheduler_unit::try_inter_warp_coissue(
+    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
+    unsigned co_issue_primary_warp_id, unsigned &available_sets,
+    unsigned &next_free_set, unsigned sort_mode) {
+  // Build a sortable list of iterators into m_next_cycle_prioritized_warps.
+  // Index 0..N-1 into a snapshot vector for deterministic ordering.
+  std::vector<shd_warp_t *> cand_warps;
+  cand_warps.reserve(m_next_cycle_prioritized_warps.size());
+  for (std::vector<shd_warp_t *>::const_iterator iter =
+           m_next_cycle_prioritized_warps.begin();
+       iter != m_next_cycle_prioritized_warps.end(); iter++) {
+    if (*iter) cand_warps.push_back(*iter);
+  }
+
+  if (sort_mode == 1) {
+    // fewest-lanes: sort by popcount(active_mask) ascending. For
+    // candidates without a ready inst, sort them last (treat as +inf).
+    std::stable_sort(cand_warps.begin(), cand_warps.end(),
+                     [this](shd_warp_t *a, shd_warp_t *b) {
+                       const warp_inst_t *ia = a->ibuffer_next_inst();
+                       const warp_inst_t *ib = b->ibuffer_next_inst();
+                       unsigned pa = ia ? m_shader
+                                              ->get_active_mask(
+                                                  a->get_warp_id(), ia)
+                                              .count()
+                                        : 33u;
+                       unsigned pb = ib ? m_shader
+                                              ->get_active_mask(
+                                                  b->get_warp_id(), ib)
+                                              .count()
+                                        : 33u;
+                       return pa < pb;
+                     });
+  }
+
+  // For sort_mode==2 (same-PC), do two passes: same-PC first, then
+  // arbitrary. For sort_mode==0/1, one pass over cand_warps.
+  unsigned num_passes = (sort_mode == 2) ? 2 : 1;
+  unsigned primary_pc = 0;
+  if (sort_mode == 2) primary_pc = co_issue_composite->pc;
+
+  // Track warps already picked so pass-1 of same-PC mode doesn't revisit
+  // them (they've already had ibuffer_step'd; a second visit would
+  // co-issue a second instruction from the same warp in one scheduler
+  // cycle, which can lead to deadlocks).
+  std::set<unsigned> picked_warps;
+
+  for (unsigned pass = 0; pass < num_passes && available_sets > 0; pass++) {
+    for (std::vector<shd_warp_t *>::iterator iter2 = cand_warps.begin();
+         iter2 != cand_warps.end() && available_sets > 0; iter2++) {
+      if ((*iter2) == NULL || (*iter2)->done_exit()) continue;
+      unsigned cand_warp_id = (*iter2)->get_warp_id();
+
+      // Skip the primary warp (already issued).
+      if (cand_warp_id == co_issue_primary_warp_id) continue;
+      if (picked_warps.count(cand_warp_id)) continue;
+
+      // Basic eligibility.
+      if (warp(cand_warp_id).ibuffer_empty()) continue;
+      if (warp(cand_warp_id).waiting()) continue;
+
+      // AWARE reconvergence eligibility.
+      bool simt_ok = true;
+      if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
+        simt_ok = warp(cand_warp_id).valid() &&
+                  !warp(cand_warp_id).blocked() &&
+                  !warp(cand_warp_id).pending_reconvergence() &&
+                  !warp(cand_warp_id).virtualized();
+      }
+      if (!simt_ok) continue;
+
+      const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
+      if (!cand_inst) continue;
+      if (!warp(cand_warp_id).ibuffer_next_valid()) continue;
+
+      // Control hazard check.
+      unsigned cand_pc, cand_rpc;
+      m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
+                                        &cand_rpc);
+      if (cand_pc != cand_inst->pc) continue;
+
+      // Scoreboard check.
+      if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) {
+        m_stats->coissue_denied_by_scoreboard[get_sid()]++;
+        continue;
+      }
+
+      // FU type match.
+      if (classify_fu_type(cand_inst) != co_issue_fu_type) {
+        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+        continue;
+      }
+
+      // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any.
+      if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc)
+        continue;
+      // Don't re-issue the same warp in pass 1 (it would already have
+      // been picked in pass 0 if it qualified; but ibuffer_next_inst
+      // has advanced for picks from pass 0, so this is self-consistent).
+
+      const active_mask_t &cand_mask =
+          m_shader->get_active_mask(cand_warp_id, cand_inst);
+
+      unsigned cand_sets_needed = 0;
+      if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
+                                      cand_warp_id, available_sets,
+                                      &cand_sets_needed)) {
+        m_stats->coissue_denied_by_no_sets[get_sid()]++;
+        continue;
+      }
+
+      printf(
+          "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
+          "CO-ISSUED with primary warp %u (%u sets)\n",
+          m_shader->get_gpu()->gpu_sim_cycle +
+              m_shader->get_gpu()->gpu_tot_sim_cycle,
+          get_sid(), m_id, cand_warp_id, co_issue_primary_warp_id,
+          cand_sets_needed);
+
+      m_shader->co_issue_warp(co_issue_composite, cand_inst, cand_mask,
+                              cand_warp_id, m_id, next_free_set);
+
+      available_sets -= cand_sets_needed;
+      next_free_set += cand_sets_needed;
+      warp(cand_warp_id).ibuffer_step();
+      picked_warps.insert(cand_warp_id);
+      m_stats->inter_warp_coissue_events[get_sid()]++;
+    }
+  }
+}
+
+void scheduler_unit::try_intra_warp_coissue(
+    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
+    unsigned co_issue_primary_warp_id, unsigned &available_sets,
+    unsigned &next_free_set) {
+  // Only meaningful under AWARE_RECONVERGENCE (where secondary I-buf
+  // slots exist at all).
+  if (m_shader->m_config->model != AWARE_RECONVERGENCE) return;
+
+  const unsigned set_width = m_shader->m_config->simd_set_width;
+  const unsigned primary_warp_id = co_issue_primary_warp_id;
+
+  for (unsigned sec_slot = 2; sec_slot < 4 && available_sets > 0;
+       sec_slot++) {
+    if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue;
+
+    const warp_inst_t *sec_inst =
+        warp(primary_warp_id).ibuffer_slot_inst(sec_slot);
+    if (!sec_inst) continue;
+
+    unsigned sec_split_id =
+        warp(primary_warp_id).ibuffer_slot_split_id(sec_slot);
+    const active_mask_t &sec_mask =
+        warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot);
+
+    if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
+      warp(primary_warp_id).ibuffer_flush_half(1);
+      m_scoreboard->clearSecondary(primary_warp_id);
+      break;
+    }
+
+    unsigned split_pc;
+    simt_mask_t split_mask;
+    m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc,
+                              &split_mask);
+    if (split_pc != sec_inst->pc) {
+      warp(primary_warp_id).ibuffer_flush_half(1);
+      m_scoreboard->clearSecondary(primary_warp_id);
+      break;
+    }
+
+    if (classify_fu_type(sec_inst) != co_issue_fu_type) {
+      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+      continue;
+    }
+
+    if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) {
+      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
+      continue;
+    }
+
+    unsigned sec_active = sec_mask.count();
+    unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width;
+    if (sec_sets_needed > available_sets) {
+      m_stats->coissue_denied_by_no_sets[get_sid()]++;
+      continue;
+    }
+
+    printf(
+        "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
+        "split %u CO-ISSUED with primary split (%u sets)\n",
+        m_shader->get_gpu()->gpu_sim_cycle +
+            m_shader->get_gpu()->gpu_tot_sim_cycle,
+        get_sid(), m_id, primary_warp_id, sec_split_id, sec_sets_needed);
+
+    m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask,
+                            primary_warp_id, m_id, next_free_set,
+                            sec_split_id);
+
+    available_sets -= sec_sets_needed;
+    next_free_set += sec_sets_needed;
+    m_stats->intra_warp_coissue_events[get_sid()]++;
+
+    warp(primary_warp_id).ibuffer_free_slot(sec_slot);
+
+    if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
+      warp(primary_warp_id).ibuffer_flush_half(1);
+      m_scoreboard->clearSecondary(primary_warp_id);
+      break;
+    }
+  }
+}
+
+void scheduler_unit::try_utilization_max_coissue(
+    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
+    unsigned co_issue_primary_warp_id, unsigned &available_sets,
+    unsigned &next_free_set) {
+  // Build a unified pool of {inter, intra} candidates that each pass
+  // eligibility filters, then sort by density desc and greedy-pack.
+  struct coissue_cand_t {
+    bool is_intra;
+    unsigned warp_id;
+    unsigned split_id;      // valid iff is_intra
+    const warp_inst_t *inst;
+    active_mask_t mask;
+    unsigned sec_slot;      // valid iff is_intra (2 or 3)
+    unsigned active_count;
+    unsigned sets_needed;
+  };
+
+  std::vector<coissue_cand_t> pool;
+  const unsigned set_width = m_shader->m_config->simd_set_width;
+
+  // --- Inter-warp candidates ---
+  for (std::vector<shd_warp_t *>::const_iterator iter =
+           m_next_cycle_prioritized_warps.begin();
+       iter != m_next_cycle_prioritized_warps.end(); iter++) {
+    if ((*iter) == NULL || (*iter)->done_exit()) continue;
+    unsigned cand_warp_id = (*iter)->get_warp_id();
+    if (cand_warp_id == co_issue_primary_warp_id) continue;
+    if (warp(cand_warp_id).ibuffer_empty()) continue;
+    if (warp(cand_warp_id).waiting()) continue;
+
+    bool simt_ok = true;
+    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
+      simt_ok = warp(cand_warp_id).valid() &&
+                !warp(cand_warp_id).blocked() &&
+                !warp(cand_warp_id).pending_reconvergence() &&
+                !warp(cand_warp_id).virtualized();
+    }
+    if (!simt_ok) continue;
+
+    const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
+    if (!cand_inst) continue;
+    if (!warp(cand_warp_id).ibuffer_next_valid()) continue;
+
+    unsigned cand_pc, cand_rpc;
+    m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
+                                      &cand_rpc);
+    if (cand_pc != cand_inst->pc) continue;
+
+    if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) {
+      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
+      continue;
+    }
+    if (classify_fu_type(cand_inst) != co_issue_fu_type) {
+      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+      continue;
+    }
+
+    const active_mask_t &cand_mask =
+        m_shader->get_active_mask(cand_warp_id, cand_inst);
+    unsigned needed = 0;
+    if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
+                                    cand_warp_id, available_sets, &needed)) {
+      m_stats->coissue_denied_by_no_sets[get_sid()]++;
+      continue;
+    }
+
+    coissue_cand_t c;
+    c.is_intra = false;
+    c.warp_id = cand_warp_id;
+    c.split_id = (unsigned)-1;
+    c.inst = cand_inst;
+    c.mask = cand_mask;
+    c.sec_slot = 0;
+    c.active_count = cand_mask.count();
+    c.sets_needed = needed;
+    pool.push_back(c);
+  }
+
+  // --- Intra-warp candidates (primary's secondary slots 2..3) ---
+  if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
+    const unsigned primary_warp_id = co_issue_primary_warp_id;
+    for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) {
+      if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue;
+      const warp_inst_t *sec_inst =
+          warp(primary_warp_id).ibuffer_slot_inst(sec_slot);
+      if (!sec_inst) continue;
+      unsigned sec_split_id =
+          warp(primary_warp_id).ibuffer_slot_split_id(sec_slot);
+      const active_mask_t &sec_mask =
+          warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot);
+
+      if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) continue;
+      unsigned split_pc;
+      simt_mask_t split_mask;
+      m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc,
+                                &split_mask);
+      if (split_pc != sec_inst->pc) continue;
+      if (classify_fu_type(sec_inst) != co_issue_fu_type) {
+        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
+        continue;
+      }
+      if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) {
+        m_stats->coissue_denied_by_scoreboard[get_sid()]++;
+        continue;
+      }
+
+      unsigned sec_active = sec_mask.count();
+      unsigned sec_needed = (sec_active + set_width - 1) / set_width;
+      if (sec_needed > available_sets) {
+        m_stats->coissue_denied_by_no_sets[get_sid()]++;
+        continue;
+      }
+
+      coissue_cand_t c;
+      c.is_intra = true;
+      c.warp_id = primary_warp_id;
+      c.split_id = sec_split_id;
+      c.inst = sec_inst;
+      c.mask = sec_mask;
+      c.sec_slot = sec_slot;
+      c.active_count = sec_active;
+      c.sets_needed = sec_needed;
+      pool.push_back(c);
+    }
+  }
+
+  // --- Density sort: active_count / sets_needed desc ------------------
+  // Integer-safe: a.active * b.sets > b.active * a.sets.
+  // Ties broken by active_count desc, then warp_id asc for determinism.
+  std::stable_sort(pool.begin(), pool.end(),
+                   [](const coissue_cand_t &a, const coissue_cand_t &b) {
+                     unsigned long lhs = (unsigned long)a.active_count *
+                                         (unsigned long)b.sets_needed;
+                     unsigned long rhs = (unsigned long)b.active_count *
+                                         (unsigned long)a.sets_needed;
+                     if (lhs != rhs) return lhs > rhs;
+                     if (a.active_count != b.active_count)
+                       return a.active_count > b.active_count;
+                     return a.warp_id < b.warp_id;
+                   });
+
+  // --- Greedy pack ----------------------------------------------------
+  for (std::vector<coissue_cand_t>::iterator it = pool.begin();
+       it != pool.end() && available_sets > 0; it++) {
+    coissue_cand_t &c = *it;
+    // Re-check feasibility: prior picks may have filled sets that this
+    // candidate overlaps (only matters for compaction_mode != 2).
+    unsigned needed = 0;
+    if (c.is_intra) {
+      if (c.sets_needed > available_sets) continue;
+      needed = c.sets_needed;
+    } else {
+      if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask,
+                                      c.warp_id, available_sets, &needed))
+        continue;
+    }
+
+    if (c.is_intra) {
+      printf(
+          "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
+          "split %u CO-ISSUED with primary split (%u sets)\n",
+          m_shader->get_gpu()->gpu_sim_cycle +
+              m_shader->get_gpu()->gpu_tot_sim_cycle,
+          get_sid(), m_id, c.warp_id, c.split_id, needed);
+
+      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
+                              m_id, next_free_set, c.split_id);
+
+      available_sets -= needed;
+      next_free_set += needed;
+      m_stats->intra_warp_coissue_events[get_sid()]++;
+      warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
+      if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
+        warp(c.warp_id).ibuffer_flush_half(1);
+        m_scoreboard->clearSecondary(c.warp_id);
+        // The whole secondary half was just flushed — skip any remaining
+        // intra-warp picks for this warp. Inter-warp picks from other
+        // warps can still proceed.
+        for (std::vector<coissue_cand_t>::iterator jt = it + 1;
+             jt != pool.end();) {
+          if (jt->is_intra && jt->warp_id == c.warp_id)
+            jt = pool.erase(jt);
+          else
+            ++jt;
+        }
+      }
+    } else {
+      printf(
+          "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
+          "CO-ISSUED with primary warp %u (%u sets)\n",
+          m_shader->get_gpu()->gpu_sim_cycle +
+              m_shader->get_gpu()->gpu_tot_sim_cycle,
+          get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed);
+
+      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
+                              m_id, next_free_set);
+      available_sets -= needed;
+      next_free_set += needed;
+      m_stats->inter_warp_coissue_events[get_sid()]++;
+      warp(c.warp_id).ibuffer_step();
+    }
+  }
+}
+
 void scheduler_unit::cycle() {
   SCHED_DPRINTF("scheduler_unit::cycle()\n");
   bool valid_inst =
@@ -1866,13 +2394,12 @@ void scheduler_unit::cycle() {
     }
   }
 
-  // SIMD set co-issue pass: after issuing primary instruction, try to pack
-  // instructions from other warps into unused SIMD sets (same FU type).
+  // SIMD set co-issue dispatch: after issuing the primary instruction,
+  // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority.
   // Only for non-MEM FUs (LDST excluded from partitioning).
   if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL &&
       co_issue_fu_type != exec_unit_type_t::MEM &&
       co_issue_fu_type != exec_unit_type_t::NONE) {
-    // Count available (unused) sets and find next free set
     available_sets = 0;
     next_free_set = 0;
     const std::vector<simd_set_info> &primary_sets =
@@ -1881,7 +2408,7 @@ void scheduler_unit::cycle() {
       if (!primary_sets[s].valid)
         available_sets++;
       else
-        next_free_set = s + 1;  // track end of used sets (for compaction)
+        next_free_set = s + 1;
     }
 
     if (available_sets > 0) {
@@ -1891,245 +2418,69 @@ void scheduler_unit::cycle() {
                  m_shader->get_gpu()->gpu_tot_sim_cycle,
              get_sid(), m_id, available_sets, co_issue_primary_warp_id);
 
-      unsigned set_width = m_shader->m_config->simd_set_width;
-
-      // Scan other warps for co-issue candidates
-      for (std::vector<shd_warp_t *>::const_iterator iter2 =
-               m_next_cycle_prioritized_warps.begin();
-           iter2 != m_next_cycle_prioritized_warps.end() && available_sets > 0;
-           iter2++) {
-        if ((*iter2) == NULL || (*iter2)->done_exit()) continue;
-        unsigned cand_warp_id = (*iter2)->get_warp_id();
-
-        // Skip the primary warp (already issued)
-        if (cand_warp_id == co_issue_primary_warp_id) continue;
-
-        // Check basic eligibility
-        if (warp(cand_warp_id).ibuffer_empty()) continue;
-        if (warp(cand_warp_id).waiting()) continue;
-
-        // AWARE reconvergence eligibility
-        bool simt_ok = true;
-        if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
-          simt_ok = warp(cand_warp_id).valid() &&
-                    !warp(cand_warp_id).blocked() &&
-                    !warp(cand_warp_id).pending_reconvergence() &&
-                    !warp(cand_warp_id).virtualized();
-        }
-        if (!simt_ok) continue;
-
-        const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
-        if (!cand_inst) continue;
-        if (!warp(cand_warp_id).ibuffer_next_valid()) continue;
-
-        // Control hazard check
-        unsigned cand_pc, cand_rpc;
-        m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
-                                          &cand_rpc);
-        if (cand_pc != cand_inst->pc) continue;
-
-        // Scoreboard check for the candidate warp
-        if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) continue;
-
-        // Check candidate targets the same FU type as the primary
-        exec_unit_type_t cand_fu_type = exec_unit_type_t::NONE;
-        if ((cand_inst->op == LOAD_OP) || (cand_inst->op == STORE_OP) ||
-            (cand_inst->op == MEMORY_BARRIER_OP) ||
-            (cand_inst->op == TENSOR_CORE_LOAD_OP) ||
-            (cand_inst->op == TENSOR_CORE_STORE_OP)) {
-          cand_fu_type = exec_unit_type_t::MEM;
-        } else if (cand_inst->op == SP_OP ||
-                   (cand_inst->op != DP_OP && cand_inst->op != SFU_OP &&
-                    cand_inst->op != ALU_SFU_OP &&
-                    cand_inst->op != TENSOR_CORE_OP &&
-                    cand_inst->op < SPEC_UNIT_START_ID &&
-                    m_shader->m_config->gpgpu_num_int_units == 0)) {
-          cand_fu_type = exec_unit_type_t::SP;
-        } else if (cand_inst->op != SP_OP && cand_inst->op != DP_OP &&
-                   cand_inst->op != SFU_OP && cand_inst->op != ALU_SFU_OP &&
-                   cand_inst->op != TENSOR_CORE_OP &&
-                   cand_inst->op < SPEC_UNIT_START_ID &&
-                   m_shader->m_config->gpgpu_num_int_units > 0) {
-          cand_fu_type = exec_unit_type_t::INT;
-        } else if (cand_inst->op == DP_OP) {
-          cand_fu_type = exec_unit_type_t::DP;
-        } else if (cand_inst->op == SFU_OP || cand_inst->op == ALU_SFU_OP) {
-          cand_fu_type = exec_unit_type_t::SFU;
-        } else if (cand_inst->op == TENSOR_CORE_OP) {
-          cand_fu_type = exec_unit_type_t::TENSOR;
-        } else if (cand_inst->op >= SPEC_UNIT_START_ID) {
-          cand_fu_type = exec_unit_type_t::SPECIALIZED;
-        }
-        if (cand_fu_type != co_issue_fu_type) continue;
-
-        // Get candidate's active mask
-        const active_mask_t &cand_mask =
-            m_shader->get_active_mask(cand_warp_id, cand_inst);
-
-        unsigned cand_sets_needed;
-        bool can_co_issue;
-
-        if (m_shader->m_config->gpgpu_compaction_mode == 2) {
-          // Full compaction: just count sets needed, no overlap check.
-          // Compacted placement uses start_set, so no two sets ever alias.
-          unsigned cand_active = cand_mask.count();
-          cand_sets_needed = (cand_active + set_width - 1) / set_width;
-          can_co_issue = (cand_sets_needed <= available_sets);
-        } else {
-          // No compaction or XOR-static: build temp sets with the selected
-          // mapping and check set-level overlap vs. primary.
-          warp_inst_t cand_temp(m_shader->m_config);
-          cand_temp = *cand_inst;
-          cand_temp.issue(cand_mask, cand_warp_id, 0, 0, 0, 0);
-          if (m_shader->m_config->gpgpu_compaction_mode == 1) {
-            cand_temp.compute_simd_sets_xor_static(
-                m_shader->m_config->gpgpu_num_simd_sets,
-                m_shader->m_config->simd_set_width);
-          } else {
-            cand_temp.compute_simd_sets(
-                m_shader->m_config->gpgpu_num_simd_sets,
-                m_shader->m_config->simd_set_width);
-          }
-          can_co_issue =
-              !warp_inst_t::simd_sets_overlap(
-                  co_issue_composite->get_simd_sets(),
-                  cand_temp.get_simd_sets());
-          cand_sets_needed = cand_temp.num_active_simd_sets();
-          if (cand_sets_needed > available_sets) can_co_issue = false;
-        }
-
-        if (!can_co_issue) continue;
-
-        printf("SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
-               "CO-ISSUED with primary warp %u (%u sets)\n",
-               m_shader->get_gpu()->gpu_sim_cycle +
-                   m_shader->get_gpu()->gpu_tot_sim_cycle,
-               get_sid(), m_id, cand_warp_id, co_issue_primary_warp_id,
-               cand_sets_needed);
-
-        // Co-issue: functional execution, SIMT update, scoreboard, merge
-        m_shader->co_issue_warp(co_issue_composite, cand_inst, cand_mask,
-                                cand_warp_id, m_id, next_free_set);
-
-        available_sets -= cand_sets_needed;
-        next_free_set += cand_sets_needed;
-
-        // Advance candidate warp's ibuffer
-        warp(cand_warp_id).ibuffer_step();
+      switch (m_shader->m_config->gpgpu_co_issue_priority) {
+        case 0:  // greedy (utilization-max, unified pool)
+          try_utilization_max_coissue(
+              co_issue_composite, co_issue_fu_type,
+              co_issue_primary_warp_id, available_sets, next_free_set);
+          break;
+        case 1:  // intra-first
+          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set);
+          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set, /*sort_mode=*/0);
+          break;
+        case 3:  // fewest-lanes first (inter), then intra
+          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set, /*sort_mode=*/1);
+          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set);
+          break;
+        case 4:  // same-PC (inter), then intra
+          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set, /*sort_mode=*/2);
+          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set);
+          break;
+        case 2:  // inter-first (default, matches pre-Change-4)
+        default:
+          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set, /*sort_mode=*/0);
+          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
+                                 co_issue_primary_warp_id, available_sets,
+                                 next_free_set);
+          break;
       }
     }
   }
 
-  // INTRA-WARP co-issue: check same warp's secondary I-Buffer half (slots 2-3)
-  // for divergent split instructions that can be co-issued.
-  // Only in AWARE mode with SIMD partitioning enabled.
+  // Change 5: end-of-cycle SIMD partitioning stats. Only collect when
+  // partitioning is enabled and a composite was issued this cycle.
   if (m_shader->m_config->gpgpu_simd_partitioning &&
-      m_shader->m_config->model == AWARE_RECONVERGENCE &&
-      co_issue_composite != NULL && available_sets > 0 &&
-      co_issue_fu_type != exec_unit_type_t::MEM &&
-      co_issue_fu_type != exec_unit_type_t::NONE) {
-    unsigned primary_warp_id = co_issue_primary_warp_id;
-    unsigned set_width = m_shader->m_config->simd_set_width;
-
-    // Check slots 2-3 (secondary half) for valid instructions
-    for (unsigned sec_slot = 2;
-         sec_slot < 4 && available_sets > 0; sec_slot++) {
-      if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue;
-
-      const warp_inst_t *sec_inst =
-          warp(primary_warp_id).ibuffer_slot_inst(sec_slot);
-      if (!sec_inst) continue;
-
-      unsigned sec_split_id =
-          warp(primary_warp_id).ibuffer_slot_split_id(sec_slot);
-      const active_mask_t &sec_mask =
-          warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot);
-
-      // Verify the split is still valid in the splits table
-      if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
-        warp(primary_warp_id).ibuffer_flush_half(1);
-        m_scoreboard->clearSecondary(primary_warp_id);
-        break;
-      }
-
-      // Verify the split's PC still matches the instruction's PC
-      unsigned split_pc;
-      simt_mask_t split_mask;
-      m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc,
-                               &split_mask);
-      if (split_pc != sec_inst->pc) {
-        warp(primary_warp_id).ibuffer_flush_half(1);
-        m_scoreboard->clearSecondary(primary_warp_id);
-        break;
-      }
-
-      // Check same FU type (reuse the FU type determination logic)
-      exec_unit_type_t sec_fu_type = exec_unit_type_t::NONE;
-      if ((sec_inst->op == LOAD_OP) || (sec_inst->op == STORE_OP) ||
-          (sec_inst->op == MEMORY_BARRIER_OP) ||
-          (sec_inst->op == TENSOR_CORE_LOAD_OP) ||
-          (sec_inst->op == TENSOR_CORE_STORE_OP)) {
-        sec_fu_type = exec_unit_type_t::MEM;
-      } else if (sec_inst->op == SP_OP ||
-                 (sec_inst->op != DP_OP && sec_inst->op != SFU_OP &&
-                  sec_inst->op != ALU_SFU_OP &&
-                  sec_inst->op != TENSOR_CORE_OP &&
-                  sec_inst->op < SPEC_UNIT_START_ID &&
-                  m_shader->m_config->gpgpu_num_int_units == 0)) {
-        sec_fu_type = exec_unit_type_t::SP;
-      } else if (sec_inst->op != SP_OP && sec_inst->op != DP_OP &&
-                 sec_inst->op != SFU_OP && sec_inst->op != ALU_SFU_OP &&
-                 sec_inst->op != TENSOR_CORE_OP &&
-                 sec_inst->op < SPEC_UNIT_START_ID &&
-                 m_shader->m_config->gpgpu_num_int_units > 0) {
-        sec_fu_type = exec_unit_type_t::INT;
-      } else if (sec_inst->op == DP_OP) {
-        sec_fu_type = exec_unit_type_t::DP;
-      } else if (sec_inst->op == SFU_OP || sec_inst->op == ALU_SFU_OP) {
-        sec_fu_type = exec_unit_type_t::SFU;
-      } else if (sec_inst->op == TENSOR_CORE_OP) {
-        sec_fu_type = exec_unit_type_t::TENSOR;
-      } else if (sec_inst->op >= SPEC_UNIT_START_ID) {
-        sec_fu_type = exec_unit_type_t::SPECIALIZED;
-      }
-      if (sec_fu_type != co_issue_fu_type) continue;
-
-      // Secondary scoreboard check: dependencies within secondary stream
-      if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst))
-        continue;
-
-      // Compute sets needed
-      unsigned sec_active = sec_mask.count();
-      unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width;
-      if (sec_sets_needed > available_sets) continue;
-
-      printf("SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
-             "split %u CO-ISSUED with primary split (%u sets)\n",
-             m_shader->get_gpu()->gpu_sim_cycle +
-                 m_shader->get_gpu()->gpu_tot_sim_cycle,
-             get_sid(), m_id, primary_warp_id, sec_split_id,
-             sec_sets_needed);
-
-      // Co-issue with targeted SIMT update (pass split_id)
-      m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask,
-                              primary_warp_id, m_id, next_free_set,
-                              sec_split_id);
-
-      available_sets -= sec_sets_needed;
-      next_free_set += sec_sets_needed;
-
-      // Free the secondary I-buffer slot (don't call ibuffer_free/step,
-      // use direct slot free since this is not the m_next pointer)
-      warp(primary_warp_id).ibuffer_free_slot(sec_slot);
-
-      // After SIMT update, check if the split was invalidated (diverged)
-      if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
-        // Split was invalidated by the update — flush the entire secondary half
-        warp(primary_warp_id).ibuffer_flush_half(1);
-        m_scoreboard->clearSecondary(primary_warp_id);
-        break;
+      co_issue_composite != NULL) {
+    unsigned valid_sets = 0;
+    unsigned active_lanes = 0;
+    const std::vector<simd_set_info> &sets =
+        co_issue_composite->get_simd_sets();
+    for (unsigned s = 0; s < sets.size(); s++) {
+      if (sets[s].valid) {
+        valid_sets++;
+        active_lanes += sets[s].set_active_mask.count();
       }
     }
+    unsigned sid = get_sid();
+    m_stats->coissue_composite_cycles[sid]++;
+    m_stats->coissue_total_active_lanes[sid] += active_lanes;
+    unsigned nbins = m_shader->m_config->gpgpu_num_simd_sets + 1;
+    if (valid_sets >= nbins) valid_sets = nbins - 1;  // safety clamp
+    m_stats->simd_sets_used_histogram[sid * nbins + valid_sets]++;
   }
 
   // issue stall statistics:
diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 3d1aea9..70de529 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -550,6 +550,50 @@ class scheduler_unit {  // this can be copied freely, so can be used in std
       const std::vector<shd_warp_t *>::const_iterator &prioritized_iter);
   inline int get_sid() const;
 
+  // --- Change 4: co-issue scheduler helpers ----------------------------
+  // Classify an instruction's FU type (the 25-line op→exec_unit_type_t
+  // switch that was duplicated across the primary pick, inter-warp pass,
+  // and intra-warp pass).
+  exec_unit_type_t classify_fu_type(const warp_inst_t *inst) const;
+
+  // Check whether `cand_inst` can co-issue alongside `composite` given
+  // `available_sets`. Writes the candidate's sets_needed to *sets_needed
+  // on success. Uses the current compaction_mode to decide between the
+  // count-only path (full compaction) and the overlap-check path
+  // (identity / XOR-static).
+  bool check_coissue_feasibility(const warp_inst_t *composite,
+                                 const warp_inst_t *cand_inst,
+                                 const active_mask_t &cand_mask,
+                                 unsigned cand_warp_id,
+                                 unsigned available_sets,
+                                 unsigned *sets_needed);
+
+  // Runs the inter-warp co-issue pass. sort_mode:
+  //   0 = round-robin (iterate m_next_cycle_prioritized_warps as-is)
+  //   1 = fewest-lanes first (sort by popcount(active_mask) ascending)
+  //   2 = same-PC first, fall through to arbitrary within the cycle
+  void try_inter_warp_coissue(warp_inst_t *co_issue_composite,
+                              exec_unit_type_t co_issue_fu_type,
+                              unsigned co_issue_primary_warp_id,
+                              unsigned &available_sets,
+                              unsigned &next_free_set,
+                              unsigned sort_mode);
+
+  // Runs the intra-warp co-issue pass (primary warp's secondary slots).
+  void try_intra_warp_coissue(warp_inst_t *co_issue_composite,
+                              exec_unit_type_t co_issue_fu_type,
+                              unsigned co_issue_primary_warp_id,
+                              unsigned &available_sets,
+                              unsigned &next_free_set);
+
+  // Unified intra+inter candidate pool, density-sorted greedy packing.
+  // Approximates optimal SIMD-lane utilization for the cycle.
+  void try_utilization_max_coissue(warp_inst_t *co_issue_composite,
+                                   exec_unit_type_t co_issue_fu_type,
+                                   unsigned co_issue_primary_warp_id,
+                                   unsigned &available_sets,
+                                   unsigned &next_free_set);
+
  protected:
   shd_warp_t &warp(int i);
 
@@ -2039,6 +2083,18 @@ struct shader_core_stats_pod {
   unsigned *gpgpu_n_shmem_bank_access;
   long *n_simt_to_mem;  // Interconnect power stats
   long *n_mem_to_simt;
+
+  // Change 5: SIMD partitioning co-issue counters. All per-core arrays
+  // of size num_shader(); histogram is flat size num_shader() *
+  // (gpgpu_num_simd_sets + 1).
+  unsigned long long *inter_warp_coissue_events;
+  unsigned long long *intra_warp_coissue_events;
+  unsigned long long *coissue_denied_by_fu_mismatch;
+  unsigned long long *coissue_denied_by_scoreboard;
+  unsigned long long *coissue_denied_by_no_sets;
+  unsigned long long *coissue_total_active_lanes;
+  unsigned long long *coissue_composite_cycles;
+  unsigned long long *simd_sets_used_histogram;
 };
 
 class shader_core_stats : public shader_core_stats_pod {
@@ -2151,6 +2207,27 @@ class shader_core_stats : public shader_core_stats_pod {
 
     m_shader_dynamic_warp_issue_distro.resize(config->num_shader());
     m_shader_warp_slot_issue_distro.resize(config->num_shader());
+
+    // Change 5: SIMD partitioning co-issue counters
+    inter_warp_coissue_events = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    intra_warp_coissue_events = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    coissue_denied_by_fu_mismatch = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    coissue_denied_by_scoreboard = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    coissue_denied_by_no_sets = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    coissue_total_active_lanes = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    coissue_composite_cycles = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    // Histogram bins: 0..gpgpu_num_simd_sets (inclusive). When
+    // partitioning is off, gpgpu_num_simd_sets == 1 so we allocate 2 bins.
+    unsigned nbins = config->gpgpu_num_simd_sets + 1;
+    simd_sets_used_histogram = (unsigned long long *)calloc(
+        (size_t)config->num_shader() * nbins, sizeof(unsigned long long));
   }
 
   ~shader_core_stats() {
@@ -2203,6 +2280,15 @@ class shader_core_stats : public shader_core_stats_pod {
     free(m_n_diverge);
     free(shader_cycle_distro);
     free(last_shader_cycle_distro);
+    // Change 5: SIMD partitioning co-issue counters
+    free(inter_warp_coissue_events);
+    free(intra_warp_coissue_events);
+    free(coissue_denied_by_fu_mismatch);
+    free(coissue_denied_by_scoreboard);
+    free(coissue_denied_by_no_sets);
+    free(coissue_total_active_lanes);
+    free(coissue_composite_cycles);
+    free(simd_sets_used_histogram);
   }
 
   void new_grid() {}