From 966b202df2441204386bad4d8bcd2c1d47aedd36 Mon Sep 17 00:00:00 2001 From: Davit Grigoryan Date: Sat, 18 Apr 2026 21:00:03 +0000 Subject: add more scheduling policies; update stats counters --- src/gpgpu-sim/gpu-sim.cc | 5 +- src/gpgpu-sim/shader.cc | 823 +++++++++++++++++++++++++++++++++-------------- src/gpgpu-sim/shader.h | 86 +++++ 3 files changed, 676 insertions(+), 238 deletions(-) diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index dd3bba7..7b947fb 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -700,8 +700,9 @@ void shader_core_config::reg_options(class OptionParser *opp) { opp, "-gpgpu_co_issue_priority", OPT_UINT32, &gpgpu_co_issue_priority, "Co-issue scheduler: 0=greedy (utilization-max), 1=intra-first, " - "2=inter-first, 3=fewest-lanes, 4=same-PC (default 0).", - "0"); + "2=inter-first (matches pre-Change-4 behavior), 3=fewest-lanes, " + "4=same-PC (default 2).", + "2"); for (unsigned j = 0; j < SPECIALIZED_UNIT_NUM; ++j) { std::stringstream ss; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index ffbb0c6..921fa60 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -756,6 +756,47 @@ void shader_core_stats::print(FILE *fout) const { m_outgoing_traffic_stats->print(fout); m_incoming_traffic_stats->print(fout); + + // Change 5: SIMD partitioning co-issue stats (aggregated across cores) + { + unsigned long long tot_inter = 0, tot_intra = 0; + unsigned long long tot_denied_fu = 0, tot_denied_sb = 0, tot_denied_ns = 0; + unsigned long long tot_active_lanes = 0, tot_composite_cycles = 0; + for (unsigned i = 0; i < m_config->num_shader(); i++) { + tot_inter += inter_warp_coissue_events[i]; + tot_intra += intra_warp_coissue_events[i]; + tot_denied_fu += coissue_denied_by_fu_mismatch[i]; + tot_denied_sb += coissue_denied_by_scoreboard[i]; + tot_denied_ns += coissue_denied_by_no_sets[i]; + tot_active_lanes += coissue_total_active_lanes[i]; + tot_composite_cycles += coissue_composite_cycles[i]; + } + fprintf(fout, "SIMD Partitioning Co-issue Stats:\n"); + fprintf(fout, "gpu_tot_inter_warp_coissue = %llu\n", tot_inter); + fprintf(fout, "gpu_tot_intra_warp_coissue = %llu\n", tot_intra); + fprintf(fout, "gpu_tot_coissue_composite_cycles = %llu\n", + tot_composite_cycles); + fprintf(fout, "gpu_tot_coissue_denied_fu_mismatch = %llu\n", tot_denied_fu); + fprintf(fout, "gpu_tot_coissue_denied_scoreboard = %llu\n", tot_denied_sb); + fprintf(fout, "gpu_tot_coissue_denied_no_sets = %llu\n", tot_denied_ns); + double avg_lanes = tot_composite_cycles > 0 + ? (double)tot_active_lanes / + (double)tot_composite_cycles + : 0.0; + fprintf(fout, "gpu_tot_avg_active_lanes_per_coissue_cycle = %.3f\n", + avg_lanes); + + unsigned nbins = m_config->gpgpu_num_simd_sets + 1; + fprintf(fout, "gpu_tot_simd_sets_used_histogram = ["); + for (unsigned b = 0; b < nbins; b++) { + unsigned long long tot_bin = 0; + for (unsigned c = 0; c < m_config->num_shader(); c++) { + tot_bin += simd_sets_used_histogram[c * nbins + b]; + } + fprintf(fout, "%s%llu", b == 0 ? "" : ", ", tot_bin); + } + fprintf(fout, "]\n"); + } } void shader_core_stats::event_warp_issued(unsigned s_id, unsigned warp_id, @@ -1525,6 +1566,493 @@ void scheduler_unit::order_by_priority( } } +// --------------------------------------------------------------------------- +// Change 4: co-issue scheduler helpers. +// --------------------------------------------------------------------------- + +exec_unit_type_t scheduler_unit::classify_fu_type( + const warp_inst_t *inst) const { + if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) || + (inst->op == MEMORY_BARRIER_OP) || + (inst->op == TENSOR_CORE_LOAD_OP) || + (inst->op == TENSOR_CORE_STORE_OP)) { + return exec_unit_type_t::MEM; + } else if (inst->op == SP_OP || + (inst->op != DP_OP && inst->op != SFU_OP && + inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP && + inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units == 0)) { + return exec_unit_type_t::SP; + } else if (inst->op != SP_OP && inst->op != DP_OP && inst->op != SFU_OP && + inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP && + inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units > 0) { + return exec_unit_type_t::INT; + } else if (inst->op == DP_OP) { + return exec_unit_type_t::DP; + } else if (inst->op == SFU_OP || inst->op == ALU_SFU_OP) { + return exec_unit_type_t::SFU; + } else if (inst->op == TENSOR_CORE_OP) { + return exec_unit_type_t::TENSOR; + } else if (inst->op >= SPEC_UNIT_START_ID) { + return exec_unit_type_t::SPECIALIZED; + } + return exec_unit_type_t::NONE; +} + +bool scheduler_unit::check_coissue_feasibility( + const warp_inst_t *composite, const warp_inst_t *cand_inst, + const active_mask_t &cand_mask, unsigned cand_warp_id, + unsigned available_sets, unsigned *sets_needed) { + const unsigned num_sets = m_shader->m_config->gpgpu_num_simd_sets; + const unsigned set_width = m_shader->m_config->simd_set_width; + + if (m_shader->m_config->gpgpu_compaction_mode == 2) { + // Full compaction: just count sets needed; placement starts at + // next_free_set so no overlap is possible. + unsigned active = cand_mask.count(); + unsigned needed = (active + set_width - 1) / set_width; + if (needed > available_sets) return false; + *sets_needed = needed; + return true; + } + + // No compaction or XOR-static: build temp sets with the selected + // mapping and check set-level overlap vs. the composite. + warp_inst_t cand_temp(m_shader->m_config); + cand_temp = *cand_inst; + cand_temp.issue(cand_mask, cand_warp_id, 0, 0, 0, 0); + if (m_shader->m_config->gpgpu_compaction_mode == 1) { + cand_temp.compute_simd_sets_xor_static(num_sets, set_width); + } else { + cand_temp.compute_simd_sets(num_sets, set_width); + } + if (warp_inst_t::simd_sets_overlap(composite->get_simd_sets(), + cand_temp.get_simd_sets())) + return false; + unsigned needed = cand_temp.num_active_simd_sets(); + if (needed > available_sets) return false; + *sets_needed = needed; + return true; +} + +void scheduler_unit::try_inter_warp_coissue( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set, unsigned sort_mode) { + // Build a sortable list of iterators into m_next_cycle_prioritized_warps. + // Index 0..N-1 into a snapshot vector for deterministic ordering. + std::vector cand_warps; + cand_warps.reserve(m_next_cycle_prioritized_warps.size()); + for (std::vector::const_iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != m_next_cycle_prioritized_warps.end(); iter++) { + if (*iter) cand_warps.push_back(*iter); + } + + if (sort_mode == 1) { + // fewest-lanes: sort by popcount(active_mask) ascending. For + // candidates without a ready inst, sort them last (treat as +inf). + std::stable_sort(cand_warps.begin(), cand_warps.end(), + [this](shd_warp_t *a, shd_warp_t *b) { + const warp_inst_t *ia = a->ibuffer_next_inst(); + const warp_inst_t *ib = b->ibuffer_next_inst(); + unsigned pa = ia ? m_shader + ->get_active_mask( + a->get_warp_id(), ia) + .count() + : 33u; + unsigned pb = ib ? m_shader + ->get_active_mask( + b->get_warp_id(), ib) + .count() + : 33u; + return pa < pb; + }); + } + + // For sort_mode==2 (same-PC), do two passes: same-PC first, then + // arbitrary. For sort_mode==0/1, one pass over cand_warps. + unsigned num_passes = (sort_mode == 2) ? 2 : 1; + unsigned primary_pc = 0; + if (sort_mode == 2) primary_pc = co_issue_composite->pc; + + // Track warps already picked so pass-1 of same-PC mode doesn't revisit + // them (they've already had ibuffer_step'd; a second visit would + // co-issue a second instruction from the same warp in one scheduler + // cycle, which can lead to deadlocks). + std::set picked_warps; + + for (unsigned pass = 0; pass < num_passes && available_sets > 0; pass++) { + for (std::vector::iterator iter2 = cand_warps.begin(); + iter2 != cand_warps.end() && available_sets > 0; iter2++) { + if ((*iter2) == NULL || (*iter2)->done_exit()) continue; + unsigned cand_warp_id = (*iter2)->get_warp_id(); + + // Skip the primary warp (already issued). + if (cand_warp_id == co_issue_primary_warp_id) continue; + if (picked_warps.count(cand_warp_id)) continue; + + // Basic eligibility. + if (warp(cand_warp_id).ibuffer_empty()) continue; + if (warp(cand_warp_id).waiting()) continue; + + // AWARE reconvergence eligibility. + bool simt_ok = true; + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + } + if (!simt_ok) continue; + + const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); + if (!cand_inst) continue; + if (!warp(cand_warp_id).ibuffer_next_valid()) continue; + + // Control hazard check. + unsigned cand_pc, cand_rpc; + m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, + &cand_rpc); + if (cand_pc != cand_inst->pc) continue; + + // Scoreboard check. + if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + + // FU type match. + if (classify_fu_type(cand_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + + // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any. + if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) + continue; + // Don't re-issue the same warp in pass 1 (it would already have + // been picked in pass 0 if it qualified; but ibuffer_next_inst + // has advanced for picks from pass 0, so this is self-consistent). + + const active_mask_t &cand_mask = + m_shader->get_active_mask(cand_warp_id, cand_inst); + + unsigned cand_sets_needed = 0; + if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, available_sets, + &cand_sets_needed)) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " + "CO-ISSUED with primary warp %u (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, cand_warp_id, co_issue_primary_warp_id, + cand_sets_needed); + + m_shader->co_issue_warp(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, m_id, next_free_set); + + available_sets -= cand_sets_needed; + next_free_set += cand_sets_needed; + warp(cand_warp_id).ibuffer_step(); + picked_warps.insert(cand_warp_id); + m_stats->inter_warp_coissue_events[get_sid()]++; + } + } +} + +void scheduler_unit::try_intra_warp_coissue( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set) { + // Only meaningful under AWARE_RECONVERGENCE (where secondary I-buf + // slots exist at all). + if (m_shader->m_config->model != AWARE_RECONVERGENCE) return; + + const unsigned set_width = m_shader->m_config->simd_set_width; + const unsigned primary_warp_id = co_issue_primary_warp_id; + + for (unsigned sec_slot = 2; sec_slot < 4 && available_sets > 0; + sec_slot++) { + if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue; + + const warp_inst_t *sec_inst = + warp(primary_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + + unsigned sec_split_id = + warp(primary_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot); + + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + warp(primary_warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(primary_warp_id); + break; + } + + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) { + warp(primary_warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(primary_warp_id); + break; + } + + if (classify_fu_type(sec_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + + if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + + unsigned sec_active = sec_mask.count(); + unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width; + if (sec_sets_needed > available_sets) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, primary_warp_id, sec_split_id, sec_sets_needed); + + m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask, + primary_warp_id, m_id, next_free_set, + sec_split_id); + + available_sets -= sec_sets_needed; + next_free_set += sec_sets_needed; + m_stats->intra_warp_coissue_events[get_sid()]++; + + warp(primary_warp_id).ibuffer_free_slot(sec_slot); + + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + warp(primary_warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(primary_warp_id); + break; + } + } +} + +void scheduler_unit::try_utilization_max_coissue( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set) { + // Build a unified pool of {inter, intra} candidates that each pass + // eligibility filters, then sort by density desc and greedy-pack. + struct coissue_cand_t { + bool is_intra; + unsigned warp_id; + unsigned split_id; // valid iff is_intra + const warp_inst_t *inst; + active_mask_t mask; + unsigned sec_slot; // valid iff is_intra (2 or 3) + unsigned active_count; + unsigned sets_needed; + }; + + std::vector pool; + const unsigned set_width = m_shader->m_config->simd_set_width; + + // --- Inter-warp candidates --- + for (std::vector::const_iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != m_next_cycle_prioritized_warps.end(); iter++) { + if ((*iter) == NULL || (*iter)->done_exit()) continue; + unsigned cand_warp_id = (*iter)->get_warp_id(); + if (cand_warp_id == co_issue_primary_warp_id) continue; + if (warp(cand_warp_id).ibuffer_empty()) continue; + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = true; + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + } + if (!simt_ok) continue; + + const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); + if (!cand_inst) continue; + if (!warp(cand_warp_id).ibuffer_next_valid()) continue; + + unsigned cand_pc, cand_rpc; + m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, + &cand_rpc); + if (cand_pc != cand_inst->pc) continue; + + if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + if (classify_fu_type(cand_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + + const active_mask_t &cand_mask = + m_shader->get_active_mask(cand_warp_id, cand_inst); + unsigned needed = 0; + if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, available_sets, &needed)) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = false; + c.warp_id = cand_warp_id; + c.split_id = (unsigned)-1; + c.inst = cand_inst; + c.mask = cand_mask; + c.sec_slot = 0; + c.active_count = cand_mask.count(); + c.sets_needed = needed; + pool.push_back(c); + } + + // --- Intra-warp candidates (primary's secondary slots 2..3) --- + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + const unsigned primary_warp_id = co_issue_primary_warp_id; + for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) { + if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue; + const warp_inst_t *sec_inst = + warp(primary_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + unsigned sec_split_id = + warp(primary_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot); + + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) continue; + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) continue; + if (classify_fu_type(sec_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + + unsigned sec_active = sec_mask.count(); + unsigned sec_needed = (sec_active + set_width - 1) / set_width; + if (sec_needed > available_sets) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = true; + c.warp_id = primary_warp_id; + c.split_id = sec_split_id; + c.inst = sec_inst; + c.mask = sec_mask; + c.sec_slot = sec_slot; + c.active_count = sec_active; + c.sets_needed = sec_needed; + pool.push_back(c); + } + } + + // --- Density sort: active_count / sets_needed desc ------------------ + // Integer-safe: a.active * b.sets > b.active * a.sets. + // Ties broken by active_count desc, then warp_id asc for determinism. + std::stable_sort(pool.begin(), pool.end(), + [](const coissue_cand_t &a, const coissue_cand_t &b) { + unsigned long lhs = (unsigned long)a.active_count * + (unsigned long)b.sets_needed; + unsigned long rhs = (unsigned long)b.active_count * + (unsigned long)a.sets_needed; + if (lhs != rhs) return lhs > rhs; + if (a.active_count != b.active_count) + return a.active_count > b.active_count; + return a.warp_id < b.warp_id; + }); + + // --- Greedy pack ---------------------------------------------------- + for (std::vector::iterator it = pool.begin(); + it != pool.end() && available_sets > 0; it++) { + coissue_cand_t &c = *it; + // Re-check feasibility: prior picks may have filled sets that this + // candidate overlaps (only matters for compaction_mode != 2). + unsigned needed = 0; + if (c.is_intra) { + if (c.sets_needed > available_sets) continue; + needed = c.sets_needed; + } else { + if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask, + c.warp_id, available_sets, &needed)) + continue; + } + + if (c.is_intra) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, c.split_id, needed); + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set, c.split_id); + + available_sets -= needed; + next_free_set += needed; + m_stats->intra_warp_coissue_events[get_sid()]++; + warp(c.warp_id).ibuffer_free_slot(c.sec_slot); + if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { + warp(c.warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(c.warp_id); + // The whole secondary half was just flushed — skip any remaining + // intra-warp picks for this warp. Inter-warp picks from other + // warps can still proceed. + for (std::vector::iterator jt = it + 1; + jt != pool.end();) { + if (jt->is_intra && jt->warp_id == c.warp_id) + jt = pool.erase(jt); + else + ++jt; + } + } + } else { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " + "CO-ISSUED with primary warp %u (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed); + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set); + available_sets -= needed; + next_free_set += needed; + m_stats->inter_warp_coissue_events[get_sid()]++; + warp(c.warp_id).ibuffer_step(); + } + } +} + void scheduler_unit::cycle() { SCHED_DPRINTF("scheduler_unit::cycle()\n"); bool valid_inst = @@ -1866,13 +2394,12 @@ void scheduler_unit::cycle() { } } - // SIMD set co-issue pass: after issuing primary instruction, try to pack - // instructions from other warps into unused SIMD sets (same FU type). + // SIMD set co-issue dispatch: after issuing the primary instruction, + // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority. // Only for non-MEM FUs (LDST excluded from partitioning). if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL && co_issue_fu_type != exec_unit_type_t::MEM && co_issue_fu_type != exec_unit_type_t::NONE) { - // Count available (unused) sets and find next free set available_sets = 0; next_free_set = 0; const std::vector &primary_sets = @@ -1881,7 +2408,7 @@ void scheduler_unit::cycle() { if (!primary_sets[s].valid) available_sets++; else - next_free_set = s + 1; // track end of used sets (for compaction) + next_free_set = s + 1; } if (available_sets > 0) { @@ -1891,245 +2418,69 @@ void scheduler_unit::cycle() { m_shader->get_gpu()->gpu_tot_sim_cycle, get_sid(), m_id, available_sets, co_issue_primary_warp_id); - unsigned set_width = m_shader->m_config->simd_set_width; - - // Scan other warps for co-issue candidates - for (std::vector::const_iterator iter2 = - m_next_cycle_prioritized_warps.begin(); - iter2 != m_next_cycle_prioritized_warps.end() && available_sets > 0; - iter2++) { - if ((*iter2) == NULL || (*iter2)->done_exit()) continue; - unsigned cand_warp_id = (*iter2)->get_warp_id(); - - // Skip the primary warp (already issued) - if (cand_warp_id == co_issue_primary_warp_id) continue; - - // Check basic eligibility - if (warp(cand_warp_id).ibuffer_empty()) continue; - if (warp(cand_warp_id).waiting()) continue; - - // AWARE reconvergence eligibility - bool simt_ok = true; - if (m_shader->m_config->model == AWARE_RECONVERGENCE) { - simt_ok = warp(cand_warp_id).valid() && - !warp(cand_warp_id).blocked() && - !warp(cand_warp_id).pending_reconvergence() && - !warp(cand_warp_id).virtualized(); - } - if (!simt_ok) continue; - - const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); - if (!cand_inst) continue; - if (!warp(cand_warp_id).ibuffer_next_valid()) continue; - - // Control hazard check - unsigned cand_pc, cand_rpc; - m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, - &cand_rpc); - if (cand_pc != cand_inst->pc) continue; - - // Scoreboard check for the candidate warp - if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) continue; - - // Check candidate targets the same FU type as the primary - exec_unit_type_t cand_fu_type = exec_unit_type_t::NONE; - if ((cand_inst->op == LOAD_OP) || (cand_inst->op == STORE_OP) || - (cand_inst->op == MEMORY_BARRIER_OP) || - (cand_inst->op == TENSOR_CORE_LOAD_OP) || - (cand_inst->op == TENSOR_CORE_STORE_OP)) { - cand_fu_type = exec_unit_type_t::MEM; - } else if (cand_inst->op == SP_OP || - (cand_inst->op != DP_OP && cand_inst->op != SFU_OP && - cand_inst->op != ALU_SFU_OP && - cand_inst->op != TENSOR_CORE_OP && - cand_inst->op < SPEC_UNIT_START_ID && - m_shader->m_config->gpgpu_num_int_units == 0)) { - cand_fu_type = exec_unit_type_t::SP; - } else if (cand_inst->op != SP_OP && cand_inst->op != DP_OP && - cand_inst->op != SFU_OP && cand_inst->op != ALU_SFU_OP && - cand_inst->op != TENSOR_CORE_OP && - cand_inst->op < SPEC_UNIT_START_ID && - m_shader->m_config->gpgpu_num_int_units > 0) { - cand_fu_type = exec_unit_type_t::INT; - } else if (cand_inst->op == DP_OP) { - cand_fu_type = exec_unit_type_t::DP; - } else if (cand_inst->op == SFU_OP || cand_inst->op == ALU_SFU_OP) { - cand_fu_type = exec_unit_type_t::SFU; - } else if (cand_inst->op == TENSOR_CORE_OP) { - cand_fu_type = exec_unit_type_t::TENSOR; - } else if (cand_inst->op >= SPEC_UNIT_START_ID) { - cand_fu_type = exec_unit_type_t::SPECIALIZED; - } - if (cand_fu_type != co_issue_fu_type) continue; - - // Get candidate's active mask - const active_mask_t &cand_mask = - m_shader->get_active_mask(cand_warp_id, cand_inst); - - unsigned cand_sets_needed; - bool can_co_issue; - - if (m_shader->m_config->gpgpu_compaction_mode == 2) { - // Full compaction: just count sets needed, no overlap check. - // Compacted placement uses start_set, so no two sets ever alias. - unsigned cand_active = cand_mask.count(); - cand_sets_needed = (cand_active + set_width - 1) / set_width; - can_co_issue = (cand_sets_needed <= available_sets); - } else { - // No compaction or XOR-static: build temp sets with the selected - // mapping and check set-level overlap vs. primary. - warp_inst_t cand_temp(m_shader->m_config); - cand_temp = *cand_inst; - cand_temp.issue(cand_mask, cand_warp_id, 0, 0, 0, 0); - if (m_shader->m_config->gpgpu_compaction_mode == 1) { - cand_temp.compute_simd_sets_xor_static( - m_shader->m_config->gpgpu_num_simd_sets, - m_shader->m_config->simd_set_width); - } else { - cand_temp.compute_simd_sets( - m_shader->m_config->gpgpu_num_simd_sets, - m_shader->m_config->simd_set_width); - } - can_co_issue = - !warp_inst_t::simd_sets_overlap( - co_issue_composite->get_simd_sets(), - cand_temp.get_simd_sets()); - cand_sets_needed = cand_temp.num_active_simd_sets(); - if (cand_sets_needed > available_sets) can_co_issue = false; - } - - if (!can_co_issue) continue; - - printf("SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " - "CO-ISSUED with primary warp %u (%u sets)\n", - m_shader->get_gpu()->gpu_sim_cycle + - m_shader->get_gpu()->gpu_tot_sim_cycle, - get_sid(), m_id, cand_warp_id, co_issue_primary_warp_id, - cand_sets_needed); - - // Co-issue: functional execution, SIMT update, scoreboard, merge - m_shader->co_issue_warp(co_issue_composite, cand_inst, cand_mask, - cand_warp_id, m_id, next_free_set); - - available_sets -= cand_sets_needed; - next_free_set += cand_sets_needed; - - // Advance candidate warp's ibuffer - warp(cand_warp_id).ibuffer_step(); + switch (m_shader->m_config->gpgpu_co_issue_priority) { + case 0: // greedy (utilization-max, unified pool) + try_utilization_max_coissue( + co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, next_free_set); + break; + case 1: // intra-first + try_intra_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set); + try_inter_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set, /*sort_mode=*/0); + break; + case 3: // fewest-lanes first (inter), then intra + try_inter_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set, /*sort_mode=*/1); + try_intra_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set); + break; + case 4: // same-PC (inter), then intra + try_inter_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set, /*sort_mode=*/2); + try_intra_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set); + break; + case 2: // inter-first (default, matches pre-Change-4) + default: + try_inter_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set, /*sort_mode=*/0); + try_intra_warp_coissue(co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, + next_free_set); + break; } } } - // INTRA-WARP co-issue: check same warp's secondary I-Buffer half (slots 2-3) - // for divergent split instructions that can be co-issued. - // Only in AWARE mode with SIMD partitioning enabled. + // Change 5: end-of-cycle SIMD partitioning stats. Only collect when + // partitioning is enabled and a composite was issued this cycle. if (m_shader->m_config->gpgpu_simd_partitioning && - m_shader->m_config->model == AWARE_RECONVERGENCE && - co_issue_composite != NULL && available_sets > 0 && - co_issue_fu_type != exec_unit_type_t::MEM && - co_issue_fu_type != exec_unit_type_t::NONE) { - unsigned primary_warp_id = co_issue_primary_warp_id; - unsigned set_width = m_shader->m_config->simd_set_width; - - // Check slots 2-3 (secondary half) for valid instructions - for (unsigned sec_slot = 2; - sec_slot < 4 && available_sets > 0; sec_slot++) { - if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue; - - const warp_inst_t *sec_inst = - warp(primary_warp_id).ibuffer_slot_inst(sec_slot); - if (!sec_inst) continue; - - unsigned sec_split_id = - warp(primary_warp_id).ibuffer_slot_split_id(sec_slot); - const active_mask_t &sec_mask = - warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot); - - // Verify the split is still valid in the splits table - if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { - warp(primary_warp_id).ibuffer_flush_half(1); - m_scoreboard->clearSecondary(primary_warp_id); - break; - } - - // Verify the split's PC still matches the instruction's PC - unsigned split_pc; - simt_mask_t split_mask; - m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc, - &split_mask); - if (split_pc != sec_inst->pc) { - warp(primary_warp_id).ibuffer_flush_half(1); - m_scoreboard->clearSecondary(primary_warp_id); - break; - } - - // Check same FU type (reuse the FU type determination logic) - exec_unit_type_t sec_fu_type = exec_unit_type_t::NONE; - if ((sec_inst->op == LOAD_OP) || (sec_inst->op == STORE_OP) || - (sec_inst->op == MEMORY_BARRIER_OP) || - (sec_inst->op == TENSOR_CORE_LOAD_OP) || - (sec_inst->op == TENSOR_CORE_STORE_OP)) { - sec_fu_type = exec_unit_type_t::MEM; - } else if (sec_inst->op == SP_OP || - (sec_inst->op != DP_OP && sec_inst->op != SFU_OP && - sec_inst->op != ALU_SFU_OP && - sec_inst->op != TENSOR_CORE_OP && - sec_inst->op < SPEC_UNIT_START_ID && - m_shader->m_config->gpgpu_num_int_units == 0)) { - sec_fu_type = exec_unit_type_t::SP; - } else if (sec_inst->op != SP_OP && sec_inst->op != DP_OP && - sec_inst->op != SFU_OP && sec_inst->op != ALU_SFU_OP && - sec_inst->op != TENSOR_CORE_OP && - sec_inst->op < SPEC_UNIT_START_ID && - m_shader->m_config->gpgpu_num_int_units > 0) { - sec_fu_type = exec_unit_type_t::INT; - } else if (sec_inst->op == DP_OP) { - sec_fu_type = exec_unit_type_t::DP; - } else if (sec_inst->op == SFU_OP || sec_inst->op == ALU_SFU_OP) { - sec_fu_type = exec_unit_type_t::SFU; - } else if (sec_inst->op == TENSOR_CORE_OP) { - sec_fu_type = exec_unit_type_t::TENSOR; - } else if (sec_inst->op >= SPEC_UNIT_START_ID) { - sec_fu_type = exec_unit_type_t::SPECIALIZED; - } - if (sec_fu_type != co_issue_fu_type) continue; - - // Secondary scoreboard check: dependencies within secondary stream - if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) - continue; - - // Compute sets needed - unsigned sec_active = sec_mask.count(); - unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width; - if (sec_sets_needed > available_sets) continue; - - printf("SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " - "split %u CO-ISSUED with primary split (%u sets)\n", - m_shader->get_gpu()->gpu_sim_cycle + - m_shader->get_gpu()->gpu_tot_sim_cycle, - get_sid(), m_id, primary_warp_id, sec_split_id, - sec_sets_needed); - - // Co-issue with targeted SIMT update (pass split_id) - m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask, - primary_warp_id, m_id, next_free_set, - sec_split_id); - - available_sets -= sec_sets_needed; - next_free_set += sec_sets_needed; - - // Free the secondary I-buffer slot (don't call ibuffer_free/step, - // use direct slot free since this is not the m_next pointer) - warp(primary_warp_id).ibuffer_free_slot(sec_slot); - - // After SIMT update, check if the split was invalidated (diverged) - if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { - // Split was invalidated by the update — flush the entire secondary half - warp(primary_warp_id).ibuffer_flush_half(1); - m_scoreboard->clearSecondary(primary_warp_id); - break; + co_issue_composite != NULL) { + unsigned valid_sets = 0; + unsigned active_lanes = 0; + const std::vector &sets = + co_issue_composite->get_simd_sets(); + for (unsigned s = 0; s < sets.size(); s++) { + if (sets[s].valid) { + valid_sets++; + active_lanes += sets[s].set_active_mask.count(); } } + unsigned sid = get_sid(); + m_stats->coissue_composite_cycles[sid]++; + m_stats->coissue_total_active_lanes[sid] += active_lanes; + unsigned nbins = m_shader->m_config->gpgpu_num_simd_sets + 1; + if (valid_sets >= nbins) valid_sets = nbins - 1; // safety clamp + m_stats->simd_sets_used_histogram[sid * nbins + valid_sets]++; } // issue stall statistics: diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 3d1aea9..70de529 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -550,6 +550,50 @@ class scheduler_unit { // this can be copied freely, so can be used in std const std::vector::const_iterator &prioritized_iter); inline int get_sid() const; + // --- Change 4: co-issue scheduler helpers ---------------------------- + // Classify an instruction's FU type (the 25-line op→exec_unit_type_t + // switch that was duplicated across the primary pick, inter-warp pass, + // and intra-warp pass). + exec_unit_type_t classify_fu_type(const warp_inst_t *inst) const; + + // Check whether `cand_inst` can co-issue alongside `composite` given + // `available_sets`. Writes the candidate's sets_needed to *sets_needed + // on success. Uses the current compaction_mode to decide between the + // count-only path (full compaction) and the overlap-check path + // (identity / XOR-static). + bool check_coissue_feasibility(const warp_inst_t *composite, + const warp_inst_t *cand_inst, + const active_mask_t &cand_mask, + unsigned cand_warp_id, + unsigned available_sets, + unsigned *sets_needed); + + // Runs the inter-warp co-issue pass. sort_mode: + // 0 = round-robin (iterate m_next_cycle_prioritized_warps as-is) + // 1 = fewest-lanes first (sort by popcount(active_mask) ascending) + // 2 = same-PC first, fall through to arbitrary within the cycle + void try_inter_warp_coissue(warp_inst_t *co_issue_composite, + exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, + unsigned &available_sets, + unsigned &next_free_set, + unsigned sort_mode); + + // Runs the intra-warp co-issue pass (primary warp's secondary slots). + void try_intra_warp_coissue(warp_inst_t *co_issue_composite, + exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, + unsigned &available_sets, + unsigned &next_free_set); + + // Unified intra+inter candidate pool, density-sorted greedy packing. + // Approximates optimal SIMD-lane utilization for the cycle. + void try_utilization_max_coissue(warp_inst_t *co_issue_composite, + exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, + unsigned &available_sets, + unsigned &next_free_set); + protected: shd_warp_t &warp(int i); @@ -2039,6 +2083,18 @@ struct shader_core_stats_pod { unsigned *gpgpu_n_shmem_bank_access; long *n_simt_to_mem; // Interconnect power stats long *n_mem_to_simt; + + // Change 5: SIMD partitioning co-issue counters. All per-core arrays + // of size num_shader(); histogram is flat size num_shader() * + // (gpgpu_num_simd_sets + 1). + unsigned long long *inter_warp_coissue_events; + unsigned long long *intra_warp_coissue_events; + unsigned long long *coissue_denied_by_fu_mismatch; + unsigned long long *coissue_denied_by_scoreboard; + unsigned long long *coissue_denied_by_no_sets; + unsigned long long *coissue_total_active_lanes; + unsigned long long *coissue_composite_cycles; + unsigned long long *simd_sets_used_histogram; }; class shader_core_stats : public shader_core_stats_pod { @@ -2151,6 +2207,27 @@ class shader_core_stats : public shader_core_stats_pod { m_shader_dynamic_warp_issue_distro.resize(config->num_shader()); m_shader_warp_slot_issue_distro.resize(config->num_shader()); + + // Change 5: SIMD partitioning co-issue counters + inter_warp_coissue_events = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + intra_warp_coissue_events = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + coissue_denied_by_fu_mismatch = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + coissue_denied_by_scoreboard = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + coissue_denied_by_no_sets = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + coissue_total_active_lanes = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + coissue_composite_cycles = (unsigned long long *)calloc( + config->num_shader(), sizeof(unsigned long long)); + // Histogram bins: 0..gpgpu_num_simd_sets (inclusive). When + // partitioning is off, gpgpu_num_simd_sets == 1 so we allocate 2 bins. + unsigned nbins = config->gpgpu_num_simd_sets + 1; + simd_sets_used_histogram = (unsigned long long *)calloc( + (size_t)config->num_shader() * nbins, sizeof(unsigned long long)); } ~shader_core_stats() { @@ -2203,6 +2280,15 @@ class shader_core_stats : public shader_core_stats_pod { free(m_n_diverge); free(shader_cycle_distro); free(last_shader_cycle_distro); + // Change 5: SIMD partitioning co-issue counters + free(inter_warp_coissue_events); + free(intra_warp_coissue_events); + free(coissue_denied_by_fu_mismatch); + free(coissue_denied_by_scoreboard); + free(coissue_denied_by_no_sets); + free(coissue_total_active_lanes); + free(coissue_composite_cycles); + free(simd_sets_used_histogram); } void new_grid() {} -- cgit v1.3