diff options
Diffstat (limited to 'src/gpgpu-sim/shader.cc')
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 966 |
1 files changed, 966 insertions, 0 deletions
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 9a47996..e9c2e19 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -2583,6 +2583,957 @@ void scheduler_unit::try_utilization_max_coissue( } } +void scheduler_unit::try_utilization_max_coissue_window( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set) { + // K-bounded clone of try_utilization_max_coissue. Both candidate scan + // loops cap iteration at the first K warps in priority order, where + // K = gpgpu_num_simd_sets. Models a HW priority encoder that can only + // poll K warps' ibuffer slots per cycle, not all in-flight warps. + struct coissue_cand_t { + bool is_intra; + unsigned warp_id; + unsigned split_id; // valid iff is_intra + const warp_inst_t *inst; + active_mask_t mask; + unsigned sec_slot; // valid iff is_intra (2 or 3) + unsigned active_count; + unsigned sets_needed; + }; + + std::vector<coissue_cand_t> pool; + const unsigned set_width = m_shader->m_config->simd_set_width; + // Window size: explicit override if non-zero, else fall back to the + // number of SIMD sets. Lets us model K != num_simd_sets without + // disturbing the partitioning math (which requires warp_size % + // num_simd_sets == 0). + const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0) + ? m_shader->m_config->gpgpu_coissue_window_K + : m_shader->m_config->gpgpu_num_simd_sets; + + // Bound the iteration to the first K warps (or fewer if the prioritized + // list is shorter). Computed once and reused for both scan passes. + std::vector<shd_warp_t *>::const_iterator wlist_end = + m_next_cycle_prioritized_warps.begin(); + std::advance( + wlist_end, + std::min((size_t)K, m_next_cycle_prioritized_warps.size())); + + // --- Inter-warp candidates (first K warps' slot 0) --- + for (std::vector<shd_warp_t *>::const_iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != wlist_end; iter++) { + if ((*iter) == NULL || (*iter)->done_exit()) continue; + unsigned cand_warp_id = (*iter)->get_warp_id(); + if (cand_warp_id == co_issue_primary_warp_id) continue; + if (warp(cand_warp_id).ibuffer_empty()) continue; + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = true; + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + } + if (!simt_ok) continue; + + const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); + if (!cand_inst) continue; + if (!warp(cand_warp_id).ibuffer_next_valid()) continue; + + unsigned cand_pc, cand_rpc; + m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, + &cand_rpc); + if (cand_pc != cand_inst->pc) continue; + + bool sb_collision_inter2; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + const active_mask_t &cand_mask_pre = + m_shader->get_active_mask(cand_warp_id, cand_inst); + sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id, + cand_inst, + cand_mask_pre); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision_inter2 = m_scoreboard->checkCollisionSlot( + cand_warp_id, /*slot=*/0, cand_inst); + } else { + sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id, + cand_inst); + } + if (sb_collision_inter2) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + if (classify_fu_type(cand_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = cand_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + + const active_mask_t &cand_mask = + m_shader->get_active_mask(cand_warp_id, cand_inst); + unsigned needed = 0; + if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, available_sets, &needed)) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = false; + c.warp_id = cand_warp_id; + c.split_id = (unsigned)-1; + c.inst = cand_inst; + c.mask = cand_mask; + c.sec_slot = 0; + c.active_count = cand_mask.count(); + c.sets_needed = needed; + pool.push_back(c); + } + + // --- Intra-warp candidates (first K warps' secondary slots 2..3) --- + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + for (std::vector<shd_warp_t *>::const_iterator iter = + m_next_cycle_prioritized_warps.begin(); + iter != wlist_end; iter++) { + if ((*iter) == NULL || (*iter)->done_exit()) continue; + unsigned cand_warp_id = (*iter)->get_warp_id(); + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + if (!simt_ok) continue; + + for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) { + if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue; + const warp_inst_t *sec_inst = + warp(cand_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + unsigned sec_split_id = + warp(cand_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot); + + if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue; + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) continue; + + bool dup_with_warp = false; + for (unsigned k = 0; k < pool.size(); k++) { + if (pool[k].warp_id == cand_warp_id) { + dup_with_warp = true; + break; + } + } + if (dup_with_warp) continue; + if (classify_fu_type(sec_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = sec_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + active_mask_t sec_mask_eff; + for (unsigned t = 0; t < MAX_WARP_SIZE; t++) { + if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t); + } + if (!sec_mask_eff.any()) continue; + + bool sb_collision; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id, + sec_inst, + sec_mask_eff); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id, + /*slot=*/1, + sec_inst); + } else { + sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id, + sec_inst); + } + if (sb_collision) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + unsigned sec_active = sec_mask_eff.count(); + unsigned sec_needed = (sec_active + set_width - 1) / set_width; + if (sec_needed > available_sets) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = true; + c.warp_id = cand_warp_id; + c.split_id = sec_split_id; + c.inst = sec_inst; + c.mask = sec_mask_eff; + c.sec_slot = sec_slot; + c.active_count = sec_active; + c.sets_needed = sec_needed; + pool.push_back(c); + } + } + } + + // --- Density sort: active_count / sets_needed desc ------------------ + std::stable_sort(pool.begin(), pool.end(), + [](const coissue_cand_t &a, const coissue_cand_t &b) { + unsigned long lhs = (unsigned long)a.active_count * + (unsigned long)b.sets_needed; + unsigned long rhs = (unsigned long)b.active_count * + (unsigned long)a.sets_needed; + if (lhs != rhs) return lhs > rhs; + if (a.active_count != b.active_count) + return a.active_count > b.active_count; + return a.warp_id < b.warp_id; + }); + + // --- Greedy pack ---------------------------------------------------- + for (std::vector<coissue_cand_t>::iterator it = pool.begin(); + it != pool.end() && available_sets > 0; it++) { + coissue_cand_t &c = *it; + unsigned needed = 0; + if (c.is_intra) { + if (c.sets_needed > available_sets) continue; + needed = c.sets_needed; + } else { + if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask, + c.warp_id, available_sets, &needed)) + continue; + } + + if (c.is_intra) { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets) [window]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, c.split_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set, c.split_id); + + available_sets -= needed; + next_free_set += needed; + m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; + warp(c.warp_id).ibuffer_free_slot(c.sec_slot); + if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { + warp(c.warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(c.warp_id); + for (std::vector<coissue_cand_t>::iterator jt = it + 1; + jt != pool.end();) { + if (jt->is_intra && jt->warp_id == c.warp_id) + jt = pool.erase(jt); + else + ++jt; + } + } + } else { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " + "CO-ISSUED with primary warp %u (%u sets) [window]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set); + available_sets -= needed; + next_free_set += needed; + m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; + warp(c.warp_id).ibuffer_step(); + } + } +} + +void scheduler_unit::try_utilization_max_coissue_window_centered( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set) { + // Centered K-bounded clone. Both candidate scan loops iterate + // m_supervised_warps over a window of K positions centered on the + // primary issuer's index (K/2 before, K/2 after; clamped at array + // boundaries). Models a HW priority encoder local to the primary's + // natural warp_id position rather than tied to scheduler priority. + struct coissue_cand_t { + bool is_intra; + unsigned warp_id; + unsigned split_id; + const warp_inst_t *inst; + active_mask_t mask; + unsigned sec_slot; + unsigned active_count; + unsigned sets_needed; + }; + + std::vector<coissue_cand_t> pool; + const unsigned set_width = m_shader->m_config->simd_set_width; + const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0) + ? m_shader->m_config->gpgpu_coissue_window_K + : m_shader->m_config->gpgpu_num_simd_sets; + const unsigned half = K / 2; + const size_t n = m_supervised_warps.size(); + + // Find primary's index in m_supervised_warps. If not found (shouldn't + // happen — primary is one of the supervised warps), bail out. + size_t primary_idx = n; + for (size_t i = 0; i < n; i++) { + if (m_supervised_warps[i] != NULL && + m_supervised_warps[i]->get_warp_id() == co_issue_primary_warp_id) { + primary_idx = i; + break; + } + } + if (primary_idx >= n) return; + + // Symmetric window [primary_idx - half, primary_idx + half]; primary + // itself is filtered out inside the loop body. Clamped at boundaries. + const size_t lo = + (primary_idx > half) ? (primary_idx - half) : (size_t)0; + const size_t hi = + std::min(n, primary_idx + half + 1); + + // --- Inter-warp candidates --- + for (size_t i = lo; i < hi; i++) { + shd_warp_t *w = m_supervised_warps[i]; + if (w == NULL || w->done_exit()) continue; + unsigned cand_warp_id = w->get_warp_id(); + if (cand_warp_id == co_issue_primary_warp_id) continue; + if (warp(cand_warp_id).ibuffer_empty()) continue; + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = true; + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + } + if (!simt_ok) continue; + + const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); + if (!cand_inst) continue; + if (!warp(cand_warp_id).ibuffer_next_valid()) continue; + + unsigned cand_pc, cand_rpc; + m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, + &cand_rpc); + if (cand_pc != cand_inst->pc) continue; + + bool sb_collision_inter2; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + const active_mask_t &cand_mask_pre = + m_shader->get_active_mask(cand_warp_id, cand_inst); + sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id, + cand_inst, + cand_mask_pre); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision_inter2 = m_scoreboard->checkCollisionSlot( + cand_warp_id, /*slot=*/0, cand_inst); + } else { + sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id, + cand_inst); + } + if (sb_collision_inter2) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + if (classify_fu_type(cand_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = cand_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + + const active_mask_t &cand_mask = + m_shader->get_active_mask(cand_warp_id, cand_inst); + unsigned needed = 0; + if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, available_sets, &needed)) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = false; + c.warp_id = cand_warp_id; + c.split_id = (unsigned)-1; + c.inst = cand_inst; + c.mask = cand_mask; + c.sec_slot = 0; + c.active_count = cand_mask.count(); + c.sets_needed = needed; + pool.push_back(c); + } + + // --- Intra-warp candidates --- + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + for (size_t i = lo; i < hi; i++) { + shd_warp_t *w = m_supervised_warps[i]; + if (w == NULL || w->done_exit()) continue; + unsigned cand_warp_id = w->get_warp_id(); + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + if (!simt_ok) continue; + + for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) { + if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue; + const warp_inst_t *sec_inst = + warp(cand_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + unsigned sec_split_id = + warp(cand_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot); + + if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue; + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) continue; + + bool dup_with_warp = false; + for (unsigned k = 0; k < pool.size(); k++) { + if (pool[k].warp_id == cand_warp_id) { + dup_with_warp = true; + break; + } + } + if (dup_with_warp) continue; + if (classify_fu_type(sec_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = sec_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + active_mask_t sec_mask_eff; + for (unsigned t = 0; t < MAX_WARP_SIZE; t++) { + if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t); + } + if (!sec_mask_eff.any()) continue; + + bool sb_collision; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id, + sec_inst, + sec_mask_eff); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id, + /*slot=*/1, + sec_inst); + } else { + sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id, + sec_inst); + } + if (sb_collision) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + unsigned sec_active = sec_mask_eff.count(); + unsigned sec_needed = (sec_active + set_width - 1) / set_width; + if (sec_needed > available_sets) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = true; + c.warp_id = cand_warp_id; + c.split_id = sec_split_id; + c.inst = sec_inst; + c.mask = sec_mask_eff; + c.sec_slot = sec_slot; + c.active_count = sec_active; + c.sets_needed = sec_needed; + pool.push_back(c); + } + } + } + + // --- Density sort --- + std::stable_sort(pool.begin(), pool.end(), + [](const coissue_cand_t &a, const coissue_cand_t &b) { + unsigned long lhs = (unsigned long)a.active_count * + (unsigned long)b.sets_needed; + unsigned long rhs = (unsigned long)b.active_count * + (unsigned long)a.sets_needed; + if (lhs != rhs) return lhs > rhs; + if (a.active_count != b.active_count) + return a.active_count > b.active_count; + return a.warp_id < b.warp_id; + }); + + // --- Greedy pack --- + for (std::vector<coissue_cand_t>::iterator it = pool.begin(); + it != pool.end() && available_sets > 0; it++) { + coissue_cand_t &c = *it; + unsigned needed = 0; + if (c.is_intra) { + if (c.sets_needed > available_sets) continue; + needed = c.sets_needed; + } else { + if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask, + c.warp_id, available_sets, &needed)) + continue; + } + + if (c.is_intra) { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets) [centered]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, c.split_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set, c.split_id); + + available_sets -= needed; + next_free_set += needed; + m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; + warp(c.warp_id).ibuffer_free_slot(c.sec_slot); + if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { + warp(c.warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(c.warp_id); + for (std::vector<coissue_cand_t>::iterator jt = it + 1; + jt != pool.end();) { + if (jt->is_intra && jt->warp_id == c.warp_id) + jt = pool.erase(jt); + else + ++jt; + } + } + } else { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " + "CO-ISSUED with primary warp %u (%u sets) [centered]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set); + available_sets -= needed; + next_free_set += needed; + m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; + warp(c.warp_id).ibuffer_step(); + } + } +} + +void scheduler_unit::try_utilization_max_coissue_window_before( + warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type, + unsigned co_issue_primary_warp_id, unsigned &available_sets, + unsigned &next_free_set) { + // All-before-primary K-bounded clone. Scans (K-1) warps strictly + // before the primary issuer's index in m_supervised_warps order. + // Asymmetric counterpart to mode 6 (centered). + struct coissue_cand_t { + bool is_intra; + unsigned warp_id; + unsigned split_id; + const warp_inst_t *inst; + active_mask_t mask; + unsigned sec_slot; + unsigned active_count; + unsigned sets_needed; + }; + + std::vector<coissue_cand_t> pool; + const unsigned set_width = m_shader->m_config->simd_set_width; + const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0) + ? m_shader->m_config->gpgpu_coissue_window_K + : m_shader->m_config->gpgpu_num_simd_sets; + const unsigned before = (K > 0) ? (K - 1) : 0; + const size_t n = m_supervised_warps.size(); + + size_t primary_idx = n; + for (size_t i = 0; i < n; i++) { + if (m_supervised_warps[i] != NULL && + m_supervised_warps[i]->get_warp_id() == co_issue_primary_warp_id) { + primary_idx = i; + break; + } + } + if (primary_idx >= n) return; + + const size_t lo = + (primary_idx > before) ? (primary_idx - before) : (size_t)0; + const size_t hi = primary_idx; // exclusive: primary itself not scanned + + // --- Inter-warp candidates --- + for (size_t i = lo; i < hi; i++) { + shd_warp_t *w = m_supervised_warps[i]; + if (w == NULL || w->done_exit()) continue; + unsigned cand_warp_id = w->get_warp_id(); + if (cand_warp_id == co_issue_primary_warp_id) continue; + if (warp(cand_warp_id).ibuffer_empty()) continue; + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = true; + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + } + if (!simt_ok) continue; + + const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst(); + if (!cand_inst) continue; + if (!warp(cand_warp_id).ibuffer_next_valid()) continue; + + unsigned cand_pc, cand_rpc; + m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc, + &cand_rpc); + if (cand_pc != cand_inst->pc) continue; + + bool sb_collision_inter2; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + const active_mask_t &cand_mask_pre = + m_shader->get_active_mask(cand_warp_id, cand_inst); + sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id, + cand_inst, + cand_mask_pre); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision_inter2 = m_scoreboard->checkCollisionSlot( + cand_warp_id, /*slot=*/0, cand_inst); + } else { + sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id, + cand_inst); + } + if (sb_collision_inter2) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + if (classify_fu_type(cand_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(cand_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = cand_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + + const active_mask_t &cand_mask = + m_shader->get_active_mask(cand_warp_id, cand_inst); + unsigned needed = 0; + if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask, + cand_warp_id, available_sets, &needed)) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = false; + c.warp_id = cand_warp_id; + c.split_id = (unsigned)-1; + c.inst = cand_inst; + c.mask = cand_mask; + c.sec_slot = 0; + c.active_count = cand_mask.count(); + c.sets_needed = needed; + pool.push_back(c); + } + + // --- Intra-warp candidates --- + if (m_shader->m_config->model == AWARE_RECONVERGENCE) { + for (size_t i = lo; i < hi; i++) { + shd_warp_t *w = m_supervised_warps[i]; + if (w == NULL || w->done_exit()) continue; + unsigned cand_warp_id = w->get_warp_id(); + if (warp(cand_warp_id).waiting()) continue; + + bool simt_ok = warp(cand_warp_id).valid() && + !warp(cand_warp_id).blocked() && + !warp(cand_warp_id).pending_reconvergence() && + !warp(cand_warp_id).virtualized(); + if (!simt_ok) continue; + + for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) { + if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue; + const warp_inst_t *sec_inst = + warp(cand_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + unsigned sec_split_id = + warp(cand_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot); + + if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue; + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) continue; + + bool dup_with_warp = false; + for (unsigned k = 0; k < pool.size(); k++) { + if (pool[k].warp_id == cand_warp_id) { + dup_with_warp = true; + break; + } + } + if (dup_with_warp) continue; + if (classify_fu_type(sec_inst) != co_issue_fu_type) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM && + mem_coissue_disallowed(sec_inst)) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + if (co_issue_fu_type == exec_unit_type_t::MEM) { + static const bool allow_mixed = + (getenv("MEMCOV2_ALLOW_MIXED") != NULL); + enum _memory_space_t pspace = co_issue_composite->space.get_type(); + enum _memory_space_t cspace = sec_inst->space.get_type(); + bool p_shared = warp_inst_t::is_shared_bucket_space(pspace); + bool c_shared = warp_inst_t::is_shared_bucket_space(cspace); + if (!allow_mixed && p_shared != c_shared) { + m_stats->coissue_denied_by_fu_mismatch[get_sid()]++; + m_stats->mem_coissue_denied_filter[get_sid()]++; + continue; + } + } + active_mask_t sec_mask_eff; + for (unsigned t = 0; t < MAX_WARP_SIZE; t++) { + if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t); + } + if (!sec_mask_eff.any()) continue; + + bool sb_collision; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id, + sec_inst, + sec_mask_eff); + } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) { + sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id, + /*slot=*/1, + sec_inst); + } else { + sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id, + sec_inst); + } + if (sb_collision) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } + unsigned sec_active = sec_mask_eff.count(); + unsigned sec_needed = (sec_active + set_width - 1) / set_width; + if (sec_needed > available_sets) { + m_stats->coissue_denied_by_no_sets[get_sid()]++; + continue; + } + + coissue_cand_t c; + c.is_intra = true; + c.warp_id = cand_warp_id; + c.split_id = sec_split_id; + c.inst = sec_inst; + c.mask = sec_mask_eff; + c.sec_slot = sec_slot; + c.active_count = sec_active; + c.sets_needed = sec_needed; + pool.push_back(c); + } + } + } + + // --- Density sort --- + std::stable_sort(pool.begin(), pool.end(), + [](const coissue_cand_t &a, const coissue_cand_t &b) { + unsigned long lhs = (unsigned long)a.active_count * + (unsigned long)b.sets_needed; + unsigned long rhs = (unsigned long)b.active_count * + (unsigned long)a.sets_needed; + if (lhs != rhs) return lhs > rhs; + if (a.active_count != b.active_count) + return a.active_count > b.active_count; + return a.warp_id < b.warp_id; + }); + + // --- Greedy pack --- + for (std::vector<coissue_cand_t>::iterator it = pool.begin(); + it != pool.end() && available_sets > 0; it++) { + coissue_cand_t &c = *it; + unsigned needed = 0; + if (c.is_intra) { + if (c.sets_needed > available_sets) continue; + needed = c.sets_needed; + } else { + if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask, + c.warp_id, available_sets, &needed)) + continue; + } + + if (c.is_intra) { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets) [before]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, c.split_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set, c.split_id); + + available_sets -= needed; + next_free_set += needed; + m_stats->intra_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_intra_events[get_sid()]++; + warp(c.warp_id).ibuffer_free_slot(c.sec_slot); + if (!m_shader->is_split_valid(c.warp_id, c.split_id)) { + warp(c.warp_id).ibuffer_flush_half(1); + m_scoreboard->clearSecondary(c.warp_id); + for (std::vector<coissue_cand_t>::iterator jt = it + 1; + jt != pool.end();) { + if (jt->is_intra && jt->warp_id == c.warp_id) + jt = pool.erase(jt); + else + ++jt; + } + } + } else { + if (m_shader->m_config->gpgpu_simd_partitioning_debug) { + printf( + "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u " + "CO-ISSUED with primary warp %u (%u sets) [before]\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed); + } + + m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id, + m_id, next_free_set); + available_sets -= needed; + next_free_set += needed; + m_stats->inter_warp_coissue_events[get_sid()]++; + if (co_issue_fu_type == exec_unit_type_t::MEM) + m_stats->mem_coissue_inter_events[get_sid()]++; + warp(c.warp_id).ibuffer_step(); + } + } +} + void scheduler_unit::cycle() { SCHED_DPRINTF("scheduler_unit::cycle()\n"); bool valid_inst = @@ -3005,6 +3956,21 @@ void scheduler_unit::cycle() { co_issue_primary_warp_id, available_sets, next_free_set); break; + case 5: // greedy K-bounded (utilization-max over first K warps) + try_utilization_max_coissue_window( + co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, next_free_set); + break; + case 6: // greedy K-bounded centered (K/2 before + K/2 after primary) + try_utilization_max_coissue_window_centered( + co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, next_free_set); + break; + case 7: // greedy K-bounded all-before (K-1 warps before primary) + try_utilization_max_coissue_window_before( + co_issue_composite, co_issue_fu_type, + co_issue_primary_warp_id, available_sets, next_free_set); + break; case 2: // inter-first (default, matches pre-Change-4) default: try_inter_warp_coissue(co_issue_composite, co_issue_fu_type, |