From 2c3524751b9b175514970c768637266bbed08767 Mon Sep 17 00:00:00 2001 From: Davit Grigoryan Date: Mon, 13 Apr 2026 02:19:54 +0000 Subject: impl intra-warp co-issuing --- src/abstract_hardware_model.cc | 63 +++++++ src/abstract_hardware_model.h | 35 ++++ src/gpgpu-sim/shader.cc | 377 ++++++++++++++++++++++++++++++++++------- src/gpgpu-sim/shader.h | 129 ++++++++++++-- 4 files changed, 532 insertions(+), 72 deletions(-) (limited to 'src') diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index deebfd6..9262a92 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -1440,6 +1440,69 @@ const simt_mask_t &simt_splits_table::get_active_mask(unsigned num) { return m_splits_table[num].m_active_mask; } +// SIMD lane partitioning: secondary split access and targeted operations + +bool simt_splits_table::get_secondary_split_info(unsigned *pc, unsigned *rpc, + unsigned *split_id, + simt_mask_t *mask) const { + if (m_fifo_queue.size() < 2) return false; + auto it = m_fifo_queue.begin(); + ++it; // second entry + unsigned sec_entry = it->m_st_entry; + auto st_it = m_splits_table.find(sec_entry); + if (st_it == m_splits_table.end() || !st_it->second.m_valid) return false; + if (st_it->second.m_blocked || st_it->second.m_virtual) return false; + *pc = st_it->second.m_pc; + *rpc = st_it->second.m_recvg_pc; + *split_id = sec_entry; + *mask = st_it->second.m_active_mask; + return true; +} + +bool simt_splits_table::is_split_valid(unsigned split_id) const { + auto it = m_splits_table.find(split_id); + return it != m_splits_table.end() && it->second.m_valid; +} + +void simt_splits_table::get_split_info(unsigned split_id, unsigned *pc, + simt_mask_t *mask) const { + auto it = m_splits_table.find(split_id); + assert(it != m_splits_table.end() && it->second.m_valid); + *pc = it->second.m_pc; + *mask = it->second.m_active_mask; +} + +void simt_splits_table::update_pc_for_split(unsigned split_id, + address_type new_pc) { + assert(m_splits_table.find(split_id) != m_splits_table.end()); + assert(m_splits_table[split_id].m_valid); + m_splits_table[split_id].m_pc = new_pc; +} + +bool simt_splits_table::move_split_to_front(unsigned split_id) { + if (m_fifo_queue.empty()) return false; + if (m_fifo_queue.front().m_st_entry == split_id) { + m_active_split = split_id; + return true; // already at front + } + // Find and remove from current position + fifo_entry target; + bool found = false; + for (auto it = m_fifo_queue.begin(); it != m_fifo_queue.end(); ++it) { + if (it->m_st_entry == (int)split_id) { + target = *it; + m_fifo_queue.erase(it); + found = true; + break; + } + } + if (!found) return false; + // Insert at front and make active + m_fifo_queue.push_front(target); + m_active_split = split_id; + return true; +} + void simt_splits_table::get_pdom_splits_entry_info(unsigned num, unsigned *pc, unsigned *rpc) { assert((m_splits_table.find(num) != m_splits_table.end()) && diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index d3f0797..4a655b3 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -1527,6 +1527,18 @@ class simt_splits_table { bool blocked(); unsigned address_to_entry(warp_inst_t inst); + unsigned get_active_split_id() const { return m_active_split; } + + // SIMD lane partitioning: secondary split access and targeted operations + bool has_secondary_split() const { return m_fifo_queue.size() >= 2; } + bool get_secondary_split_info(unsigned *pc, unsigned *rpc, unsigned *split_id, + simt_mask_t *mask) const; + bool is_split_valid(unsigned split_id) const; + void get_split_info(unsigned split_id, unsigned *pc, + simt_mask_t *mask) const; + void update_pc_for_split(unsigned split_id, address_type new_pc); + bool move_split_to_front(unsigned split_id); + protected: unsigned m_warp_size; unsigned m_warp_id; @@ -1664,6 +1676,29 @@ class simt_tables { tmp_active_mask, type, recvged); } + unsigned get_active_split_id() const { + return m_simt_splits_table->get_active_split_id(); + } + // SIMD lane partitioning: secondary split access and targeted operations + bool has_secondary_split() const { + return m_simt_splits_table->has_secondary_split(); + } + bool get_secondary_split_info(unsigned *pc, unsigned *rpc, unsigned *split_id, + simt_mask_t *mask) const { + return m_simt_splits_table->get_secondary_split_info(pc, rpc, split_id, + mask); + } + bool is_split_valid(unsigned split_id) const { + return m_simt_splits_table->is_split_valid(split_id); + } + void get_split_info(unsigned split_id, unsigned *pc, + simt_mask_t *mask) const { + m_simt_splits_table->get_split_info(split_id, pc, mask); + } + bool move_split_to_front(unsigned split_id) { + return m_simt_splits_table->move_split_to_front(split_id); + } + private: unsigned m_warp_id; unsigned m_warp_size; diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index b0b1371..43e86b6 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -910,41 +910,46 @@ const active_mask_t &exec_shader_core_ctx::get_active_mask( } void shader_core_ctx::decode() { - if (m_inst_fetch_buffer.m_valid) { - // decode 1 or 2 instructions and place them into ibuffer - address_type pc = m_inst_fetch_buffer.m_pc; - const warp_inst_t *pI1 = get_next_inst(m_inst_fetch_buffer.m_warp_id, pc); + // Helper lambda to decode a fetch buffer into a specific I-Buffer half + auto decode_fetch_buffer = [&](ifetch_buffer_t &fbuf) { + if (!fbuf.m_valid) return; + unsigned wid = fbuf.m_warp_id; + unsigned half = fbuf.m_ibuffer_half; + unsigned slot_base = half * 2; // IBUFFER_HALF_SIZE + address_type pc = fbuf.m_pc; + + const warp_inst_t *pI1 = get_next_inst(wid, pc); if (pI1) { - m_warp[m_inst_fetch_buffer.m_warp_id]->ibuffer_fill(0, pI1); - m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline(); + m_warp[wid]->ibuffer_fill(slot_base, pI1, fbuf.m_split_id, + fbuf.m_split_mask); + m_warp[wid]->inc_inst_in_pipeline(); m_stats->m_num_decoded_insn[m_sid]++; - if ((pI1->oprnd_type == INT_OP) || - (pI1->oprnd_type == UN_OP)) { // these counters get added up in mcPat - // to compute scheduler power + if ((pI1->oprnd_type == INT_OP) || (pI1->oprnd_type == UN_OP)) m_stats->m_num_INTdecoded_insn[m_sid]++; - } else if (pI1->oprnd_type == FP_OP) { + else if (pI1->oprnd_type == FP_OP) m_stats->m_num_FPdecoded_insn[m_sid]++; - } - const warp_inst_t *pI2 = - get_next_inst(m_inst_fetch_buffer.m_warp_id, pc + pI1->isize); + + const warp_inst_t *pI2 = get_next_inst(wid, pc + pI1->isize); if (pI2) { - m_warp[m_inst_fetch_buffer.m_warp_id]->ibuffer_fill(1, pI2); - m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline(); + m_warp[wid]->ibuffer_fill(slot_base + 1, pI2, fbuf.m_split_id, + fbuf.m_split_mask); + m_warp[wid]->inc_inst_in_pipeline(); m_stats->m_num_decoded_insn[m_sid]++; - if ((pI1->oprnd_type == INT_OP) || - (pI1->oprnd_type == UN_OP)) { // these counters get added up in - // mcPat to compute scheduler power + if ((pI2->oprnd_type == INT_OP) || (pI2->oprnd_type == UN_OP)) m_stats->m_num_INTdecoded_insn[m_sid]++; - } else if (pI2->oprnd_type == FP_OP) { + else if (pI2->oprnd_type == FP_OP) m_stats->m_num_FPdecoded_insn[m_sid]++; - } } } - m_inst_fetch_buffer.m_valid = false; - } + fbuf.m_valid = false; + }; + + decode_fetch_buffer(m_inst_fetch_buffer); + decode_fetch_buffer(m_inst_fetch_buffer_secondary); } void shader_core_ctx::fetch() { + // --- Primary fetch: fills I-Buffer half 0 --- if (!m_inst_fetch_buffer.m_valid) { if (m_L1I->access_ready()) { mem_fetch *mf = m_L1I->next_access(); @@ -953,16 +958,23 @@ void shader_core_ctx::fetch() { ifetch_buffer_t(m_warp[mf->get_wid()]->get_pc(), mf->get_access_size(), mf->get_wid()); assert(m_warp[mf->get_wid()]->get_pc() == - (mf->get_addr() - - PROGRAM_MEM_START)); // Verify that we got the instruction we - // were expecting. + (mf->get_addr() - PROGRAM_MEM_START)); m_inst_fetch_buffer.m_valid = true; + // Tag with split info for AWARE mode + if (m_config->model == AWARE_RECONVERGENCE) { + unsigned wid = mf->get_wid(); + unsigned spc, srpc; + m_simt_tables[wid]->get_pdom_active_split_info(&spc, &srpc); + m_inst_fetch_buffer.m_split_id = + m_simt_tables[wid]->get_active_split_id(); + m_inst_fetch_buffer.m_split_mask = + m_simt_tables[wid]->get_active_mask(); + } + m_inst_fetch_buffer.m_ibuffer_half = 0; m_warp[mf->get_wid()]->set_last_fetch(m_gpu->gpu_sim_cycle); delete mf; } else { - // find an active warp with space in instruction buffer that is not - // already waiting on a cache miss and get next 1-2 instructions from - // i-cache... + // find an active warp with space in instruction buffer for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { unsigned warp_id = (m_last_warp_fetched + 1 + i) % m_config->max_warps_per_shader; @@ -995,14 +1007,15 @@ void shader_core_ctx::fetch() { assert(m_active_warps >= 0); } - // this code fetches instructions from the i-cache or generates memory + // this code fetches instructions from the i-cache bool simt_conditions = true; if (m_config->model == AWARE_RECONVERGENCE) simt_conditions = !is_virtualized(warp_id); + // Changed: use ibuffer_half_empty(0) instead of ibuffer_empty() if (simt_conditions && !m_warp[warp_id]->functional_done() && !m_warp[warp_id]->imiss_pending() && - m_warp[warp_id]->ibuffer_empty()) { + m_warp[warp_id]->ibuffer_half_empty(0)) { address_type pc; pc = m_warp[warp_id]->get_pc(); address_type ppc = pc + PROGRAM_MEM_START; @@ -1012,8 +1025,6 @@ void shader_core_ctx::fetch() { if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz()) nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block); - // TODO: replace with use of allocator - // mem_fetch *mf = m_mem_fetch_allocator->alloc() mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx); mem_fetch *mf = new mem_fetch( acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE, @@ -1036,6 +1047,14 @@ void shader_core_ctx::fetch() { } else if (status == HIT) { m_last_warp_fetched = warp_id; m_inst_fetch_buffer = ifetch_buffer_t(pc, nbytes, warp_id); + // Tag with split info + if (m_config->model == AWARE_RECONVERGENCE) { + m_inst_fetch_buffer.m_split_id = + m_simt_tables[warp_id]->get_active_split_id(); + m_inst_fetch_buffer.m_split_mask = + m_simt_tables[warp_id]->get_active_mask(); + } + m_inst_fetch_buffer.m_ibuffer_half = 0; m_warp[warp_id]->set_last_fetch(m_gpu->gpu_sim_cycle); delete mf; } else { @@ -1049,6 +1068,74 @@ void shader_core_ctx::fetch() { } } + // --- Secondary fetch: fills I-Buffer half 1 from a non-overlapping split --- + // Only in AWARE mode with SIMD partitioning enabled. + // HIT-only: if I-Cache misses, skip (no miss tracking for secondary). + if (m_config->model == AWARE_RECONVERGENCE && + m_config->gpgpu_simd_partitioning && + !m_inst_fetch_buffer_secondary.m_valid) { + // Find the warp that just had primary fetch (if any), or scan for a warp + // whose secondary half is empty and has a secondary split + for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { + unsigned warp_id = + (m_last_warp_fetched + i) % m_config->max_warps_per_shader; + + if (m_warp[warp_id]->functional_done()) continue; + if (!m_warp[warp_id]->ibuffer_half_empty(1)) continue; + if (is_virtualized(warp_id)) continue; + + // Check if this warp has a secondary split with non-overlapping mask + unsigned sec_pc, sec_rpc, sec_split_id; + simt_mask_t sec_mask; + if (!has_secondary_split(warp_id)) continue; + if (!get_secondary_split_info(warp_id, &sec_pc, &sec_rpc, &sec_split_id, + &sec_mask)) + continue; + + // Non-overlap check: secondary mask must not overlap with half 0's mask + if (m_warp[warp_id]->ibuffer_half_assigned(0)) { + active_mask_t half0_mask = m_warp[warp_id]->ibuffer_half_mask(0); + if ((sec_mask & half0_mask).any()) continue; // overlapping, skip + } + + // Attempt I-Cache access (HIT-only for secondary) + address_type ppc = sec_pc + PROGRAM_MEM_START; + unsigned nbytes = 16; + unsigned offset_in_block = + sec_pc & (m_config->m_L1I_config.get_line_sz() - 1); + if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz()) + nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block); + + mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx); + mem_fetch *mf = new mem_fetch( + acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE, + warp_id, m_sid, m_tpc, m_memory_config, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + std::list events; + enum cache_request_status status; + if (m_config->perfect_inst_const_cache) { + status = HIT; + } else { + status = m_L1I->access((new_addr_type)ppc, mf, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, + events); + } + + if (status == HIT) { + m_inst_fetch_buffer_secondary = ifetch_buffer_t( + sec_pc, nbytes, warp_id, 1, sec_split_id, sec_mask); + // Assign I-Buffer half 1 to this split + m_warp[warp_id]->ibuffer_assign_half(1, sec_split_id, sec_mask); + delete mf; + break; // one secondary fetch per cycle + } else { + // MISS or RESERVATION_FAIL: silently skip, no pending state + delete mf; + // Don't break — try next warp + } + } + } + m_L1I->cycle(); } @@ -1214,9 +1301,12 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, const warp_inst_t *next_inst, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id, - unsigned start_set) { - // Free the co-issued warp's I-buffer entry - m_warp[warp_id]->ibuffer_free(); + unsigned start_set, unsigned split_id) { + // For intra-warp co-issue: don't free ibuffer here (caller frees the slot) + // For inter-warp co-issue: free the co-issued warp's I-buffer entry + if (split_id == (unsigned)-1) { + m_warp[warp_id]->ibuffer_free(); + } assert(next_inst->valid()); // Create a temporary instruction for functional execution @@ -1237,11 +1327,33 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, // Functional execution for the co-issued warp's threads func_exec_inst(temp_inst); - // Update SIMT divergence structures for the co-issued warp - updateSIMTDivergenceStructures(warp_id, &temp_inst); + // Update SIMT divergence structures + if (split_id != (unsigned)-1 && + m_config->model == AWARE_RECONVERGENCE) { + // Intra-warp co-issue: move the target split to FIFO front first + bool moved = m_simt_tables[warp_id]->move_split_to_front(split_id); + if (moved && m_simt_tables[warp_id]->is_split_valid(split_id)) { + updateSIMTDivergenceStructures(warp_id, &temp_inst); + } + // If move failed or split invalid: skip SIMT update (split was + // invalidated by another operation during this cycle) + } else { + updateSIMTDivergenceStructures(warp_id, &temp_inst); + } - // Reserve scoreboard for the co-issued warp - m_scoreboard->reserveRegisters(&temp_inst); + // Reserve scoreboard for the co-issued warp. + // For intra-warp co-issue (same warp_id), skip reservation — the primary + // instruction already reserved for this warp, and the scoreboard tracks at + // warp granularity. Both splits may write to the same register number + // (which is safe since they operate on exclusive threads), but the + // scoreboard would abort on a duplicate reservation. + if (split_id == (unsigned)-1) { + // Inter-warp co-issue: different warp_id, always safe to reserve + m_scoreboard->reserveRegisters(&temp_inst); + } + // For intra-warp: we skip reservation. The primary's reservation covers + // shared registers, and non-shared registers will still be released at + // writeback (the release tolerates releasing non-reserved registers). // Set next PC for the co-issued warp m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize); @@ -1398,6 +1510,8 @@ void scheduler_unit::cycle() { exec_unit_type_t co_issue_fu_type = exec_unit_type_t::NONE; register_set *co_issue_reg_set = NULL; unsigned co_issue_primary_warp_id = (unsigned)-1; + unsigned available_sets = 0; + unsigned next_free_set = 0; order_warps(); for (std::vector::const_iterator iter = @@ -1728,8 +1842,8 @@ void scheduler_unit::cycle() { co_issue_fu_type != exec_unit_type_t::MEM && co_issue_fu_type != exec_unit_type_t::NONE) { // Count available (unused) sets and find next free set - unsigned available_sets = 0; - unsigned next_free_set = 0; + available_sets = 0; + next_free_set = 0; const std::vector &primary_sets = co_issue_composite->get_simd_sets(); for (unsigned s = 0; s < primary_sets.size(); s++) { @@ -1866,6 +1980,111 @@ void scheduler_unit::cycle() { } } + // INTRA-WARP co-issue: check same warp's secondary I-Buffer half (slots 2-3) + // for divergent split instructions that can be co-issued. + // Only in AWARE mode with SIMD partitioning enabled. + if (m_shader->m_config->gpgpu_simd_partitioning && + m_shader->m_config->model == AWARE_RECONVERGENCE && + co_issue_composite != NULL && available_sets > 0 && + co_issue_fu_type != exec_unit_type_t::MEM && + co_issue_fu_type != exec_unit_type_t::NONE) { + unsigned primary_warp_id = co_issue_primary_warp_id; + unsigned set_width = m_shader->m_config->simd_set_width; + + // Check slots 2-3 (secondary half) for valid instructions + for (unsigned sec_slot = 2; + sec_slot < 4 && available_sets > 0; sec_slot++) { + if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue; + + const warp_inst_t *sec_inst = + warp(primary_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + + unsigned sec_split_id = + warp(primary_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot); + + // Verify the split is still valid in the splits table + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + + // Verify the split's PC still matches the instruction's PC + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) { + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + + // Check same FU type (reuse the FU type determination logic) + exec_unit_type_t sec_fu_type = exec_unit_type_t::NONE; + if ((sec_inst->op == LOAD_OP) || (sec_inst->op == STORE_OP) || + (sec_inst->op == MEMORY_BARRIER_OP) || + (sec_inst->op == TENSOR_CORE_LOAD_OP) || + (sec_inst->op == TENSOR_CORE_STORE_OP)) { + sec_fu_type = exec_unit_type_t::MEM; + } else if (sec_inst->op == SP_OP || + (sec_inst->op != DP_OP && sec_inst->op != SFU_OP && + sec_inst->op != ALU_SFU_OP && + sec_inst->op != TENSOR_CORE_OP && + sec_inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units == 0)) { + sec_fu_type = exec_unit_type_t::SP; + } else if (sec_inst->op != SP_OP && sec_inst->op != DP_OP && + sec_inst->op != SFU_OP && sec_inst->op != ALU_SFU_OP && + sec_inst->op != TENSOR_CORE_OP && + sec_inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units > 0) { + sec_fu_type = exec_unit_type_t::INT; + } else if (sec_inst->op == DP_OP) { + sec_fu_type = exec_unit_type_t::DP; + } else if (sec_inst->op == SFU_OP || sec_inst->op == ALU_SFU_OP) { + sec_fu_type = exec_unit_type_t::SFU; + } else if (sec_inst->op == TENSOR_CORE_OP) { + sec_fu_type = exec_unit_type_t::TENSOR; + } else if (sec_inst->op >= SPEC_UNIT_START_ID) { + sec_fu_type = exec_unit_type_t::SPECIALIZED; + } + if (sec_fu_type != co_issue_fu_type) continue; + + // Compute sets needed + unsigned sec_active = sec_mask.count(); + unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width; + if (sec_sets_needed > available_sets) continue; + + printf("SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, primary_warp_id, sec_split_id, + sec_sets_needed); + + // Co-issue with targeted SIMT update (pass split_id) + m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask, + primary_warp_id, m_id, next_free_set, + sec_split_id); + + available_sets -= sec_sets_needed; + next_free_set += sec_sets_needed; + + // Free the secondary I-buffer slot (don't call ibuffer_free/step, + // use direct slot free since this is not the m_next pointer) + warp(primary_warp_id).ibuffer_free_slot(sec_slot); + + // After SIMT update, check if the split was invalidated (diverged) + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + // Split was invalidated by the update — flush the entire secondary half + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + } + } + // issue stall statistics: if (!valid_inst) m_stats->shader_cycle_distro[0]++; // idle or control hazard @@ -2293,29 +2512,41 @@ void shader_core_ctx::writeback() { } // Release scoreboard for primary instruction (covers the outer warp_id) m_scoreboard->releaseRegisters(pipe_reg); - // Release scoreboard for any co-issued warps (non-primary) - for (unsigned wid : unique_warp_ids) { - if (wid == warp_id) continue; // already released above - // For co-issued warps, release their destination registers - // which were reserved in co_issue_warp() via reserveRegisters - // on the temporary instruction. The temp used the co-issued - // warp's out[] registers, so we need to release those. - // Since the composite's outer out[] belongs to the primary warp, - // we look at the sets to find co-issued warp registers. - for (unsigned s = 0; s < sets.size(); s++) { - if (sets[s].valid && sets[s].warp_id == wid && - sets[s].source_inst != NULL) { - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (sets[s].source_inst->out[r] > 0) { - m_scoreboard->releaseRegister(wid, sets[s].source_inst->out[r]); - } - } + // Release scoreboard for ALL co-issued sets (inter-warp AND intra-warp) + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + unsigned set_wid = sets[s].warp_id; + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (sets[s].source_inst->out[r] > 0) { + m_scoreboard->releaseRegister(set_wid, + sets[s].source_inst->out[r]); } } } - // dec_inst_in_pipeline for each unique warp - for (unsigned wid : unique_warp_ids) { - m_warp[wid]->dec_inst_in_pipeline(); + // dec_inst_in_pipeline: once for the primary instruction, plus once + // for each UNIQUE co-issued warp (not per-set — a co-issued warp may + // span multiple sets but was only inc'd once at decode). + // For intra-warp co-issue, the co-issued split shares the same warp_id + // as the primary — dec once for primary + once for the co-issued split. + m_warp[warp_id]->dec_inst_in_pipeline(); + std::set co_issued_warps_decremented; + bool intra_warp_decremented = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + // Inter-warp: dec once per unique co-issued warp + if (co_issued_warps_decremented.find(sets[s].warp_id) == + co_issued_warps_decremented.end()) { + m_warp[sets[s].warp_id]->dec_inst_in_pipeline(); + co_issued_warps_decremented.insert(sets[s].warp_id); + } + } else { + // Intra-warp: dec once for the co-issued split (same warp_id) + if (!intra_warp_decremented) { + m_warp[warp_id]->dec_inst_in_pipeline(); + intra_warp_decremented = true; + } + } } } else { m_scoreboard->releaseRegisters(pipe_reg); @@ -4563,6 +4794,30 @@ bool shader_core_ctx::warp_valid(unsigned wid) { return m_simt_tables[wid]->valid(); } +bool shader_core_ctx::has_secondary_split(unsigned wid) { + return m_simt_tables[wid]->has_secondary_split(); +} + +bool shader_core_ctx::get_secondary_split_info(unsigned wid, unsigned *pc, + unsigned *rpc, + unsigned *split_id, + simt_mask_t *mask) { + return m_simt_tables[wid]->get_secondary_split_info(pc, rpc, split_id, mask); +} + +bool shader_core_ctx::is_split_valid(unsigned wid, unsigned split_id) { + return m_simt_tables[wid]->is_split_valid(split_id); +} + +void shader_core_ctx::get_split_info(unsigned wid, unsigned split_id, + unsigned *pc, simt_mask_t *mask) { + m_simt_tables[wid]->get_split_info(split_id, pc, mask); +} + +bool shader_core_ctx::move_split_to_front(unsigned wid, unsigned split_id) { + return m_simt_tables[wid]->move_split_to_front(split_id); +} + void shader_core_ctx::update_st_size(unsigned n) { // Stats not tracked in gpgpu-sim base; no-op. (void)n; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 31008ec..d8103ac 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -223,27 +223,102 @@ class shd_warp_t { assert(slot < IBUFFER_SIZE); m_ibuffer[slot].m_inst = pI; m_ibuffer[slot].m_valid = true; - m_next = 0; + m_ibuffer[slot].m_split_id = (unsigned)-1; + m_ibuffer[slot].m_split_mask.reset(); + if (slot < IBUFFER_HALF_SIZE) m_next = 0; // only reset for primary half + } + void ibuffer_fill(unsigned slot, const warp_inst_t *pI, unsigned split_id, + const active_mask_t &split_mask) { + assert(slot < IBUFFER_SIZE); + m_ibuffer[slot].m_inst = pI; + m_ibuffer[slot].m_valid = true; + m_ibuffer[slot].m_split_id = split_id; + m_ibuffer[slot].m_split_mask = split_mask; + if (slot < IBUFFER_HALF_SIZE) m_next = 0; // only reset for primary half } bool ibuffer_empty() const { for (unsigned i = 0; i < IBUFFER_SIZE; i++) if (m_ibuffer[i].m_valid) return false; return true; } + bool ibuffer_half_empty(unsigned half) const { + unsigned start = half * IBUFFER_HALF_SIZE; + for (unsigned i = start; i < start + IBUFFER_HALF_SIZE; i++) + if (m_ibuffer[i].m_valid) return false; + return true; + } + bool ibuffer_any_half_empty() const { + return ibuffer_half_empty(0) || ibuffer_half_empty(1); + } void ibuffer_flush() { for (unsigned i = 0; i < IBUFFER_SIZE; i++) { + if (m_ibuffer[i].m_valid) { + dec_inst_in_pipeline(); + } + m_ibuffer[i].m_inst = NULL; + m_ibuffer[i].m_valid = false; + m_ibuffer[i].m_split_id = (unsigned)-1; + } + for (unsigned h = 0; h < IBUFFER_NUM_HALVES; h++) + m_ibuffer_halves[h].m_assigned = false; + } + void ibuffer_flush_half(unsigned half) { + unsigned start = half * IBUFFER_HALF_SIZE; + for (unsigned i = start; i < start + IBUFFER_HALF_SIZE; i++) { if (m_ibuffer[i].m_valid) dec_inst_in_pipeline(); m_ibuffer[i].m_inst = NULL; m_ibuffer[i].m_valid = false; + m_ibuffer[i].m_split_id = (unsigned)-1; } + m_ibuffer_halves[half].m_assigned = false; } const warp_inst_t *ibuffer_next_inst() { return m_ibuffer[m_next].m_inst; } bool ibuffer_next_valid() { return m_ibuffer[m_next].m_valid; } + unsigned ibuffer_next_split_id() const { + return m_ibuffer[m_next].m_split_id; + } + const active_mask_t &ibuffer_next_split_mask() const { + return m_ibuffer[m_next].m_split_mask; + } void ibuffer_free() { m_ibuffer[m_next].m_inst = NULL; m_ibuffer[m_next].m_valid = false; } - void ibuffer_step() { m_next = (m_next + 1) % IBUFFER_SIZE; } + void ibuffer_step() { m_next = (m_next + 1) % IBUFFER_HALF_SIZE; } + unsigned get_ibuffer_next() const { return m_next; } + // Direct slot access for co-issue pass + bool ibuffer_slot_valid(unsigned slot) const { + return m_ibuffer[slot].m_valid; + } + const warp_inst_t *ibuffer_slot_inst(unsigned slot) const { + return m_ibuffer[slot].m_inst; + } + unsigned ibuffer_slot_split_id(unsigned slot) const { + return m_ibuffer[slot].m_split_id; + } + const active_mask_t &ibuffer_slot_split_mask(unsigned slot) const { + return m_ibuffer[slot].m_split_mask; + } + void ibuffer_free_slot(unsigned slot) { + m_ibuffer[slot].m_inst = NULL; + m_ibuffer[slot].m_valid = false; + } + // Half assignment tracking + void ibuffer_assign_half(unsigned half, unsigned split_id, + const active_mask_t &mask) { + m_ibuffer_halves[half].m_assigned_split_id = split_id; + m_ibuffer_halves[half].m_assigned_mask = mask; + m_ibuffer_halves[half].m_assigned = true; + } + bool ibuffer_half_assigned(unsigned half) const { + return m_ibuffer_halves[half].m_assigned; + } + unsigned ibuffer_half_split_id(unsigned half) const { + return m_ibuffer_halves[half].m_assigned_split_id; + } + const active_mask_t &ibuffer_half_mask(unsigned half) const { + return m_ibuffer_halves[half].m_assigned_mask; + } bool imiss_pending() const { return m_imiss_pending; } void set_imiss_pending() { m_imiss_pending = true; } @@ -285,7 +360,9 @@ class shd_warp_t { } private: - static const unsigned IBUFFER_SIZE = 2; + static const unsigned IBUFFER_SIZE = 4; + static const unsigned IBUFFER_HALF_SIZE = 2; + static const unsigned IBUFFER_NUM_HALVES = 2; class shader_core_ctx *m_shader; unsigned long long m_streamID; unsigned m_cta_id; @@ -303,13 +380,24 @@ class shd_warp_t { ibuffer_entry() { m_valid = false; m_inst = NULL; + m_split_id = (unsigned)-1; } const warp_inst_t *m_inst; bool m_valid; + unsigned m_split_id; // splits table entry this belongs to + active_mask_t m_split_mask; // the split's active mask at decode time + }; + + struct ibuffer_half_t { + ibuffer_half_t() : m_assigned(false), m_assigned_split_id((unsigned)-1) {} + bool m_assigned; + unsigned m_assigned_split_id; + active_mask_t m_assigned_mask; }; warp_inst_t m_inst_at_barrier; ibuffer_entry m_ibuffer[IBUFFER_SIZE]; + ibuffer_half_t m_ibuffer_halves[IBUFFER_NUM_HALVES]; unsigned m_next; unsigned m_n_atomic; // number of outstanding atomic operations @@ -1150,19 +1238,27 @@ struct insn_latency_info { }; struct ifetch_buffer_t { - ifetch_buffer_t() { m_valid = false; } + ifetch_buffer_t() : m_valid(false), m_ibuffer_half(0), + m_split_id((unsigned)-1) { m_split_mask.reset(); } - ifetch_buffer_t(address_type pc, unsigned nbytes, unsigned warp_id) { - m_valid = true; - m_pc = pc; - m_nbytes = nbytes; - m_warp_id = warp_id; - } + ifetch_buffer_t(address_type pc, unsigned nbytes, unsigned warp_id) + : m_valid(true), m_pc(pc), m_nbytes(nbytes), m_warp_id(warp_id), + m_ibuffer_half(0), m_split_id((unsigned)-1) { m_split_mask.reset(); } + + ifetch_buffer_t(address_type pc, unsigned nbytes, unsigned warp_id, + unsigned ibuffer_half, unsigned split_id, + const active_mask_t &split_mask) + : m_valid(true), m_pc(pc), m_nbytes(nbytes), m_warp_id(warp_id), + m_ibuffer_half(ibuffer_half), m_split_id(split_id), + m_split_mask(split_mask) {} bool m_valid; address_type m_pc; unsigned m_nbytes; unsigned m_warp_id; + unsigned m_ibuffer_half; // which I-Buffer half to fill (0 or 1) + unsigned m_split_id; // splits table entry + active_mask_t m_split_mask; // the split's active mask }; class shader_core_config; @@ -2501,6 +2597,15 @@ class shader_core_ctx : public core_t { bool pending_reconvergence(unsigned wid); bool warp_blocked(unsigned wid); bool warp_valid(unsigned wid); + // SIMD lane partitioning: secondary split access + bool has_secondary_split(unsigned wid); + bool get_secondary_split_info(unsigned wid, unsigned *pc, unsigned *rpc, + unsigned *split_id, simt_mask_t *mask); + bool is_split_valid(unsigned wid, unsigned split_id); + void get_split_info(unsigned wid, unsigned split_id, unsigned *pc, + simt_mask_t *mask); + bool move_split_to_front(unsigned wid, unsigned split_id); + simt_tables *get_simt_tables(unsigned wid) { return m_simt_tables[wid]; } bool push_to_st_response_fifo(unsigned wid, unsigned entry); bool push_to_rt_response_fifo(unsigned wid, unsigned entry); void update_st_size(unsigned n); @@ -2538,7 +2643,8 @@ class shader_core_ctx : public core_t { unsigned warp_id, unsigned sch_id); void co_issue_warp(warp_inst_t *composite, const warp_inst_t *next_inst, const active_mask_t &active_mask, unsigned warp_id, - unsigned sch_id, unsigned start_set = 0); + unsigned sch_id, unsigned start_set = 0, + unsigned split_id = (unsigned)-1); void create_front_pipeline(); void create_schedulers(); @@ -2621,6 +2727,7 @@ class shader_core_ctx : public core_t { std::vector m_warp; // per warp information array barrier_set_t m_barriers; ifetch_buffer_t m_inst_fetch_buffer; + ifetch_buffer_t m_inst_fetch_buffer_secondary; // for secondary split std::vector m_pipeline_reg; Scoreboard *m_scoreboard; opndcoll_rfu_t m_operand_collector; -- cgit v1.3