diff options
Diffstat (limited to 'src/gpgpu-sim/shader.cc')
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 377 |
1 files changed, 316 insertions, 61 deletions
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index b0b1371..43e86b6 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -910,41 +910,46 @@ const active_mask_t &exec_shader_core_ctx::get_active_mask( } void shader_core_ctx::decode() { - if (m_inst_fetch_buffer.m_valid) { - // decode 1 or 2 instructions and place them into ibuffer - address_type pc = m_inst_fetch_buffer.m_pc; - const warp_inst_t *pI1 = get_next_inst(m_inst_fetch_buffer.m_warp_id, pc); + // Helper lambda to decode a fetch buffer into a specific I-Buffer half + auto decode_fetch_buffer = [&](ifetch_buffer_t &fbuf) { + if (!fbuf.m_valid) return; + unsigned wid = fbuf.m_warp_id; + unsigned half = fbuf.m_ibuffer_half; + unsigned slot_base = half * 2; // IBUFFER_HALF_SIZE + address_type pc = fbuf.m_pc; + + const warp_inst_t *pI1 = get_next_inst(wid, pc); if (pI1) { - m_warp[m_inst_fetch_buffer.m_warp_id]->ibuffer_fill(0, pI1); - m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline(); + m_warp[wid]->ibuffer_fill(slot_base, pI1, fbuf.m_split_id, + fbuf.m_split_mask); + m_warp[wid]->inc_inst_in_pipeline(); m_stats->m_num_decoded_insn[m_sid]++; - if ((pI1->oprnd_type == INT_OP) || - (pI1->oprnd_type == UN_OP)) { // these counters get added up in mcPat - // to compute scheduler power + if ((pI1->oprnd_type == INT_OP) || (pI1->oprnd_type == UN_OP)) m_stats->m_num_INTdecoded_insn[m_sid]++; - } else if (pI1->oprnd_type == FP_OP) { + else if (pI1->oprnd_type == FP_OP) m_stats->m_num_FPdecoded_insn[m_sid]++; - } - const warp_inst_t *pI2 = - get_next_inst(m_inst_fetch_buffer.m_warp_id, pc + pI1->isize); + + const warp_inst_t *pI2 = get_next_inst(wid, pc + pI1->isize); if (pI2) { - m_warp[m_inst_fetch_buffer.m_warp_id]->ibuffer_fill(1, pI2); - m_warp[m_inst_fetch_buffer.m_warp_id]->inc_inst_in_pipeline(); + m_warp[wid]->ibuffer_fill(slot_base + 1, pI2, fbuf.m_split_id, + fbuf.m_split_mask); + m_warp[wid]->inc_inst_in_pipeline(); m_stats->m_num_decoded_insn[m_sid]++; - if ((pI1->oprnd_type == INT_OP) || - (pI1->oprnd_type == UN_OP)) { // these counters get added up in - // mcPat to compute scheduler power + if ((pI2->oprnd_type == INT_OP) || (pI2->oprnd_type == UN_OP)) m_stats->m_num_INTdecoded_insn[m_sid]++; - } else if (pI2->oprnd_type == FP_OP) { + else if (pI2->oprnd_type == FP_OP) m_stats->m_num_FPdecoded_insn[m_sid]++; - } } } - m_inst_fetch_buffer.m_valid = false; - } + fbuf.m_valid = false; + }; + + decode_fetch_buffer(m_inst_fetch_buffer); + decode_fetch_buffer(m_inst_fetch_buffer_secondary); } void shader_core_ctx::fetch() { + // --- Primary fetch: fills I-Buffer half 0 --- if (!m_inst_fetch_buffer.m_valid) { if (m_L1I->access_ready()) { mem_fetch *mf = m_L1I->next_access(); @@ -953,16 +958,23 @@ void shader_core_ctx::fetch() { ifetch_buffer_t(m_warp[mf->get_wid()]->get_pc(), mf->get_access_size(), mf->get_wid()); assert(m_warp[mf->get_wid()]->get_pc() == - (mf->get_addr() - - PROGRAM_MEM_START)); // Verify that we got the instruction we - // were expecting. + (mf->get_addr() - PROGRAM_MEM_START)); m_inst_fetch_buffer.m_valid = true; + // Tag with split info for AWARE mode + if (m_config->model == AWARE_RECONVERGENCE) { + unsigned wid = mf->get_wid(); + unsigned spc, srpc; + m_simt_tables[wid]->get_pdom_active_split_info(&spc, &srpc); + m_inst_fetch_buffer.m_split_id = + m_simt_tables[wid]->get_active_split_id(); + m_inst_fetch_buffer.m_split_mask = + m_simt_tables[wid]->get_active_mask(); + } + m_inst_fetch_buffer.m_ibuffer_half = 0; m_warp[mf->get_wid()]->set_last_fetch(m_gpu->gpu_sim_cycle); delete mf; } else { - // find an active warp with space in instruction buffer that is not - // already waiting on a cache miss and get next 1-2 instructions from - // i-cache... + // find an active warp with space in instruction buffer for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { unsigned warp_id = (m_last_warp_fetched + 1 + i) % m_config->max_warps_per_shader; @@ -995,14 +1007,15 @@ void shader_core_ctx::fetch() { assert(m_active_warps >= 0); } - // this code fetches instructions from the i-cache or generates memory + // this code fetches instructions from the i-cache bool simt_conditions = true; if (m_config->model == AWARE_RECONVERGENCE) simt_conditions = !is_virtualized(warp_id); + // Changed: use ibuffer_half_empty(0) instead of ibuffer_empty() if (simt_conditions && !m_warp[warp_id]->functional_done() && !m_warp[warp_id]->imiss_pending() && - m_warp[warp_id]->ibuffer_empty()) { + m_warp[warp_id]->ibuffer_half_empty(0)) { address_type pc; pc = m_warp[warp_id]->get_pc(); address_type ppc = pc + PROGRAM_MEM_START; @@ -1012,8 +1025,6 @@ void shader_core_ctx::fetch() { if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz()) nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block); - // TODO: replace with use of allocator - // mem_fetch *mf = m_mem_fetch_allocator->alloc() mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx); mem_fetch *mf = new mem_fetch( acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE, @@ -1036,6 +1047,14 @@ void shader_core_ctx::fetch() { } else if (status == HIT) { m_last_warp_fetched = warp_id; m_inst_fetch_buffer = ifetch_buffer_t(pc, nbytes, warp_id); + // Tag with split info + if (m_config->model == AWARE_RECONVERGENCE) { + m_inst_fetch_buffer.m_split_id = + m_simt_tables[warp_id]->get_active_split_id(); + m_inst_fetch_buffer.m_split_mask = + m_simt_tables[warp_id]->get_active_mask(); + } + m_inst_fetch_buffer.m_ibuffer_half = 0; m_warp[warp_id]->set_last_fetch(m_gpu->gpu_sim_cycle); delete mf; } else { @@ -1049,6 +1068,74 @@ void shader_core_ctx::fetch() { } } + // --- Secondary fetch: fills I-Buffer half 1 from a non-overlapping split --- + // Only in AWARE mode with SIMD partitioning enabled. + // HIT-only: if I-Cache misses, skip (no miss tracking for secondary). + if (m_config->model == AWARE_RECONVERGENCE && + m_config->gpgpu_simd_partitioning && + !m_inst_fetch_buffer_secondary.m_valid) { + // Find the warp that just had primary fetch (if any), or scan for a warp + // whose secondary half is empty and has a secondary split + for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) { + unsigned warp_id = + (m_last_warp_fetched + i) % m_config->max_warps_per_shader; + + if (m_warp[warp_id]->functional_done()) continue; + if (!m_warp[warp_id]->ibuffer_half_empty(1)) continue; + if (is_virtualized(warp_id)) continue; + + // Check if this warp has a secondary split with non-overlapping mask + unsigned sec_pc, sec_rpc, sec_split_id; + simt_mask_t sec_mask; + if (!has_secondary_split(warp_id)) continue; + if (!get_secondary_split_info(warp_id, &sec_pc, &sec_rpc, &sec_split_id, + &sec_mask)) + continue; + + // Non-overlap check: secondary mask must not overlap with half 0's mask + if (m_warp[warp_id]->ibuffer_half_assigned(0)) { + active_mask_t half0_mask = m_warp[warp_id]->ibuffer_half_mask(0); + if ((sec_mask & half0_mask).any()) continue; // overlapping, skip + } + + // Attempt I-Cache access (HIT-only for secondary) + address_type ppc = sec_pc + PROGRAM_MEM_START; + unsigned nbytes = 16; + unsigned offset_in_block = + sec_pc & (m_config->m_L1I_config.get_line_sz() - 1); + if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz()) + nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block); + + mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx); + mem_fetch *mf = new mem_fetch( + acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE, + warp_id, m_sid, m_tpc, m_memory_config, + m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle); + std::list<cache_event> events; + enum cache_request_status status; + if (m_config->perfect_inst_const_cache) { + status = HIT; + } else { + status = m_L1I->access((new_addr_type)ppc, mf, + m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, + events); + } + + if (status == HIT) { + m_inst_fetch_buffer_secondary = ifetch_buffer_t( + sec_pc, nbytes, warp_id, 1, sec_split_id, sec_mask); + // Assign I-Buffer half 1 to this split + m_warp[warp_id]->ibuffer_assign_half(1, sec_split_id, sec_mask); + delete mf; + break; // one secondary fetch per cycle + } else { + // MISS or RESERVATION_FAIL: silently skip, no pending state + delete mf; + // Don't break — try next warp + } + } + } + m_L1I->cycle(); } @@ -1214,9 +1301,12 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, const warp_inst_t *next_inst, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id, - unsigned start_set) { - // Free the co-issued warp's I-buffer entry - m_warp[warp_id]->ibuffer_free(); + unsigned start_set, unsigned split_id) { + // For intra-warp co-issue: don't free ibuffer here (caller frees the slot) + // For inter-warp co-issue: free the co-issued warp's I-buffer entry + if (split_id == (unsigned)-1) { + m_warp[warp_id]->ibuffer_free(); + } assert(next_inst->valid()); // Create a temporary instruction for functional execution @@ -1237,11 +1327,33 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, // Functional execution for the co-issued warp's threads func_exec_inst(temp_inst); - // Update SIMT divergence structures for the co-issued warp - updateSIMTDivergenceStructures(warp_id, &temp_inst); + // Update SIMT divergence structures + if (split_id != (unsigned)-1 && + m_config->model == AWARE_RECONVERGENCE) { + // Intra-warp co-issue: move the target split to FIFO front first + bool moved = m_simt_tables[warp_id]->move_split_to_front(split_id); + if (moved && m_simt_tables[warp_id]->is_split_valid(split_id)) { + updateSIMTDivergenceStructures(warp_id, &temp_inst); + } + // If move failed or split invalid: skip SIMT update (split was + // invalidated by another operation during this cycle) + } else { + updateSIMTDivergenceStructures(warp_id, &temp_inst); + } - // Reserve scoreboard for the co-issued warp - m_scoreboard->reserveRegisters(&temp_inst); + // Reserve scoreboard for the co-issued warp. + // For intra-warp co-issue (same warp_id), skip reservation — the primary + // instruction already reserved for this warp, and the scoreboard tracks at + // warp granularity. Both splits may write to the same register number + // (which is safe since they operate on exclusive threads), but the + // scoreboard would abort on a duplicate reservation. + if (split_id == (unsigned)-1) { + // Inter-warp co-issue: different warp_id, always safe to reserve + m_scoreboard->reserveRegisters(&temp_inst); + } + // For intra-warp: we skip reservation. The primary's reservation covers + // shared registers, and non-shared registers will still be released at + // writeback (the release tolerates releasing non-reserved registers). // Set next PC for the co-issued warp m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize); @@ -1398,6 +1510,8 @@ void scheduler_unit::cycle() { exec_unit_type_t co_issue_fu_type = exec_unit_type_t::NONE; register_set *co_issue_reg_set = NULL; unsigned co_issue_primary_warp_id = (unsigned)-1; + unsigned available_sets = 0; + unsigned next_free_set = 0; order_warps(); for (std::vector<shd_warp_t *>::const_iterator iter = @@ -1728,8 +1842,8 @@ void scheduler_unit::cycle() { co_issue_fu_type != exec_unit_type_t::MEM && co_issue_fu_type != exec_unit_type_t::NONE) { // Count available (unused) sets and find next free set - unsigned available_sets = 0; - unsigned next_free_set = 0; + available_sets = 0; + next_free_set = 0; const std::vector<simd_set_info> &primary_sets = co_issue_composite->get_simd_sets(); for (unsigned s = 0; s < primary_sets.size(); s++) { @@ -1866,6 +1980,111 @@ void scheduler_unit::cycle() { } } + // INTRA-WARP co-issue: check same warp's secondary I-Buffer half (slots 2-3) + // for divergent split instructions that can be co-issued. + // Only in AWARE mode with SIMD partitioning enabled. + if (m_shader->m_config->gpgpu_simd_partitioning && + m_shader->m_config->model == AWARE_RECONVERGENCE && + co_issue_composite != NULL && available_sets > 0 && + co_issue_fu_type != exec_unit_type_t::MEM && + co_issue_fu_type != exec_unit_type_t::NONE) { + unsigned primary_warp_id = co_issue_primary_warp_id; + unsigned set_width = m_shader->m_config->simd_set_width; + + // Check slots 2-3 (secondary half) for valid instructions + for (unsigned sec_slot = 2; + sec_slot < 4 && available_sets > 0; sec_slot++) { + if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue; + + const warp_inst_t *sec_inst = + warp(primary_warp_id).ibuffer_slot_inst(sec_slot); + if (!sec_inst) continue; + + unsigned sec_split_id = + warp(primary_warp_id).ibuffer_slot_split_id(sec_slot); + const active_mask_t &sec_mask = + warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot); + + // Verify the split is still valid in the splits table + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + + // Verify the split's PC still matches the instruction's PC + unsigned split_pc; + simt_mask_t split_mask; + m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc, + &split_mask); + if (split_pc != sec_inst->pc) { + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + + // Check same FU type (reuse the FU type determination logic) + exec_unit_type_t sec_fu_type = exec_unit_type_t::NONE; + if ((sec_inst->op == LOAD_OP) || (sec_inst->op == STORE_OP) || + (sec_inst->op == MEMORY_BARRIER_OP) || + (sec_inst->op == TENSOR_CORE_LOAD_OP) || + (sec_inst->op == TENSOR_CORE_STORE_OP)) { + sec_fu_type = exec_unit_type_t::MEM; + } else if (sec_inst->op == SP_OP || + (sec_inst->op != DP_OP && sec_inst->op != SFU_OP && + sec_inst->op != ALU_SFU_OP && + sec_inst->op != TENSOR_CORE_OP && + sec_inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units == 0)) { + sec_fu_type = exec_unit_type_t::SP; + } else if (sec_inst->op != SP_OP && sec_inst->op != DP_OP && + sec_inst->op != SFU_OP && sec_inst->op != ALU_SFU_OP && + sec_inst->op != TENSOR_CORE_OP && + sec_inst->op < SPEC_UNIT_START_ID && + m_shader->m_config->gpgpu_num_int_units > 0) { + sec_fu_type = exec_unit_type_t::INT; + } else if (sec_inst->op == DP_OP) { + sec_fu_type = exec_unit_type_t::DP; + } else if (sec_inst->op == SFU_OP || sec_inst->op == ALU_SFU_OP) { + sec_fu_type = exec_unit_type_t::SFU; + } else if (sec_inst->op == TENSOR_CORE_OP) { + sec_fu_type = exec_unit_type_t::TENSOR; + } else if (sec_inst->op >= SPEC_UNIT_START_ID) { + sec_fu_type = exec_unit_type_t::SPECIALIZED; + } + if (sec_fu_type != co_issue_fu_type) continue; + + // Compute sets needed + unsigned sec_active = sec_mask.count(); + unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width; + if (sec_sets_needed > available_sets) continue; + + printf("SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u " + "split %u CO-ISSUED with primary split (%u sets)\n", + m_shader->get_gpu()->gpu_sim_cycle + + m_shader->get_gpu()->gpu_tot_sim_cycle, + get_sid(), m_id, primary_warp_id, sec_split_id, + sec_sets_needed); + + // Co-issue with targeted SIMT update (pass split_id) + m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask, + primary_warp_id, m_id, next_free_set, + sec_split_id); + + available_sets -= sec_sets_needed; + next_free_set += sec_sets_needed; + + // Free the secondary I-buffer slot (don't call ibuffer_free/step, + // use direct slot free since this is not the m_next pointer) + warp(primary_warp_id).ibuffer_free_slot(sec_slot); + + // After SIMT update, check if the split was invalidated (diverged) + if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) { + // Split was invalidated by the update — flush the entire secondary half + warp(primary_warp_id).ibuffer_flush_half(1); + break; + } + } + } + // issue stall statistics: if (!valid_inst) m_stats->shader_cycle_distro[0]++; // idle or control hazard @@ -2293,29 +2512,41 @@ void shader_core_ctx::writeback() { } // Release scoreboard for primary instruction (covers the outer warp_id) m_scoreboard->releaseRegisters(pipe_reg); - // Release scoreboard for any co-issued warps (non-primary) - for (unsigned wid : unique_warp_ids) { - if (wid == warp_id) continue; // already released above - // For co-issued warps, release their destination registers - // which were reserved in co_issue_warp() via reserveRegisters - // on the temporary instruction. The temp used the co-issued - // warp's out[] registers, so we need to release those. - // Since the composite's outer out[] belongs to the primary warp, - // we look at the sets to find co-issued warp registers. - for (unsigned s = 0; s < sets.size(); s++) { - if (sets[s].valid && sets[s].warp_id == wid && - sets[s].source_inst != NULL) { - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (sets[s].source_inst->out[r] > 0) { - m_scoreboard->releaseRegister(wid, sets[s].source_inst->out[r]); - } - } + // Release scoreboard for ALL co-issued sets (inter-warp AND intra-warp) + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + unsigned set_wid = sets[s].warp_id; + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (sets[s].source_inst->out[r] > 0) { + m_scoreboard->releaseRegister(set_wid, + sets[s].source_inst->out[r]); } } } - // dec_inst_in_pipeline for each unique warp - for (unsigned wid : unique_warp_ids) { - m_warp[wid]->dec_inst_in_pipeline(); + // dec_inst_in_pipeline: once for the primary instruction, plus once + // for each UNIQUE co-issued warp (not per-set — a co-issued warp may + // span multiple sets but was only inc'd once at decode). + // For intra-warp co-issue, the co-issued split shares the same warp_id + // as the primary — dec once for primary + once for the co-issued split. + m_warp[warp_id]->dec_inst_in_pipeline(); + std::set<unsigned> co_issued_warps_decremented; + bool intra_warp_decremented = false; + for (unsigned s = 0; s < sets.size(); s++) { + if (!sets[s].valid || sets[s].source_inst == NULL) continue; + if (sets[s].warp_id != warp_id) { + // Inter-warp: dec once per unique co-issued warp + if (co_issued_warps_decremented.find(sets[s].warp_id) == + co_issued_warps_decremented.end()) { + m_warp[sets[s].warp_id]->dec_inst_in_pipeline(); + co_issued_warps_decremented.insert(sets[s].warp_id); + } + } else { + // Intra-warp: dec once for the co-issued split (same warp_id) + if (!intra_warp_decremented) { + m_warp[warp_id]->dec_inst_in_pipeline(); + intra_warp_decremented = true; + } + } } } else { m_scoreboard->releaseRegisters(pipe_reg); @@ -4563,6 +4794,30 @@ bool shader_core_ctx::warp_valid(unsigned wid) { return m_simt_tables[wid]->valid(); } +bool shader_core_ctx::has_secondary_split(unsigned wid) { + return m_simt_tables[wid]->has_secondary_split(); +} + +bool shader_core_ctx::get_secondary_split_info(unsigned wid, unsigned *pc, + unsigned *rpc, + unsigned *split_id, + simt_mask_t *mask) { + return m_simt_tables[wid]->get_secondary_split_info(pc, rpc, split_id, mask); +} + +bool shader_core_ctx::is_split_valid(unsigned wid, unsigned split_id) { + return m_simt_tables[wid]->is_split_valid(split_id); +} + +void shader_core_ctx::get_split_info(unsigned wid, unsigned split_id, + unsigned *pc, simt_mask_t *mask) { + m_simt_tables[wid]->get_split_info(split_id, pc, mask); +} + +bool shader_core_ctx::move_split_to_front(unsigned wid, unsigned split_id) { + return m_simt_tables[wid]->move_split_to_front(split_id); +} + void shader_core_ctx::update_st_size(unsigned n) { // Stats not tracked in gpgpu-sim base; no-op. (void)n; |