diff options
| -rw-r--r-- | src/abstract_hardware_model.cc | 8 | ||||
| -rw-r--r-- | src/abstract_hardware_model.h | 8 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.cc | 8 | ||||
| -rw-r--r-- | src/gpgpu-sim/scoreboard.cc | 181 | ||||
| -rw-r--r-- | src/gpgpu-sim/scoreboard.h | 50 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 277 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.h | 37 |
7 files changed, 482 insertions, 87 deletions
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 72639d4..5d95db6 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -305,6 +305,14 @@ void warp_inst_t::set_split_id_on_sets(unsigned split_id) { } } +void warp_inst_t::set_source_mask_on_sets(const active_mask_t &mask) { + for (unsigned s = 0; s < m_simd_sets.size(); s++) { + if (m_simd_sets[s].valid) { + m_simd_sets[s].source_mask = mask; + } + } +} + void warp_inst_t::merge_simd_sets( const std::vector<simd_set_info> &other_sets) { assert(m_simd_sets.size() == other_sets.size()); diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 6ce1871..6331d6b 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -1162,6 +1162,10 @@ struct simd_set_info { unsigned num_active_threads; // popcount of set_active_mask bool valid; // Whether this set has any active threads const inst_t *source_inst; // Pointer to static decoded instruction (for MIMD) + // Mode 1 (mask-aware scoreboard): full issue-time active mask of the + // source_inst that owns this set. Used at writeback to release the + // matching (reg, mask) entry. Zero/empty when source_inst is NULL. + active_mask_t source_mask; // Mixed-space MEM co-issue (v2): per-set shared state. // Populated only when source_inst's space is shared/sstarr; otherwise @@ -1176,6 +1180,7 @@ struct simd_set_info { num_active_threads(0), valid(false), source_inst(NULL), cycles(0) { set_active_mask.reset(); active_mask_in_warp.reset(); + source_mask.reset(); for (unsigned i = 0; i < MAX_WARP_SIZE; i++) thread_ids[i] = 0; } @@ -1329,6 +1334,9 @@ class warp_inst_t : public inst_t { void merge_simd_sets(const std::vector<simd_set_info> &other_sets); void set_source_inst_on_sets(const inst_t *src); void set_split_id_on_sets(unsigned split_id); + // Mode 1: stamp the source_inst's full active_mask onto each valid set + // for use at writeback (releaseRegisterMask). + void set_source_mask_on_sets(const active_mask_t &mask); static bool simd_sets_overlap(const std::vector<simd_set_info> &a, const std::vector<simd_set_info> &b); diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 1c0c6c6..4fb288b 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -720,6 +720,14 @@ void shader_core_config::reg_options(class OptionParser *opp) { "may share the LDST unit in one pipe slot; atomics, shared, tex, " "const, barriers, LDGSTS, and BRU spill/fill are still excluded.", "0"); + option_parser_register( + opp, "-gpgpu_scoreboard_mode", OPT_UINT32, &gpgpu_scoreboard_mode, + "Scoreboard implementation for intra-warp co-issue hazard tracking: " + "0=legacy (primary + secondary tables, today's behavior), " + "1=mask-aware single scoreboard (entries keyed by (reg, active_mask)), " + "2=slot-pinned per-half scoreboards with wait-for-clean at " + "divergence/reconvergence. Default 0 preserves legacy behavior.", + "0"); for (unsigned j = 0; j < SPECIALIZED_UNIT_NUM; ++j) { std::stringstream ss; diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc index 6cdcaf5..70e6ac1 100644 --- a/src/gpgpu-sim/scoreboard.cc +++ b/src/gpgpu-sim/scoreboard.cc @@ -33,13 +33,16 @@ #include "shader_trace.h" // Constructor -Scoreboard::Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t* gpu) +Scoreboard::Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t* gpu, + unsigned mode) : longopregs() { m_sid = sid; + m_mode = mode; // Initialize size of table reg_table.resize(n_warps); longopregs.resize(n_warps); sec_reg_table.resize(n_warps); + mask_reg_table.resize(n_warps); m_gpu = gpu; @@ -55,6 +58,11 @@ Scoreboard::Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t* gpu) m_sec_total_duration_cycles = 0; m_sec_completed_reservations = 0; m_sec_max_duration_cycles = 0; + m_mask_reserve_inserts = 0; + m_mask_reserve_inc_refcount = 0; + m_mask_release_match = 0; + m_mask_release_nomatch = 0; + m_mask_release_erase = 0; } // Print scoreboard contents @@ -111,15 +119,26 @@ const bool Scoreboard::islongop(unsigned warp_id, unsigned regnum) { } void Scoreboard::reserveRegisters(const class warp_inst_t* inst) { - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (inst->out[r] > 0) { - reserveRegister(inst->warp_id(), inst->out[r]); - SHADER_DPRINTF(SCOREBOARD, "Reserved register - warp:%d, reg: %d\n", - inst->warp_id(), inst->out[r]); + if (m_mode == 1) { + // Mask-aware: insert (reg, inst's active mask, ref_count++) per output. + const active_mask_t &mask = inst->get_active_mask(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + reserveRegisterMask(inst->warp_id(), inst->out[r], mask); + } + } + } else { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + reserveRegister(inst->warp_id(), inst->out[r]); + SHADER_DPRINTF(SCOREBOARD, "Reserved register - warp:%d, reg: %d\n", + inst->warp_id(), inst->out[r]); + } } } - // Keep track of long operations + // Keep track of long operations (longopregs is mode-agnostic — used as a + // hint by issue eligibility, not for hazard tracking) if (inst->is_load() && (inst->space.get_type() == global_space || inst->space.get_type() == local_space || inst->space.get_type() == param_space_kernel || @@ -138,12 +157,22 @@ void Scoreboard::reserveRegisters(const class warp_inst_t* inst) { // Release registers for an instruction void Scoreboard::releaseRegisters(const class warp_inst_t* inst) { - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (inst->out[r] > 0) { - SHADER_DPRINTF(SCOREBOARD, "Register Released - warp:%d, reg: %d\n", - inst->warp_id(), inst->out[r]); - releaseRegister(inst->warp_id(), inst->out[r]); - longopregs[inst->warp_id()].erase(inst->out[r]); + if (m_mode == 1) { + const active_mask_t &mask = inst->get_active_mask(); + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + releaseRegisterMask(inst->warp_id(), inst->out[r], mask); + longopregs[inst->warp_id()].erase(inst->out[r]); + } + } + } else { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (inst->out[r] > 0) { + SHADER_DPRINTF(SCOREBOARD, "Register Released - warp:%d, reg: %d\n", + inst->warp_id(), inst->out[r]); + releaseRegister(inst->warp_id(), inst->out[r]); + longopregs[inst->warp_id()].erase(inst->out[r]); + } } } } @@ -180,9 +209,14 @@ bool Scoreboard::checkCollision(unsigned wid, const class inst_t* inst) const { } bool Scoreboard::pendingWrites(unsigned wid) const { + if (m_mode == 1) return !mask_reg_table[wid].empty(); return !reg_table[wid].empty(); } +bool Scoreboard::pendingWritesAny(unsigned wid) const { + return pendingWrites(wid); +} + // Secondary scoreboard methods for intra-warp co-issue void Scoreboard::reserveRegisterSecondary(unsigned wid, unsigned regnum) { @@ -237,6 +271,88 @@ void Scoreboard::clearSecondary(unsigned wid) { sec_reg_table[wid].clear(); } +// ===================================================================== +// Mode 1: mask-aware scoreboard +// ===================================================================== +// +// Each reservation is a (reg, active_mask, ref_count) tuple. Identical +// (reg, mask) reservations are coalesced into one entry with ref_count +// incremented. Hazard detection is mask intersection — an instruction +// with active_mask M collides on register R iff there exists an entry +// (R, M', _) with (M & M').any(). +// +// Release is idempotent: matching entry decrements ref_count and is +// erased when ref_count hits zero; mismatch is silently ignored (so +// the existing per-set release loops, which call release once per valid +// set, work without dedup). + +void Scoreboard::reserveRegisterMask(unsigned wid, unsigned reg, + const active_mask_t &mask) { + for (auto &e : mask_reg_table[wid]) { + if (e.reg == reg && e.mask == mask) { + e.ref_count++; + m_mask_reserve_inc_refcount++; + return; + } + } + mask_resv e; + e.reg = reg; + e.mask = mask; + e.ref_count = 1; + e.resv_cycle = m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle; + mask_reg_table[wid].push_back(e); + m_mask_reserve_inserts++; +} + +void Scoreboard::releaseRegisterMask(unsigned wid, unsigned reg, + const active_mask_t &mask) { + for (auto it = mask_reg_table[wid].begin(); + it != mask_reg_table[wid].end(); ++it) { + if (it->reg == reg && it->mask == mask) { + assert(it->ref_count > 0); + it->ref_count--; + m_mask_release_match++; + if (it->ref_count == 0) { + mask_reg_table[wid].erase(it); + m_mask_release_erase++; + } + return; + } + } + m_mask_release_nomatch++; + // No matching entry — idempotent no-op (mirrors legacy releaseRegister). +} + +void Scoreboard::releaseSetReg(unsigned wid, unsigned reg, + const active_mask_t &mask, + bool is_intra_legacy) { + if (m_mode == 1) { + releaseRegisterMask(wid, reg, mask); + } else if (is_intra_legacy) { + releaseRegisterSecondary(wid, reg); + } else { + releaseRegister(wid, reg); + } +} + +bool Scoreboard::checkCollisionMask(unsigned wid, const class inst_t *inst, + const active_mask_t &mask) const { + std::set<int> inst_regs; + for (unsigned i = 0; i < inst->outcount; i++) inst_regs.insert(inst->out[i]); + for (unsigned j = 0; j < inst->incount; j++) inst_regs.insert(inst->in[j]); + if (inst->pred > 0) inst_regs.insert(inst->pred); + if (inst->ar1 > 0) inst_regs.insert(inst->ar1); + if (inst->ar2 > 0) inst_regs.insert(inst->ar2); + + for (auto reg : inst_regs) { + if (reg <= 0) continue; + for (const auto &e : mask_reg_table[wid]) { + if (e.reg == (unsigned)reg && (e.mask & mask).any()) return true; + } + } + return false; +} + // End-of-run accounting dump. void Scoreboard::dumpAccounting(FILE* out) const { unsigned long long primary_remaining = 0; @@ -268,6 +384,34 @@ void Scoreboard::dumpAccounting(FILE* out) const { m_sec_release_calls, sec_remaining, m_sec_completed_reservations, sec_avg_dur, m_sec_max_duration_cycles, m_sec_clear_calls, m_sec_clear_regs_dropped); + // Mode 1 mask-aware scoreboard accounting. + unsigned long long mask_remaining_entries = 0; + unsigned long long mask_remaining_refcount = 0; + for (unsigned w = 0; w < mask_reg_table.size(); w++) { + mask_remaining_entries += mask_reg_table[w].size(); + for (const auto &e : mask_reg_table[w]) mask_remaining_refcount += e.ref_count; + } + fprintf(out, + "mask_scoreboard_accounting sid=%u inserts=%llu inc_refcount=%llu " + "release_match=%llu release_nomatch=%llu erase=%llu " + "remaining_entries=%llu remaining_refcount=%llu\n", + m_sid, m_mask_reserve_inserts, m_mask_reserve_inc_refcount, + m_mask_release_match, m_mask_release_nomatch, m_mask_release_erase, + mask_remaining_entries, mask_remaining_refcount); + // If anything is leaked, dump per-warp leak details. + if (mask_remaining_entries > 0) { + for (unsigned w = 0; w < mask_reg_table.size(); w++) { + if (mask_reg_table[w].empty()) continue; + fprintf(out, " mask_leak sid=%u wid=%u entries=%zu:\n", m_sid, w, + mask_reg_table[w].size()); + for (const auto &e : mask_reg_table[w]) { + fprintf(out, + " reg=%u ref_count=%u resv_cycle=%llu mask_count=%lu mask=%s\n", + e.reg, e.ref_count, e.resv_cycle, e.mask.count(), + e.mask.to_string().c_str()); + } + } + } } // Read-only diagnostic. Prints the primary and secondary scoreboard @@ -290,4 +434,15 @@ void Scoreboard::dump_warp_state(FILE *out, unsigned wid) const { for (auto r : longopregs[wid]) fprintf(out, " r%u", r); fprintf(out, "\n"); } + // Mode 1: mask_reg_table for this warp. + if (wid < mask_reg_table.size() && !mask_reg_table[wid].empty()) { + fprintf(out, " mask_reg_table[%u] (size=%zu):\n", wid, + mask_reg_table[wid].size()); + for (const auto &e : mask_reg_table[wid]) { + fprintf(out, + " reg=%u ref_count=%u resv_cycle=%llu mask_count=%lu mask=%s\n", + e.reg, e.ref_count, e.resv_cycle, e.mask.count(), + e.mask.to_string().c_str()); + } + } } diff --git a/src/gpgpu-sim/scoreboard.h b/src/gpgpu-sim/scoreboard.h index 3dbb1ce..660587e 100644 --- a/src/gpgpu-sim/scoreboard.h +++ b/src/gpgpu-sim/scoreboard.h @@ -41,7 +41,12 @@ class Scoreboard { public: - Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t *gpu); + // mode: 0 = legacy (primary + secondary tables), 1 = mask-aware, + // 2 = slot-pinned. See scoreboard_redesign_plan.md. + Scoreboard(unsigned sid, unsigned n_warps, class gpgpu_t *gpu, + unsigned mode = 0); + + unsigned mode() const { return m_mode; } void reserveRegisters(const warp_inst_t *inst); void releaseRegisters(const warp_inst_t *inst); @@ -52,12 +57,35 @@ class Scoreboard { void printContents() const; const bool islongop(unsigned warp_id, unsigned regnum); - // Secondary scoreboard for intra-warp co-issue (split-level tracking) + // Secondary scoreboard for intra-warp co-issue (split-level tracking). + // Mode 0 (legacy) only — bypassed under modes 1 and 2. void reserveRegisterSecondary(unsigned wid, unsigned regnum); void releaseRegisterSecondary(unsigned wid, unsigned regnum); bool checkCollisionSecondary(unsigned wid, const inst_t *inst) const; void clearSecondary(unsigned wid); + // Mode 1: mask-aware single scoreboard. + // Each reservation entry tracks (reg, active_mask, ref_count). Hazard + // checks return true iff any of the inst's read/written regs has an + // entry whose mask overlaps the candidate's active_mask. See + // for_claude/docs/scoreboard_redesign_plan.md for design. + void reserveRegisterMask(unsigned wid, unsigned reg, + const active_mask_t &mask); + void releaseRegisterMask(unsigned wid, unsigned reg, + const active_mask_t &mask); + bool checkCollisionMask(unsigned wid, const inst_t *inst, + const active_mask_t &mask) const; + // Mask-agnostic helper: any in-flight writes for this warp under the + // active scoreboard table. In mode 1 this checks mask_reg_table; in + // mode 0 it falls through to legacy reg_table emptiness. + bool pendingWritesAny(unsigned wid) const; + + // Per-set release dispatcher used at writeback. In mode 0, picks + // primary vs secondary release based on `is_intra_legacy`. In mode 1, + // releases (reg, mask) from mask_reg_table; `is_intra_legacy` ignored. + void releaseSetReg(unsigned wid, unsigned reg, const active_mask_t &mask, + bool is_intra_legacy); + public: // Accounting / leak-detection (called from gpu-sim end-of-run dump). // Aggregates reserve/release counts, reservation-duration histogram, @@ -74,6 +102,7 @@ class Scoreboard { int get_sid() const { return m_sid; } unsigned m_sid; + unsigned m_mode; // 0=legacy, 1=mask-aware, 2=slot-pinned // keeps track of pending writes to registers // indexed by warp id, reg_id => pending write count @@ -84,6 +113,23 @@ class Scoreboard { // Secondary path register tracking (for I-Buffer half 1, intra-warp co-issue) std::vector<std::set<unsigned> > sec_reg_table; + // Mode 1: per-warp list of mask-aware reservations. + struct mask_resv { + unsigned reg; + active_mask_t mask; + unsigned ref_count; + unsigned long long resv_cycle; + }; + std::vector<std::vector<mask_resv> > mask_reg_table; + + // Mode 1 accounting (used by dumpAccounting + dump_warp_state for leak + // detection on the deadlock path). + unsigned long long m_mask_reserve_inserts; // new entry pushes + unsigned long long m_mask_reserve_inc_refcount; // ref_count++ on existing + unsigned long long m_mask_release_match; // found and decremented + unsigned long long m_mask_release_nomatch; // no matching entry + unsigned long long m_mask_release_erase; // ref_count hit 0 + erased + class gpgpu_t *m_gpu; // ---- Accounting (per-shader, scalar; cheap to update) ---- diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index a2a77a7..ec764aa 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -191,7 +191,8 @@ void shader_core_ctx::create_front_pipeline() { } void shader_core_ctx::create_schedulers() { - m_scoreboard = new Scoreboard(m_sid, m_config->max_warps_per_shader, m_gpu); + m_scoreboard = new Scoreboard(m_sid, m_config->max_warps_per_shader, m_gpu, + m_config->gpgpu_scoreboard_mode); // scedulers // must currently occur after all inputs have been initialized. @@ -1528,21 +1529,31 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, } // Reserve scoreboard for the co-issued warp. - // For intra-warp co-issue (same warp_id), skip reservation — the primary - // instruction already reserved for this warp, and the scoreboard tracks at - // warp granularity. Both splits may write to the same register number - // (which is safe since they operate on exclusive threads), but the - // scoreboard would abort on a duplicate reservation. - if (split_id == (unsigned)-1) { - // Inter-warp co-issue: different warp_id, always safe to reserve + if (m_config->gpgpu_scoreboard_mode == 1) { + // Mode 1 (mask-aware): both inter-warp and intra-warp use the unified + // mask-aware scoreboard. The active_mask carried in temp_inst already + // differentiates the two splits' reservations — disjoint-lane writes + // get distinct (reg, mask) entries even when they share warp_id. m_scoreboard->reserveRegisters(&temp_inst); } else { - // Intra-warp co-issue: reserve on secondary scoreboard to track - // dependencies within the secondary split's instruction stream. - // Cannot use primary scoreboard (would abort on duplicate register names). - for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { - if (temp_inst.out[r] > 0) { - m_scoreboard->reserveRegisterSecondary(warp_id, temp_inst.out[r]); + // Mode 0 (legacy): + // For intra-warp co-issue (same warp_id), skip primary reservation — + // the primary instruction already reserved for this warp, and the + // scoreboard tracks at warp granularity. Both splits may write to the + // same register number (which is safe since they operate on exclusive + // threads), but the scoreboard would abort on a duplicate reservation. + if (split_id == (unsigned)-1) { + // Inter-warp co-issue: different warp_id, always safe to reserve + m_scoreboard->reserveRegisters(&temp_inst); + } else { + // Intra-warp co-issue: reserve on secondary scoreboard to track + // dependencies within the secondary split's instruction stream. + // Cannot use primary scoreboard (would abort on duplicate register + // names). + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { + if (temp_inst.out[r] > 0) { + m_scoreboard->reserveRegisterSecondary(warp_id, temp_inst.out[r]); + } } } } @@ -1555,6 +1566,10 @@ void shader_core_ctx::co_issue_warp(warp_inst_t *composite, // between primary and secondary-split sets that share warp_id). temp_inst.set_source_inst_on_sets(next_inst); temp_inst.set_split_id_on_sets(split_id); + // Mode 1: stamp the source's full issue mask so per-set writeback can + // release the matching mask-aware reservation. Cheap and harmless in + // mode 0 (field is unused there). + temp_inst.set_source_mask_on_sets(active_mask); // Mixed-space MEM co-issue (v2): for SHARED coissuers, capture per-lane // memreqaddr + temp_inst.cycles into each valid set BEFORE merge. The @@ -1948,7 +1963,20 @@ void scheduler_unit::try_inter_warp_coissue( } // Scoreboard check. - if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) { + // Mode 1 needs the candidate's active mask for hazard intersection; + // hoist its lookup before the legacy/mode-1 dispatch. + bool sb_collision_inter; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + const active_mask_t &cand_mask_pre = + m_shader->get_active_mask(cand_warp_id, cand_inst); + sb_collision_inter = m_scoreboard->checkCollisionMask(cand_warp_id, + cand_inst, + cand_mask_pre); + } else { + sb_collision_inter = m_scoreboard->checkCollision(cand_warp_id, + cand_inst); + } + if (sb_collision_inter) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; } @@ -2091,21 +2119,30 @@ void scheduler_unit::try_intra_warp_coissue( } } - if (m_scoreboard->checkCollisionSecondary(primary_warp_id, sec_inst)) { - m_stats->coissue_denied_by_scoreboard[get_sid()]++; - continue; - } - // Stale-mask guard: see commentary on the same fix in // try_utilization_max_coissue. Cached `sec_mask` may diverge from // current `split_mask` (lane re-bucketing or split-ID reuse). Dispatch // only the intersection — lanes in BOTH masks are guaranteed to have - // their per-thread PC at sec_inst->pc. + // their per-thread PC at sec_inst->pc. Hoisted before the scoreboard + // check so mode 1 can use the effective mask for hazard detection. active_mask_t sec_mask_eff; for (unsigned t = 0; t < MAX_WARP_SIZE; t++) { if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t); } if (!sec_mask_eff.any()) continue; + + bool sb_collision; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision = m_scoreboard->checkCollisionMask(primary_warp_id, + sec_inst, sec_mask_eff); + } else { + sb_collision = m_scoreboard->checkCollisionSecondary(primary_warp_id, + sec_inst); + } + if (sb_collision) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } unsigned sec_active = sec_mask_eff.count(); unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width; if (sec_sets_needed > available_sets) { @@ -2190,7 +2227,18 @@ void scheduler_unit::try_utilization_max_coissue( &cand_rpc); if (cand_pc != cand_inst->pc) continue; - if (m_scoreboard->checkCollision(cand_warp_id, cand_inst)) { + bool sb_collision_inter2; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + const active_mask_t &cand_mask_pre = + m_shader->get_active_mask(cand_warp_id, cand_inst); + sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id, + cand_inst, + cand_mask_pre); + } else { + sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id, + cand_inst); + } + if (sb_collision_inter2) { m_stats->coissue_denied_by_scoreboard[get_sid()]++; continue; } @@ -2326,11 +2374,6 @@ void scheduler_unit::try_utilization_max_coissue( continue; } } - if (m_scoreboard->checkCollisionSecondary(cand_warp_id, sec_inst)) { - m_stats->coissue_denied_by_scoreboard[get_sid()]++; - continue; - } - // Stale-mask guard: the cached `sec_mask` is from secondary-fetch // time and may not match the current splits-table mask. Two // failure modes: @@ -2344,12 +2387,27 @@ void scheduler_unit::try_utilization_max_coissue( // cached mask is stale. // Both reduce to: trust only lanes present in BOTH masks. The // ibuffer slot ownership invariant guarantees lanes in `split_mask` - // are at `split_pc == sec_inst->pc`. + // are at `split_pc == sec_inst->pc`. Hoisted before the scoreboard + // check so mode 1 can use the effective mask for hazard detection. active_mask_t sec_mask_eff; for (unsigned t = 0; t < MAX_WARP_SIZE; t++) { if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t); } if (!sec_mask_eff.any()) continue; + + bool sb_collision; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id, + sec_inst, + sec_mask_eff); + } else { + sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id, + sec_inst); + } + if (sb_collision) { + m_stats->coissue_denied_by_scoreboard[get_sid()]++; + continue; + } unsigned sec_active = sec_mask_eff.count(); unsigned sec_needed = (sec_active + set_width - 1) / set_width; if (sec_needed > available_sets) { @@ -2548,15 +2606,23 @@ void scheduler_unit::cycle() { m_scoreboard->clearSecondary(warp_id); } else { valid_inst = true; - if (!m_scoreboard->checkCollision(warp_id, pI)) { + // Mode 1 needs the warp's active mask for hazard intersection; + // hoist it before the scoreboard check. + const active_mask_t &active_mask = + m_shader->get_active_mask(warp_id, pI); + bool sb_collision_primary; + if (m_shader->m_config->gpgpu_scoreboard_mode == 1) { + sb_collision_primary = + m_scoreboard->checkCollisionMask(warp_id, pI, active_mask); + } else { + sb_collision_primary = m_scoreboard->checkCollision(warp_id, pI); + } + if (!sb_collision_primary) { SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) passes scoreboard\n", (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id()); ready_inst = true; - const active_mask_t &active_mask = - m_shader->get_active_mask(warp_id, pI); - assert(warp(warp_id).inst_in_pipeline()); if ((pI->op == LOAD_OP) || (pI->op == STORE_OP) || @@ -3330,14 +3396,19 @@ void shader_core_ctx::writeback() { } // Release scoreboard for primary instruction (covers the outer warp_id) m_scoreboard->releaseRegisters(pipe_reg); - // Release scoreboard for ALL co-issued sets (inter-warp AND intra-warp) + // Release scoreboard for ALL co-issued sets (inter-warp AND intra-warp). + // Dispatcher: mode 0 -> primary release; mode 1 -> mask-aware release + // using the set's stamped source_mask. Idempotent; safe to call once + // per set even when multiple sets share the same source_inst. for (unsigned s = 0; s < sets.size(); s++) { if (!sets[s].valid || sets[s].source_inst == NULL) continue; unsigned set_wid = sets[s].warp_id; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { if (sets[s].source_inst->out[r] > 0) { - m_scoreboard->releaseRegister(set_wid, - sets[s].source_inst->out[r]); + m_scoreboard->releaseSetReg(set_wid, + sets[s].source_inst->out[r], + sets[s].source_mask, + /*is_intra_legacy=*/false); } } } @@ -3359,12 +3430,14 @@ void shader_core_ctx::writeback() { co_issued_warps_decremented.insert(sets[s].warp_id); } } else { - // Intra-warp: release from secondary scoreboard and dec once + // Intra-warp: release from secondary scoreboard (mode 0) or + // mask-aware (mode 1) and dec once. if (!intra_warp_decremented) { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { if (sets[s].source_inst->out[r] > 0) { - m_scoreboard->releaseRegisterSecondary( - sets[s].warp_id, sets[s].source_inst->out[r]); + m_scoreboard->releaseSetReg( + sets[s].warp_id, sets[s].source_inst->out[r], + sets[s].source_mask, /*is_intra_legacy=*/true); } } m_warp[warp_id]->dec_inst_in_pipeline(); @@ -3532,6 +3605,11 @@ mem_stage_stall_type ldst_unit::process_cache_access( [src.out_inst->out[r]]--; else m_pending_writes[src.wid][src.out_inst->out[r]]--; + // Mode 1: per-mask decrement + release when the inst's own + // accesses fully drain (independent of other in-flight insts' + // contributions to the aggregate counter). + dec_mask_pw_and_maybe_release(src.wid, src.out_inst->out[r], + src.source_mask); } // release LDGSTS (primary-only: LDGSTS is excluded from MEM co-issue) @@ -3695,7 +3773,8 @@ void ldst_unit::L1_latency_queue_cycle() { if (!still_pending) { m_pending_writes_secondary[src.wid][src_split_mf].erase( reg); - m_scoreboard->releaseRegisterSecondary(src.wid, reg); + m_scoreboard->releaseSetReg(src.wid, reg, src.source_mask, + /*is_intra_legacy=*/true); any_completed = true; } } else { @@ -3703,10 +3782,14 @@ void ldst_unit::L1_latency_queue_cycle() { unsigned still_pending = --m_pending_writes[src.wid][reg]; if (!still_pending) { m_pending_writes[src.wid].erase(reg); - m_scoreboard->releaseRegister(src.wid, reg); + m_scoreboard->releaseSetReg(src.wid, reg, src.source_mask, + /*is_intra_legacy=*/false); any_completed = true; } } + // Mode 1: per-mask release fires per inst, decoupled from + // the aggregate counter above. + dec_mask_pw_and_maybe_release(src.wid, reg, src.source_mask); } } if (any_completed) m_core->warp_inst_complete(mf_next->get_inst()); @@ -4259,17 +4342,41 @@ ldst_unit::ldst_unit(mem_fetch_interface *icnt, mem_config, stats, sid, tpc); } +void ldst_unit::dec_mask_pw_and_maybe_release(unsigned wid, unsigned reg, + const active_mask_t &mask) { + if (m_core->get_config()->gpgpu_scoreboard_mode != 1) return; + unsigned long mk = mask.to_ulong(); + auto wit = m_pending_writes_mask.find(wid); + if (wit == m_pending_writes_mask.end()) return; + auto rit = wit->second.find(reg); + if (rit == wit->second.end()) return; + auto mit = rit->second.find(mk); + if (mit == rit->second.end()) return; + assert(mit->second > 0); + if (--(mit->second) == 0) { + rit->second.erase(mit); + if (rit->second.empty()) { + wit->second.erase(rit); + if (wit->second.empty()) m_pending_writes_mask.erase(wit); + } + m_scoreboard->releaseRegisterMask(wid, reg, mask); + } +} + ldst_unit::mem_src_t ldst_unit::resolve_source( const warp_inst_t &inst, unsigned access_src_wid, unsigned access_src_split_id) const { mem_src_t r; r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid; r.out_inst = &inst; + r.source_mask.reset(); // Primary access: (wid == primary AND split_id == -1). Return early with // out_inst = composite (primary's out[]). Note: split_id is the // discriminator — a coissuer can have the same warp_id as primary when // it came from the primary warp's own secondary ibuffer slot. if (r.wid == inst.warp_id() && access_src_split_id == (unsigned)-1) { + // Primary's own mask is on the composite itself. + r.source_mask = inst.get_active_mask(); return r; } if (!inst.has_simd_sets()) return r; @@ -4280,6 +4387,7 @@ ldst_unit::mem_src_t ldst_unit::resolve_source( if (sets[s].warp_id == r.wid && sets[s].split_id == access_src_split_id) { r.out_inst = sets[s].source_inst; + r.source_mask = sets[s].source_mask; break; } } @@ -4487,9 +4595,19 @@ void ldst_unit::issue(register_set ®_set) { unsigned n_primary = n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)]; if (n_primary > 0) { + const bool mode1 = + (m_core->get_config()->gpgpu_scoreboard_mode == 1); + unsigned long pri_mask_key = + mode1 ? inst->get_active_mask().to_ulong() : 0; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = inst->out[r]; - if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_primary; + if (reg_id > 0) { + m_pending_writes[primary_wid][reg_id] += n_primary; + if (mode1) { + m_pending_writes_mask[primary_wid][reg_id][pri_mask_key] += + n_primary; + } + } } } } @@ -4513,6 +4631,10 @@ void ldst_unit::issue(register_set ®_set) { // is_intra = "uses secondary scoreboard map" = split came from a // secondary ibuffer slot. Not tied to warp_id == primary_wid. bool is_intra = (sets[s].split_id != (unsigned)-1); + const bool mode1 = + (m_core->get_config()->gpgpu_scoreboard_mode == 1); + unsigned long set_mask_key = + mode1 ? sets[s].source_mask.to_ulong() : 0; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id > 0) { @@ -4522,15 +4644,29 @@ void ldst_unit::issue(register_set ®_set) { } else { m_pending_writes[sets[s].warp_id][reg_id] += n_src; } + if (mode1) { + m_pending_writes_mask[sets[s].warp_id][reg_id][set_mask_key] += + n_src; + } } } } } else if (primary_is_tracked_load) { // Legacy (non-co-issue) path: single-warp pending_writes. unsigned n_accesses = inst->accessq_count(); + const bool mode1 = + (m_core->get_config()->gpgpu_scoreboard_mode == 1); + unsigned long pri_mask_key = + mode1 ? inst->get_active_mask().to_ulong() : 0; for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = inst->out[r]; - if (reg_id > 0) m_pending_writes[primary_wid][reg_id] += n_accesses; + if (reg_id > 0) { + m_pending_writes[primary_wid][reg_id] += n_accesses; + if (mode1) { + m_pending_writes_mask[primary_wid][reg_id][pri_mask_key] += + n_accesses; + } + } } } @@ -4631,7 +4767,9 @@ void ldst_unit::writeback() { if (!still_pending) { m_pending_writes_secondary[src_i.wid][src_split_i] .erase(reg); - m_scoreboard->releaseRegisterSecondary(src_i.wid, reg); + m_scoreboard->releaseSetReg(src_i.wid, reg, + src_i.source_mask, + /*is_intra_legacy=*/true); insn_completed = true; } } else { @@ -4640,10 +4778,14 @@ void ldst_unit::writeback() { --m_pending_writes[src_i.wid][reg]; if (!still_pending) { m_pending_writes[src_i.wid].erase(reg); - m_scoreboard->releaseRegister(src_i.wid, reg); + m_scoreboard->releaseSetReg(src_i.wid, reg, + src_i.source_mask, + /*is_intra_legacy=*/false); insn_completed = true; } } + // Mode 1: per-mask release fires per inst. + dec_mask_pw_and_maybe_release(src_i.wid, reg, src_i.source_mask); } else if (m_next_wb.m_is_ldgsts) { // LDGSTS excluded from co-issue; source_list always size<=1 m_pending_ldgsts[primary_wid][m_next_wb.pc] @@ -4663,7 +4805,9 @@ void ldst_unit::writeback() { (void)is_intra_coissued; // unused on shared path for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { if (src.out_inst->out[r] > 0) { - m_scoreboard->releaseRegister(src.wid, src.out_inst->out[r]); + m_scoreboard->releaseSetReg(src.wid, src.out_inst->out[r], + src.source_mask, + /*is_intra_legacy=*/false); insn_completed = true; } } @@ -4682,16 +4826,13 @@ void ldst_unit::writeback() { sets[s].split_id); if (released.count(key)) continue; released.insert(key); - // Secondary slot coissuer → secondary scoreboard map. + // Secondary slot coissuer → secondary scoreboard map (mode 0). bool coissued_intra = (sets[s].split_id != (unsigned)-1); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { if (sets[s].source_inst->out[r] > 0) { - if (coissued_intra) - m_scoreboard->releaseRegisterSecondary( - sets[s].warp_id, sets[s].source_inst->out[r]); - else - m_scoreboard->releaseRegister( - sets[s].warp_id, sets[s].source_inst->out[r]); + m_scoreboard->releaseSetReg( + sets[s].warp_id, sets[s].source_inst->out[r], + sets[s].source_mask, /*is_intra_legacy=*/coissued_intra); insn_completed = true; } } @@ -5013,10 +5154,9 @@ void ldst_unit::cycle() { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg = sets[s].source_inst->out[r]; if (reg == 0) continue; - if (is_intra) - m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg); - else - m_scoreboard->releaseRegister(sets[s].warp_id, reg); + m_scoreboard->releaseSetReg(sets[s].warp_id, reg, + sets[s].source_mask, + /*is_intra_legacy=*/is_intra); } } pipe_reg.set_shared_side_released(true); @@ -5188,10 +5328,9 @@ void ldst_unit::cycle() { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id == 0) continue; - if (is_intra) - m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, reg_id); - else - m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id, + sets[s].source_mask, + /*is_intra_legacy=*/is_intra); } } } @@ -5331,16 +5470,14 @@ void ldst_unit::cycle() { sets[s].split_id); if (released.count(key)) continue; released.insert(key); - // Secondary slot → secondary scoreboard map. + // Secondary slot → secondary scoreboard map (mode 0). bool is_intra = (sets[s].split_id != (unsigned)-1); for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id == 0) continue; - if (is_intra) - m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, - reg_id); - else - m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id, + sets[s].source_mask, + /*is_intra_legacy=*/is_intra); } } } @@ -5423,11 +5560,9 @@ void ldst_unit::cycle() { for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = sets[s].source_inst->out[r]; if (reg_id == 0) continue; - if (is_intra) - m_scoreboard->releaseRegisterSecondary(sets[s].warp_id, - reg_id); - else - m_scoreboard->releaseRegister(sets[s].warp_id, reg_id); + m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id, + sets[s].source_mask, + /*is_intra_legacy=*/is_intra); } } } diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 51ed214..ebcfc8d 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -1659,13 +1659,27 @@ class ldst_unit : public pipelined_simd_unit { mem_stage_stall_type process_memory_access_queue_l1cache(l1_cache *cache, warp_inst_t &inst); + // Mode 1 helper: decrement m_pending_writes_mask[wid][reg][mask] and, if + // it hits zero, release the (reg, mask) entry in the mask-aware + // scoreboard. No-op in mode 0. Inline here so the compiler can elide the + // mode-0 path entirely. + void dec_mask_pw_and_maybe_release(unsigned wid, unsigned reg, + const active_mask_t &mask); + // MEM co-issue: resolve which source warp + instruction an access/mem_fetch // belongs to (for routing pending_writes / scoreboard release back to the // originating warp under composites). Returns {src_wid, src_out_inst}. // For non-composite or primary-source accesses, returns {primary_wid, &inst}. // split_id differentiates intra-warp co-issued sets (same wid as primary) // from the primary set itself; use (unsigned)-1 when not applicable. - struct mem_src_t { unsigned wid; const inst_t *out_inst; }; + struct mem_src_t { + unsigned wid; + const inst_t *out_inst; + // Mode 1: source_inst's full issue mask, looked up from the + // matching simd_set's source_mask. Empty for primary access (which + // uses pipe_reg's own get_active_mask() at release time). + active_mask_t source_mask; + }; mem_src_t resolve_source(const warp_inst_t &inst, unsigned access_src_wid, unsigned access_src_split_id) const; @@ -1721,6 +1735,19 @@ class ldst_unit : public pipelined_simd_unit { std::map<unsigned /*split_id*/, std::map<unsigned /*reg*/, unsigned /*count*/>>> m_pending_writes_secondary; + // Mode 1 (mask-aware scoreboard) pending-writes counter, keyed by + // (wid, reg, mask). Mode 0's m_pending_writes aggregates across all + // in-flight insts to the same (wid, reg), which is fine for mode 0's + // single-entry-per-(wid,reg) scoreboard but causes leaks under mode 1's + // per-mask scoreboard: when the aggregate hits zero only the + // last-retiring access's mask gets a scoreboard release; other masks + // leak. This per-mask counter fires the mask-aware release at the + // correct time for each inst. Updated only when mode==1. + std::map<unsigned /*wid*/, + std::map<unsigned /*reg*/, + std::map<unsigned long /*mask.to_ulong()*/, + unsigned /*count*/>>> + m_pending_writes_mask; unsigned m_writeback_arb; // round-robin arbiter for writeback contention // between L1T, L1C, shared unsigned m_num_writeback_clients; @@ -1851,6 +1878,13 @@ class shader_core_config : public core_config { gpgpu_co_issue_priority); abort(); } + if (gpgpu_scoreboard_mode > 2) { + fprintf(stderr, + "GPGPU-Sim: invalid -gpgpu_scoreboard_mode %u; must be " + "0 (legacy), 1 (mask-aware), or 2 (slot-pinned)\n", + gpgpu_scoreboard_mode); + abort(); + } if (gpgpu_enable_compaction && gpgpu_compaction_mode == 0) { gpgpu_compaction_mode = 2; } @@ -2014,6 +2048,7 @@ class shader_core_config : public core_config { unsigned gpgpu_co_issue_priority; // 0=greedy..4=same-PC bool gpgpu_simd_partitioning_debug; // verbose SIMD_SETS printf gate bool gpgpu_mem_coissue; // enable MEM/LDST co-issue path + unsigned gpgpu_scoreboard_mode; // 0=legacy, 1=mask-aware, 2=slot-pinned unsigned n_simt_cores_per_cluster; unsigned n_simt_clusters; |