diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/cuda-sim/cuda-sim.cc | 9 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.cc | 56 | ||||
| -rw-r--r-- | src/gpgpu-sim/gpu-sim.h | 5 | ||||
| -rw-r--r-- | src/gpgpu-sim/scoreboard.cc | 22 | ||||
| -rw-r--r-- | src/gpgpu-sim/scoreboard.h | 5 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.cc | 116 | ||||
| -rw-r--r-- | src/gpgpu-sim/shader.h | 9 |
7 files changed, 221 insertions, 1 deletions
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index fb72f25..2a88e5a 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1157,6 +1157,15 @@ void ptx_instruction::pre_decode() { break; } + // Pre-tag atomics on the static template so co-issue eligibility checks + // (mem_coissue_disallowed in shader.cc) can detect them at candidate-scan + // time. Without this, m_isatomic only gets set during per-instance + // functional execution (add_callback path), which is too late — the + // intra co-issue scan reads the SHARED template's flag and would + // mistakenly admit an atomic into a composite, leaving the warp's + // n_atomic counter unbalanced and the warp stuck in waiting(). + if (m_opcode == ATOM_OP) m_isatomic = true; + set_opcode_and_latency(); set_bar_type(); // Get register operands diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index c858c50..1c0c6c6 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -1342,7 +1342,52 @@ void gpgpu_sim::print_stats(unsigned long long streamID) { } } +// Static pointer + signal handler for MEMCO_DBG_DEADLOCK. When a +// SIGSEGV / SIGABRT fires before the deadlock detector triggers, dump +// per-warp state for the most recently active gpgpu_sim and re-raise. +static gpgpu_sim *g_dbg_sim_for_handler = NULL; +static volatile int g_dbg_sim_handler_in_progress = 0; +static void memco_dbg_signal_handler(int sig) { + if (g_dbg_sim_handler_in_progress) { + // already dumping; if we got here again, give up and re-raise the + // default to avoid infinite recursion in handler. + signal(sig, SIG_DFL); + raise(sig); + return; + } + g_dbg_sim_handler_in_progress = 1; + fprintf(stdout, + "\n\n[DEADLOCK_DUMP] caught signal %d — dumping warp state " + "before crash\n", sig); + fflush(stdout); + if (g_dbg_sim_for_handler) g_dbg_sim_for_handler->dbg_dump_all_warps(); + fflush(stdout); + signal(sig, SIG_DFL); + raise(sig); +} + +void gpgpu_sim::dbg_dump_all_warps() { + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (m_cluster[i]->get_not_completed() > 0) { + m_cluster[i]->dump_warps_at_deadlock(stdout); + } + } +} + void gpgpu_sim::deadlock_check() { + // Lazy installer for the signal handler (only when MEMCO_DBG_DEADLOCK + // is set). Once installed, future signals are caught. + static bool sig_handler_installed = false; + if (!sig_handler_installed && getenv("MEMCO_DBG_DEADLOCK") != NULL) { + sig_handler_installed = true; + g_dbg_sim_for_handler = this; + signal(SIGSEGV, memco_dbg_signal_handler); + signal(SIGABRT, memco_dbg_signal_handler); + } else if (getenv("MEMCO_DBG_DEADLOCK") != NULL) { + // Keep the static pointer up to date in case multiple gpgpu_sim + // instances are created (rare). + g_dbg_sim_for_handler = this; + } if (m_config.gpu_deadlock_detect && gpu_deadlock) { fflush(stdout); printf( @@ -1382,6 +1427,17 @@ void gpgpu_sim::deadlock_check() { printf( "\nRe-run the simulator in gdb and use debug routines in .gdbinit to " "debug this\n"); + // MEMCO_DBG_DEADLOCK: dump per-warp state for any non-empty cluster + // before aborting. Helps localize which specific warp(s) are stuck + // and why (scoreboard / simt-table / ibuffer state). + if (getenv("MEMCO_DBG_DEADLOCK") != NULL) { + for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) { + if (m_cluster[i]->get_not_completed() > 0) { + m_cluster[i]->dump_warps_at_deadlock(stdout); + } + } + fflush(stdout); + } fflush(stdout); abort(); } diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 98e52f4..5da92a8 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -601,6 +601,11 @@ class gpgpu_sim : public gpgpu_t { void print_stats(unsigned long long streamID); void update_stats(); void deadlock_check(); + // MEMCO_DBG_DEADLOCK diagnostic — invoked from signal handler to dump + // per-warp state for all clusters with un-completed threads. Public so + // the static signal-handler thunk in gpu-sim.cc can call it through + // the saved gpgpu_sim* pointer. + void dbg_dump_all_warps(); void inc_completed_cta() { gpu_completed_cta++; } void get_pdom_stack_top_info(unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc); diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc index bcaea56..6cdcaf5 100644 --- a/src/gpgpu-sim/scoreboard.cc +++ b/src/gpgpu-sim/scoreboard.cc @@ -269,3 +269,25 @@ void Scoreboard::dumpAccounting(FILE* out) const { sec_avg_dur, m_sec_max_duration_cycles, m_sec_clear_calls, m_sec_clear_regs_dropped); } + +// Read-only diagnostic. Prints the primary and secondary scoreboard +// register sets for one warp. Called from deadlock_check() under +// MEMCO_DBG_DEADLOCK to help localize a stuck warp. +void Scoreboard::dump_warp_state(FILE *out, unsigned wid) const { + if (wid >= reg_table.size()) return; + fprintf(out, " primary_reg_table[%u] (size=%zu):", wid, + reg_table[wid].size()); + for (auto r : reg_table[wid]) fprintf(out, " r%u", r); + fprintf(out, "\n"); + fprintf(out, " sec_reg_table[%u] (size=%zu):", wid, + sec_reg_table[wid].size()); + for (auto r : sec_reg_table[wid]) fprintf(out, " r%u", r); + fprintf(out, "\n"); + // Long-op registers — relevant for stuck loads + if (wid < longopregs.size() && !longopregs[wid].empty()) { + fprintf(out, " longopregs[%u] (size=%zu):", wid, + longopregs[wid].size()); + for (auto r : longopregs[wid]) fprintf(out, " r%u", r); + fprintf(out, "\n"); + } +} diff --git a/src/gpgpu-sim/scoreboard.h b/src/gpgpu-sim/scoreboard.h index 1de81dd..3dbb1ce 100644 --- a/src/gpgpu-sim/scoreboard.h +++ b/src/gpgpu-sim/scoreboard.h @@ -64,6 +64,11 @@ class Scoreboard { // and any registers still reserved at the end of simulation. void dumpAccounting(FILE *out) const; + // Read-only inspector — dumps the primary and secondary reg sets for one + // warp. Called from the deadlock-detection path under MEMCO_DBG_DEADLOCK + // env var. No state mutation. + void dump_warp_state(FILE *out, unsigned wid) const; + private: void reserveRegister(unsigned wid, unsigned regnum); int get_sid() const { return m_sid; } diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 9a1658d..a2a77a7 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -6469,6 +6469,15 @@ void shader_core_ctx::set_max_cta(const kernel_info_t &kernel) { } void shader_core_ctx::decrement_atomic_count(unsigned wid, unsigned n) { + static const bool dbg = + (getenv("MEMCO_DBG_DEADLOCK") != NULL); + if (dbg) { + fprintf(stdout, + "[ATOMIC_DEC] sid=%u wid=%u dec=%u prev=%u new=%u\n", + m_sid, wid, n, m_warp[wid]->get_n_atomic(), + m_warp[wid]->get_n_atomic() - n); + fflush(stdout); + } assert(m_warp[wid]->get_n_atomic() >= n); m_warp[wid]->dec_n_atomic(n); } @@ -7189,6 +7198,95 @@ void simt_core_cluster::print_not_completed(FILE *fp) const { } } +// Read-only diagnostic — dispatches a per-warp state dump for every core +// in this cluster that still has un-completed threads. Called from +// gpu-sim.cc::deadlock_check() under MEMCO_DBG_DEADLOCK env var. +void simt_core_cluster::dump_warps_at_deadlock(FILE *fp) { + for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) { + if (m_core[i]->get_not_completed() == 0) continue; + m_core[i]->dump_warps_at_deadlock(fp); + } +} + +// Read-only diagnostic — for each warp on this core, prints flags, +// ibuffer slots, simt-table state, and scoreboard reservations. Helps +// pinpoint which warp is stuck and why when the deadlock detector fires. +// No state mutation: only invokes accessors. +void shader_core_ctx::dump_warps_at_deadlock(FILE *fout) { + fprintf(fout, "[DEADLOCK_DUMP] === core sid=%u not_completed=%u ===\n", + m_sid, get_not_completed()); + for (unsigned wid = 0; wid < m_warp.size(); wid++) { + if (m_warp[wid] == NULL) continue; + shd_warp_t *w = m_warp[wid]; + bool valid_f = w->valid(); + bool done = w->done_exit(); + bool waiting = w->waiting(); + bool blocked = w->blocked(); + bool prec = w->pending_reconvergence(); + bool virt = w->virtualized(); + bool ib_empty = w->ibuffer_empty(); + // Skip warps with nothing interesting (done or fully empty + no + // scoreboard activity). The stuck warp will have ib_empty=false OR + // active scoreboard regs — keep those prominent. + if (done && ib_empty) continue; + fprintf(fout, + "[DEADLOCK_DUMP] wid=%u valid=%d done_exit=%d waiting=%d " + "blocked=%d pending_rec=%d virt=%d ibuf_empty=%d active_split=%u\n", + wid, (int)valid_f, (int)done, (int)waiting, (int)blocked, + (int)prec, (int)virt, (int)ib_empty, + (m_simt_tables && m_simt_tables[wid]) + ? m_simt_tables[wid]->get_active_split_id() + : 0); + // Why is waiting() true? Show each branch's contribution. + if (waiting) { + bool fdone = w->functional_done(); + bool wb = warp_waiting_at_barrier(wid); + bool wmb = warp_waiting_at_mem_barrier(wid); + unsigned natomic = w->get_n_atomic(); + fprintf(fout, + "[DEADLOCK_DUMP] waiting_reason: functional_done=%d " + "barrier=%d mem_barrier=%d n_atomic=%u\n", + (int)fdone, (int)wb, (int)wmb, natomic); + } + // ibuffer slots 0..3 (IBUFFER_SIZE=4, hardcoded since the constant is + // private to shd_warp_t) + for (unsigned slot = 0; slot < 4; slot++) { + if (!w->ibuffer_slot_valid(slot)) { + fprintf(fout, "[DEADLOCK_DUMP] ibuffer[%u]: <empty>\n", slot); + continue; + } + const warp_inst_t *pI = w->ibuffer_slot_inst(slot); + unsigned split_id = w->ibuffer_slot_split_id(slot); + const active_mask_t &mask = w->ibuffer_slot_split_mask(slot); + fprintf(fout, + "[DEADLOCK_DUMP] ibuffer[%u]: pc=0x%lx op=%d " + "split_id=%d mask_count=%u\n", + slot, pI ? (unsigned long)pI->pc : 0ul, + pI ? (int)pI->op : -1, (int)split_id, + (unsigned)mask.count()); + } + // simt_tables summary — use safe accessors only (full print() can + // segfault on corrupted state, which is likely with a stuck warp). + if (m_simt_tables && m_simt_tables[wid]) { + simt_tables *st = m_simt_tables[wid]; + unsigned active_pc = 0, active_rpc = 0; + st->get_pdom_active_split_info(&active_pc, &active_rpc); + fprintf(fout, + "[DEADLOCK_DUMP] simt: active_split_id=%u active_pc=0x%lx " + "active_rpc=0x%lx ST_size=%u RT_size=%u blocked=%d " + "pending_recvg=%d virt=%d valid=%d\n", + st->get_active_split_id(), + (unsigned long)active_pc, (unsigned long)active_rpc, + st->getSTsize(), st->getRTsize(), + (int)st->blocked(), (int)st->is_pending_reconvergence(), + (int)st->is_virtualized(), (int)st->valid()); + } + // scoreboard + if (m_scoreboard) m_scoreboard->dump_warp_state(fout, wid); + } + fflush(fout); +} + float simt_core_cluster::get_current_occupancy( unsigned long long &active, unsigned long long &total) const { float aggregate = 0.f; @@ -7569,7 +7667,23 @@ void simt_core_cluster::get_L1T_sub_stats(struct cache_sub_stats &css) const { void exec_shader_core_ctx::checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) { - if (inst.isatomic()) m_warp[inst.warp_id()]->inc_n_atomic(); + if (inst.isatomic()) { + static const bool dbg = + (getenv("MEMCO_DBG_DEADLOCK") != NULL); + unsigned wid = inst.warp_id(); + if (dbg) { + fprintf(stdout, + "[ATOMIC_INC] sid=%u wid=%u pc=0x%lx tid=%u prev=%u new=%u " + "dbg_path=%u space=%d\n", + m_sid, wid, (unsigned long)inst.pc, tid, + m_warp[wid]->get_n_atomic(), + m_warp[wid]->get_n_atomic() + 1, + inst.get_dbg_path(), + (int)inst.space.get_type()); + fflush(stdout); + } + m_warp[wid]->inc_n_atomic(); + } if (inst.space.is_local() && (inst.is_load() || inst.is_store())) { new_addr_type localaddrs[MAX_ACCESSES_PER_INSN_PER_THREAD]; unsigned num_addrs; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index f25ba0e..51ed214 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -2503,6 +2503,12 @@ class shader_core_ctx : public core_t { const shader_core_config *config, const memory_config *mem_config, shader_core_stats *stats); + // Read-only diagnostic — dumps every warp's state (flags, ibuffer, + // simt_tables, scoreboard) to help localize a deadlocked warp. Called + // from gpu-sim.cc deadlock_check() under MEMCO_DBG_DEADLOCK env var. + // Public so simt_core_cluster can invoke without relying on friendship. + void dump_warps_at_deadlock(FILE *fout); + // used by simt_core_cluster: // modifiers void cycle(); @@ -3110,6 +3116,9 @@ class simt_core_cluster { unsigned max_cta(const kernel_info_t &kernel); unsigned get_not_completed() const; void print_not_completed(FILE *fp) const; + // Read-only diagnostic — dumps each non-empty core's per-warp state + // (see shader_core_ctx::dump_warps_at_deadlock). + void dump_warps_at_deadlock(FILE *fp); unsigned get_n_active_cta() const; unsigned get_n_active_sms() const; gpgpu_sim *get_gpu() { return m_gpu; } |