fix double sb from fifo rotations during intra coissue

1 files changed, 231 insertions, 20 deletions

diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index 7b208e8..ac5cd09 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1119,18 +1119,35 @@ void shader_core_ctx::fetch() {
         if (simt_conditions && !m_warp[warp_id]->functional_done() &&
             !m_warp[warp_id]->imiss_pending() &&
             m_warp[warp_id]->ibuffer_half_empty(0)) {
-          // Mode 2 (slot-pinned): if the next split to fetch from differs
-          // from half 0's previously assigned owner, wait for the slot's
-          // scoreboard to drain before reassigning. This is the "wait
-          // for clean at divergence/reconvergence/split-rotation" gate.
+          // Mode 2 (slot-pinned): drain gate at fetch time. Two trigger
+          // conditions: (1) FIFO membership change set by
+          // simt_tables::update() (divergence/reconvergence/CALL/RET);
+          // (2) slot 0 about to be reassigned to a different split than
+          // its previous owner — this catches co-issue's
+          // move_split_to_front rotations, where the secondary split
+          // (with pending writes in slot 1) becomes slot 0's new owner.
+          // Without (2), the same split's writes are stranded in slot 1
+          // while slot 0 issues reads of those same regs (real RAW).
+          // Both conditions require BOTH slots clean before fetch
+          // proceeds, so cross-slot pending writes from prior owners
+          // drain regardless of which slot is being reassigned.
           if (m_config->gpgpu_scoreboard_mode == 2 &&
               m_config->model == AWARE_RECONVERGENCE) {
             unsigned new_split_id =
                 m_simt_tables[warp_id]->get_active_split_id();
-            if (m_warp[warp_id]->ibuffer_half_assigned(0) &&
-                m_warp[warp_id]->ibuffer_half_split_id(0) != new_split_id &&
-                !m_scoreboard->slotClean(warp_id, 0)) {
-              continue;  // drain pending — try next warp
+            bool slot0_reassigned =
+                m_warp[warp_id]->ibuffer_half_assigned(0) &&
+                m_warp[warp_id]->ibuffer_half_split_id(0) != new_split_id;
+            bool fifo_changed =
+                m_simt_tables[warp_id]->div_recv_drain_pending();
+            if (slot0_reassigned || fifo_changed) {
+              if (!m_scoreboard->slotClean(warp_id, 0) ||
+                  !m_scoreboard->slotClean(warp_id, 1)) {
+                continue;  // drain pending — try next warp
+              }
+              if (fifo_changed) {
+                m_simt_tables[warp_id]->clear_div_recv_drain_pending();
+              }
             }
           }
           address_type pc;
@@ -1225,13 +1242,25 @@ void shader_core_ctx::fetch() {
         if ((sec_mask & half0_mask).any()) continue;  // overlapping, skip
       }
 
-      // Mode 2: if half 1's previous owner differs from this candidate,
-      // wait for SB[1] to drain before reassigning.
-      if (m_config->gpgpu_scoreboard_mode == 2 &&
-          m_warp[warp_id]->ibuffer_half_assigned(1) &&
-          m_warp[warp_id]->ibuffer_half_split_id(1) != sec_split_id &&
-          !m_scoreboard->slotClean(warp_id, 1)) {
-        continue;  // drain pending — try next warp
+      // Mode 2: drain gate at secondary fetch (mirrors the primary-fetch
+      // gate above). Two trigger conditions: (1) FIFO membership change
+      // (div_recv_drain_pending); (2) slot 1 reassignment to a different
+      // split than its previous owner. Both require BOTH slots clean.
+      if (m_config->gpgpu_scoreboard_mode == 2) {
+        bool slot1_reassigned =
+            m_warp[warp_id]->ibuffer_half_assigned(1) &&
+            m_warp[warp_id]->ibuffer_half_split_id(1) != sec_split_id;
+        bool fifo_changed =
+            m_simt_tables[warp_id]->div_recv_drain_pending();
+        if (slot1_reassigned || fifo_changed) {
+          if (!m_scoreboard->slotClean(warp_id, 0) ||
+              !m_scoreboard->slotClean(warp_id, 1)) {
+            continue;  // drain pending — try next warp
+          }
+          if (fifo_changed) {
+            m_simt_tables[warp_id]->clear_div_recv_drain_pending();
+          }
+        }
       }
 
       // Attempt I-Cache access (HIT-only for secondary)
@@ -1996,6 +2025,24 @@ void scheduler_unit::try_inter_warp_coissue(
         m_stats->coissue_skipped_simt_blocked[get_sid()]++;
         continue;
       }
+      // Mode 2 (slot-pinned scoreboard): if the candidate warp has a
+      // pending FIFO-membership-change drain, refuse to inter-coissue
+      // with it. Otherwise an instruction fetched pre-divergence (with
+      // a now-stale broader mask) could issue with the post-divergence
+      // narrower active mask, hitting cross-slot RAWs against pending
+      // writes from the broader mask. Same gate as the primary issue
+      // path's drain check, applied to the candidate.
+      if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+          m_shader->m_config->model == AWARE_RECONVERGENCE &&
+          m_shader->get_simt_tables(cand_warp_id)
+              ->div_recv_drain_pending()) {
+        if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
+            !m_scoreboard->slotClean(cand_warp_id, 1)) {
+          continue;
+        }
+        m_shader->get_simt_tables(cand_warp_id)
+            ->clear_div_recv_drain_pending();
+      }
 
       const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
       if (!cand_inst) {
@@ -2288,6 +2335,18 @@ void scheduler_unit::try_utilization_max_coissue(
                 !warp(cand_warp_id).virtualized();
     }
     if (!simt_ok) continue;
+    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
+    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+        m_shader->m_config->model == AWARE_RECONVERGENCE &&
+        m_shader->get_simt_tables(cand_warp_id)
+            ->div_recv_drain_pending()) {
+      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
+          !m_scoreboard->slotClean(cand_warp_id, 1)) {
+        continue;
+      }
+      m_shader->get_simt_tables(cand_warp_id)
+          ->clear_div_recv_drain_pending();
+    }
 
     const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
     if (!cand_inst) continue;
@@ -2306,8 +2365,24 @@ void scheduler_unit::try_utilization_max_coissue(
                                                              cand_inst,
                                                              cand_mask_pre);
     } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
-      sb_collision_inter2 = m_scoreboard->checkCollisionSlot(
+      // OR check: stall on either slot — see commentary at primary
+      // issue path.
+      bool s0c = m_scoreboard->checkCollisionSlot(
           cand_warp_id, /*slot=*/0, cand_inst);
+      bool s1c = m_scoreboard->checkCollisionSlot(
+          cand_warp_id, /*slot=*/1, cand_inst);
+      sb_collision_inter2 = s0c || s1c;
+      const active_mask_t &cand_mask_for_diag =
+          m_shader->get_active_mask(cand_warp_id, cand_inst);
+      m_scoreboard->m_xslot_inter_coissue_checks++;
+      if (!s0c && s1c) {
+        m_scoreboard->m_xslot_inter_coissue_would_stall++;
+      }
+      if (!s0c && !s1c &&
+          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
+                                              cand_mask_for_diag)) {
+        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
+      }
     } else {
       sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                          cand_inst);
@@ -2656,6 +2731,18 @@ void scheduler_unit::try_utilization_max_coissue_window(
                 !warp(cand_warp_id).virtualized();
     }
     if (!simt_ok) continue;
+    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
+    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+        m_shader->m_config->model == AWARE_RECONVERGENCE &&
+        m_shader->get_simt_tables(cand_warp_id)
+            ->div_recv_drain_pending()) {
+      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
+          !m_scoreboard->slotClean(cand_warp_id, 1)) {
+        continue;
+      }
+      m_shader->get_simt_tables(cand_warp_id)
+          ->clear_div_recv_drain_pending();
+    }
 
     const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
     if (!cand_inst) continue;
@@ -2674,8 +2761,24 @@ void scheduler_unit::try_utilization_max_coissue_window(
                                                              cand_inst,
                                                              cand_mask_pre);
     } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
-      sb_collision_inter2 = m_scoreboard->checkCollisionSlot(
+      // OR check: stall on either slot — see commentary at primary
+      // issue path.
+      bool s0c = m_scoreboard->checkCollisionSlot(
           cand_warp_id, /*slot=*/0, cand_inst);
+      bool s1c = m_scoreboard->checkCollisionSlot(
+          cand_warp_id, /*slot=*/1, cand_inst);
+      sb_collision_inter2 = s0c || s1c;
+      const active_mask_t &cand_mask_for_diag =
+          m_shader->get_active_mask(cand_warp_id, cand_inst);
+      m_scoreboard->m_xslot_inter_coissue_checks++;
+      if (!s0c && s1c) {
+        m_scoreboard->m_xslot_inter_coissue_would_stall++;
+      }
+      if (!s0c && !s1c &&
+          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
+                                              cand_mask_for_diag)) {
+        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
+      }
     } else {
       sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                          cand_inst);
@@ -2989,6 +3092,18 @@ void scheduler_unit::try_utilization_max_coissue_window_centered(
                 !warp(cand_warp_id).virtualized();
     }
     if (!simt_ok) continue;
+    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
+    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+        m_shader->m_config->model == AWARE_RECONVERGENCE &&
+        m_shader->get_simt_tables(cand_warp_id)
+            ->div_recv_drain_pending()) {
+      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
+          !m_scoreboard->slotClean(cand_warp_id, 1)) {
+        continue;
+      }
+      m_shader->get_simt_tables(cand_warp_id)
+          ->clear_div_recv_drain_pending();
+    }
 
     const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
     if (!cand_inst) continue;
@@ -3007,8 +3122,24 @@ void scheduler_unit::try_utilization_max_coissue_window_centered(
                                                              cand_inst,
                                                              cand_mask_pre);
     } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
-      sb_collision_inter2 = m_scoreboard->checkCollisionSlot(
+      // OR check: stall on either slot — see commentary at primary
+      // issue path.
+      bool s0c = m_scoreboard->checkCollisionSlot(
           cand_warp_id, /*slot=*/0, cand_inst);
+      bool s1c = m_scoreboard->checkCollisionSlot(
+          cand_warp_id, /*slot=*/1, cand_inst);
+      sb_collision_inter2 = s0c || s1c;
+      const active_mask_t &cand_mask_for_diag =
+          m_shader->get_active_mask(cand_warp_id, cand_inst);
+      m_scoreboard->m_xslot_inter_coissue_checks++;
+      if (!s0c && s1c) {
+        m_scoreboard->m_xslot_inter_coissue_would_stall++;
+      }
+      if (!s0c && !s1c &&
+          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
+                                              cand_mask_for_diag)) {
+        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
+      }
     } else {
       sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                          cand_inst);
@@ -3314,6 +3445,18 @@ void scheduler_unit::try_utilization_max_coissue_window_before(
                 !warp(cand_warp_id).virtualized();
     }
     if (!simt_ok) continue;
+    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
+    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+        m_shader->m_config->model == AWARE_RECONVERGENCE &&
+        m_shader->get_simt_tables(cand_warp_id)
+            ->div_recv_drain_pending()) {
+      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
+          !m_scoreboard->slotClean(cand_warp_id, 1)) {
+        continue;
+      }
+      m_shader->get_simt_tables(cand_warp_id)
+          ->clear_div_recv_drain_pending();
+    }
 
     const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
     if (!cand_inst) continue;
@@ -3332,8 +3475,24 @@ void scheduler_unit::try_utilization_max_coissue_window_before(
                                                              cand_inst,
                                                              cand_mask_pre);
     } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
-      sb_collision_inter2 = m_scoreboard->checkCollisionSlot(
+      // OR check: stall on either slot — see commentary at primary
+      // issue path.
+      bool s0c = m_scoreboard->checkCollisionSlot(
           cand_warp_id, /*slot=*/0, cand_inst);
+      bool s1c = m_scoreboard->checkCollisionSlot(
+          cand_warp_id, /*slot=*/1, cand_inst);
+      sb_collision_inter2 = s0c || s1c;
+      const active_mask_t &cand_mask_for_diag =
+          m_shader->get_active_mask(cand_warp_id, cand_inst);
+      m_scoreboard->m_xslot_inter_coissue_checks++;
+      if (!s0c && s1c) {
+        m_scoreboard->m_xslot_inter_coissue_would_stall++;
+      }
+      if (!s0c && !s1c &&
+          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
+                                              cand_mask_for_diag)) {
+        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
+      }
     } else {
       sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                          cand_inst);
@@ -3640,6 +3799,23 @@ void scheduler_unit::cycle() {
                         !warp(warp_id).virtualized();
     }
 
+    // Mode 2 (slot-pinned scoreboard) drain gate at issue time.
+    // Fetch-only drain leaks: ibuffer entries already-fetched pre-
+    // divergence may match the post-divergence active PC and issue
+    // with the new (subset) active mask, reading regs whose pending
+    // writes carry the FULL pre-divergence mask → real cross-slot
+    // RAW. Block all issues from this warp until both slot
+    // scoreboards drain. Flag clears here once observed clean.
+    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
+        m_shader->m_config->model == AWARE_RECONVERGENCE &&
+        m_shader->get_simt_tables(warp_id)->div_recv_drain_pending()) {
+      if (!m_scoreboard->slotClean(warp_id, 0) ||
+          !m_scoreboard->slotClean(warp_id, 1)) {
+        continue;  // drain pending — try next warp
+      }
+      m_shader->get_simt_tables(warp_id)->clear_div_recv_drain_pending();
+    }
+
     while (simt_conditions && !warp(warp_id).waiting() &&
            !warp(warp_id).ibuffer_empty() && (checked < max_issue) &&
            (checked <= issued) && (issued < max_issue)) {
@@ -3682,8 +3858,43 @@ void scheduler_unit::cycle() {
             sb_collision_primary =
                 m_scoreboard->checkCollisionMask(warp_id, pI, active_mask);
           } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
-            sb_collision_primary = m_scoreboard->checkCollisionSlot(
+            // Slot-pinned: issue stalls if the inst's regs have pending
+            // writes in EITHER slot. The rotation case (split moves
+            // from slot 1 to slot 0 via co-issue's move_split_to_front)
+            // strands prior writes in slot 1 while the same split's
+            // next instructions issue from slot 0 — same-split
+            // cross-slot RAW. Drain gates at fetch reduce this but
+            // can't catch all rotation+pre-fetched-ibuffer interactions.
+            // The OR check is mask-blind (no false positives from
+            // lane-disjoint splits in steady-state divergent
+            // execution? — yes there are: see m_xslot_primary_issue_would_stall).
+            bool slot0_coll = m_scoreboard->checkCollisionSlot(
                 warp_id, /*slot=*/0, pI);
+            bool slot1_coll = m_scoreboard->checkCollisionSlot(
+                warp_id, /*slot=*/1, pI);
+            sb_collision_primary = slot0_coll || slot1_coll;
+            // Diagnostic kept for now: count would_stall (unmasked)
+            // and real_would_stall (mask-aware shadow). With OR check
+            // active, _real_would_stall should drop to 0 — any real
+            // RAW would also fire one of slot0/slot1 collision.
+            m_scoreboard->m_xslot_primary_issue_checks++;
+            if (!slot0_coll && slot1_coll) {
+              m_scoreboard->m_xslot_primary_issue_would_stall++;
+            }
+            if (!slot0_coll && !slot1_coll &&
+                m_scoreboard->checkCollisionShadow(warp_id, pI,
+                                                    active_mask)) {
+              m_scoreboard->m_xslot_primary_issue_real_would_stall++;
+              static unsigned long long s_traced = 0;
+              if (s_traced < 50 && getenv("XSLOT_TRACE")) {
+                unsigned long long cyc =
+                    m_shader->get_gpu()->gpu_sim_cycle +
+                    m_shader->get_gpu()->gpu_tot_sim_cycle;
+                m_scoreboard->dumpShadowOverlap(warp_id, pI, active_mask,
+                                                 stderr, "primary", cyc);
+                s_traced++;
+              }
+            }
           } else {
             sb_collision_primary = m_scoreboard->checkCollision(warp_id, pI);
           }