co-issue global mem requests

1 files changed, 61 insertions, 1 deletions

diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h
index 2800b81..cd03aec 100644
--- a/src/gpgpu-sim/shader.h
+++ b/src/gpgpu-sim/shader.h
@@ -556,6 +556,13 @@ class scheduler_unit {  // this can be copied freely, so can be used in std
   // and intra-warp pass).
   exec_unit_type_t classify_fu_type(const warp_inst_t *inst) const;
 
+  // MEM co-issue safety filter. Returns true if the given inst must NOT
+  // participate (as primary or co-issuer) in a MEM composite. v1 excludes
+  // atomics, shared-space ops (deferred per-set bank-conflict rework),
+  // texture/const caches (line-based coalesce model not per-set-aware),
+  // MEMORY_BARRIER_OP, LDGSTS, and BRU spill/fill requests.
+  bool mem_coissue_disallowed(const warp_inst_t *inst) const;
+
   // Check whether `cand_inst` can co-issue alongside `composite` given
   // `available_sets`. Writes the candidate's sets_needed to *sets_needed
   // on success. Uses the current compaction_mode to decide between the
@@ -1648,6 +1655,17 @@ class ldst_unit : public pipelined_simd_unit {
                                                    warp_inst_t &inst);
   mem_stage_stall_type process_memory_access_queue_l1cache(l1_cache *cache,
                                                            warp_inst_t &inst);
+
+  // MEM co-issue: resolve which source warp + instruction an access/mem_fetch
+  // belongs to (for routing pending_writes / scoreboard release back to the
+  // originating warp under composites). Returns {src_wid, src_out_inst}.
+  // For non-composite or primary-source accesses, returns {primary_wid, &inst}.
+  // split_id differentiates intra-warp co-issued sets (same wid as primary)
+  // from the primary set itself; use (unsigned)-1 when not applicable.
+  struct mem_src_t { unsigned wid; const inst_t *out_inst; };
+  mem_src_t resolve_source(const warp_inst_t &inst,
+                            unsigned access_src_wid,
+                            unsigned access_src_split_id) const;
   gpgpu_sim *m_gpu;
 
   const memory_config *m_memory_config;
@@ -1669,6 +1687,22 @@ class ldst_unit : public pipelined_simd_unit {
 
   mem_fetch *m_next_global;
   warp_inst_t m_next_wb;
+  // MEM co-issue: source warp + split_id for the mem_fetch that fed m_next_wb.
+  // Used by writeback() to decrement the correct source's pending_writes /
+  // release the correct source's scoreboard registers (primary vs secondary).
+  // Set at each "m_next_wb = ..." assignment; defaults to composite primary.
+  unsigned m_next_wb_src_wid;
+  unsigned m_next_wb_src_split_id;
+  // MEM co-issue intra-warp pending writes: primary and intra-warp-coissued
+  // may write the same register number (R5) for disjoint thread subsets.
+  // The existing m_pending_writes[wid][reg] is single-counter per (wid,reg)
+  // and would conflate the two — leading to premature / delayed scoreboard
+  // release (deadlock). Use a separate map keyed by (wid, split_id, reg)
+  // for intra-warp-coissued accesses so counts stay distinct.
+  std::map<unsigned /*wid*/,
+           std::map<unsigned /*split_id*/,
+                    std::map<unsigned /*reg*/, unsigned /*count*/>>>
+      m_pending_writes_secondary;
   unsigned m_writeback_arb;  // round-robin arbiter for writeback contention
                              // between L1T, L1C, shared
   unsigned m_num_writeback_clients;
@@ -1961,6 +1995,7 @@ class shader_core_config : public core_config {
   unsigned gpgpu_opndcoll_read_latency; // extra cycles to collector
   unsigned gpgpu_co_issue_priority;     // 0=greedy..4=same-PC
   bool gpgpu_simd_partitioning_debug;   // verbose SIMD_SETS printf gate
+  bool gpgpu_mem_coissue;               // enable MEM/LDST co-issue path
 
   unsigned n_simt_cores_per_cluster;
   unsigned n_simt_clusters;
@@ -2112,6 +2147,14 @@ struct shader_core_stats_pod {
   unsigned long long *coissue_skipped_no_inst;
   unsigned long long *coissue_skipped_pc_mismatch;
   unsigned long long *coissue_skipped_samepc_pass0;
+
+  // MEM co-issue specific counters (the subset of coissue_* events where
+  // the composite's FU type is MEM). Useful for diagnosing whether the
+  // -gpgpu_mem_coissue flag is actually firing and how much of the total
+  // co-issue activity is memory-backed on a given workload.
+  unsigned long long *mem_coissue_inter_events;
+  unsigned long long *mem_coissue_intra_events;
+  unsigned long long *mem_coissue_denied_filter;  // safety filter rejects
 };
 
 class shader_core_stats : public shader_core_stats_pod {
@@ -2269,6 +2312,14 @@ class shader_core_stats : public shader_core_stats_pod {
         config->num_shader(), sizeof(unsigned long long));
     coissue_skipped_samepc_pass0 = (unsigned long long *)calloc(
         config->num_shader(), sizeof(unsigned long long));
+
+    // MEM co-issue-specific counters
+    mem_coissue_inter_events = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    mem_coissue_intra_events = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
+    mem_coissue_denied_filter = (unsigned long long *)calloc(
+        config->num_shader(), sizeof(unsigned long long));
   }
 
   ~shader_core_stats() {
@@ -2341,6 +2392,9 @@ class shader_core_stats : public shader_core_stats_pod {
     free(coissue_skipped_no_inst);
     free(coissue_skipped_pc_mismatch);
     free(coissue_skipped_samepc_pass0);
+    free(mem_coissue_inter_events);
+    free(mem_coissue_intra_events);
+    free(mem_coissue_denied_filter);
   }
 
   void new_grid() {}
@@ -2404,10 +2458,16 @@ class shader_core_mem_fetch_allocator : public mem_fetch_allocator {
   mem_fetch *alloc(const warp_inst_t &inst, const mem_access_t &access,
                    unsigned long long cycle) const {
     warp_inst_t inst_copy = inst;
+    // If the access was tagged with a source warp_id (MEM co-issue),
+    // stamp the mem_fetch with that wid so writeback routes back to the
+    // correct warp's scoreboard / pending_writes entry. Otherwise fall
+    // back to the composite's primary warp_id (legacy behavior).
+    unsigned wid = access.get_source_wid();
+    if (wid == (unsigned)-1) wid = inst.warp_id();
     mem_fetch *mf = new mem_fetch(
         access, &inst_copy, inst.get_streamID(),
         access.is_write() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE,
-        inst.warp_id(), m_core_id, m_cluster_id, m_memory_config, cycle);
+        wid, m_core_id, m_cluster_id, m_memory_config, cycle);
     return mf;
   }