// Copyright (c) 2009-2021, Tor M. Aamodt, Wilson W.L. Fung, Ali Bakhoda,
// George L. Yuan, Andrew Turner, Inderpreet Singh, Vijay Kandiah, Nikos
// Hardavellas, Mahmoud Khairy, Junrui Pan, Timothy G. Rogers The University of
// British Columbia, Northwestern University, Purdue University All rights
// reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this
//    list of conditions and the following disclaimer;
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution;
// 3. Neither the names of The University of British Columbia, Northwestern
//    University nor the names of their contributors may be used to
//    endorse or promote products derived from this software without specific
//    prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

#include "shader.h"
#include <float.h>
#include <limits.h>
#include <string.h>
#include <array>
#include "../../libcuda/gpgpu_context.h"
#include "../cuda-sim/cuda-sim.h"
#include "../cuda-sim/ptx-stats.h"
#include "../cuda-sim/ptx_sim.h"
#include "../statwrapper.h"
#include "addrdec.h"
#include "dram.h"
#include "gpu-misc.h"
#include "gpu-sim.h"
#include "icnt_wrapper.h"
#include "mem_fetch.h"
#include "mem_latency_stat.h"
#include "shader_trace.h"
#include "stat-tool.h"
#include "traffic_breakdown.h"
#include "visualizer.h"

#define PRIORITIZE_MSHR_OVER_WB 1
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#define MIN(a, b) (((a) < (b)) ? (a) : (b))

// MEMCO v3: diagnostic globals aggregating inter-set merge activity across
// all ldst_units. Printed at gpgpu termination.
unsigned long long g_memcov3_n_merges_global = 0;
unsigned long long g_memcov3_n_merged_sources_global = 0;

mem_fetch *shader_core_mem_fetch_allocator::alloc(
    new_addr_type addr, mem_access_type type, unsigned size, bool wr,
    unsigned long long cycle, unsigned long long streamID) const {
  mem_access_t access(type, addr, size, wr, m_memory_config->gpgpu_ctx);
  mem_fetch *mf = new mem_fetch(
      access, NULL, streamID, wr ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, -1,
      m_core_id, m_cluster_id, m_memory_config, cycle);
  return mf;
}

mem_fetch *shader_core_mem_fetch_allocator::alloc(
    new_addr_type addr, mem_access_type type, const active_mask_t &active_mask,
    const mem_access_byte_mask_t &byte_mask,
    const mem_access_sector_mask_t &sector_mask, unsigned size, bool wr,
    unsigned long long cycle, unsigned wid, unsigned sid, unsigned tpc,
    mem_fetch *original_mf, unsigned long long streamID) const {
  mem_access_t access(type, addr, size, wr, active_mask, byte_mask, sector_mask,
                      m_memory_config->gpgpu_ctx);
  mem_fetch *mf = new mem_fetch(
      access, NULL, streamID, wr ? WRITE_PACKET_SIZE : READ_PACKET_SIZE, wid,
      m_core_id, m_cluster_id, m_memory_config, cycle, original_mf);
  return mf;
}
/////////////////////////////////////////////////////////////////////////////

std::list<unsigned> shader_core_ctx::get_regs_written(const inst_t &fvt) const {
  std::list<unsigned> result;
  for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) {
    int reg_num = fvt.arch_reg.dst[op];  // this math needs to match that used
                                         // in function_info::ptx_decode_inst
    if (reg_num >= 0)                    // valid register
      result.push_back(reg_num);
  }
  return result;
}

void exec_shader_core_ctx::create_shd_warp() {
  m_warp.resize(m_config->max_warps_per_shader);
  for (unsigned k = 0; k < m_config->max_warps_per_shader; ++k) {
    m_warp[k] = new shd_warp_t(this, m_config->warp_size);
  }
}

void shader_core_ctx::create_front_pipeline() {
  // pipeline_stages is the sum of normal pipeline stages and specialized_unit
  // stages * 2 (for ID and EX)
  unsigned total_pipeline_stages =
      N_PIPELINE_STAGES + m_config->m_specialized_unit.size() * 2;
  m_pipeline_reg.reserve(total_pipeline_stages);
  for (int j = 0; j < N_PIPELINE_STAGES; j++) {
    m_pipeline_reg.push_back(
        register_set(m_config->pipe_widths[j], pipeline_stage_name_decode[j]));
  }
  for (unsigned j = 0; j < m_config->m_specialized_unit.size(); j++) {
    m_pipeline_reg.push_back(
        register_set(m_config->m_specialized_unit[j].id_oc_spec_reg_width,
                     m_config->m_specialized_unit[j].name));
    m_config->m_specialized_unit[j].ID_OC_SPEC_ID = m_pipeline_reg.size() - 1;
    m_specilized_dispatch_reg.push_back(
        &m_pipeline_reg[m_pipeline_reg.size() - 1]);
  }
  for (unsigned j = 0; j < m_config->m_specialized_unit.size(); j++) {
    m_pipeline_reg.push_back(
        register_set(m_config->m_specialized_unit[j].oc_ex_spec_reg_width,
                     m_config->m_specialized_unit[j].name));
    m_config->m_specialized_unit[j].OC_EX_SPEC_ID = m_pipeline_reg.size() - 1;
  }

  if (m_config->sub_core_model) {
    // in subcore model, each scheduler should has its own issue register, so
    // ensure num scheduler = reg width
    assert(m_config->gpgpu_num_sched_per_core ==
           m_pipeline_reg[ID_OC_SP].get_size());
    assert(m_config->gpgpu_num_sched_per_core ==
           m_pipeline_reg[ID_OC_SFU].get_size());
    assert(m_config->gpgpu_num_sched_per_core ==
           m_pipeline_reg[ID_OC_MEM].get_size());
    if (m_config->gpgpu_tensor_core_avail)
      assert(m_config->gpgpu_num_sched_per_core ==
             m_pipeline_reg[ID_OC_TENSOR_CORE].get_size());
    if (m_config->gpgpu_num_dp_units > 0)
      assert(m_config->gpgpu_num_sched_per_core ==
             m_pipeline_reg[ID_OC_DP].get_size());
    if (m_config->gpgpu_num_int_units > 0)
      assert(m_config->gpgpu_num_sched_per_core ==
             m_pipeline_reg[ID_OC_INT].get_size());
    for (unsigned j = 0; j < m_config->m_specialized_unit.size(); j++) {
      if (m_config->m_specialized_unit[j].num_units > 0)
        assert(m_config->gpgpu_num_sched_per_core ==
               m_config->m_specialized_unit[j].id_oc_spec_reg_width);
    }
  }

  m_threadState = (thread_ctx_t *)calloc(sizeof(thread_ctx_t),
                                         m_config->n_thread_per_shader);

  m_not_completed = 0;
  m_active_threads.reset();
  m_n_active_cta = 0;
  for (unsigned i = 0; i < MAX_CTA_PER_SHADER; i++) m_cta_status[i] = 0;
  for (unsigned i = 0; i < m_config->n_thread_per_shader; i++) {
    m_thread[i] = NULL;
    m_threadState[i].m_cta_id = -1;
    m_threadState[i].m_active = false;
  }

  // m_icnt = new shader_memory_interface(this,cluster);
  if (m_memory_config->SST_mode) {
    m_icnt = new sst_memory_interface(
        this, static_cast<sst_simt_core_cluster *>(m_cluster));
  } else if (m_config->gpgpu_perfect_mem) {
    m_icnt = new perfect_memory_interface(this, m_cluster);
  } else {
    m_icnt = new shader_memory_interface(this, m_cluster);
  }
  m_mem_fetch_allocator =
      new shader_core_mem_fetch_allocator(m_sid, m_tpc, m_memory_config);

  // fetch
  m_last_warp_fetched = 0;

#define STRSIZE 1024
  char name[STRSIZE];
  snprintf(name, STRSIZE, "L1I_%03d", m_sid);
  m_L1I = new read_only_cache(name, m_config->m_L1I_config, m_sid,
                              get_shader_instruction_cache_id(), m_icnt,
                              IN_L1I_MISS_QUEUE, OTHER_GPU_CACHE, m_gpu);
}

void shader_core_ctx::create_schedulers() {
  m_scoreboard = new Scoreboard(m_sid, m_config->max_warps_per_shader, m_gpu,
                                m_config->gpgpu_scoreboard_mode);

  // scedulers
  // must currently occur after all inputs have been initialized.
  std::string sched_config = m_config->gpgpu_scheduler_string;
  const concrete_scheduler scheduler =
      sched_config.find("lrr") != std::string::npos ? CONCRETE_SCHEDULER_LRR
      : sched_config.find("two_level_active") != std::string::npos
          ? CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE
      : sched_config.find("gto") != std::string::npos ? CONCRETE_SCHEDULER_GTO
      : sched_config.find("rrr") != std::string::npos ? CONCRETE_SCHEDULER_RRR
      : sched_config.find("old") != std::string::npos
          ? CONCRETE_SCHEDULER_OLDEST_FIRST
      : sched_config.find("warp_limiting") != std::string::npos
          ? CONCRETE_SCHEDULER_WARP_LIMITING
          : NUM_CONCRETE_SCHEDULERS;
  assert(scheduler != NUM_CONCRETE_SCHEDULERS);

  for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++) {
    switch (scheduler) {
      case CONCRETE_SCHEDULER_LRR:
        schedulers.push_back(new lrr_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i));
        break;
      case CONCRETE_SCHEDULER_TWO_LEVEL_ACTIVE:
        schedulers.push_back(new two_level_active_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i, m_config->gpgpu_scheduler_string));
        break;
      case CONCRETE_SCHEDULER_GTO:
        schedulers.push_back(new gto_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i));
        break;
      case CONCRETE_SCHEDULER_RRR:
        schedulers.push_back(new rrr_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i));
        break;
      case CONCRETE_SCHEDULER_OLDEST_FIRST:
        schedulers.push_back(new oldest_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i));
        break;
      case CONCRETE_SCHEDULER_WARP_LIMITING:
        schedulers.push_back(new swl_scheduler(
            m_stats, this, m_scoreboard, m_simt_stack, &m_warp,
            &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP],
            &m_pipeline_reg[ID_OC_SFU], &m_pipeline_reg[ID_OC_INT],
            &m_pipeline_reg[ID_OC_TENSOR_CORE], m_specilized_dispatch_reg,
            &m_pipeline_reg[ID_OC_MEM], i, m_config->gpgpu_scheduler_string));
        break;
      default:
        abort();
    };
  }

  for (unsigned i = 0; i < m_warp.size(); i++) {
    // distribute i's evenly though schedulers;
    schedulers[i % m_config->gpgpu_num_sched_per_core]->add_supervised_warp_id(
        i);
  }
  for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; ++i) {
    schedulers[i]->done_adding_supervised_warps();
  }
}

void shader_core_ctx::create_exec_pipeline() {
  // op collector configuration
  enum { SP_CUS, DP_CUS, SFU_CUS, TENSOR_CORE_CUS, INT_CUS, MEM_CUS, GEN_CUS };

  opndcoll_rfu_t::port_vector_t in_ports;
  opndcoll_rfu_t::port_vector_t out_ports;
  opndcoll_rfu_t::uint_vector_t cu_sets;

  // configure generic collectors
  m_operand_collector.add_cu_set(
      GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen,
      m_config->gpgpu_operand_collector_num_out_ports_gen);

  for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_gen;
       i++) {
    in_ports.push_back(&m_pipeline_reg[ID_OC_SP]);
    in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
    in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]);
    out_ports.push_back(&m_pipeline_reg[OC_EX_SP]);
    out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
    out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]);
    if (m_config->gpgpu_tensor_core_avail) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]);
    }
    if (m_config->gpgpu_num_dp_units > 0) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
    }
    if (m_config->gpgpu_num_int_units > 0) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_INT]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_INT]);
    }
    if (m_config->m_specialized_unit.size() > 0) {
      for (unsigned j = 0; j < m_config->m_specialized_unit.size(); ++j) {
        in_ports.push_back(
            &m_pipeline_reg[m_config->m_specialized_unit[j].ID_OC_SPEC_ID]);
        out_ports.push_back(
            &m_pipeline_reg[m_config->m_specialized_unit[j].OC_EX_SPEC_ID]);
      }
    }
    cu_sets.push_back((unsigned)GEN_CUS);
    m_operand_collector.add_port(in_ports, out_ports, cu_sets);
    in_ports.clear(), out_ports.clear(), cu_sets.clear();
  }

  if (m_config->enable_specialized_operand_collector) {
    m_operand_collector.add_cu_set(
        SP_CUS, m_config->gpgpu_operand_collector_num_units_sp,
        m_config->gpgpu_operand_collector_num_out_ports_sp);
    m_operand_collector.add_cu_set(
        DP_CUS, m_config->gpgpu_operand_collector_num_units_dp,
        m_config->gpgpu_operand_collector_num_out_ports_dp);
    m_operand_collector.add_cu_set(
        TENSOR_CORE_CUS,
        m_config->gpgpu_operand_collector_num_units_tensor_core,
        m_config->gpgpu_operand_collector_num_out_ports_tensor_core);
    m_operand_collector.add_cu_set(
        SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu,
        m_config->gpgpu_operand_collector_num_out_ports_sfu);
    m_operand_collector.add_cu_set(
        MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem,
        m_config->gpgpu_operand_collector_num_out_ports_mem);
    m_operand_collector.add_cu_set(
        INT_CUS, m_config->gpgpu_operand_collector_num_units_int,
        m_config->gpgpu_operand_collector_num_out_ports_int);

    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp;
         i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_SP]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_SP]);
      cu_sets.push_back((unsigned)SP_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }

    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp;
         i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_DP]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_DP]);
      cu_sets.push_back((unsigned)DP_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }

    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu;
         i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]);
      cu_sets.push_back((unsigned)SFU_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }

    for (unsigned i = 0;
         i < m_config->gpgpu_operand_collector_num_in_ports_tensor_core; i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]);
      cu_sets.push_back((unsigned)TENSOR_CORE_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }

    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem;
         i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]);
      cu_sets.push_back((unsigned)MEM_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }

    for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_int;
         i++) {
      in_ports.push_back(&m_pipeline_reg[ID_OC_INT]);
      out_ports.push_back(&m_pipeline_reg[OC_EX_INT]);
      cu_sets.push_back((unsigned)INT_CUS);
      cu_sets.push_back((unsigned)GEN_CUS);
      m_operand_collector.add_port(in_ports, out_ports, cu_sets);
      in_ports.clear(), out_ports.clear(), cu_sets.clear();
    }
  }

  m_operand_collector.init(m_config->gpgpu_num_reg_banks, this);

  m_num_function_units =
      m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units +
      m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units +
      m_config->gpgpu_num_int_units + m_config->m_specialized_unit_num +
      1;  // sp_unit, sfu, dp, tensor, int, ldst_unit
  // m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ];
  // m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ];

  // m_fu = new simd_function_unit*[m_num_function_units];

  for (unsigned k = 0; k < m_config->gpgpu_num_sp_units; k++) {
    m_fu.push_back(new sp_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
    m_dispatch_port.push_back(ID_OC_SP);
    m_issue_port.push_back(OC_EX_SP);
  }

  for (unsigned k = 0; k < m_config->gpgpu_num_dp_units; k++) {
    m_fu.push_back(new dp_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
    m_dispatch_port.push_back(ID_OC_DP);
    m_issue_port.push_back(OC_EX_DP);
  }
  for (unsigned k = 0; k < m_config->gpgpu_num_int_units; k++) {
    m_fu.push_back(new int_unit(&m_pipeline_reg[EX_WB], m_config, this, k));
    m_dispatch_port.push_back(ID_OC_INT);
    m_issue_port.push_back(OC_EX_INT);
  }

  for (unsigned k = 0; k < m_config->gpgpu_num_sfu_units; k++) {
    m_fu.push_back(new sfu(&m_pipeline_reg[EX_WB], m_config, this, k));
    m_dispatch_port.push_back(ID_OC_SFU);
    m_issue_port.push_back(OC_EX_SFU);
  }

  for (unsigned k = 0; k < m_config->gpgpu_num_tensor_core_units; k++) {
    m_fu.push_back(new tensor_core(&m_pipeline_reg[EX_WB], m_config, this, k));
    m_dispatch_port.push_back(ID_OC_TENSOR_CORE);
    m_issue_port.push_back(OC_EX_TENSOR_CORE);
  }

  for (unsigned j = 0; j < m_config->m_specialized_unit.size(); j++) {
    for (unsigned k = 0; k < m_config->m_specialized_unit[j].num_units; k++) {
      m_fu.push_back(new specialized_unit(
          &m_pipeline_reg[EX_WB], m_config, this, SPEC_UNIT_START_ID + j,
          m_config->m_specialized_unit[j].name,
          m_config->m_specialized_unit[j].latency, k));
      m_dispatch_port.push_back(m_config->m_specialized_unit[j].ID_OC_SPEC_ID);
      m_issue_port.push_back(m_config->m_specialized_unit[j].OC_EX_SPEC_ID);
    }
  }

  m_ldst_unit = new ldst_unit(m_icnt, m_mem_fetch_allocator, this,
                              &m_operand_collector, m_scoreboard, m_config,
                              m_memory_config, m_stats, m_sid, m_tpc, m_gpu);
  m_fu.push_back(m_ldst_unit);
  m_dispatch_port.push_back(ID_OC_MEM);
  m_issue_port.push_back(OC_EX_MEM);

  assert(m_num_function_units == m_fu.size() and
         m_fu.size() == m_dispatch_port.size() and
         m_fu.size() == m_issue_port.size());

  // there are as many result buses as the width of the EX_WB stage
  num_result_bus = m_config->pipe_widths[EX_WB];
  for (unsigned i = 0; i < num_result_bus; i++) {
    this->m_result_bus.push_back(new std::bitset<MAX_ALU_LATENCY>());
  }

  if (m_config->model == AWARE_RECONVERGENCE)
    updateSIMTDivergenceStructuresInitialization();
}

shader_core_ctx::shader_core_ctx(class gpgpu_sim *gpu,
                                 class simt_core_cluster *cluster,
                                 unsigned shader_id, unsigned tpc_id,
                                 const shader_core_config *config,
                                 const memory_config *mem_config,
                                 shader_core_stats *stats)
    : core_t(gpu, NULL, config->warp_size, config->n_thread_per_shader),
      m_barriers(this, config->max_warps_per_shader, config->max_cta_per_core,
                 config->max_barriers_per_cta, config->warp_size),
      m_active_warps(0),
      m_dynamic_warp_id(0) {
  m_cluster = cluster;
  m_config = config;
  m_memory_config = mem_config;
  m_stats = stats;
  // unsigned warp_size = config->warp_size;
  Issue_Prio = 0;

  m_sid = shader_id;
  m_tpc = tpc_id;

  if (get_gpu()->get_config().g_power_simulation_enabled) {
    scaling_coeffs = get_gpu()->get_scaling_coeffs();
  }

  m_last_inst_gpu_sim_cycle = 0;
  m_last_inst_gpu_tot_sim_cycle = 0;

  // Jin: for concurrent kernels on a SM
  m_occupied_n_threads = 0;
  m_occupied_shmem = 0;
  m_occupied_regs = 0;
  m_occupied_ctas = 0;
  m_occupied_hwtid.reset();
  m_occupied_cta_to_hwtid.clear();
}

void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread,
                             bool reset_not_completed) {
  if (reset_not_completed) {
    m_not_completed = 0;
    m_active_threads.reset();

    // Jin: for concurrent kernels on a SM
    m_occupied_n_threads = 0;
    m_occupied_shmem = 0;
    m_occupied_regs = 0;
    m_occupied_ctas = 0;
    m_occupied_hwtid.reset();
    m_occupied_cta_to_hwtid.clear();
    m_active_warps = 0;
  }
  for (unsigned i = start_thread; i < end_thread; i++) {
    m_threadState[i].n_insn = 0;
    m_threadState[i].m_cta_id = -1;
  }
  for (unsigned i = start_thread / m_config->warp_size;
       i < end_thread / m_config->warp_size; ++i) {
    m_warp[i]->reset();
    if (m_config->model == POST_DOMINATOR) {
      m_simt_stack[i]->reset();
    } else {
      m_simt_tables[i]->reset();
    }
  }
}

void shader_core_ctx::init_warps(unsigned cta_id, unsigned start_thread,
                                 unsigned end_thread, unsigned ctaid,
                                 int cta_size, kernel_info_t &kernel) {
  address_type start_pc = next_pc(start_thread);
  unsigned kernel_id = kernel.get_uid();
  unsigned start_warp = start_thread / m_config->warp_size;
  unsigned warp_per_cta = cta_size / m_config->warp_size;
  unsigned end_warp = end_thread / m_config->warp_size +
                      ((end_thread % m_config->warp_size) ? 1 : 0);
  for (unsigned i = start_warp; i < end_warp; ++i) {
    unsigned n_active = 0;
    simt_mask_t active_threads;
    for (unsigned t = 0; t < m_config->warp_size; t++) {
      unsigned hwtid = i * m_config->warp_size + t;
      if (hwtid < end_thread) {
        n_active++;
        assert(!m_active_threads.test(hwtid));
        m_active_threads.set(hwtid);
        active_threads.set(t);
      }
    }

    if (m_config->model == POST_DOMINATOR) {
      m_simt_stack[i]->launch(start_pc, active_threads);

      if (m_gpu->resume_option == 1 && kernel_id == m_gpu->resume_kernel &&
          ctaid >= m_gpu->resume_CTA && ctaid < m_gpu->checkpoint_CTA_t) {
        char fname[2048];
        snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt",
                 i % warp_per_cta, ctaid);
        unsigned pc, rpc;
        m_simt_stack[i]->resume(fname);
        m_simt_stack[i]->get_pdom_stack_top_info(&pc, &rpc);
        for (unsigned t = 0; t < m_config->warp_size; t++) {
          if (m_thread != NULL) {
            m_thread[i * m_config->warp_size + t]->set_npc(pc);
            m_thread[i * m_config->warp_size + t]->update_pc();
          }
        }
        start_pc = pc;
      }
    } else {
      m_simt_tables[i]->launch(start_pc, active_threads);
    }

    m_warp[i]->init(start_pc, cta_id, i, active_threads, m_dynamic_warp_id,
                    kernel.get_streamID());
    ++m_dynamic_warp_id;
    m_not_completed += n_active;
    ++m_active_warps;
  }
}

// return the next pc of a thread
address_type shader_core_ctx::next_pc(int tid) const {
  if (tid == -1) return -1;
  ptx_thread_info *the_thread = m_thread[tid];
  if (the_thread == NULL) return -1;
  return the_thread
      ->get_pc();  // PC should already be updatd to next PC at this point (was
                   // set in shader_decode() last time thread ran)
}

void gpgpu_sim::get_pdom_stack_top_info(unsigned sid, unsigned tid,
                                        unsigned *pc, unsigned *rpc) {
  unsigned cluster_id = m_shader_config->sid_to_cluster(sid);
  m_cluster[cluster_id]->get_pdom_stack_top_info(sid, tid, pc, rpc);
}

void shader_core_ctx::get_pdom_stack_top_info(unsigned tid, unsigned *pc,
                                              unsigned *rpc) const {
  unsigned warp_id = tid / m_config->warp_size;
  if (m_config->model == POST_DOMINATOR) {
    m_simt_stack[warp_id]->get_pdom_stack_top_info(pc, rpc);
  } else {
    m_simt_tables[warp_id]->get_pdom_active_split_info(pc, rpc);
  }
}

float shader_core_ctx::get_current_occupancy(unsigned long long &active,
                                             unsigned long long &total) const {
  // To match the achieved_occupancy in nvprof, only SMs that are active are
  // counted toward the occupancy.
  if (m_active_warps > 0) {
    total += m_warp.size();
    active += m_active_warps;
    return float(active) / float(total);
  } else {
    return 0;
  }
}

void shader_core_stats::print(FILE *fout) const {
  unsigned long long thread_icount_uarch = 0;
  unsigned long long warp_icount_uarch = 0;

  for (unsigned i = 0; i < m_config->num_shader(); i++) {
    thread_icount_uarch += m_num_sim_insn[i];
    warp_icount_uarch += m_num_sim_winsn[i];
  }
  fprintf(fout, "gpgpu_n_tot_thrd_icount = %lld\n", thread_icount_uarch);
  fprintf(fout, "gpgpu_n_tot_w_icount = %lld\n", warp_icount_uarch);

  fprintf(fout, "gpgpu_n_stall_shd_mem = %d\n", gpgpu_n_stall_shd_mem);
  fprintf(fout, "gpgpu_n_mem_read_local = %d\n", gpgpu_n_mem_read_local);
  fprintf(fout, "gpgpu_n_mem_write_local = %d\n", gpgpu_n_mem_write_local);
  fprintf(fout, "gpgpu_n_mem_read_global = %d\n", gpgpu_n_mem_read_global);
  fprintf(fout, "gpgpu_n_mem_write_global = %d\n", gpgpu_n_mem_write_global);
  fprintf(fout, "gpgpu_n_mem_texture = %d\n", gpgpu_n_mem_texture);
  fprintf(fout, "gpgpu_n_mem_const = %d\n", gpgpu_n_mem_const);

  fprintf(fout, "gpgpu_n_load_insn  = %d\n", gpgpu_n_load_insn);
  fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn);
  fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn);
  fprintf(fout, "gpgpu_n_sstarr_insn = %d\n", gpgpu_n_sstarr_insn);
  fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn);
  fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn);
  fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn);

  fprintf(fout, "gpgpu_n_shmem_bkconflict = %d\n", gpgpu_n_shmem_bkconflict);
  fprintf(fout, "gpgpu_n_l1cache_bkconflict = %d\n",
          gpgpu_n_l1cache_bkconflict);

  fprintf(fout, "gpgpu_n_intrawarp_mshr_merge = %d\n",
          gpgpu_n_intrawarp_mshr_merge);
  fprintf(fout, "gpgpu_n_cmem_portconflict = %d\n", gpgpu_n_cmem_portconflict);

  fprintf(fout, "gpgpu_stall_shd_mem[c_mem][resource_stall] = %d\n",
          gpu_stall_shd_mem_breakdown[C_MEM][BK_CONF]);
  // fprintf(fout, "gpgpu_stall_shd_mem[c_mem][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[C_MEM][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[c_mem][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[C_MEM][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[c_mem][data_port_stall] = %d\n",
  // gpu_stall_shd_mem_breakdown[C_MEM][DATA_PORT_STALL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[t_mem][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[T_MEM][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[t_mem][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[T_MEM][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[t_mem][data_port_stall] = %d\n",
  // gpu_stall_shd_mem_breakdown[T_MEM][DATA_PORT_STALL]);
  fprintf(fout, "gpgpu_stall_shd_mem[s_mem][bk_conf] = %d\n",
          gpu_stall_shd_mem_breakdown[S_MEM][BK_CONF]);
  fprintf(
      fout, "gpgpu_stall_shd_mem[gl_mem][resource_stall] = %d\n",
      gpu_stall_shd_mem_breakdown[G_MEM_LD][BK_CONF] +
          gpu_stall_shd_mem_breakdown[G_MEM_ST][BK_CONF] +
          gpu_stall_shd_mem_breakdown[L_MEM_LD][BK_CONF] +
          gpu_stall_shd_mem_breakdown[L_MEM_ST][BK_CONF]);  // coalescing stall
                                                            // at data cache
  fprintf(
      fout, "gpgpu_stall_shd_mem[gl_mem][coal_stall] = %d\n",
      gpu_stall_shd_mem_breakdown[G_MEM_LD][COAL_STALL] +
          gpu_stall_shd_mem_breakdown[G_MEM_ST][COAL_STALL] +
          gpu_stall_shd_mem_breakdown[L_MEM_LD][COAL_STALL] +
          gpu_stall_shd_mem_breakdown[L_MEM_ST]
                                     [COAL_STALL]);  // coalescing stall + bank
                                                     // conflict at data cache
  fprintf(fout, "gpgpu_stall_shd_mem[gl_mem][data_port_stall] = %d\n",
          gpu_stall_shd_mem_breakdown[G_MEM_LD][DATA_PORT_STALL] +
              gpu_stall_shd_mem_breakdown[G_MEM_ST][DATA_PORT_STALL] +
              gpu_stall_shd_mem_breakdown[L_MEM_LD][DATA_PORT_STALL] +
              gpu_stall_shd_mem_breakdown[L_MEM_ST]
                                         [DATA_PORT_STALL]);  // data port stall
                                                              // at data cache
  // fprintf(fout, "gpgpu_stall_shd_mem[g_mem_ld][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_LD][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_ld][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_LD][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_ld][wb_icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_ld][wb_rsrv_fail] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_LD][WB_CACHE_RSRV_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_st][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_ST][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_st][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_ST][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_st][wb_icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[g_mem_st][wb_rsrv_fail] = %d\n",
  // gpu_stall_shd_mem_breakdown[G_MEM_ST][WB_CACHE_RSRV_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_LD][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_LD][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_LD][WB_CACHE_RSRV_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_st][mshr_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_ST][MSHR_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_st][icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_ST][ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][wb_icnt_rc] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_ICNT_RC_FAIL]); fprintf(fout,
  // "gpgpu_stall_shd_mem[l_mem_ld][wb_rsrv_fail] = %d\n",
  // gpu_stall_shd_mem_breakdown[L_MEM_ST][WB_CACHE_RSRV_FAIL]);

  fprintf(fout, "gpu_reg_bank_conflict_stalls = %d\n",
          gpu_reg_bank_conflict_stalls);

  fprintf(fout, "Warp Occupancy Distribution:\n");
  fprintf(fout, "Stall:%d\t", shader_cycle_distro[2]);
  fprintf(fout, "W0_Idle:%d\t", shader_cycle_distro[0]);
  fprintf(fout, "W0_Scoreboard:%d", shader_cycle_distro[1]);
  for (unsigned i = 3; i < m_config->warp_size + 3; i++)
    fprintf(fout, "\tW%d:%d", i - 2, shader_cycle_distro[i]);
  fprintf(fout, "\n");
  fprintf(fout, "single_issue_nums: ");
  for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++)
    fprintf(fout, "WS%d:%d\t", i, single_issue_nums[i]);
  fprintf(fout, "\n");
  fprintf(fout, "dual_issue_nums: ");
  for (unsigned i = 0; i < m_config->gpgpu_num_sched_per_core; i++)
    fprintf(fout, "WS%d:%d\t", i, dual_issue_nums[i]);
  fprintf(fout, "\n");

  m_outgoing_traffic_stats->print(fout);
  m_incoming_traffic_stats->print(fout);

  // Change 5: SIMD partitioning co-issue stats (aggregated across cores)
  {
    unsigned long long tot_inter = 0, tot_intra = 0;
    unsigned long long tot_denied_fu = 0, tot_denied_sb = 0, tot_denied_ns = 0;
    unsigned long long tot_active_lanes = 0, tot_composite_cycles = 0;
    for (unsigned i = 0; i < m_config->num_shader(); i++) {
      tot_inter += inter_warp_coissue_events[i];
      tot_intra += intra_warp_coissue_events[i];
      tot_denied_fu += coissue_denied_by_fu_mismatch[i];
      tot_denied_sb += coissue_denied_by_scoreboard[i];
      tot_denied_ns += coissue_denied_by_no_sets[i];
      tot_active_lanes += coissue_total_active_lanes[i];
      tot_composite_cycles += coissue_composite_cycles[i];
    }
    fprintf(fout, "SIMD Partitioning Co-issue Stats:\n");
    fprintf(fout, "gpu_tot_inter_warp_coissue = %llu\n", tot_inter);
    fprintf(fout, "gpu_tot_intra_warp_coissue = %llu\n", tot_intra);
    fprintf(fout, "gpu_tot_coissue_composite_cycles = %llu\n",
            tot_composite_cycles);
    fprintf(fout, "gpu_tot_coissue_denied_fu_mismatch = %llu\n", tot_denied_fu);
    fprintf(fout, "gpu_tot_coissue_denied_scoreboard = %llu\n", tot_denied_sb);
    fprintf(fout, "gpu_tot_coissue_denied_no_sets = %llu\n", tot_denied_ns);
    double avg_lanes = tot_composite_cycles > 0
                           ? (double)tot_active_lanes /
                                 (double)tot_composite_cycles
                           : 0.0;
    fprintf(fout, "gpu_tot_avg_active_lanes_per_coissue_cycle = %.3f\n",
            avg_lanes);

    unsigned nbins = m_config->gpgpu_num_simd_sets + 1;
    fprintf(fout, "gpu_tot_simd_sets_used_histogram = [");
    for (unsigned b = 0; b < nbins; b++) {
      unsigned long long tot_bin = 0;
      for (unsigned c = 0; c < m_config->num_shader(); c++) {
        tot_bin += simd_sets_used_histogram[c * nbins + b];
      }
      fprintf(fout, "%s%llu", b == 0 ? "" : ", ", tot_bin);
    }
    fprintf(fout, "]\n");

    // Diagnostic counters (per-filter breakdown of inter-warp co-issue
    // candidate rejections — quantifies the "silent" rejections that
    // precede the coissue_denied_by_* counters).
    unsigned long long tot_seen = 0, tot_null = 0, tot_done = 0,
                       tot_primary = 0, tot_picked = 0,
                       tot_iempty = 0, tot_waiting = 0, tot_simt = 0,
                       tot_noinst = 0, tot_pcm = 0, tot_spc0 = 0;
    for (unsigned i = 0; i < m_config->num_shader(); i++) {
      tot_seen += coissue_cand_seen[i];
      tot_null += coissue_skipped_null[i];
      tot_done += coissue_skipped_done_exit[i];
      tot_primary += coissue_skipped_is_primary[i];
      tot_picked += coissue_skipped_picked_earlier[i];
      tot_iempty += coissue_skipped_ibuffer_empty[i];
      tot_waiting += coissue_skipped_waiting[i];
      tot_simt += coissue_skipped_simt_blocked[i];
      tot_noinst += coissue_skipped_no_inst[i];
      tot_pcm += coissue_skipped_pc_mismatch[i];
      tot_spc0 += coissue_skipped_samepc_pass0[i];
    }
    fprintf(fout, "gpu_tot_coissue_cand_seen = %llu\n", tot_seen);
    fprintf(fout, "gpu_tot_coissue_skipped_null = %llu\n", tot_null);
    fprintf(fout, "gpu_tot_coissue_skipped_done_exit = %llu\n", tot_done);
    fprintf(fout, "gpu_tot_coissue_skipped_is_primary = %llu\n", tot_primary);
    fprintf(fout, "gpu_tot_coissue_skipped_picked_earlier = %llu\n",
            tot_picked);
    fprintf(fout, "gpu_tot_coissue_skipped_ibuffer_empty = %llu\n",
            tot_iempty);
    fprintf(fout, "gpu_tot_coissue_skipped_waiting = %llu\n", tot_waiting);
    fprintf(fout, "gpu_tot_coissue_skipped_simt_blocked = %llu\n", tot_simt);
    fprintf(fout, "gpu_tot_coissue_skipped_no_inst = %llu\n", tot_noinst);
    fprintf(fout, "gpu_tot_coissue_skipped_pc_mismatch = %llu\n", tot_pcm);
    fprintf(fout, "gpu_tot_coissue_skipped_samepc_pass0 = %llu\n", tot_spc0);

    // MEM co-issue-specific subtotals
    unsigned long long tot_mem_inter = 0, tot_mem_intra = 0,
                       tot_mem_denied = 0;
    for (unsigned i = 0; i < m_config->num_shader(); i++) {
      tot_mem_inter += mem_coissue_inter_events[i];
      tot_mem_intra += mem_coissue_intra_events[i];
      tot_mem_denied += mem_coissue_denied_filter[i];
    }
    fprintf(fout, "gpu_tot_mem_coissue_inter_events = %llu\n", tot_mem_inter);
    fprintf(fout, "gpu_tot_mem_coissue_intra_events = %llu\n", tot_mem_intra);
    fprintf(fout, "gpu_tot_mem_coissue_denied_filter = %llu\n",
            tot_mem_denied);
  }
}

void shader_core_stats::event_warp_issued(unsigned s_id, unsigned warp_id,
                                          unsigned num_issued,
                                          unsigned dynamic_warp_id) {
  assert(warp_id <= m_config->max_warps_per_shader);
  for (unsigned i = 0; i < num_issued; ++i) {
    if (m_shader_dynamic_warp_issue_distro[s_id].size() <= dynamic_warp_id) {
      m_shader_dynamic_warp_issue_distro[s_id].resize(dynamic_warp_id + 1);
    }
    ++m_shader_dynamic_warp_issue_distro[s_id][dynamic_warp_id];
    if (m_shader_warp_slot_issue_distro[s_id].size() <= warp_id) {
      m_shader_warp_slot_issue_distro[s_id].resize(warp_id + 1);
    }
    ++m_shader_warp_slot_issue_distro[s_id][warp_id];
  }
}

void shader_core_stats::visualizer_print(gzFile visualizer_file) {
  // warp divergence breakdown
  gzprintf(visualizer_file, "WarpDivergenceBreakdown:");
  unsigned int total = 0;
  unsigned int cf =
      (m_config->gpgpu_warpdistro_shader == -1) ? m_config->num_shader() : 1;
  gzprintf(visualizer_file, " %d",
           (shader_cycle_distro[0] - last_shader_cycle_distro[0]) / cf);
  gzprintf(visualizer_file, " %d",
           (shader_cycle_distro[1] - last_shader_cycle_distro[1]) / cf);
  gzprintf(visualizer_file, " %d",
           (shader_cycle_distro[2] - last_shader_cycle_distro[2]) / cf);
  for (unsigned i = 0; i < m_config->warp_size + 3; i++) {
    if (i >= 3) {
      total += (shader_cycle_distro[i] - last_shader_cycle_distro[i]);
      if (((i - 3) % (m_config->warp_size / 8)) ==
          ((m_config->warp_size / 8) - 1)) {
        gzprintf(visualizer_file, " %d", total / cf);
        total = 0;
      }
    }
    last_shader_cycle_distro[i] = shader_cycle_distro[i];
  }
  gzprintf(visualizer_file, "\n");

  gzprintf(visualizer_file, "ctas_completed: %d\n", ctas_completed);
  ctas_completed = 0;
  // warp issue breakdown
  unsigned sid = m_config->gpgpu_warp_issue_shader;
  unsigned count = 0;
  unsigned warp_id_issued_sum = 0;
  gzprintf(visualizer_file, "WarpIssueSlotBreakdown:");
  if (m_shader_warp_slot_issue_distro[sid].size() > 0) {
    for (std::vector<unsigned>::const_iterator iter =
             m_shader_warp_slot_issue_distro[sid].begin();
         iter != m_shader_warp_slot_issue_distro[sid].end(); iter++, count++) {
      unsigned diff = count < m_last_shader_warp_slot_issue_distro.size()
                          ? *iter - m_last_shader_warp_slot_issue_distro[count]
                          : *iter;
      gzprintf(visualizer_file, " %d", diff);
      warp_id_issued_sum += diff;
    }
    m_last_shader_warp_slot_issue_distro = m_shader_warp_slot_issue_distro[sid];
  } else {
    gzprintf(visualizer_file, " 0");
  }
  gzprintf(visualizer_file, "\n");

#define DYNAMIC_WARP_PRINT_RESOLUTION 32
  unsigned total_issued_this_resolution = 0;
  unsigned dynamic_id_issued_sum = 0;
  count = 0;
  gzprintf(visualizer_file, "WarpIssueDynamicIdBreakdown:");
  if (m_shader_dynamic_warp_issue_distro[sid].size() > 0) {
    for (std::vector<unsigned>::const_iterator iter =
             m_shader_dynamic_warp_issue_distro[sid].begin();
         iter != m_shader_dynamic_warp_issue_distro[sid].end();
         iter++, count++) {
      unsigned diff =
          count < m_last_shader_dynamic_warp_issue_distro.size()
              ? *iter - m_last_shader_dynamic_warp_issue_distro[count]
              : *iter;
      total_issued_this_resolution += diff;
      if ((count + 1) % DYNAMIC_WARP_PRINT_RESOLUTION == 0) {
        gzprintf(visualizer_file, " %d", total_issued_this_resolution);
        dynamic_id_issued_sum += total_issued_this_resolution;
        total_issued_this_resolution = 0;
      }
    }
    if (count % DYNAMIC_WARP_PRINT_RESOLUTION != 0) {
      gzprintf(visualizer_file, " %d", total_issued_this_resolution);
      dynamic_id_issued_sum += total_issued_this_resolution;
    }
    m_last_shader_dynamic_warp_issue_distro =
        m_shader_dynamic_warp_issue_distro[sid];
    assert(warp_id_issued_sum == dynamic_id_issued_sum);
  } else {
    gzprintf(visualizer_file, " 0");
  }
  gzprintf(visualizer_file, "\n");

  // overall cache miss rates
  gzprintf(visualizer_file, "gpgpu_n_l1cache_bkconflict: %d\n",
           gpgpu_n_l1cache_bkconflict);
  gzprintf(visualizer_file, "gpgpu_n_shmem_bkconflict: %d\n",
           gpgpu_n_shmem_bkconflict);

  // instruction count per shader core
  gzprintf(visualizer_file, "shaderinsncount:  ");
  for (unsigned i = 0; i < m_config->num_shader(); i++)
    gzprintf(visualizer_file, "%u ", m_num_sim_insn[i]);
  gzprintf(visualizer_file, "\n");
  // warp instruction count per shader core
  gzprintf(visualizer_file, "shaderwarpinsncount:  ");
  for (unsigned i = 0; i < m_config->num_shader(); i++)
    gzprintf(visualizer_file, "%u ", m_num_sim_winsn[i]);
  gzprintf(visualizer_file, "\n");
  // warp divergence per shader core
  gzprintf(visualizer_file, "shaderwarpdiv: ");
  for (unsigned i = 0; i < m_config->num_shader(); i++)
    gzprintf(visualizer_file, "%u ", m_n_diverge[i]);
  gzprintf(visualizer_file, "\n");
}

#define PROGRAM_MEM_START                                      \
  0xF0000000 /* should be distinct from other memory spaces... \
                check ptx_ir.h to verify this does not overlap \
                other memory spaces */

const warp_inst_t *exec_shader_core_ctx::get_next_inst(unsigned warp_id,
                                                       address_type pc) {
  // read the inst from the functional model
  return m_gpu->gpgpu_ctx->ptx_fetch_inst(pc);
}

void exec_shader_core_ctx::get_pdom_stack_top_info(unsigned warp_id,
                                                   const warp_inst_t *pI,
                                                   unsigned *pc,
                                                   unsigned *rpc) {
  if (m_config->model == POST_DOMINATOR) {
    m_simt_stack[warp_id]->get_pdom_stack_top_info(pc, rpc);
  } else {
    m_simt_tables[warp_id]->get_pdom_active_split_info(pc, rpc);
  }
}

const active_mask_t &exec_shader_core_ctx::get_active_mask(
    unsigned warp_id, const warp_inst_t *pI) {
  if (m_config->model == POST_DOMINATOR) {
    return m_simt_stack[warp_id]->get_active_mask();
  } else {
    return m_simt_tables[warp_id]->get_active_mask();
  }
}

void shader_core_ctx::decode() {
  // Helper lambda to decode a fetch buffer into a specific I-Buffer half
  auto decode_fetch_buffer = [&](ifetch_buffer_t &fbuf) {
    if (!fbuf.m_valid) return;
    unsigned wid = fbuf.m_warp_id;
    unsigned half = fbuf.m_ibuffer_half;
    unsigned slot_base = half * 2;  // IBUFFER_HALF_SIZE
    address_type pc = fbuf.m_pc;

    const warp_inst_t *pI1 = get_next_inst(wid, pc);
    if (pI1) {
      m_warp[wid]->ibuffer_fill(slot_base, pI1, fbuf.m_split_id,
                                fbuf.m_split_mask);
      m_warp[wid]->inc_inst_in_pipeline();
      m_stats->m_num_decoded_insn[m_sid]++;
      if ((pI1->oprnd_type == INT_OP) || (pI1->oprnd_type == UN_OP))
        m_stats->m_num_INTdecoded_insn[m_sid]++;
      else if (pI1->oprnd_type == FP_OP)
        m_stats->m_num_FPdecoded_insn[m_sid]++;

      const warp_inst_t *pI2 = get_next_inst(wid, pc + pI1->isize);
      if (pI2) {
        m_warp[wid]->ibuffer_fill(slot_base + 1, pI2, fbuf.m_split_id,
                                  fbuf.m_split_mask);
        m_warp[wid]->inc_inst_in_pipeline();
        m_stats->m_num_decoded_insn[m_sid]++;
        if ((pI2->oprnd_type == INT_OP) || (pI2->oprnd_type == UN_OP))
          m_stats->m_num_INTdecoded_insn[m_sid]++;
        else if (pI2->oprnd_type == FP_OP)
          m_stats->m_num_FPdecoded_insn[m_sid]++;
      }
    }
    fbuf.m_valid = false;
  };

  decode_fetch_buffer(m_inst_fetch_buffer);
  decode_fetch_buffer(m_inst_fetch_buffer_secondary);
}

void shader_core_ctx::fetch() {
  // --- Primary fetch: fills I-Buffer half 0 ---
  if (!m_inst_fetch_buffer.m_valid) {
    if (m_L1I->access_ready()) {
      mem_fetch *mf = m_L1I->next_access();
      m_warp[mf->get_wid()]->clear_imiss_pending();
      m_inst_fetch_buffer =
          ifetch_buffer_t(m_warp[mf->get_wid()]->get_pc(),
                          mf->get_access_size(), mf->get_wid());
      assert(m_warp[mf->get_wid()]->get_pc() ==
             (mf->get_addr() - PROGRAM_MEM_START));
      m_inst_fetch_buffer.m_valid = true;
      // Tag with split info for AWARE mode
      if (m_config->model == AWARE_RECONVERGENCE) {
        unsigned wid = mf->get_wid();
        unsigned spc, srpc;
        m_simt_tables[wid]->get_pdom_active_split_info(&spc, &srpc);
        m_inst_fetch_buffer.m_split_id =
            m_simt_tables[wid]->get_active_split_id();
        m_inst_fetch_buffer.m_split_mask =
            m_simt_tables[wid]->get_active_mask();
        // Mode 2: record half 0's owner.
        if (m_config->gpgpu_scoreboard_mode == 2) {
          active_mask_t mask_aw;
          const simt_mask_t &m = m_simt_tables[wid]->get_active_mask();
          for (unsigned t = 0; t < MAX_WARP_SIZE; t++)
            if (m.test(t)) mask_aw.set(t);
          m_warp[wid]->ibuffer_assign_half(0, m_inst_fetch_buffer.m_split_id,
                                           mask_aw);
        }
      }
      m_inst_fetch_buffer.m_ibuffer_half = 0;
      m_warp[mf->get_wid()]->set_last_fetch(m_gpu->gpu_sim_cycle);
      delete mf;
    } else {
      // find an active warp with space in instruction buffer
      for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) {
        unsigned warp_id =
            (m_last_warp_fetched + 1 + i) % m_config->max_warps_per_shader;

        // this code checks if this warp has finished executing and can be
        // reclaimed
        if (m_warp[warp_id]->hardware_done() &&
            !m_scoreboard->pendingWrites(warp_id) &&
            !m_warp[warp_id]->done_exit()) {
          bool did_exit = false;
          for (unsigned t = 0; t < m_config->warp_size; t++) {
            unsigned tid = warp_id * m_config->warp_size + t;
            if (m_threadState[tid].m_active == true) {
              m_threadState[tid].m_active = false;
              unsigned cta_id = m_warp[warp_id]->get_cta_id();
              if (m_thread[tid] == NULL) {
                register_cta_thread_exit(cta_id,
                                         m_warp[warp_id]->get_kernel_info());
              } else {
                register_cta_thread_exit(cta_id,
                                         &(m_thread[tid]->get_kernel()));
              }
              m_not_completed -= 1;
              m_active_threads.reset(tid);
              did_exit = true;
            }
          }
          if (did_exit) m_warp[warp_id]->set_done_exit();
          --m_active_warps;
          assert(m_active_warps >= 0);
        }

        // this code fetches instructions from the i-cache
        bool simt_conditions = true;
        if (m_config->model == AWARE_RECONVERGENCE)
          simt_conditions = !is_virtualized(warp_id);

        // Changed: use ibuffer_half_empty(0) instead of ibuffer_empty()
        if (simt_conditions && !m_warp[warp_id]->functional_done() &&
            !m_warp[warp_id]->imiss_pending() &&
            m_warp[warp_id]->ibuffer_half_empty(0)) {
          // Mode 2 (slot-pinned): drain gate at fetch time. Two trigger
          // conditions: (1) FIFO membership change set by
          // simt_tables::update() (divergence/reconvergence/CALL/RET);
          // (2) slot 0 about to be reassigned to a different split than
          // its previous owner — this catches co-issue's
          // move_split_to_front rotations, where the secondary split
          // (with pending writes in slot 1) becomes slot 0's new owner.
          // Without (2), the same split's writes are stranded in slot 1
          // while slot 0 issues reads of those same regs (real RAW).
          // Both conditions require BOTH slots clean before fetch
          // proceeds, so cross-slot pending writes from prior owners
          // drain regardless of which slot is being reassigned.
          if (m_config->gpgpu_scoreboard_mode == 2 &&
              m_config->model == AWARE_RECONVERGENCE) {
            unsigned new_split_id =
                m_simt_tables[warp_id]->get_active_split_id();
            bool slot0_reassigned =
                m_warp[warp_id]->ibuffer_half_assigned(0) &&
                m_warp[warp_id]->ibuffer_half_split_id(0) != new_split_id;
            bool fifo_changed =
                m_simt_tables[warp_id]->div_recv_drain_pending();
            if (slot0_reassigned || fifo_changed) {
              if (!m_scoreboard->slotClean(warp_id, 0) ||
                  !m_scoreboard->slotClean(warp_id, 1)) {
                continue;  // drain pending — try next warp
              }
              if (fifo_changed) {
                m_simt_tables[warp_id]->clear_div_recv_drain_pending();
              }
            }
          }
          address_type pc;
          pc = m_warp[warp_id]->get_pc();
          address_type ppc = pc + PROGRAM_MEM_START;
          unsigned nbytes = 16;
          unsigned offset_in_block =
              pc & (m_config->m_L1I_config.get_line_sz() - 1);
          if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz())
            nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block);

          mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx);
          mem_fetch *mf = new mem_fetch(
              acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE,
              warp_id, m_sid, m_tpc, m_memory_config,
              m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
          std::list<cache_event> events;
          enum cache_request_status status;
          if (m_config->perfect_inst_const_cache) {
            status = HIT;
            shader_cache_access_log(m_sid, INSTRUCTION, 0);
          } else
            status = m_L1I->access(
                (new_addr_type)ppc, mf,
                m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle, events);

          if (status == MISS) {
            m_last_warp_fetched = warp_id;
            m_warp[warp_id]->set_imiss_pending();
            m_warp[warp_id]->set_last_fetch(m_gpu->gpu_sim_cycle);
          } else if (status == HIT) {
            m_last_warp_fetched = warp_id;
            m_inst_fetch_buffer = ifetch_buffer_t(pc, nbytes, warp_id);
            // Tag with split info
            if (m_config->model == AWARE_RECONVERGENCE) {
              m_inst_fetch_buffer.m_split_id =
                  m_simt_tables[warp_id]->get_active_split_id();
              m_inst_fetch_buffer.m_split_mask =
                  m_simt_tables[warp_id]->get_active_mask();
              // Mode 2: record half 0's owner for the wait-for-clean gate.
              if (m_config->gpgpu_scoreboard_mode == 2) {
                active_mask_t mask_aw;
                const simt_mask_t &m =
                    m_simt_tables[warp_id]->get_active_mask();
                for (unsigned t = 0; t < MAX_WARP_SIZE; t++)
                  if (m.test(t)) mask_aw.set(t);
                m_warp[warp_id]->ibuffer_assign_half(
                    0, m_inst_fetch_buffer.m_split_id, mask_aw);
              }
            }
            m_inst_fetch_buffer.m_ibuffer_half = 0;
            m_warp[warp_id]->set_last_fetch(m_gpu->gpu_sim_cycle);
            delete mf;
          } else {
            m_last_warp_fetched = warp_id;
            assert(status == RESERVATION_FAIL);
            delete mf;
          }
          break;
        }
      }
    }
  }

  // --- Secondary fetch: fills I-Buffer half 1 from a non-overlapping split ---
  // Only in AWARE mode with SIMD partitioning enabled.
  // HIT-only: if I-Cache misses, skip (no miss tracking for secondary).
  if (m_config->model == AWARE_RECONVERGENCE &&
      m_config->gpgpu_simd_partitioning &&
      !m_inst_fetch_buffer_secondary.m_valid) {
    // Find the warp that just had primary fetch (if any), or scan for a warp
    // whose secondary half is empty and has a secondary split
    for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) {
      unsigned warp_id =
          (m_last_warp_fetched + i) % m_config->max_warps_per_shader;

      if (m_warp[warp_id]->functional_done()) continue;
      if (!m_warp[warp_id]->ibuffer_half_empty(1)) continue;
      if (is_virtualized(warp_id)) continue;

      // Check if this warp has a secondary split with non-overlapping mask
      unsigned sec_pc, sec_rpc, sec_split_id;
      simt_mask_t sec_mask;
      if (!has_secondary_split(warp_id)) continue;
      if (!get_secondary_split_info(warp_id, &sec_pc, &sec_rpc, &sec_split_id,
                                    &sec_mask))
        continue;

      // Non-overlap check: secondary mask must not overlap with half 0's mask
      if (m_warp[warp_id]->ibuffer_half_assigned(0)) {
        active_mask_t half0_mask = m_warp[warp_id]->ibuffer_half_mask(0);
        if ((sec_mask & half0_mask).any()) continue;  // overlapping, skip
      }

      // Mode 2: drain gate at secondary fetch (mirrors the primary-fetch
      // gate above). Two trigger conditions: (1) FIFO membership change
      // (div_recv_drain_pending); (2) slot 1 reassignment to a different
      // split than its previous owner. Both require BOTH slots clean.
      if (m_config->gpgpu_scoreboard_mode == 2) {
        bool slot1_reassigned =
            m_warp[warp_id]->ibuffer_half_assigned(1) &&
            m_warp[warp_id]->ibuffer_half_split_id(1) != sec_split_id;
        bool fifo_changed =
            m_simt_tables[warp_id]->div_recv_drain_pending();
        if (slot1_reassigned || fifo_changed) {
          if (!m_scoreboard->slotClean(warp_id, 0) ||
              !m_scoreboard->slotClean(warp_id, 1)) {
            continue;  // drain pending — try next warp
          }
          if (fifo_changed) {
            m_simt_tables[warp_id]->clear_div_recv_drain_pending();
          }
        }
      }

      // Attempt I-Cache access (HIT-only for secondary)
      address_type ppc = sec_pc + PROGRAM_MEM_START;
      unsigned nbytes = 16;
      unsigned offset_in_block =
          sec_pc & (m_config->m_L1I_config.get_line_sz() - 1);
      if ((offset_in_block + nbytes) > m_config->m_L1I_config.get_line_sz())
        nbytes = (m_config->m_L1I_config.get_line_sz() - offset_in_block);

      mem_access_t acc(INST_ACC_R, ppc, nbytes, false, m_gpu->gpgpu_ctx);
      mem_fetch *mf = new mem_fetch(
          acc, NULL, m_warp[warp_id]->get_streamID(), READ_PACKET_SIZE,
          warp_id, m_sid, m_tpc, m_memory_config,
          m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
      std::list<cache_event> events;
      enum cache_request_status status;
      if (m_config->perfect_inst_const_cache) {
        status = HIT;
      } else {
        status = m_L1I->access((new_addr_type)ppc, mf,
                               m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle,
                               events);
      }

      if (status == HIT) {
        m_inst_fetch_buffer_secondary = ifetch_buffer_t(
            sec_pc, nbytes, warp_id, 1, sec_split_id, sec_mask);
        // Assign I-Buffer half 1 to this split
        m_warp[warp_id]->ibuffer_assign_half(1, sec_split_id, sec_mask);
        static const bool dbg_pc_enabled_sec = (getenv("MEMCO_DBG_PC") != NULL);
        if (dbg_pc_enabled_sec && warp_id == 0) {
          fprintf(stderr,
                  "[SEC_FETCH_DBG] cycle=%llu warp=%u sec_split_id=%u "
                  "sec_pc=0x%lx sec_mask=%s\n",
                  (unsigned long long)(m_gpu->gpu_sim_cycle +
                                       m_gpu->gpu_tot_sim_cycle),
                  warp_id, sec_split_id, (unsigned long)sec_pc,
                  sec_mask.to_string().c_str());
          fflush(stderr);
        }
        delete mf;
        break;  // one secondary fetch per cycle
      } else {
        // MISS or RESERVATION_FAIL: silently skip, no pending state
        delete mf;
        // Don't break — try next warp
      }
    }
  }

  m_L1I->cycle();
}

void exec_shader_core_ctx::execute_warp_inst_per_set(warp_inst_t &inst) {
  // Execute each SIMD set's threads independently
  const std::vector<simd_set_info> &sets = inst.get_simd_sets();
  unsigned warp_id = inst.warp_id();
  unsigned set_width = m_config->simd_set_width;

  for (unsigned s = 0; s < sets.size(); s++) {
    const simd_set_info &set = sets[s];
    if (!set.valid) continue;

    // Execute each active thread in this set
    for (unsigned lane = 0; lane < set_width; lane++) {
      if (set.set_active_mask.test(lane)) {
        unsigned t = set.thread_ids[lane];  // original thread position (0-31)
        unsigned hw_tid = m_warp_size * warp_id + t;
        assert(inst.active(t));  // thread must be active in original mask
        m_thread[hw_tid]->ptx_exec_inst(inst, t);
        checkExecutionStatusAndUpdate(inst, t, hw_tid);
      }
    }
  }
}

void exec_shader_core_ctx::func_exec_inst(warp_inst_t &inst) {
  if (m_config->gpgpu_simd_partitioning && inst.has_simd_sets()) {
    execute_warp_inst_per_set(inst);
  } else {
    execute_warp_inst_t(inst);
  }
  if (inst.is_load() || inst.is_store()) {
    inst.generate_mem_accesses();
    // inst.print_m_accessq();
  }
}

warp_inst_t *shader_core_ctx::issue_warp(register_set &pipe_reg_set,
                                         const warp_inst_t *next_inst,
                                         const active_mask_t &active_mask,
                                         unsigned warp_id, unsigned sch_id) {
  warp_inst_t **pipe_reg =
      pipe_reg_set.get_free(m_config->sub_core_model, sch_id);
  assert(pipe_reg);

  m_warp[warp_id]->ibuffer_free();
  assert(next_inst->valid());
  **pipe_reg = *next_inst;  // static instruction information
  (*pipe_reg)->issue(
      active_mask, warp_id, m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle,
      m_warp[warp_id]->get_dynamic_warp_id(), sch_id,
      m_warp[warp_id]->get_streamID());  // dynamic instruction information
  (*pipe_reg)->set_dbg_path(0);
  (*pipe_reg)->set_dbg_split_id(-1);
  (*pipe_reg)->set_dbg_source_inst(next_inst);
  // Mode 2: primary issue always comes through ibuffer half 0.
  (*pipe_reg)->set_ibuffer_half_id(0);
  static const bool dbg_pc_enabled_pri = (getenv("MEMCO_DBG_PC") != NULL);
  if (dbg_pc_enabled_pri && warp_id == 0 &&
      m_config->model == AWARE_RECONVERGENCE) {
    unsigned active_split_id = m_simt_tables[warp_id]->get_active_split_id();
    fprintf(stderr,
            "[PRI_ISSUE_DBG] cycle=%llu warp=%u front_split_id=%u "
            "inst_pc=0x%lx active_mask=%s\n",
            (unsigned long long)(m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle),
            warp_id, active_split_id, (unsigned long)next_inst->pc,
            active_mask.to_string().c_str());
    fflush(stderr);
  }
  m_stats->shader_cycle_distro[2 + (*pipe_reg)->active_count()]++;
  // Compute SIMD set assignments before functional execution
  if (m_config->gpgpu_simd_partitioning) {
    switch (m_config->gpgpu_compaction_mode) {
      case 1:  // xor-static
        (*pipe_reg)->compute_simd_sets_xor_static(
            m_config->gpgpu_num_simd_sets, m_config->simd_set_width, 0);
        break;
      case 2:  // full
        (*pipe_reg)->compute_simd_sets_compacted(
            m_config->gpgpu_num_simd_sets, m_config->simd_set_width, 0);
        break;
      default:  // 0 = none
        (*pipe_reg)->compute_simd_sets(m_config->gpgpu_num_simd_sets,
                                       m_config->simd_set_width);
        break;
    }
  }
  func_exec_inst(**pipe_reg);

  // Add LDGSTS instructions into a buffer
  unsigned int ldgdepbar_id = m_warp[warp_id]->m_ldgdepbar_id;
  if (next_inst->m_is_ldgsts) {
    if (m_warp[warp_id]->m_ldgdepbar_buf.size() == ldgdepbar_id + 1) {
      m_warp[warp_id]->m_ldgdepbar_buf[ldgdepbar_id].push_back(*next_inst);
    } else {
      assert(m_warp[warp_id]->m_ldgdepbar_buf.size() < ldgdepbar_id + 1);
      std::vector<warp_inst_t> l;
      l.push_back(*next_inst);
      m_warp[warp_id]->m_ldgdepbar_buf.push_back(l);
    }
    // If the mask of the instruction is all 0, then the address is also 0,
    // so that there's no need to check through the writeback
    if (next_inst->get_active_mask() == 0) {
      (m_warp[warp_id]->m_ldgdepbar_buf.back()).back().pc = -1;
    }
  }

  bool split_reaches_barrier = false;
  if (next_inst->op == BARRIER_OP) {
    m_warp[warp_id]->store_info_of_last_inst_at_barrier(*pipe_reg);
    if (m_config->model == POST_DOMINATOR) {
      m_barriers.warp_reaches_barrier(m_warp[warp_id]->get_cta_id(), warp_id,
                                      const_cast<warp_inst_t *>(next_inst));
    } else {
      // AWARE_RECONVERGENCE: warp reaches barrier only when all its splits do
      split_reaches_barrier = true;
      bool warp_reaches_barrier =
          m_simt_tables[warp_id]->split_reaches_barrier(next_inst->pc);
      if (warp_reaches_barrier) {
        split_reaches_barrier = false;
        m_barriers.warp_reaches_barrier(m_warp[warp_id]->get_cta_id(), warp_id,
                                        const_cast<warp_inst_t *>(next_inst));
        if (m_barriers.warps_count_at_barrier(m_warp[warp_id]->get_cta_id()) ==
            0) {
          unsigned n = m_config->n_thread_per_shader / m_config->warp_size;
          for (unsigned i = 0; i < n; i++) {
            if (m_warp[i]->get_cta_id() == m_warp[warp_id]->get_cta_id())
              m_simt_tables[i]->release_barrier();
          }
        }
      }
    }
  } else if (next_inst->op == MEMORY_BARRIER_OP) {
    m_warp[warp_id]->set_membar();
  } else if (next_inst->m_is_ldgdepbar) {  // Add for LDGDEPBAR
    m_warp[warp_id]->m_ldgdepbar_id++;
    // If there are no added LDGSTS, insert an empty vector
    if (m_warp[warp_id]->m_ldgdepbar_buf.size() != ldgdepbar_id + 1) {
      assert(m_warp[warp_id]->m_ldgdepbar_buf.size() < ldgdepbar_id + 1);
      std::vector<warp_inst_t> l;
      m_warp[warp_id]->m_ldgdepbar_buf.push_back(l);
    }
  } else if (next_inst->m_is_depbar) {  // Add for DEPBAR
    // Set to true immediately when a DEPBAR instruction is met
    m_warp[warp_id]->m_waiting_ldgsts = true;
    m_warp[warp_id]->m_depbar_group =
        next_inst->m_depbar_group_no;  // set in trace_driven.cc

    // Record the last group that's possbily being monitored by this DEPBAR
    // instr
    m_warp[warp_id]->m_depbar_start_id = m_warp[warp_id]->m_ldgdepbar_id - 1;

    // Record the last group that's actually being monitored by this DEPBAR
    // instr
    unsigned int end_group =
        m_warp[warp_id]->m_ldgdepbar_id - m_warp[warp_id]->m_depbar_group;

    // Check for the case that the LDGSTSs monitored have finished when
    // encountering the DEPBAR instruction
    bool done_flag = true;
    for (int i = 0; i < end_group; i++) {
      for (int j = 0; j < m_warp[warp_id]->m_ldgdepbar_buf[i].size(); j++) {
        if (m_warp[warp_id]->m_ldgdepbar_buf[i][j].pc != -1) {
          done_flag = false;
          goto UpdateDEPBAR;
        }
      }
    }

  UpdateDEPBAR:
    if (done_flag) {
      if (m_warp[warp_id]->m_waiting_ldgsts) {
        m_warp[warp_id]->m_waiting_ldgsts = false;
      }
    }
  }

  updateSIMTDivergenceStructures(warp_id, *pipe_reg);

  m_scoreboard->reserveRegisters(*pipe_reg);
  m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize);
  if (split_reaches_barrier) m_simt_tables[warp_id]->push_back();
  return *pipe_reg;
}

void shader_core_ctx::co_issue_warp(warp_inst_t *composite,
                                    const warp_inst_t *next_inst,
                                    const active_mask_t &active_mask,
                                    unsigned warp_id, unsigned sch_id,
                                    unsigned start_set, unsigned split_id) {
  // For intra-warp co-issue: don't free ibuffer here (caller frees the slot)
  // For inter-warp co-issue: free the co-issued warp's I-buffer entry
  if (split_id == (unsigned)-1) {
    m_warp[warp_id]->ibuffer_free();
  }
  assert(next_inst->valid());

  // Create a temporary instruction for functional execution
  warp_inst_t temp_inst = *next_inst;
  temp_inst.issue(active_mask, warp_id,
                  m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle,
                  m_warp[warp_id]->get_dynamic_warp_id(), sch_id,
                  m_warp[warp_id]->get_streamID());
  // Debug path tag: 1=intra (split_id != -1), 2=inter (split_id == -1).
  // MEM-coissue (path=3) is set by caller after this if applicable; here
  // we use intra/inter based on split_id which the caller passes.
  temp_inst.set_dbg_path(split_id != (unsigned)-1 ? 1u : 2u);
  temp_inst.set_dbg_split_id(split_id != (unsigned)-1 ? (int)split_id : -1);
  temp_inst.set_dbg_source_inst(next_inst);
  // Mode 2: stamp the source's owning ibuffer half. Intra-warp co-issue
  // came from primary's slot 1 (secondary half); inter-warp from the
  // donor's slot 0 (donor's own primary half).
  temp_inst.set_ibuffer_half_id(split_id != (unsigned)-1 ? 1u : 0u);

  // Debug: for intra co-issue, compare cached active_mask (from
  // ibuffer slot) against current splits-table mask + warp's
  // m_active_threads. Logs only when MEMCO_DBG_PC env var is set,
  // and when the cached mask differs from the current split mask
  // OR includes any thread no longer in m_active_threads.
  static const bool dbg_pc_enabled = (getenv("MEMCO_DBG_PC") != NULL);
  if (dbg_pc_enabled && m_config->model == AWARE_RECONVERGENCE &&
      split_id != (unsigned)-1) {
    if (m_simt_tables[warp_id]->is_split_valid(split_id)) {
      unsigned cur_pc;
      simt_mask_t cur_mask;
      m_simt_tables[warp_id]->get_split_info(split_id, &cur_pc, &cur_mask);
      bool mask_drift = false;
      for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
        if (active_mask.test(t) != cur_mask.test(t)) mask_drift = true;
      }
      if (mask_drift || cur_pc != next_inst->pc) {
        fprintf(stderr,
                "[CO_ISSUE_DRIFT] cycle=%llu warp=%u INTRA split=%u "
                "next_inst_pc=0x%lx cur_pc=0x%lx cached_mask=%s "
                "cur_split_mask=%s drift=%d pc_drift=%d\n",
                (unsigned long long)(m_gpu->gpu_sim_cycle +
                                     m_gpu->gpu_tot_sim_cycle),
                warp_id, split_id, (unsigned long)next_inst->pc,
                (unsigned long)cur_pc,
                active_mask.to_string().c_str(),
                cur_mask.to_string().c_str(),
                mask_drift ? 1 : 0,
                (cur_pc != next_inst->pc) ? 1 : 0);
        fflush(stderr);
      }
    } else {
      fprintf(stderr,
              "[CO_ISSUE_INVALID] cycle=%llu warp=%u INTRA split=%u "
              "next_inst_pc=0x%lx cached_mask=%s\n",
              (unsigned long long)(m_gpu->gpu_sim_cycle +
                                   m_gpu->gpu_tot_sim_cycle),
              warp_id, split_id, (unsigned long)next_inst->pc,
              active_mask.to_string().c_str());
      fflush(stderr);
    }
  }

  // Compute SIMD sets for the co-issued instruction
  switch (m_config->gpgpu_compaction_mode) {
    case 1:  // xor-static
      temp_inst.compute_simd_sets_xor_static(m_config->gpgpu_num_simd_sets,
                                             m_config->simd_set_width,
                                             start_set);
      break;
    case 2:  // full
      temp_inst.compute_simd_sets_compacted(m_config->gpgpu_num_simd_sets,
                                            m_config->simd_set_width,
                                            start_set);
      break;
    default:  // 0 = none
      temp_inst.compute_simd_sets(m_config->gpgpu_num_simd_sets,
                                  m_config->simd_set_width);
      break;
  }

  // Functional execution for the co-issued warp's threads
  func_exec_inst(temp_inst);

  // Update SIMT divergence structures
  if (split_id != (unsigned)-1 &&
      m_config->model == AWARE_RECONVERGENCE) {
    // Intra-warp co-issue: move the target split to FIFO front first
    bool moved = m_simt_tables[warp_id]->move_split_to_front(split_id);
    if (moved && m_simt_tables[warp_id]->is_split_valid(split_id)) {
      updateSIMTDivergenceStructures(warp_id, &temp_inst);
    }
    // If move failed or split invalid: skip SIMT update (split was
    // invalidated by another operation during this cycle)
  } else {
    updateSIMTDivergenceStructures(warp_id, &temp_inst);
  }

  // Reserve scoreboard for the co-issued warp.
  if (m_config->gpgpu_scoreboard_mode == 1 ||
      m_config->gpgpu_scoreboard_mode == 2) {
    // Mode 1 (mask-aware): both inter and intra use unified mask-aware
    // scoreboard; mask differentiates entries.
    // Mode 2 (slot-pinned): both inter and intra reserve on their
    // respective slot's scoreboard via temp_inst's set ibuffer_half_id
    // (set above to 0 for inter, 1 for intra).
    m_scoreboard->reserveRegisters(&temp_inst);
  } else {
    // Mode 0 (legacy):
    // For intra-warp co-issue (same warp_id), skip primary reservation —
    // the primary instruction already reserved for this warp, and the
    // scoreboard tracks at warp granularity. Both splits may write to the
    // same register number (which is safe since they operate on exclusive
    // threads), but the scoreboard would abort on a duplicate reservation.
    if (split_id == (unsigned)-1) {
      // Inter-warp co-issue: different warp_id, always safe to reserve
      m_scoreboard->reserveRegisters(&temp_inst);
    } else {
      // Intra-warp co-issue: reserve on secondary scoreboard to track
      // dependencies within the secondary split's instruction stream.
      // Cannot use primary scoreboard (would abort on duplicate register
      // names).
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
        if (temp_inst.out[r] > 0) {
          m_scoreboard->reserveRegisterSecondary(warp_id, temp_inst.out[r]);
        }
      }
    }
  }

  // Set next PC for the co-issued warp
  m_warp[warp_id]->set_next_pc(next_inst->pc + next_inst->isize);

  // Tag the co-issued sets with their source instruction + split_id for
  // writeback routing (intra-warp composites need split_id to disambiguate
  // between primary and secondary-split sets that share warp_id).
  temp_inst.set_source_inst_on_sets(next_inst);
  temp_inst.set_split_id_on_sets(split_id);
  // Mode 1: stamp the source's full issue mask so per-set writeback can
  // release the matching mask-aware reservation. Cheap and harmless in
  // mode 0 (field is unused there).
  temp_inst.set_source_mask_on_sets(active_mask);
  // Mode 2: stamp the source's owning ibuffer half (0 for inter-warp
  // donor's primary half, 1 for intra-warp's secondary half) so per-set
  // writeback can pick the right slot scoreboard.
  temp_inst.set_source_slot_on_sets(split_id != (unsigned)-1 ? 1u : 0u);

  // Mixed-space MEM co-issue (v2): for SHARED coissuers, capture per-lane
  // memreqaddr + temp_inst.cycles into each valid set BEFORE merge. The
  // captured state survives temp_inst's destruction at function exit and
  // is used by per-set shared_cycle + any per-set bank-conflict recompute
  // at retirement. No-op for global coissuers (shared_cycle and the
  // retirement walk gate on source_inst->space). Guarded on is_load ||
  // is_store to skip non-memory ops.
  if ((temp_inst.is_load() || temp_inst.is_store()) &&
      (next_inst->space.get_type() == shared_space ||
       next_inst->space.get_type() == sstarr_space)) {
    unsigned set_width = m_config->simd_set_width;
    std::vector<simd_set_info> &sets = temp_inst.get_simd_sets_mutable();
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid) continue;
      temp_inst.capture_per_set_shared_state(sets[s], set_width);
    }
  }

  // For MEM co-issue: splice temp_inst's already-coalesced mem accesses
  // into the composite, stamping each with source (warp_id, split_id) so
  // the LDST unit routes pending_writes + scoreboard release back to the
  // originating warp/split. This must happen BEFORE merge_simd_sets
  // because we also need the temp_inst's own simd_sets to still carry
  // valid flags here; but the accessq itself is independent.
  if (temp_inst.is_load() || temp_inst.is_store()) {
    unsigned src_wid = warp_id;
    unsigned src_split = split_id;  // -1 for inter-warp
    while (!temp_inst.accessq_empty()) {
      mem_access_t acc = temp_inst.accessq_back();
      acc.set_source_wid(src_wid);
      acc.set_source_split_id(src_split);
      composite->inject_mem_acccesses(acc);
      temp_inst.accessq_pop_back();
    }
  }

  // Merge the co-issued instruction's valid sets into the composite
  composite->merge_simd_sets(temp_inst.get_simd_sets());

  // Use max latency across all co-issued instructions for result bus reservation
  if (temp_inst.latency > composite->latency) {
    composite->latency = temp_inst.latency;
  }
}

void shader_core_ctx::issue() {
  // Ensure fair round robin issu between schedulers
  unsigned j;
  for (unsigned i = 0; i < schedulers.size(); i++) {
    j = (Issue_Prio + i) % schedulers.size();
    schedulers[j]->cycle();
  }
  Issue_Prio = (Issue_Prio + 1) % schedulers.size();

  // really is issue;
  // for (unsigned i = 0; i < schedulers.size(); i++) {
  //    schedulers[i]->cycle();
  //}
}

shd_warp_t &scheduler_unit::warp(int i) { return *((*m_warp)[i]); }

/**
 * A general function to order things in a Loose Round Robin way. The simplist
 * use of this function would be to implement a loose RR scheduler between all
 * the warps assigned to this core. A more sophisticated usage would be to order
 * a set of "fetch groups" in a RR fashion. In the first case, the templated
 * class variable would be a simple unsigned int representing the warp_id.  In
 * the 2lvl case, T could be a struct or a list representing a set of warp_ids.
 * @param result_list: The resultant list the caller wants returned.  This list
 * is cleared and then populated in a loose round robin way
 * @param input_list: The list of things that should be put into the
 * result_list. For a simple scheduler this can simply be the m_supervised_warps
 * list.
 * @param last_issued_from_input:  An iterator pointing the last member in the
 * input_list that issued. Since this function orders in a RR fashion, the
 * object pointed to by this iterator will be last in the prioritization list
 * @param num_warps_to_add: The number of warps you want the scheudler to pick
 * between this cycle. Normally, this will be all the warps availible on the
 * core, i.e. m_supervised_warps.size(). However, a more sophisticated scheduler
 * may wish to limit this number. If the number if < m_supervised_warps.size(),
 * then only the warps with highest RR priority will be placed in the
 * result_list.
 */
template <class T>
void scheduler_unit::order_lrr(
    std::vector<T> &result_list, const typename std::vector<T> &input_list,
    const typename std::vector<T>::const_iterator &last_issued_from_input,
    unsigned num_warps_to_add) {
  assert(num_warps_to_add <= input_list.size());
  result_list.clear();
  typename std::vector<T>::const_iterator iter =
      (last_issued_from_input == input_list.end()) ? input_list.begin()
                                                   : last_issued_from_input + 1;

  for (unsigned count = 0; count < num_warps_to_add; ++iter, ++count) {
    if (iter == input_list.end()) {
      iter = input_list.begin();
    }
    result_list.push_back(*iter);
  }
}

template <class T>
void scheduler_unit::order_rrr(
    std::vector<T> &result_list, const typename std::vector<T> &input_list,
    const typename std::vector<T>::const_iterator &last_issued_from_input,
    unsigned num_warps_to_add) {
  result_list.clear();

  if (m_num_issued_last_cycle > 0 || warp(m_current_turn_warp).done_exit() ||
      warp(m_current_turn_warp).waiting()) {
    std::vector<shd_warp_t *>::const_iterator iter =
        (last_issued_from_input == input_list.end())
            ? input_list.begin()
            : last_issued_from_input + 1;
    for (unsigned count = 0; count < num_warps_to_add; ++iter, ++count) {
      if (iter == input_list.end()) {
        iter = input_list.begin();
      }
      unsigned warp_id = (*iter)->get_warp_id();
      if (!(*iter)->done_exit() && !(*iter)->waiting()) {
        result_list.push_back(*iter);
        m_current_turn_warp = warp_id;
        break;
      }
    }
  } else {
    result_list.push_back(&warp(m_current_turn_warp));
  }
}
/**
 * A general function to order things in an priority-based way.
 * The core usage of the function is similar to order_lrr.
 * The explanation of the additional parameters (beyond order_lrr) explains the
 * further extensions.
 * @param ordering: An enum that determines how the age function will be treated
 * in prioritization see the definition of OrderingType.
 * @param priority_function: This function is used to sort the input_list.  It
 * is passed to stl::sort as the sorting fucntion. So, if you wanted to sort a
 * list of integer warp_ids with the oldest warps having the most priority, then
 * the priority_function would compare the age of the two warps.
 */
template <class T>
void scheduler_unit::order_by_priority(
    std::vector<T> &result_list, const typename std::vector<T> &input_list,
    const typename std::vector<T>::const_iterator &last_issued_from_input,
    unsigned num_warps_to_add, OrderingType ordering,
    bool (*priority_func)(T lhs, T rhs)) {
  assert(num_warps_to_add <= input_list.size());
  result_list.clear();
  typename std::vector<T> temp = input_list;

  if (ORDERING_GREEDY_THEN_PRIORITY_FUNC == ordering) {
    T greedy_value = *last_issued_from_input;
    result_list.push_back(greedy_value);

    std::sort(temp.begin(), temp.end(), priority_func);
    typename std::vector<T>::iterator iter = temp.begin();
    for (unsigned count = 0; count < num_warps_to_add; ++count, ++iter) {
      if (*iter != greedy_value) {
        result_list.push_back(*iter);
      }
    }
  } else if (ORDERED_PRIORITY_FUNC_ONLY == ordering) {
    std::sort(temp.begin(), temp.end(), priority_func);
    typename std::vector<T>::iterator iter = temp.begin();
    for (unsigned count = 0; count < num_warps_to_add; ++count, ++iter) {
      result_list.push_back(*iter);
    }
  } else {
    fprintf(stderr, "Unknown ordering - %d\n", ordering);
    abort();
  }
}

// ---------------------------------------------------------------------------
// Change 4: co-issue scheduler helpers.
// ---------------------------------------------------------------------------

bool scheduler_unit::mem_coissue_disallowed(const warp_inst_t *inst) const {
  if (!inst) return true;
  // Exclude TENSOR_CORE loads/stores (quirky pipeline), memory barriers,
  // LDGSTS, and BRU spill/fill. These remain MEM-bucketed at classify time
  // but are rejected from co-issue.
  if (inst->op == TENSOR_CORE_LOAD_OP || inst->op == TENSOR_CORE_STORE_OP)
    return true;
  if (inst->op == MEMORY_BARRIER_OP) return true;
  if (inst->m_is_ldgsts) return true;
  if (inst->is_bru_st_spill_request() || inst->is_bru_rt_spill_request() ||
      inst->is_bru_st_fill_request() || inst->is_bru_rt_fill_request())
    return true;
  // Exclude atomics (decrement_atomic_count ties to a single warp).
  // Mixed-space MEM co-issue (v2): allow shared_space + sstarr_space in
  // addition to the original global/local/param_local. Cross-bucket
  // composites are rejected at the candidate-feasibility check in
  // try_utilization_max_coissue (same_space only). MEMCOV2_DISABLE_MIXED=1
  // tightens to v1 (global/local only) for A/B testing.
  if (inst->isatomic()) return true;
  static const bool disable_shared =
      (getenv("MEMCOV2_DISABLE_MIXED") != NULL);
  memory_space_t space = inst->space;
  enum _memory_space_t t = space.get_type();
  bool in_global_bucket = warp_inst_t::is_global_bucket_space(t);
  bool in_shared_bucket = warp_inst_t::is_shared_bucket_space(t);
  if (!in_global_bucket && (disable_shared || !in_shared_bucket))
    return true;
  return false;
}

exec_unit_type_t scheduler_unit::classify_fu_type(
    const warp_inst_t *inst) const {
  if ((inst->op == LOAD_OP) || (inst->op == STORE_OP) ||
      (inst->op == MEMORY_BARRIER_OP) ||
      (inst->op == TENSOR_CORE_LOAD_OP) ||
      (inst->op == TENSOR_CORE_STORE_OP)) {
    return exec_unit_type_t::MEM;
  } else if (inst->op == SP_OP ||
             (inst->op != DP_OP && inst->op != SFU_OP &&
              inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP &&
              inst->op < SPEC_UNIT_START_ID &&
              m_shader->m_config->gpgpu_num_int_units == 0)) {
    return exec_unit_type_t::SP;
  } else if (inst->op != SP_OP && inst->op != DP_OP && inst->op != SFU_OP &&
             inst->op != ALU_SFU_OP && inst->op != TENSOR_CORE_OP &&
             inst->op < SPEC_UNIT_START_ID &&
             m_shader->m_config->gpgpu_num_int_units > 0) {
    return exec_unit_type_t::INT;
  } else if (inst->op == DP_OP) {
    return exec_unit_type_t::DP;
  } else if (inst->op == SFU_OP || inst->op == ALU_SFU_OP) {
    return exec_unit_type_t::SFU;
  } else if (inst->op == TENSOR_CORE_OP) {
    return exec_unit_type_t::TENSOR;
  } else if (inst->op >= SPEC_UNIT_START_ID) {
    return exec_unit_type_t::SPECIALIZED;
  }
  return exec_unit_type_t::NONE;
}

bool scheduler_unit::check_coissue_feasibility(
    const warp_inst_t *composite, const warp_inst_t *cand_inst,
    const active_mask_t &cand_mask, unsigned cand_warp_id,
    unsigned available_sets, unsigned *sets_needed) {
  const unsigned num_sets = m_shader->m_config->gpgpu_num_simd_sets;
  const unsigned set_width = m_shader->m_config->simd_set_width;

  if (m_shader->m_config->gpgpu_compaction_mode == 2) {
    // Full compaction: just count sets needed; placement starts at
    // next_free_set so no overlap is possible.
    unsigned active = cand_mask.count();
    unsigned needed = (active + set_width - 1) / set_width;
    if (needed > available_sets) return false;
    *sets_needed = needed;
    return true;
  }

  // No compaction or XOR-static: build temp sets with the selected
  // mapping and check set-level overlap vs. the composite.
  warp_inst_t cand_temp(m_shader->m_config);
  cand_temp = *cand_inst;
  cand_temp.issue(cand_mask, cand_warp_id, 0, 0, 0, 0);
  if (m_shader->m_config->gpgpu_compaction_mode == 1) {
    cand_temp.compute_simd_sets_xor_static(num_sets, set_width);
  } else {
    cand_temp.compute_simd_sets(num_sets, set_width);
  }
  if (warp_inst_t::simd_sets_overlap(composite->get_simd_sets(),
                                      cand_temp.get_simd_sets()))
    return false;
  unsigned needed = cand_temp.num_active_simd_sets();
  if (needed > available_sets) return false;
  *sets_needed = needed;
  return true;
}

void scheduler_unit::try_inter_warp_coissue(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set, unsigned sort_mode) {
  // Build a sortable list of iterators into m_next_cycle_prioritized_warps.
  // Index 0..N-1 into a snapshot vector for deterministic ordering.
  std::vector<shd_warp_t *> cand_warps;
  cand_warps.reserve(m_next_cycle_prioritized_warps.size());
  for (std::vector<shd_warp_t *>::const_iterator iter =
           m_next_cycle_prioritized_warps.begin();
       iter != m_next_cycle_prioritized_warps.end(); iter++) {
    if (*iter) cand_warps.push_back(*iter);
  }

  if (sort_mode == 1) {
    // fewest-lanes: sort by popcount(active_mask) ascending. For
    // candidates without a ready inst, sort them last (treat as +inf).
    std::stable_sort(cand_warps.begin(), cand_warps.end(),
                     [this](shd_warp_t *a, shd_warp_t *b) {
                       const warp_inst_t *ia = a->ibuffer_next_inst();
                       const warp_inst_t *ib = b->ibuffer_next_inst();
                       unsigned pa = ia ? m_shader
                                              ->get_active_mask(
                                                  a->get_warp_id(), ia)
                                              .count()
                                        : 33u;
                       unsigned pb = ib ? m_shader
                                              ->get_active_mask(
                                                  b->get_warp_id(), ib)
                                              .count()
                                        : 33u;
                       return pa < pb;
                     });
  }

  // For sort_mode==2 (same-PC), do two passes: same-PC first, then
  // arbitrary. For sort_mode==0/1, one pass over cand_warps.
  unsigned num_passes = (sort_mode == 2) ? 2 : 1;
  unsigned primary_pc = 0;
  if (sort_mode == 2) primary_pc = co_issue_composite->pc;

  // Track warps already picked so pass-1 of same-PC mode doesn't revisit
  // them (they've already had ibuffer_step'd; a second visit would
  // co-issue a second instruction from the same warp in one scheduler
  // cycle, which can lead to deadlocks).
  std::set<unsigned> picked_warps;

  for (unsigned pass = 0; pass < num_passes && available_sets > 0; pass++) {
    for (std::vector<shd_warp_t *>::iterator iter2 = cand_warps.begin();
         iter2 != cand_warps.end() && available_sets > 0; iter2++) {
      // Count every candidate-warp iteration as "seen" so per-filter
      // skip counts are interpretable as a fraction.
      m_stats->coissue_cand_seen[get_sid()]++;
      if ((*iter2) == NULL) {
        m_stats->coissue_skipped_null[get_sid()]++;
        continue;
      }
      if ((*iter2)->done_exit()) {
        m_stats->coissue_skipped_done_exit[get_sid()]++;
        continue;
      }
      unsigned cand_warp_id = (*iter2)->get_warp_id();

      // Skip the primary warp (already issued).
      if (cand_warp_id == co_issue_primary_warp_id) {
        m_stats->coissue_skipped_is_primary[get_sid()]++;
        continue;
      }
      if (picked_warps.count(cand_warp_id)) {
        m_stats->coissue_skipped_picked_earlier[get_sid()]++;
        continue;
      }

      // Basic eligibility.
      if (warp(cand_warp_id).ibuffer_empty()) {
        m_stats->coissue_skipped_ibuffer_empty[get_sid()]++;
        continue;
      }
      if (warp(cand_warp_id).waiting()) {
        m_stats->coissue_skipped_waiting[get_sid()]++;
        continue;
      }

      // AWARE reconvergence eligibility.
      bool simt_ok = true;
      if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
        simt_ok = warp(cand_warp_id).valid() &&
                  !warp(cand_warp_id).blocked() &&
                  !warp(cand_warp_id).pending_reconvergence() &&
                  !warp(cand_warp_id).virtualized();
      }
      if (!simt_ok) {
        m_stats->coissue_skipped_simt_blocked[get_sid()]++;
        continue;
      }
      // Mode 2 (slot-pinned scoreboard): if the candidate warp has a
      // pending FIFO-membership-change drain, refuse to inter-coissue
      // with it. Otherwise an instruction fetched pre-divergence (with
      // a now-stale broader mask) could issue with the post-divergence
      // narrower active mask, hitting cross-slot RAWs against pending
      // writes from the broader mask. Same gate as the primary issue
      // path's drain check, applied to the candidate.
      if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
          m_shader->m_config->model == AWARE_RECONVERGENCE &&
          m_shader->get_simt_tables(cand_warp_id)
              ->div_recv_drain_pending()) {
        if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
            !m_scoreboard->slotClean(cand_warp_id, 1)) {
          continue;
        }
        m_shader->get_simt_tables(cand_warp_id)
            ->clear_div_recv_drain_pending();
      }

      const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
      if (!cand_inst) {
        m_stats->coissue_skipped_no_inst[get_sid()]++;
        continue;
      }
      if (!warp(cand_warp_id).ibuffer_next_valid()) {
        m_stats->coissue_skipped_no_inst[get_sid()]++;
        continue;
      }

      // Control hazard check.
      unsigned cand_pc, cand_rpc;
      m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
                                        &cand_rpc);
      if (cand_pc != cand_inst->pc) {
        m_stats->coissue_skipped_pc_mismatch[get_sid()]++;
        continue;
      }

      // Scoreboard check.
      // Mode 1 needs the candidate's active mask for hazard intersection.
      // Mode 2 checks the donor warp's slot 0 (its primary half).
      bool sb_collision_inter;
      if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
        const active_mask_t &cand_mask_pre =
            m_shader->get_active_mask(cand_warp_id, cand_inst);
        sb_collision_inter = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                              cand_inst,
                                                              cand_mask_pre);
      } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
        sb_collision_inter = m_scoreboard->checkCollisionSlot(
            cand_warp_id, /*slot=*/0, cand_inst);
      } else {
        sb_collision_inter = m_scoreboard->checkCollision(cand_warp_id,
                                                          cand_inst);
      }
      if (sb_collision_inter) {
        m_stats->coissue_denied_by_scoreboard[get_sid()]++;
        continue;
      }

      // FU type match.
      if (classify_fu_type(cand_inst) != co_issue_fu_type) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        continue;
      }

      // MEM co-issue safety filter.
      if (co_issue_fu_type == exec_unit_type_t::MEM &&
          mem_coissue_disallowed(cand_inst)) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
      // Same-bucket (Stage 4): primary and candidate must share bucket.
      // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging.
      if (co_issue_fu_type == exec_unit_type_t::MEM) {
        static const bool allow_mixed =
            (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
        enum _memory_space_t pspace = co_issue_composite->space.get_type();
        enum _memory_space_t cspace = cand_inst->space.get_type();
        bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
        bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
        if (!allow_mixed && p_shared != c_shared) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          m_stats->mem_coissue_denied_filter[get_sid()]++;
          continue;
        }
      }

      // same-PC mode: pass 0 requires pc==primary_pc; pass 1 takes any.
      if (sort_mode == 2 && pass == 0 && cand_inst->pc != primary_pc) {
        m_stats->coissue_skipped_samepc_pass0[get_sid()]++;
        continue;
      }

      const active_mask_t &cand_mask =
          m_shader->get_active_mask(cand_warp_id, cand_inst);

      unsigned cand_sets_needed = 0;
      if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
                                      cand_warp_id, available_sets,
                                      &cand_sets_needed)) {
        m_stats->coissue_denied_by_no_sets[get_sid()]++;
        continue;
      }

      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
            "CO-ISSUED with primary warp %u (%u sets)\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, cand_warp_id, co_issue_primary_warp_id,
            cand_sets_needed);
      }

      m_shader->co_issue_warp(co_issue_composite, cand_inst, cand_mask,
                              cand_warp_id, m_id, next_free_set);

      available_sets -= cand_sets_needed;
      next_free_set += cand_sets_needed;
      warp(cand_warp_id).ibuffer_step();
      picked_warps.insert(cand_warp_id);
      m_stats->inter_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_inter_events[get_sid()]++;
    }
  }
}

void scheduler_unit::try_intra_warp_coissue(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set) {
  // Only meaningful under AWARE_RECONVERGENCE (where secondary I-buf
  // slots exist at all).
  if (m_shader->m_config->model != AWARE_RECONVERGENCE) return;

  const unsigned set_width = m_shader->m_config->simd_set_width;
  const unsigned primary_warp_id = co_issue_primary_warp_id;

  for (unsigned sec_slot = 2; sec_slot < 4 && available_sets > 0;
       sec_slot++) {
    if (!warp(primary_warp_id).ibuffer_slot_valid(sec_slot)) continue;

    const warp_inst_t *sec_inst =
        warp(primary_warp_id).ibuffer_slot_inst(sec_slot);
    if (!sec_inst) continue;

    unsigned sec_split_id =
        warp(primary_warp_id).ibuffer_slot_split_id(sec_slot);
    const active_mask_t &sec_mask =
        warp(primary_warp_id).ibuffer_slot_split_mask(sec_slot);

    if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
      warp(primary_warp_id).ibuffer_flush_half(1);
      m_scoreboard->clearSecondary(primary_warp_id);
      break;
    }

    unsigned split_pc;
    simt_mask_t split_mask;
    m_shader->get_split_info(primary_warp_id, sec_split_id, &split_pc,
                              &split_mask);
    if (split_pc != sec_inst->pc) {
      warp(primary_warp_id).ibuffer_flush_half(1);
      m_scoreboard->clearSecondary(primary_warp_id);
      break;
    }

    if (classify_fu_type(sec_inst) != co_issue_fu_type) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      continue;
    }

    // MEM co-issue safety filter.
    if (co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(sec_inst)) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      m_stats->mem_coissue_denied_filter[get_sid()]++;
      continue;
    }
    // Same-bucket (Stage 4): primary and candidate must share bucket.
    // MEMCOV2_ALLOW_MIXED=1 bypasses this filter for Stage 5 debugging.
    if (co_issue_fu_type == exec_unit_type_t::MEM) {
      static const bool allow_mixed =
          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
      enum _memory_space_t pspace = co_issue_composite->space.get_type();
      enum _memory_space_t cspace = sec_inst->space.get_type();
      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
      if (!allow_mixed && p_shared != c_shared) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
    }

    // Stale-mask guard: see commentary on the same fix in
    // try_utilization_max_coissue. Cached `sec_mask` may diverge from
    // current `split_mask` (lane re-bucketing or split-ID reuse). Dispatch
    // only the intersection — lanes in BOTH masks are guaranteed to have
    // their per-thread PC at sec_inst->pc. Hoisted before the scoreboard
    // check so mode 1 can use the effective mask for hazard detection.
    active_mask_t sec_mask_eff;
    for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
      if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t);
    }
    if (!sec_mask_eff.any()) continue;

    bool sb_collision;
    if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
      sb_collision = m_scoreboard->checkCollisionMask(primary_warp_id,
                                                      sec_inst, sec_mask_eff);
    } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
      // Intra-warp candidates check primary warp's slot 1 (secondary half).
      sb_collision = m_scoreboard->checkCollisionSlot(primary_warp_id,
                                                      /*slot=*/1, sec_inst);
    } else {
      sb_collision = m_scoreboard->checkCollisionSecondary(primary_warp_id,
                                                           sec_inst);
    }
    if (sb_collision) {
      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
      continue;
    }
    unsigned sec_active = sec_mask_eff.count();
    unsigned sec_sets_needed = (sec_active + set_width - 1) / set_width;
    if (sec_sets_needed > available_sets) {
      m_stats->coissue_denied_by_no_sets[get_sid()]++;
      continue;
    }

    if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
      printf(
          "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
          "split %u CO-ISSUED with primary split (%u sets)\n",
          m_shader->get_gpu()->gpu_sim_cycle +
              m_shader->get_gpu()->gpu_tot_sim_cycle,
          get_sid(), m_id, primary_warp_id, sec_split_id, sec_sets_needed);
    }

    m_shader->co_issue_warp(co_issue_composite, sec_inst, sec_mask_eff,
                            primary_warp_id, m_id, next_free_set,
                            sec_split_id);

    available_sets -= sec_sets_needed;
    next_free_set += sec_sets_needed;
    m_stats->intra_warp_coissue_events[get_sid()]++;
    if (co_issue_fu_type == exec_unit_type_t::MEM)
      m_stats->mem_coissue_intra_events[get_sid()]++;

    warp(primary_warp_id).ibuffer_free_slot(sec_slot);

    if (!m_shader->is_split_valid(primary_warp_id, sec_split_id)) {
      warp(primary_warp_id).ibuffer_flush_half(1);
      m_scoreboard->clearSecondary(primary_warp_id);
      break;
    }

    // At most one intra coissue per primary per cycle. Mirror the
    // dup_with_warp invariant enforced in try_utilization_max_coissue
    // (see commentary at the intra scan of that function): issuing two
    // AWARE updates against the same warp in one cycle is unsafe — the
    // first co_issue_warp call mutates the splits-table, and the second
    // candidate's cached split_id can refer to a stale split by then,
    // leading to instructions whose scoreboard reservations are never
    // released (deadlock observed under bfs-wlw with priority=1/2).
    break;
  }
}

void scheduler_unit::try_utilization_max_coissue(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set) {
  // Build a unified pool of {inter, intra} candidates that each pass
  // eligibility filters, then sort by density desc and greedy-pack.
  struct coissue_cand_t {
    bool is_intra;
    unsigned warp_id;
    unsigned split_id;      // valid iff is_intra
    const warp_inst_t *inst;
    active_mask_t mask;
    unsigned sec_slot;      // valid iff is_intra (2 or 3)
    unsigned active_count;
    unsigned sets_needed;
  };

  std::vector<coissue_cand_t> pool;
  const unsigned set_width = m_shader->m_config->simd_set_width;

  // --- Inter-warp candidates ---
  for (std::vector<shd_warp_t *>::const_iterator iter =
           m_next_cycle_prioritized_warps.begin();
       iter != m_next_cycle_prioritized_warps.end(); iter++) {
    if ((*iter) == NULL || (*iter)->done_exit()) continue;
    unsigned cand_warp_id = (*iter)->get_warp_id();
    if (cand_warp_id == co_issue_primary_warp_id) continue;
    if (warp(cand_warp_id).ibuffer_empty()) continue;
    if (warp(cand_warp_id).waiting()) continue;

    bool simt_ok = true;
    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
      simt_ok = warp(cand_warp_id).valid() &&
                !warp(cand_warp_id).blocked() &&
                !warp(cand_warp_id).pending_reconvergence() &&
                !warp(cand_warp_id).virtualized();
    }
    if (!simt_ok) continue;
    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
        m_shader->m_config->model == AWARE_RECONVERGENCE &&
        m_shader->get_simt_tables(cand_warp_id)
            ->div_recv_drain_pending()) {
      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
          !m_scoreboard->slotClean(cand_warp_id, 1)) {
        continue;
      }
      m_shader->get_simt_tables(cand_warp_id)
          ->clear_div_recv_drain_pending();
    }

    const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
    if (!cand_inst) continue;
    if (!warp(cand_warp_id).ibuffer_next_valid()) continue;

    unsigned cand_pc, cand_rpc;
    m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
                                      &cand_rpc);
    if (cand_pc != cand_inst->pc) continue;

    bool sb_collision_inter2;
    if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
      const active_mask_t &cand_mask_pre =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                             cand_inst,
                                                             cand_mask_pre);
    } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
      // OR check: stall on either slot — see commentary at primary
      // issue path.
      bool s0c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/0, cand_inst);
      bool s1c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/1, cand_inst);
      sb_collision_inter2 = s0c || s1c;
      const active_mask_t &cand_mask_for_diag =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      m_scoreboard->m_xslot_inter_coissue_checks++;
      if (!s0c && s1c) {
        m_scoreboard->m_xslot_inter_coissue_would_stall++;
      }
      if (!s0c && !s1c &&
          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
                                              cand_mask_for_diag)) {
        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
      }
    } else {
      sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                         cand_inst);
    }
    if (sb_collision_inter2) {
      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
      continue;
    }
    if (classify_fu_type(cand_inst) != co_issue_fu_type) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(cand_inst)) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      m_stats->mem_coissue_denied_filter[get_sid()]++;
      continue;
    }
    // Mixed-space MEM co-issue (v2, same-bucket only): primary and
    // candidate must be in the same space bucket (shared or global).
    // Cross-bucket composites are disallowed here — Stage 5 will add a
    // mixed-retire path. MEMCOV2_ALLOW_MIXED=1 bypasses for debugging.
    if (co_issue_fu_type == exec_unit_type_t::MEM) {
      static const bool allow_mixed =
          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
      enum _memory_space_t pspace = co_issue_composite->space.get_type();
      enum _memory_space_t cspace = cand_inst->space.get_type();
      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
      if (!allow_mixed && p_shared != c_shared) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
    }

    const active_mask_t &cand_mask =
        m_shader->get_active_mask(cand_warp_id, cand_inst);
    unsigned needed = 0;
    if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
                                    cand_warp_id, available_sets, &needed)) {
      m_stats->coissue_denied_by_no_sets[get_sid()]++;
      continue;
    }

    coissue_cand_t c;
    c.is_intra = false;
    c.warp_id = cand_warp_id;
    c.split_id = (unsigned)-1;
    c.inst = cand_inst;
    c.mask = cand_mask;
    c.sec_slot = 0;
    c.active_count = cand_mask.count();
    c.sets_needed = needed;
    pool.push_back(c);
  }

  // --- Intra-warp candidates (all warps' secondary slots 2..3) ---
  if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
    for (std::vector<shd_warp_t *>::const_iterator iter =
             m_next_cycle_prioritized_warps.begin();
         iter != m_next_cycle_prioritized_warps.end(); iter++) {
      if ((*iter) == NULL || (*iter)->done_exit()) continue;
      unsigned cand_warp_id = (*iter)->get_warp_id();
      if (warp(cand_warp_id).waiting()) continue;

      bool simt_ok = warp(cand_warp_id).valid() &&
                     !warp(cand_warp_id).blocked() &&
                     !warp(cand_warp_id).pending_reconvergence() &&
                     !warp(cand_warp_id).virtualized();
      if (!simt_ok) continue;

      for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) {
        if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue;
        const warp_inst_t *sec_inst =
            warp(cand_warp_id).ibuffer_slot_inst(sec_slot);
        if (!sec_inst) continue;
        unsigned sec_split_id =
            warp(cand_warp_id).ibuffer_slot_split_id(sec_slot);
        const active_mask_t &sec_mask =
            warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot);

        if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue;
        unsigned split_pc;
        simt_mask_t split_mask;
        m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc,
                                  &split_mask);
        if (split_pc != sec_inst->pc) continue;

        // At most one greedy-pool candidate per warp. Two reasons:
        //   (a) Dup with primary half: when the FIFO front split moves to
        //       match a split already cached in this warp's secondary half
        //       (slots 2/3), primary fetch refills slot 0 with the same
        //       (split_id, pc) that the secondary slot already holds. The
        //       inter scan above would have admitted slot 0; admitting the
        //       secondary slot here too double-issues the same warp/split/pc,
        //       advancing per-thread PCs twice and tripping
        //       `pc == inst.pc` in cuda-sim.cc:1797.
        //   (b) Order-of-operations within the pool: when multiple
        //       candidates target the same warp, the earlier one's
        //       co_issue_warp call mutates the warp's splits-table (move,
        //       update, possibly split-id reuse). The later candidate's
        //       cached split_id can refer to a different split by then.
        //       simt_tables::update on the second candidate then asserts
        //       `top_pc == next_inst_pc` because FRONT.pc has changed.
        // Restricting to one candidate per warp keeps the scheduler from
        // issuing two AWARE updates against the same warp in one cycle.
        bool dup_with_warp = false;
        for (unsigned k = 0; k < pool.size(); k++) {
          if (pool[k].warp_id == cand_warp_id) {
            dup_with_warp = true;
            break;
          }
        }
        if (dup_with_warp) continue;
        if (classify_fu_type(sec_inst) != co_issue_fu_type) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM &&
            mem_coissue_disallowed(sec_inst)) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          m_stats->mem_coissue_denied_filter[get_sid()]++;
          continue;
        }
        // Same-bucket check: reject cross-bucket intra candidates.
        // MEMCOV2_ALLOW_MIXED=1 bypasses for Stage 5 debugging.
        if (co_issue_fu_type == exec_unit_type_t::MEM) {
          static const bool allow_mixed =
              (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
          enum _memory_space_t pspace = co_issue_composite->space.get_type();
          enum _memory_space_t cspace = sec_inst->space.get_type();
          bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
          bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
          if (!allow_mixed && p_shared != c_shared) {
            m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
            m_stats->mem_coissue_denied_filter[get_sid()]++;
            continue;
          }
        }
        // Stale-mask guard: the cached `sec_mask` is from secondary-fetch
        // time and may not match the current splits-table mask. Two
        // failure modes:
        //   1. AWARE re-bucketing moves lanes out of this split (mask
        //      shrinks) while keeping the split's PC stable. Cached lanes
        //      that left have advanced their per-thread PC; dispatching
        //      them at the cached PC trips cuda-sim.cc:1797.
        //   2. Split-ID reuse: the original split was invalidated and the
        //      ID slot got recycled for a new split that happens to be at
        //      the same PC. The current mask is from the *new* split; the
        //      cached mask is stale.
        // Both reduce to: trust only lanes present in BOTH masks. The
        // ibuffer slot ownership invariant guarantees lanes in `split_mask`
        // are at `split_pc == sec_inst->pc`. Hoisted before the scoreboard
        // check so mode 1 can use the effective mask for hazard detection.
        active_mask_t sec_mask_eff;
        for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
          if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t);
        }
        if (!sec_mask_eff.any()) continue;

        bool sb_collision;
        if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
          sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                          sec_inst,
                                                          sec_mask_eff);
        } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
          sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id,
                                                          /*slot=*/1,
                                                          sec_inst);
        } else {
          sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id,
                                                               sec_inst);
        }
        if (sb_collision) {
          m_stats->coissue_denied_by_scoreboard[get_sid()]++;
          continue;
        }
        unsigned sec_active = sec_mask_eff.count();
        // Use the same feasibility test as inter coissue. Under
        // compaction != 2 partitions are fixed by thread_id, so the
        // coissuer's set positions can overlap the composite's even
        // when (active/set_width) <= available_sets. merge_simd_sets
        // silently drops overlapping coissuer sets while the access
        // was already injected, leaving m_pending_writes_secondary
        // unincremented but later decremented at writeback (underflow
        // on sb0/sb2, deadlock on sb1).
        unsigned sec_needed = 0;
        if (!check_coissue_feasibility(co_issue_composite, sec_inst,
                                        sec_mask_eff, cand_warp_id,
                                        available_sets, &sec_needed)) {
          m_stats->coissue_denied_by_no_sets[get_sid()]++;
          continue;
        }

        coissue_cand_t c;
        c.is_intra = true;
        c.warp_id = cand_warp_id;
        c.split_id = sec_split_id;
        c.inst = sec_inst;
        c.mask = sec_mask_eff;
        c.sec_slot = sec_slot;
        c.active_count = sec_active;
        c.sets_needed = sec_needed;
        pool.push_back(c);
      }
    }
  }

  // --- Density sort: active_count / sets_needed desc ------------------
  // Integer-safe: a.active * b.sets > b.active * a.sets.
  // Ties broken by active_count desc, then warp_id asc for determinism.
  std::stable_sort(pool.begin(), pool.end(),
                   [](const coissue_cand_t &a, const coissue_cand_t &b) {
                     unsigned long lhs = (unsigned long)a.active_count *
                                         (unsigned long)b.sets_needed;
                     unsigned long rhs = (unsigned long)b.active_count *
                                         (unsigned long)a.sets_needed;
                     if (lhs != rhs) return lhs > rhs;
                     if (a.active_count != b.active_count)
                       return a.active_count > b.active_count;
                     return a.warp_id < b.warp_id;
                   });

  // --- Greedy pack ----------------------------------------------------
  for (std::vector<coissue_cand_t>::iterator it = pool.begin();
       it != pool.end() && available_sets > 0; it++) {
    coissue_cand_t &c = *it;
    // Re-check feasibility: prior picks may have filled sets that this
    // candidate overlaps (only matters for compaction_mode != 2). Both
    // intra and inter must re-evaluate against the *current* composite
    // — the cached c.sets_needed from admission can be stale once an
    // earlier pool entry has merged its sets in.
    unsigned needed = 0;
    if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask,
                                    c.warp_id, available_sets, &needed))
      continue;

    if (c.is_intra) {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
            "split %u CO-ISSUED with primary split (%u sets)\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, c.split_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set, c.split_id);

      available_sets -= needed;
      next_free_set += needed;
      m_stats->intra_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_intra_events[get_sid()]++;
      warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
      if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
        warp(c.warp_id).ibuffer_flush_half(1);
        m_scoreboard->clearSecondary(c.warp_id);
        // The whole secondary half was just flushed — skip any remaining
        // intra-warp picks for this warp. Inter-warp picks from other
        // warps can still proceed.
        for (std::vector<coissue_cand_t>::iterator jt = it + 1;
             jt != pool.end();) {
          if (jt->is_intra && jt->warp_id == c.warp_id)
            jt = pool.erase(jt);
          else
            ++jt;
        }
      }
    } else {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
            "CO-ISSUED with primary warp %u (%u sets)\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set);
      available_sets -= needed;
      next_free_set += needed;
      m_stats->inter_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_inter_events[get_sid()]++;
      warp(c.warp_id).ibuffer_step();
    }
  }
}

void scheduler_unit::try_utilization_max_coissue_window(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set) {
  // K-bounded clone of try_utilization_max_coissue. Both candidate scan
  // loops cap iteration at the first K warps in priority order, where
  // K = gpgpu_num_simd_sets. Models a HW priority encoder that can only
  // poll K warps' ibuffer slots per cycle, not all in-flight warps.
  struct coissue_cand_t {
    bool is_intra;
    unsigned warp_id;
    unsigned split_id;      // valid iff is_intra
    const warp_inst_t *inst;
    active_mask_t mask;
    unsigned sec_slot;      // valid iff is_intra (2 or 3)
    unsigned active_count;
    unsigned sets_needed;
  };

  std::vector<coissue_cand_t> pool;
  const unsigned set_width = m_shader->m_config->simd_set_width;
  // Window size: explicit override if non-zero, else fall back to the
  // number of SIMD sets. Lets us model K != num_simd_sets without
  // disturbing the partitioning math (which requires warp_size %
  // num_simd_sets == 0).
  const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0)
                         ? m_shader->m_config->gpgpu_coissue_window_K
                         : m_shader->m_config->gpgpu_num_simd_sets;

  // Bound the iteration to the first K warps (or fewer if the prioritized
  // list is shorter). Computed once and reused for both scan passes.
  std::vector<shd_warp_t *>::const_iterator wlist_end =
      m_next_cycle_prioritized_warps.begin();
  std::advance(
      wlist_end,
      std::min((size_t)K, m_next_cycle_prioritized_warps.size()));

  // --- Inter-warp candidates (first K warps' slot 0) ---
  for (std::vector<shd_warp_t *>::const_iterator iter =
           m_next_cycle_prioritized_warps.begin();
       iter != wlist_end; iter++) {
    if ((*iter) == NULL || (*iter)->done_exit()) continue;
    unsigned cand_warp_id = (*iter)->get_warp_id();
    if (cand_warp_id == co_issue_primary_warp_id) continue;
    if (warp(cand_warp_id).ibuffer_empty()) continue;
    if (warp(cand_warp_id).waiting()) continue;

    bool simt_ok = true;
    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
      simt_ok = warp(cand_warp_id).valid() &&
                !warp(cand_warp_id).blocked() &&
                !warp(cand_warp_id).pending_reconvergence() &&
                !warp(cand_warp_id).virtualized();
    }
    if (!simt_ok) continue;
    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
        m_shader->m_config->model == AWARE_RECONVERGENCE &&
        m_shader->get_simt_tables(cand_warp_id)
            ->div_recv_drain_pending()) {
      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
          !m_scoreboard->slotClean(cand_warp_id, 1)) {
        continue;
      }
      m_shader->get_simt_tables(cand_warp_id)
          ->clear_div_recv_drain_pending();
    }

    const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
    if (!cand_inst) continue;
    if (!warp(cand_warp_id).ibuffer_next_valid()) continue;

    unsigned cand_pc, cand_rpc;
    m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
                                      &cand_rpc);
    if (cand_pc != cand_inst->pc) continue;

    bool sb_collision_inter2;
    if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
      const active_mask_t &cand_mask_pre =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                             cand_inst,
                                                             cand_mask_pre);
    } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
      // OR check: stall on either slot — see commentary at primary
      // issue path.
      bool s0c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/0, cand_inst);
      bool s1c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/1, cand_inst);
      sb_collision_inter2 = s0c || s1c;
      const active_mask_t &cand_mask_for_diag =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      m_scoreboard->m_xslot_inter_coissue_checks++;
      if (!s0c && s1c) {
        m_scoreboard->m_xslot_inter_coissue_would_stall++;
      }
      if (!s0c && !s1c &&
          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
                                              cand_mask_for_diag)) {
        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
      }
    } else {
      sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                         cand_inst);
    }
    if (sb_collision_inter2) {
      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
      continue;
    }
    if (classify_fu_type(cand_inst) != co_issue_fu_type) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(cand_inst)) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      m_stats->mem_coissue_denied_filter[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM) {
      static const bool allow_mixed =
          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
      enum _memory_space_t pspace = co_issue_composite->space.get_type();
      enum _memory_space_t cspace = cand_inst->space.get_type();
      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
      if (!allow_mixed && p_shared != c_shared) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
    }

    const active_mask_t &cand_mask =
        m_shader->get_active_mask(cand_warp_id, cand_inst);
    unsigned needed = 0;
    if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
                                    cand_warp_id, available_sets, &needed)) {
      m_stats->coissue_denied_by_no_sets[get_sid()]++;
      continue;
    }

    coissue_cand_t c;
    c.is_intra = false;
    c.warp_id = cand_warp_id;
    c.split_id = (unsigned)-1;
    c.inst = cand_inst;
    c.mask = cand_mask;
    c.sec_slot = 0;
    c.active_count = cand_mask.count();
    c.sets_needed = needed;
    pool.push_back(c);
  }

  // --- Intra-warp candidates (first K warps' secondary slots 2..3) ---
  if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
    for (std::vector<shd_warp_t *>::const_iterator iter =
             m_next_cycle_prioritized_warps.begin();
         iter != wlist_end; iter++) {
      if ((*iter) == NULL || (*iter)->done_exit()) continue;
      unsigned cand_warp_id = (*iter)->get_warp_id();
      if (warp(cand_warp_id).waiting()) continue;

      bool simt_ok = warp(cand_warp_id).valid() &&
                     !warp(cand_warp_id).blocked() &&
                     !warp(cand_warp_id).pending_reconvergence() &&
                     !warp(cand_warp_id).virtualized();
      if (!simt_ok) continue;

      for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) {
        if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue;
        const warp_inst_t *sec_inst =
            warp(cand_warp_id).ibuffer_slot_inst(sec_slot);
        if (!sec_inst) continue;
        unsigned sec_split_id =
            warp(cand_warp_id).ibuffer_slot_split_id(sec_slot);
        const active_mask_t &sec_mask =
            warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot);

        if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue;
        unsigned split_pc;
        simt_mask_t split_mask;
        m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc,
                                  &split_mask);
        if (split_pc != sec_inst->pc) continue;

        bool dup_with_warp = false;
        for (unsigned k = 0; k < pool.size(); k++) {
          if (pool[k].warp_id == cand_warp_id) {
            dup_with_warp = true;
            break;
          }
        }
        if (dup_with_warp) continue;
        if (classify_fu_type(sec_inst) != co_issue_fu_type) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM &&
            mem_coissue_disallowed(sec_inst)) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          m_stats->mem_coissue_denied_filter[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM) {
          static const bool allow_mixed =
              (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
          enum _memory_space_t pspace = co_issue_composite->space.get_type();
          enum _memory_space_t cspace = sec_inst->space.get_type();
          bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
          bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
          if (!allow_mixed && p_shared != c_shared) {
            m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
            m_stats->mem_coissue_denied_filter[get_sid()]++;
            continue;
          }
        }
        active_mask_t sec_mask_eff;
        for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
          if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t);
        }
        if (!sec_mask_eff.any()) continue;

        bool sb_collision;
        if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
          sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                          sec_inst,
                                                          sec_mask_eff);
        } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
          sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id,
                                                          /*slot=*/1,
                                                          sec_inst);
        } else {
          sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id,
                                                               sec_inst);
        }
        if (sb_collision) {
          m_stats->coissue_denied_by_scoreboard[get_sid()]++;
          continue;
        }
        unsigned sec_active = sec_mask_eff.count();
        // Use the same feasibility test as inter coissue. Under
        // compaction != 2 partitions are fixed by thread_id, so the
        // coissuer's set positions can overlap the composite's even
        // when (active/set_width) <= available_sets. merge_simd_sets
        // silently drops overlapping coissuer sets while the access
        // was already injected, leaving m_pending_writes_secondary
        // unincremented but later decremented at writeback (underflow
        // on sb0/sb2, deadlock on sb1).
        unsigned sec_needed = 0;
        if (!check_coissue_feasibility(co_issue_composite, sec_inst,
                                        sec_mask_eff, cand_warp_id,
                                        available_sets, &sec_needed)) {
          m_stats->coissue_denied_by_no_sets[get_sid()]++;
          continue;
        }

        coissue_cand_t c;
        c.is_intra = true;
        c.warp_id = cand_warp_id;
        c.split_id = sec_split_id;
        c.inst = sec_inst;
        c.mask = sec_mask_eff;
        c.sec_slot = sec_slot;
        c.active_count = sec_active;
        c.sets_needed = sec_needed;
        pool.push_back(c);
      }
    }
  }

  // --- Density sort: active_count / sets_needed desc ------------------
  std::stable_sort(pool.begin(), pool.end(),
                   [](const coissue_cand_t &a, const coissue_cand_t &b) {
                     unsigned long lhs = (unsigned long)a.active_count *
                                         (unsigned long)b.sets_needed;
                     unsigned long rhs = (unsigned long)b.active_count *
                                         (unsigned long)a.sets_needed;
                     if (lhs != rhs) return lhs > rhs;
                     if (a.active_count != b.active_count)
                       return a.active_count > b.active_count;
                     return a.warp_id < b.warp_id;
                   });

  // --- Greedy pack ----------------------------------------------------
  for (std::vector<coissue_cand_t>::iterator it = pool.begin();
       it != pool.end() && available_sets > 0; it++) {
    coissue_cand_t &c = *it;
    unsigned needed = 0;
    if (c.is_intra) {
      if (c.sets_needed > available_sets) continue;
      needed = c.sets_needed;
    } else {
      if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask,
                                      c.warp_id, available_sets, &needed))
        continue;
    }

    if (c.is_intra) {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
            "split %u CO-ISSUED with primary split (%u sets) [window]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, c.split_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set, c.split_id);

      available_sets -= needed;
      next_free_set += needed;
      m_stats->intra_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_intra_events[get_sid()]++;
      warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
      if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
        warp(c.warp_id).ibuffer_flush_half(1);
        m_scoreboard->clearSecondary(c.warp_id);
        for (std::vector<coissue_cand_t>::iterator jt = it + 1;
             jt != pool.end();) {
          if (jt->is_intra && jt->warp_id == c.warp_id)
            jt = pool.erase(jt);
          else
            ++jt;
        }
      }
    } else {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
            "CO-ISSUED with primary warp %u (%u sets) [window]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set);
      available_sets -= needed;
      next_free_set += needed;
      m_stats->inter_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_inter_events[get_sid()]++;
      warp(c.warp_id).ibuffer_step();
    }
  }
}

void scheduler_unit::try_utilization_max_coissue_window_centered(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set) {
  // Centered K-bounded clone. Both candidate scan loops iterate
  // m_supervised_warps over a window of K positions centered on the
  // primary issuer's index (K/2 before, K/2 after; clamped at array
  // boundaries). Models a HW priority encoder local to the primary's
  // natural warp_id position rather than tied to scheduler priority.
  struct coissue_cand_t {
    bool is_intra;
    unsigned warp_id;
    unsigned split_id;
    const warp_inst_t *inst;
    active_mask_t mask;
    unsigned sec_slot;
    unsigned active_count;
    unsigned sets_needed;
  };

  std::vector<coissue_cand_t> pool;
  const unsigned set_width = m_shader->m_config->simd_set_width;
  const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0)
                         ? m_shader->m_config->gpgpu_coissue_window_K
                         : m_shader->m_config->gpgpu_num_simd_sets;
  const unsigned half = K / 2;
  const size_t n = m_supervised_warps.size();

  // Find primary's index in m_supervised_warps. If not found (shouldn't
  // happen — primary is one of the supervised warps), bail out.
  size_t primary_idx = n;
  for (size_t i = 0; i < n; i++) {
    if (m_supervised_warps[i] != NULL &&
        m_supervised_warps[i]->get_warp_id() == co_issue_primary_warp_id) {
      primary_idx = i;
      break;
    }
  }
  if (primary_idx >= n) return;

  // Symmetric window [primary_idx - half, primary_idx + half]; primary
  // itself is filtered out inside the loop body. Clamped at boundaries.
  const size_t lo =
      (primary_idx > half) ? (primary_idx - half) : (size_t)0;
  const size_t hi =
      std::min(n, primary_idx + half + 1);

  // --- Inter-warp candidates ---
  for (size_t i = lo; i < hi; i++) {
    shd_warp_t *w = m_supervised_warps[i];
    if (w == NULL || w->done_exit()) continue;
    unsigned cand_warp_id = w->get_warp_id();
    if (cand_warp_id == co_issue_primary_warp_id) continue;
    if (warp(cand_warp_id).ibuffer_empty()) continue;
    if (warp(cand_warp_id).waiting()) continue;

    bool simt_ok = true;
    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
      simt_ok = warp(cand_warp_id).valid() &&
                !warp(cand_warp_id).blocked() &&
                !warp(cand_warp_id).pending_reconvergence() &&
                !warp(cand_warp_id).virtualized();
    }
    if (!simt_ok) continue;
    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
        m_shader->m_config->model == AWARE_RECONVERGENCE &&
        m_shader->get_simt_tables(cand_warp_id)
            ->div_recv_drain_pending()) {
      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
          !m_scoreboard->slotClean(cand_warp_id, 1)) {
        continue;
      }
      m_shader->get_simt_tables(cand_warp_id)
          ->clear_div_recv_drain_pending();
    }

    const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
    if (!cand_inst) continue;
    if (!warp(cand_warp_id).ibuffer_next_valid()) continue;

    unsigned cand_pc, cand_rpc;
    m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
                                      &cand_rpc);
    if (cand_pc != cand_inst->pc) continue;

    bool sb_collision_inter2;
    if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
      const active_mask_t &cand_mask_pre =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                             cand_inst,
                                                             cand_mask_pre);
    } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
      // OR check: stall on either slot — see commentary at primary
      // issue path.
      bool s0c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/0, cand_inst);
      bool s1c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/1, cand_inst);
      sb_collision_inter2 = s0c || s1c;
      const active_mask_t &cand_mask_for_diag =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      m_scoreboard->m_xslot_inter_coissue_checks++;
      if (!s0c && s1c) {
        m_scoreboard->m_xslot_inter_coissue_would_stall++;
      }
      if (!s0c && !s1c &&
          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
                                              cand_mask_for_diag)) {
        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
      }
    } else {
      sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                         cand_inst);
    }
    if (sb_collision_inter2) {
      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
      continue;
    }
    if (classify_fu_type(cand_inst) != co_issue_fu_type) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(cand_inst)) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      m_stats->mem_coissue_denied_filter[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM) {
      static const bool allow_mixed =
          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
      enum _memory_space_t pspace = co_issue_composite->space.get_type();
      enum _memory_space_t cspace = cand_inst->space.get_type();
      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
      if (!allow_mixed && p_shared != c_shared) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
    }

    const active_mask_t &cand_mask =
        m_shader->get_active_mask(cand_warp_id, cand_inst);
    unsigned needed = 0;
    if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
                                    cand_warp_id, available_sets, &needed)) {
      m_stats->coissue_denied_by_no_sets[get_sid()]++;
      continue;
    }

    coissue_cand_t c;
    c.is_intra = false;
    c.warp_id = cand_warp_id;
    c.split_id = (unsigned)-1;
    c.inst = cand_inst;
    c.mask = cand_mask;
    c.sec_slot = 0;
    c.active_count = cand_mask.count();
    c.sets_needed = needed;
    pool.push_back(c);
  }

  // --- Intra-warp candidates ---
  if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
    for (size_t i = lo; i < hi; i++) {
      shd_warp_t *w = m_supervised_warps[i];
      if (w == NULL || w->done_exit()) continue;
      unsigned cand_warp_id = w->get_warp_id();
      if (warp(cand_warp_id).waiting()) continue;

      bool simt_ok = warp(cand_warp_id).valid() &&
                     !warp(cand_warp_id).blocked() &&
                     !warp(cand_warp_id).pending_reconvergence() &&
                     !warp(cand_warp_id).virtualized();
      if (!simt_ok) continue;

      for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) {
        if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue;
        const warp_inst_t *sec_inst =
            warp(cand_warp_id).ibuffer_slot_inst(sec_slot);
        if (!sec_inst) continue;
        unsigned sec_split_id =
            warp(cand_warp_id).ibuffer_slot_split_id(sec_slot);
        const active_mask_t &sec_mask =
            warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot);

        if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue;
        unsigned split_pc;
        simt_mask_t split_mask;
        m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc,
                                  &split_mask);
        if (split_pc != sec_inst->pc) continue;

        bool dup_with_warp = false;
        for (unsigned k = 0; k < pool.size(); k++) {
          if (pool[k].warp_id == cand_warp_id) {
            dup_with_warp = true;
            break;
          }
        }
        if (dup_with_warp) continue;
        if (classify_fu_type(sec_inst) != co_issue_fu_type) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM &&
            mem_coissue_disallowed(sec_inst)) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          m_stats->mem_coissue_denied_filter[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM) {
          static const bool allow_mixed =
              (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
          enum _memory_space_t pspace = co_issue_composite->space.get_type();
          enum _memory_space_t cspace = sec_inst->space.get_type();
          bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
          bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
          if (!allow_mixed && p_shared != c_shared) {
            m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
            m_stats->mem_coissue_denied_filter[get_sid()]++;
            continue;
          }
        }
        active_mask_t sec_mask_eff;
        for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
          if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t);
        }
        if (!sec_mask_eff.any()) continue;

        bool sb_collision;
        if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
          sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                          sec_inst,
                                                          sec_mask_eff);
        } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
          sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id,
                                                          /*slot=*/1,
                                                          sec_inst);
        } else {
          sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id,
                                                               sec_inst);
        }
        if (sb_collision) {
          m_stats->coissue_denied_by_scoreboard[get_sid()]++;
          continue;
        }
        unsigned sec_active = sec_mask_eff.count();
        // Use the same feasibility test as inter coissue. Under
        // compaction != 2 partitions are fixed by thread_id, so the
        // coissuer's set positions can overlap the composite's even
        // when (active/set_width) <= available_sets. merge_simd_sets
        // silently drops overlapping coissuer sets while the access
        // was already injected, leaving m_pending_writes_secondary
        // unincremented but later decremented at writeback (underflow
        // on sb0/sb2, deadlock on sb1).
        unsigned sec_needed = 0;
        if (!check_coissue_feasibility(co_issue_composite, sec_inst,
                                        sec_mask_eff, cand_warp_id,
                                        available_sets, &sec_needed)) {
          m_stats->coissue_denied_by_no_sets[get_sid()]++;
          continue;
        }

        coissue_cand_t c;
        c.is_intra = true;
        c.warp_id = cand_warp_id;
        c.split_id = sec_split_id;
        c.inst = sec_inst;
        c.mask = sec_mask_eff;
        c.sec_slot = sec_slot;
        c.active_count = sec_active;
        c.sets_needed = sec_needed;
        pool.push_back(c);
      }
    }
  }

  // --- Density sort ---
  std::stable_sort(pool.begin(), pool.end(),
                   [](const coissue_cand_t &a, const coissue_cand_t &b) {
                     unsigned long lhs = (unsigned long)a.active_count *
                                         (unsigned long)b.sets_needed;
                     unsigned long rhs = (unsigned long)b.active_count *
                                         (unsigned long)a.sets_needed;
                     if (lhs != rhs) return lhs > rhs;
                     if (a.active_count != b.active_count)
                       return a.active_count > b.active_count;
                     return a.warp_id < b.warp_id;
                   });

  // --- Greedy pack ---
  for (std::vector<coissue_cand_t>::iterator it = pool.begin();
       it != pool.end() && available_sets > 0; it++) {
    coissue_cand_t &c = *it;
    unsigned needed = 0;
    if (c.is_intra) {
      if (c.sets_needed > available_sets) continue;
      needed = c.sets_needed;
    } else {
      if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask,
                                      c.warp_id, available_sets, &needed))
        continue;
    }

    if (c.is_intra) {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
            "split %u CO-ISSUED with primary split (%u sets) [centered]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, c.split_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set, c.split_id);

      available_sets -= needed;
      next_free_set += needed;
      m_stats->intra_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_intra_events[get_sid()]++;
      warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
      if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
        warp(c.warp_id).ibuffer_flush_half(1);
        m_scoreboard->clearSecondary(c.warp_id);
        for (std::vector<coissue_cand_t>::iterator jt = it + 1;
             jt != pool.end();) {
          if (jt->is_intra && jt->warp_id == c.warp_id)
            jt = pool.erase(jt);
          else
            ++jt;
        }
      }
    } else {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
            "CO-ISSUED with primary warp %u (%u sets) [centered]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set);
      available_sets -= needed;
      next_free_set += needed;
      m_stats->inter_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_inter_events[get_sid()]++;
      warp(c.warp_id).ibuffer_step();
    }
  }
}

void scheduler_unit::try_utilization_max_coissue_window_before(
    warp_inst_t *co_issue_composite, exec_unit_type_t co_issue_fu_type,
    unsigned co_issue_primary_warp_id, unsigned &available_sets,
    unsigned &next_free_set) {
  // All-before-primary K-bounded clone. Scans (K-1) warps strictly
  // before the primary issuer's index in m_supervised_warps order.
  // Asymmetric counterpart to mode 6 (centered).
  struct coissue_cand_t {
    bool is_intra;
    unsigned warp_id;
    unsigned split_id;
    const warp_inst_t *inst;
    active_mask_t mask;
    unsigned sec_slot;
    unsigned active_count;
    unsigned sets_needed;
  };

  std::vector<coissue_cand_t> pool;
  const unsigned set_width = m_shader->m_config->simd_set_width;
  const unsigned K = (m_shader->m_config->gpgpu_coissue_window_K > 0)
                         ? m_shader->m_config->gpgpu_coissue_window_K
                         : m_shader->m_config->gpgpu_num_simd_sets;
  const unsigned before = (K > 0) ? (K - 1) : 0;
  const size_t n = m_supervised_warps.size();

  size_t primary_idx = n;
  for (size_t i = 0; i < n; i++) {
    if (m_supervised_warps[i] != NULL &&
        m_supervised_warps[i]->get_warp_id() == co_issue_primary_warp_id) {
      primary_idx = i;
      break;
    }
  }
  if (primary_idx >= n) return;

  const size_t lo =
      (primary_idx > before) ? (primary_idx - before) : (size_t)0;
  const size_t hi = primary_idx;  // exclusive: primary itself not scanned

  // --- Inter-warp candidates ---
  for (size_t i = lo; i < hi; i++) {
    shd_warp_t *w = m_supervised_warps[i];
    if (w == NULL || w->done_exit()) continue;
    unsigned cand_warp_id = w->get_warp_id();
    if (cand_warp_id == co_issue_primary_warp_id) continue;
    if (warp(cand_warp_id).ibuffer_empty()) continue;
    if (warp(cand_warp_id).waiting()) continue;

    bool simt_ok = true;
    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
      simt_ok = warp(cand_warp_id).valid() &&
                !warp(cand_warp_id).blocked() &&
                !warp(cand_warp_id).pending_reconvergence() &&
                !warp(cand_warp_id).virtualized();
    }
    if (!simt_ok) continue;
    // Mode 2 sb drain gate (mirrors try_inter_warp_coissue).
    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
        m_shader->m_config->model == AWARE_RECONVERGENCE &&
        m_shader->get_simt_tables(cand_warp_id)
            ->div_recv_drain_pending()) {
      if (!m_scoreboard->slotClean(cand_warp_id, 0) ||
          !m_scoreboard->slotClean(cand_warp_id, 1)) {
        continue;
      }
      m_shader->get_simt_tables(cand_warp_id)
          ->clear_div_recv_drain_pending();
    }

    const warp_inst_t *cand_inst = warp(cand_warp_id).ibuffer_next_inst();
    if (!cand_inst) continue;
    if (!warp(cand_warp_id).ibuffer_next_valid()) continue;

    unsigned cand_pc, cand_rpc;
    m_shader->get_pdom_stack_top_info(cand_warp_id, cand_inst, &cand_pc,
                                      &cand_rpc);
    if (cand_pc != cand_inst->pc) continue;

    bool sb_collision_inter2;
    if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
      const active_mask_t &cand_mask_pre =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      sb_collision_inter2 = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                             cand_inst,
                                                             cand_mask_pre);
    } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
      // OR check: stall on either slot — see commentary at primary
      // issue path.
      bool s0c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/0, cand_inst);
      bool s1c = m_scoreboard->checkCollisionSlot(
          cand_warp_id, /*slot=*/1, cand_inst);
      sb_collision_inter2 = s0c || s1c;
      const active_mask_t &cand_mask_for_diag =
          m_shader->get_active_mask(cand_warp_id, cand_inst);
      m_scoreboard->m_xslot_inter_coissue_checks++;
      if (!s0c && s1c) {
        m_scoreboard->m_xslot_inter_coissue_would_stall++;
      }
      if (!s0c && !s1c &&
          m_scoreboard->checkCollisionShadow(cand_warp_id, cand_inst,
                                              cand_mask_for_diag)) {
        m_scoreboard->m_xslot_inter_coissue_real_would_stall++;
      }
    } else {
      sb_collision_inter2 = m_scoreboard->checkCollision(cand_warp_id,
                                                         cand_inst);
    }
    if (sb_collision_inter2) {
      m_stats->coissue_denied_by_scoreboard[get_sid()]++;
      continue;
    }
    if (classify_fu_type(cand_inst) != co_issue_fu_type) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(cand_inst)) {
      m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
      m_stats->mem_coissue_denied_filter[get_sid()]++;
      continue;
    }
    if (co_issue_fu_type == exec_unit_type_t::MEM) {
      static const bool allow_mixed =
          (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
      enum _memory_space_t pspace = co_issue_composite->space.get_type();
      enum _memory_space_t cspace = cand_inst->space.get_type();
      bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
      bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
      if (!allow_mixed && p_shared != c_shared) {
        m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
        m_stats->mem_coissue_denied_filter[get_sid()]++;
        continue;
      }
    }

    const active_mask_t &cand_mask =
        m_shader->get_active_mask(cand_warp_id, cand_inst);
    unsigned needed = 0;
    if (!check_coissue_feasibility(co_issue_composite, cand_inst, cand_mask,
                                    cand_warp_id, available_sets, &needed)) {
      m_stats->coissue_denied_by_no_sets[get_sid()]++;
      continue;
    }

    coissue_cand_t c;
    c.is_intra = false;
    c.warp_id = cand_warp_id;
    c.split_id = (unsigned)-1;
    c.inst = cand_inst;
    c.mask = cand_mask;
    c.sec_slot = 0;
    c.active_count = cand_mask.count();
    c.sets_needed = needed;
    pool.push_back(c);
  }

  // --- Intra-warp candidates ---
  if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
    for (size_t i = lo; i < hi; i++) {
      shd_warp_t *w = m_supervised_warps[i];
      if (w == NULL || w->done_exit()) continue;
      unsigned cand_warp_id = w->get_warp_id();
      if (warp(cand_warp_id).waiting()) continue;

      bool simt_ok = warp(cand_warp_id).valid() &&
                     !warp(cand_warp_id).blocked() &&
                     !warp(cand_warp_id).pending_reconvergence() &&
                     !warp(cand_warp_id).virtualized();
      if (!simt_ok) continue;

      for (unsigned sec_slot = 2; sec_slot < 4; sec_slot++) {
        if (!warp(cand_warp_id).ibuffer_slot_valid(sec_slot)) continue;
        const warp_inst_t *sec_inst =
            warp(cand_warp_id).ibuffer_slot_inst(sec_slot);
        if (!sec_inst) continue;
        unsigned sec_split_id =
            warp(cand_warp_id).ibuffer_slot_split_id(sec_slot);
        const active_mask_t &sec_mask =
            warp(cand_warp_id).ibuffer_slot_split_mask(sec_slot);

        if (!m_shader->is_split_valid(cand_warp_id, sec_split_id)) continue;
        unsigned split_pc;
        simt_mask_t split_mask;
        m_shader->get_split_info(cand_warp_id, sec_split_id, &split_pc,
                                  &split_mask);
        if (split_pc != sec_inst->pc) continue;

        bool dup_with_warp = false;
        for (unsigned k = 0; k < pool.size(); k++) {
          if (pool[k].warp_id == cand_warp_id) {
            dup_with_warp = true;
            break;
          }
        }
        if (dup_with_warp) continue;
        if (classify_fu_type(sec_inst) != co_issue_fu_type) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM &&
            mem_coissue_disallowed(sec_inst)) {
          m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
          m_stats->mem_coissue_denied_filter[get_sid()]++;
          continue;
        }
        if (co_issue_fu_type == exec_unit_type_t::MEM) {
          static const bool allow_mixed =
              (getenv("MEMCOV2_ALLOW_MIXED") != NULL);
          enum _memory_space_t pspace = co_issue_composite->space.get_type();
          enum _memory_space_t cspace = sec_inst->space.get_type();
          bool p_shared = warp_inst_t::is_shared_bucket_space(pspace);
          bool c_shared = warp_inst_t::is_shared_bucket_space(cspace);
          if (!allow_mixed && p_shared != c_shared) {
            m_stats->coissue_denied_by_fu_mismatch[get_sid()]++;
            m_stats->mem_coissue_denied_filter[get_sid()]++;
            continue;
          }
        }
        active_mask_t sec_mask_eff;
        for (unsigned t = 0; t < MAX_WARP_SIZE; t++) {
          if (sec_mask.test(t) && split_mask.test(t)) sec_mask_eff.set(t);
        }
        if (!sec_mask_eff.any()) continue;

        bool sb_collision;
        if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
          sb_collision = m_scoreboard->checkCollisionMask(cand_warp_id,
                                                          sec_inst,
                                                          sec_mask_eff);
        } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
          sb_collision = m_scoreboard->checkCollisionSlot(cand_warp_id,
                                                          /*slot=*/1,
                                                          sec_inst);
        } else {
          sb_collision = m_scoreboard->checkCollisionSecondary(cand_warp_id,
                                                               sec_inst);
        }
        if (sb_collision) {
          m_stats->coissue_denied_by_scoreboard[get_sid()]++;
          continue;
        }
        unsigned sec_active = sec_mask_eff.count();
        // Use the same feasibility test as inter coissue. Under
        // compaction != 2 partitions are fixed by thread_id, so the
        // coissuer's set positions can overlap the composite's even
        // when (active/set_width) <= available_sets. merge_simd_sets
        // silently drops overlapping coissuer sets while the access
        // was already injected, leaving m_pending_writes_secondary
        // unincremented but later decremented at writeback (underflow
        // on sb0/sb2, deadlock on sb1).
        unsigned sec_needed = 0;
        if (!check_coissue_feasibility(co_issue_composite, sec_inst,
                                        sec_mask_eff, cand_warp_id,
                                        available_sets, &sec_needed)) {
          m_stats->coissue_denied_by_no_sets[get_sid()]++;
          continue;
        }

        coissue_cand_t c;
        c.is_intra = true;
        c.warp_id = cand_warp_id;
        c.split_id = sec_split_id;
        c.inst = sec_inst;
        c.mask = sec_mask_eff;
        c.sec_slot = sec_slot;
        c.active_count = sec_active;
        c.sets_needed = sec_needed;
        pool.push_back(c);
      }
    }
  }

  // --- Density sort ---
  std::stable_sort(pool.begin(), pool.end(),
                   [](const coissue_cand_t &a, const coissue_cand_t &b) {
                     unsigned long lhs = (unsigned long)a.active_count *
                                         (unsigned long)b.sets_needed;
                     unsigned long rhs = (unsigned long)b.active_count *
                                         (unsigned long)a.sets_needed;
                     if (lhs != rhs) return lhs > rhs;
                     if (a.active_count != b.active_count)
                       return a.active_count > b.active_count;
                     return a.warp_id < b.warp_id;
                   });

  // --- Greedy pack ---
  for (std::vector<coissue_cand_t>::iterator it = pool.begin();
       it != pool.end() && available_sets > 0; it++) {
    coissue_cand_t &c = *it;
    unsigned needed = 0;
    if (c.is_intra) {
      if (c.sets_needed > available_sets) continue;
      needed = c.sets_needed;
    } else {
      if (!check_coissue_feasibility(co_issue_composite, c.inst, c.mask,
                                      c.warp_id, available_sets, &needed))
        continue;
    }

    if (c.is_intra) {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: INTRA-WARP warp %u "
            "split %u CO-ISSUED with primary split (%u sets) [before]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, c.split_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set, c.split_id);

      available_sets -= needed;
      next_free_set += needed;
      m_stats->intra_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_intra_events[get_sid()]++;
      warp(c.warp_id).ibuffer_free_slot(c.sec_slot);
      if (!m_shader->is_split_valid(c.warp_id, c.split_id)) {
        warp(c.warp_id).ibuffer_flush_half(1);
        m_scoreboard->clearSecondary(c.warp_id);
        for (std::vector<coissue_cand_t>::iterator jt = it + 1;
             jt != pool.end();) {
          if (jt->is_intra && jt->warp_id == c.warp_id)
            jt = pool.erase(jt);
          else
            ++jt;
        }
      }
    } else {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf(
            "SIMD_SETS: cycle %llu, core %u, sched %u: warp %u "
            "CO-ISSUED with primary warp %u (%u sets) [before]\n",
            m_shader->get_gpu()->gpu_sim_cycle +
                m_shader->get_gpu()->gpu_tot_sim_cycle,
            get_sid(), m_id, c.warp_id, co_issue_primary_warp_id, needed);
      }

      m_shader->co_issue_warp(co_issue_composite, c.inst, c.mask, c.warp_id,
                              m_id, next_free_set);
      available_sets -= needed;
      next_free_set += needed;
      m_stats->inter_warp_coissue_events[get_sid()]++;
      if (co_issue_fu_type == exec_unit_type_t::MEM)
        m_stats->mem_coissue_inter_events[get_sid()]++;
      warp(c.warp_id).ibuffer_step();
    }
  }
}

void scheduler_unit::cycle() {
  SCHED_DPRINTF("scheduler_unit::cycle()\n");
  bool valid_inst =
      false;  // there was one warp with a valid instruction to issue (didn't
              // require flush due to control hazard)
  bool ready_inst = false;   // of the valid instructions, there was one not
                             // waiting for pending register writes
  bool issued_inst = false;  // of these we issued one

  // SIMD set co-issue tracking
  warp_inst_t *co_issue_composite = NULL;
  exec_unit_type_t co_issue_fu_type = exec_unit_type_t::NONE;
  register_set *co_issue_reg_set = NULL;
  unsigned co_issue_primary_warp_id = (unsigned)-1;
  unsigned available_sets = 0;
  unsigned next_free_set = 0;

  order_warps();
  for (std::vector<shd_warp_t *>::const_iterator iter =
           m_next_cycle_prioritized_warps.begin();
       iter != m_next_cycle_prioritized_warps.end(); iter++) {
    // Don't consider warps that are not yet valid
    if ((*iter) == NULL || (*iter)->done_exit()) {
      continue;
    }
    SCHED_DPRINTF("Testing (warp_id %u, dynamic_warp_id %u)\n",
                  (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
    unsigned warp_id = (*iter)->get_warp_id();
    unsigned checked = 0;
    unsigned issued = 0;
    exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE;
    unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp;
    bool diff_exec_units =
        m_shader->m_config
            ->gpgpu_dual_issue_diff_exec_units;  // In tis mode, we only allow
                                                 // dual issue to diff execution
                                                 // units (as in Maxwell and
                                                 // Pascal)

    if (warp(warp_id).ibuffer_empty())
      SCHED_DPRINTF(
          "Warp (warp_id %u, dynamic_warp_id %u) fails as ibuffer_empty\n",
          (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());

    if (warp(warp_id).waiting())
      SCHED_DPRINTF(
          "Warp (warp_id %u, dynamic_warp_id %u) fails as waiting for "
          "barrier\n",
          (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());

    bool simt_conditions = true;
    if (m_shader->m_config->model == AWARE_RECONVERGENCE) {
      simt_conditions = warp(warp_id).valid() &&
                        !warp(warp_id).blocked() &&
                        !warp(warp_id).pending_reconvergence() &&
                        !warp(warp_id).virtualized();
    }

    // Mode 2 (slot-pinned scoreboard) drain gate at issue time.
    // Fetch-only drain leaks: ibuffer entries already-fetched pre-
    // divergence may match the post-divergence active PC and issue
    // with the new (subset) active mask, reading regs whose pending
    // writes carry the FULL pre-divergence mask → real cross-slot
    // RAW. Block all issues from this warp until both slot
    // scoreboards drain. Flag clears here once observed clean.
    if (m_shader->m_config->gpgpu_scoreboard_mode == 2 &&
        m_shader->m_config->model == AWARE_RECONVERGENCE &&
        m_shader->get_simt_tables(warp_id)->div_recv_drain_pending()) {
      if (!m_scoreboard->slotClean(warp_id, 0) ||
          !m_scoreboard->slotClean(warp_id, 1)) {
        continue;  // drain pending — try next warp
      }
      m_shader->get_simt_tables(warp_id)->clear_div_recv_drain_pending();
    }

    while (simt_conditions && !warp(warp_id).waiting() &&
           !warp(warp_id).ibuffer_empty() && (checked < max_issue) &&
           (checked <= issued) && (issued < max_issue)) {
      const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst();
      // Jin: handle cdp latency;
      if (pI && pI->m_is_cdp && warp(warp_id).m_cdp_latency > 0) {
        assert(warp(warp_id).m_cdp_dummy);
        warp(warp_id).m_cdp_latency--;
        break;
      }

      bool valid = warp(warp_id).ibuffer_next_valid();
      bool warp_inst_issued = false;
      unsigned pc, rpc;
      m_shader->get_pdom_stack_top_info(warp_id, pI, &pc, &rpc);
      SCHED_DPRINTF(
          "Warp (warp_id %u, dynamic_warp_id %u) has valid instruction (%s)\n",
          (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(),
          m_shader->m_config->gpgpu_ctx->func_sim->ptx_get_insn_str(pc)
              .c_str());
      if (pI) {
        assert(valid);
        if (pc != pI->pc) {
          SCHED_DPRINTF(
              "Warp (warp_id %u, dynamic_warp_id %u) control hazard "
              "instruction flush\n",
              (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
          // control hazard
          warp(warp_id).set_next_pc(pc);
          warp(warp_id).ibuffer_flush();
          m_scoreboard->clearSecondary(warp_id);
        } else {
          valid_inst = true;
          // Mode 1 needs the warp's active mask for hazard intersection;
          // hoist it before the scoreboard check.
          const active_mask_t &active_mask =
              m_shader->get_active_mask(warp_id, pI);
          bool sb_collision_primary;
          if (m_shader->m_config->gpgpu_scoreboard_mode == 1) {
            sb_collision_primary =
                m_scoreboard->checkCollisionMask(warp_id, pI, active_mask);
          } else if (m_shader->m_config->gpgpu_scoreboard_mode == 2) {
            // Slot-pinned: issue stalls if the inst's regs have pending
            // writes in EITHER slot. The rotation case (split moves
            // from slot 1 to slot 0 via co-issue's move_split_to_front)
            // strands prior writes in slot 1 while the same split's
            // next instructions issue from slot 0 — same-split
            // cross-slot RAW. Drain gates at fetch reduce this but
            // can't catch all rotation+pre-fetched-ibuffer interactions.
            // The OR check is mask-blind (no false positives from
            // lane-disjoint splits in steady-state divergent
            // execution? — yes there are: see m_xslot_primary_issue_would_stall).
            bool slot0_coll = m_scoreboard->checkCollisionSlot(
                warp_id, /*slot=*/0, pI);
            bool slot1_coll = m_scoreboard->checkCollisionSlot(
                warp_id, /*slot=*/1, pI);
            sb_collision_primary = slot0_coll || slot1_coll;
            // Diagnostic kept for now: count would_stall (unmasked)
            // and real_would_stall (mask-aware shadow). With OR check
            // active, _real_would_stall should drop to 0 — any real
            // RAW would also fire one of slot0/slot1 collision.
            m_scoreboard->m_xslot_primary_issue_checks++;
            if (!slot0_coll && slot1_coll) {
              m_scoreboard->m_xslot_primary_issue_would_stall++;
            }
            if (!slot0_coll && !slot1_coll &&
                m_scoreboard->checkCollisionShadow(warp_id, pI,
                                                    active_mask)) {
              m_scoreboard->m_xslot_primary_issue_real_would_stall++;
              static unsigned long long s_traced = 0;
              if (s_traced < 50 && getenv("XSLOT_TRACE")) {
                unsigned long long cyc =
                    m_shader->get_gpu()->gpu_sim_cycle +
                    m_shader->get_gpu()->gpu_tot_sim_cycle;
                m_scoreboard->dumpShadowOverlap(warp_id, pI, active_mask,
                                                 stderr, "primary", cyc);
                s_traced++;
              }
            }
          } else {
            sb_collision_primary = m_scoreboard->checkCollision(warp_id, pI);
          }
          if (!sb_collision_primary) {
            SCHED_DPRINTF(
                "Warp (warp_id %u, dynamic_warp_id %u) passes scoreboard\n",
                (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
            ready_inst = true;

            assert(warp(warp_id).inst_in_pipeline());

            if ((pI->op == LOAD_OP) || (pI->op == STORE_OP) ||
                (pI->op == MEMORY_BARRIER_OP) ||
                (pI->op == TENSOR_CORE_LOAD_OP) ||
                (pI->op == TENSOR_CORE_STORE_OP)) {
              if (m_mem_out->has_free(m_shader->m_config->sub_core_model,
                                      m_id) &&
                  (!diff_exec_units ||
                   previous_issued_inst_exec_type != exec_unit_type_t::MEM)) {
                warp_inst_t *issued_pipe_reg = m_shader->issue_warp(
                    *m_mem_out, pI, active_mask, warp_id, m_id);
                // When MEM co-issue is enabled, capture the composite pointer
                // so the dispatch loop below can attempt to merge co-issuers
                // into this MEM pipeline slot. Otherwise leave co_issue_*
                // variables unset for bit-parity with the pre-MEM-coissue tree.
                if (m_shader->m_config->gpgpu_simd_partitioning &&
                    m_shader->m_config->gpgpu_mem_coissue) {
                  co_issue_composite = issued_pipe_reg;
                  co_issue_fu_type = exec_unit_type_t::MEM;
                  co_issue_reg_set = m_mem_out;
                  co_issue_primary_warp_id = warp_id;
                }
                issued++;
                issued_inst = true;
                warp_inst_issued = true;
                previous_issued_inst_exec_type = exec_unit_type_t::MEM;
              }
            } else {
              // This code need to be refactored
              if (pI->op != TENSOR_CORE_OP && pI->op != SFU_OP &&
                  pI->op != DP_OP && !(pI->op >= SPEC_UNIT_START_ID)) {
                bool execute_on_SP = false;
                bool execute_on_INT = false;

                bool sp_pipe_avail =
                    (m_shader->m_config->gpgpu_num_sp_units > 0) &&
                    m_sp_out->has_free(m_shader->m_config->sub_core_model,
                                       m_id);
                bool int_pipe_avail =
                    (m_shader->m_config->gpgpu_num_int_units > 0) &&
                    m_int_out->has_free(m_shader->m_config->sub_core_model,
                                        m_id);

                // if INT unit pipline exist, then execute ALU and INT
                // operations on INT unit and SP-FPU on SP unit (like in Volta)
                // if INT unit pipline does not exist, then execute all ALU, INT
                // and SP operations on SP unit (as in Fermi, Pascal GPUs)
                if (m_shader->m_config->gpgpu_num_int_units > 0 &&
                    int_pipe_avail && pI->op != SP_OP &&
                    !(diff_exec_units &&
                      previous_issued_inst_exec_type == exec_unit_type_t::INT))
                  execute_on_INT = true;
                else if (sp_pipe_avail &&
                         (m_shader->m_config->gpgpu_num_int_units == 0 ||
                          (m_shader->m_config->gpgpu_num_int_units > 0 &&
                           pI->op == SP_OP)) &&
                         !(diff_exec_units && previous_issued_inst_exec_type ==
                                                  exec_unit_type_t::SP))
                  execute_on_SP = true;

                if (execute_on_INT || execute_on_SP) {
                  // Jin: special for CDP api
                  if (pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) {
                    assert(warp(warp_id).m_cdp_latency == 0);

                    if (pI->m_is_cdp == 1)
                      warp(warp_id).m_cdp_latency =
                          m_shader->m_config->gpgpu_ctx->func_sim
                              ->cdp_latency[pI->m_is_cdp - 1];
                    else  // cudaLaunchDeviceV2 and cudaGetParameterBufferV2
                      warp(warp_id).m_cdp_latency =
                          m_shader->m_config->gpgpu_ctx->func_sim
                              ->cdp_latency[pI->m_is_cdp - 1] +
                          m_shader->m_config->gpgpu_ctx->func_sim
                                  ->cdp_latency[pI->m_is_cdp] *
                              active_mask.count();
                    warp(warp_id).m_cdp_dummy = true;
                    break;
                  } else if (pI->m_is_cdp && warp(warp_id).m_cdp_dummy) {
                    assert(warp(warp_id).m_cdp_latency == 0);
                    warp(warp_id).m_cdp_dummy = false;
                  }
                }

                if (execute_on_SP) {
                  bool bru_avail = true;
                  if (m_shader->m_config->model == AWARE_RECONVERGENCE &&
                      pI->op == BRANCH_OP)
                    bru_avail = m_shader->branch_unit_avail(warp_id);
                  if (bru_avail) {
                  co_issue_composite = m_shader->issue_warp(
                      *m_sp_out, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::SP;
                  co_issue_reg_set = m_sp_out;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type = exec_unit_type_t::SP;
                  }
                } else if (execute_on_INT) {
                  co_issue_composite = m_shader->issue_warp(
                      *m_int_out, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::INT;
                  co_issue_reg_set = m_int_out;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type = exec_unit_type_t::INT;
                }
              } else if ((m_shader->m_config->gpgpu_num_dp_units > 0) &&
                         (pI->op == DP_OP) &&
                         !(diff_exec_units && previous_issued_inst_exec_type ==
                                                  exec_unit_type_t::DP)) {
                bool dp_pipe_avail =
                    (m_shader->m_config->gpgpu_num_dp_units > 0) &&
                    m_dp_out->has_free(m_shader->m_config->sub_core_model,
                                       m_id);

                if (dp_pipe_avail) {
                  co_issue_composite = m_shader->issue_warp(
                      *m_dp_out, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::DP;
                  co_issue_reg_set = m_dp_out;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type = exec_unit_type_t::DP;
                }
              }  // If the DP units = 0 (like in Fermi archi), then execute DP
                 // inst on SFU unit
              else if (((m_shader->m_config->gpgpu_num_dp_units == 0 &&
                         pI->op == DP_OP) ||
                        (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) &&
                       !(diff_exec_units && previous_issued_inst_exec_type ==
                                                exec_unit_type_t::SFU)) {
                bool sfu_pipe_avail =
                    (m_shader->m_config->gpgpu_num_sfu_units > 0) &&
                    m_sfu_out->has_free(m_shader->m_config->sub_core_model,
                                        m_id);

                if (sfu_pipe_avail) {
                  co_issue_composite = m_shader->issue_warp(
                      *m_sfu_out, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::SFU;
                  co_issue_reg_set = m_sfu_out;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type = exec_unit_type_t::SFU;
                }
              } else if ((pI->op == TENSOR_CORE_OP) &&
                         !(diff_exec_units && previous_issued_inst_exec_type ==
                                                  exec_unit_type_t::TENSOR)) {
                bool tensor_core_pipe_avail =
                    (m_shader->m_config->gpgpu_num_tensor_core_units > 0) &&
                    m_tensor_core_out->has_free(
                        m_shader->m_config->sub_core_model, m_id);

                if (tensor_core_pipe_avail) {
                  co_issue_composite = m_shader->issue_warp(
                      *m_tensor_core_out, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::TENSOR;
                  co_issue_reg_set = m_tensor_core_out;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type = exec_unit_type_t::TENSOR;
                }
              } else if ((pI->op >= SPEC_UNIT_START_ID) &&
                         !(diff_exec_units &&
                           previous_issued_inst_exec_type ==
                               exec_unit_type_t::SPECIALIZED)) {
                unsigned spec_id = pI->op - SPEC_UNIT_START_ID;
                assert(spec_id < m_shader->m_config->m_specialized_unit.size());
                register_set *spec_reg_set = m_spec_cores_out[spec_id];
                bool spec_pipe_avail =
                    (m_shader->m_config->m_specialized_unit[spec_id].num_units >
                     0) &&
                    spec_reg_set->has_free(m_shader->m_config->sub_core_model,
                                           m_id);

                if (spec_pipe_avail) {
                  co_issue_composite = m_shader->issue_warp(
                      *spec_reg_set, pI, active_mask, warp_id, m_id);
                  co_issue_fu_type = exec_unit_type_t::SPECIALIZED;
                  co_issue_reg_set = spec_reg_set;
                  co_issue_primary_warp_id = warp_id;
                  issued++;
                  issued_inst = true;
                  warp_inst_issued = true;
                  previous_issued_inst_exec_type =
                      exec_unit_type_t::SPECIALIZED;
                }
              }

            }  // end of else
          } else {
            SCHED_DPRINTF(
                "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n",
                (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
          }
        }
      } else if (valid) {
        // this case can happen after a return instruction in diverged warp
        SCHED_DPRINTF(
            "Warp (warp_id %u, dynamic_warp_id %u) return from diverged warp "
            "flush\n",
            (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
        warp(warp_id).set_next_pc(pc);
        warp(warp_id).ibuffer_flush();
        m_scoreboard->clearSecondary(warp_id);
      }
      if (warp_inst_issued) {
        SCHED_DPRINTF(
            "Warp (warp_id %u, dynamic_warp_id %u) issued %u instructions\n",
            (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id(), issued);
        do_on_warp_issued(warp_id, issued, iter);
      }
      checked++;
    }
    if (issued) {
      // This might be a bit inefficient, but we need to maintain
      // two ordered list for proper scheduler execution.
      // We could remove the need for this loop by associating a
      // supervised_is index with each entry in the
      // m_next_cycle_prioritized_warps vector. For now, just run through until
      // you find the right warp_id
      for (std::vector<shd_warp_t *>::const_iterator supervised_iter =
               m_supervised_warps.begin();
           supervised_iter != m_supervised_warps.end(); ++supervised_iter) {
        if (*iter == *supervised_iter) {
          m_last_supervised_issued = supervised_iter;
        }
      }
      m_num_issued_last_cycle = issued;
      if (issued == 1)
        m_stats->single_issue_nums[m_id]++;
      else if (issued > 1)
        m_stats->dual_issue_nums[m_id]++;
      else
        abort();  // issued should be > 0

      break;
    }
  }

  // SIMD set co-issue dispatch: after issuing the primary instruction,
  // run one of 5 co-issue strategies based on -gpgpu_co_issue_priority.
  // MEM co-issue is opt-in via -gpgpu_mem_coissue (the primary-issue branch
  // above leaves co_issue_composite=NULL when the flag is off, so this
  // block is skipped naturally in that case).
  if (m_shader->m_config->gpgpu_simd_partitioning && co_issue_composite != NULL &&
      co_issue_fu_type != exec_unit_type_t::NONE &&
      !(co_issue_fu_type == exec_unit_type_t::MEM &&
        mem_coissue_disallowed(co_issue_composite))) {
    available_sets = 0;
    next_free_set = 0;
    const std::vector<simd_set_info> &primary_sets =
        co_issue_composite->get_simd_sets();
    for (unsigned s = 0; s < primary_sets.size(); s++) {
      if (!primary_sets[s].valid)
        available_sets++;
      else
        next_free_set = s + 1;
    }

    if (available_sets > 0) {
      if (m_shader->m_config->gpgpu_simd_partitioning_debug) {
        printf("SIMD_SETS: cycle %llu, core %u, sched %u: %u sets available "
               "after primary warp %u\n",
               m_shader->get_gpu()->gpu_sim_cycle +
                   m_shader->get_gpu()->gpu_tot_sim_cycle,
               get_sid(), m_id, available_sets, co_issue_primary_warp_id);
      }

      switch (m_shader->m_config->gpgpu_co_issue_priority) {
        case 0:  // greedy (utilization-max, unified pool)
          try_utilization_max_coissue(
              co_issue_composite, co_issue_fu_type,
              co_issue_primary_warp_id, available_sets, next_free_set);
          break;
        case 1:  // intra-first
          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set);
          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set, /*sort_mode=*/0);
          break;
        case 3:  // fewest-lanes first (inter), then intra
          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set, /*sort_mode=*/1);
          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set);
          break;
        case 4:  // same-PC (inter), then intra
          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set, /*sort_mode=*/2);
          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set);
          break;
        case 5:  // greedy K-bounded (utilization-max over first K warps)
          try_utilization_max_coissue_window(
              co_issue_composite, co_issue_fu_type,
              co_issue_primary_warp_id, available_sets, next_free_set);
          break;
        case 6:  // greedy K-bounded centered (K/2 before + K/2 after primary)
          try_utilization_max_coissue_window_centered(
              co_issue_composite, co_issue_fu_type,
              co_issue_primary_warp_id, available_sets, next_free_set);
          break;
        case 7:  // greedy K-bounded all-before (K-1 warps before primary)
          try_utilization_max_coissue_window_before(
              co_issue_composite, co_issue_fu_type,
              co_issue_primary_warp_id, available_sets, next_free_set);
          break;
        case 2:  // inter-first (default, matches pre-Change-4)
        default:
          try_inter_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set, /*sort_mode=*/0);
          try_intra_warp_coissue(co_issue_composite, co_issue_fu_type,
                                 co_issue_primary_warp_id, available_sets,
                                 next_free_set);
          break;
      }
    }
  }

  // Change 5: end-of-cycle SIMD partitioning stats. Only collect when
  // partitioning is enabled and a composite was issued this cycle.
  if (m_shader->m_config->gpgpu_simd_partitioning &&
      co_issue_composite != NULL) {
    unsigned valid_sets = 0;
    unsigned active_lanes = 0;
    const std::vector<simd_set_info> &sets =
        co_issue_composite->get_simd_sets();
    for (unsigned s = 0; s < sets.size(); s++) {
      if (sets[s].valid) {
        valid_sets++;
        active_lanes += sets[s].set_active_mask.count();
      }
    }
    unsigned sid = get_sid();
    m_stats->coissue_composite_cycles[sid]++;
    m_stats->coissue_total_active_lanes[sid] += active_lanes;
    unsigned nbins = m_shader->m_config->gpgpu_num_simd_sets + 1;
    if (valid_sets >= nbins) valid_sets = nbins - 1;  // safety clamp
    m_stats->simd_sets_used_histogram[sid * nbins + valid_sets]++;
  }

  // issue stall statistics:
  if (!valid_inst)
    m_stats->shader_cycle_distro[0]++;  // idle or control hazard
  else if (!ready_inst)
    m_stats->shader_cycle_distro[1]++;  // waiting for RAW hazards (possibly due
                                        // to memory)
  else if (!issued_inst)
    m_stats->shader_cycle_distro[2]++;  // pipeline stalled
}

void scheduler_unit::do_on_warp_issued(
    unsigned warp_id, unsigned num_issued,
    const std::vector<shd_warp_t *>::const_iterator &prioritized_iter) {
  m_stats->event_warp_issued(m_shader->get_sid(), warp_id, num_issued,
                             warp(warp_id).get_dynamic_warp_id());
  warp(warp_id).ibuffer_step();
}

bool scheduler_unit::sort_warps_by_oldest_dynamic_id(shd_warp_t *lhs,
                                                     shd_warp_t *rhs) {
  if (rhs && lhs) {
    if (lhs->done_exit() || lhs->waiting()) {
      return false;
    } else if (rhs->done_exit() || rhs->waiting()) {
      return true;
    } else {
      return lhs->get_dynamic_warp_id() < rhs->get_dynamic_warp_id();
    }
  } else {
    return lhs < rhs;
  }
}

void lrr_scheduler::order_warps() {
  order_lrr(m_next_cycle_prioritized_warps, m_supervised_warps,
            m_last_supervised_issued, m_supervised_warps.size());
}
void rrr_scheduler::order_warps() {
  order_rrr(m_next_cycle_prioritized_warps, m_supervised_warps,
            m_last_supervised_issued, m_supervised_warps.size());
}

void gto_scheduler::order_warps() {
  order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps,
                    m_last_supervised_issued, m_supervised_warps.size(),
                    ORDERING_GREEDY_THEN_PRIORITY_FUNC,
                    scheduler_unit::sort_warps_by_oldest_dynamic_id);
}

void oldest_scheduler::order_warps() {
  order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps,
                    m_last_supervised_issued, m_supervised_warps.size(),
                    ORDERED_PRIORITY_FUNC_ONLY,
                    scheduler_unit::sort_warps_by_oldest_dynamic_id);
}

void two_level_active_scheduler::do_on_warp_issued(
    unsigned warp_id, unsigned num_issued,
    const std::vector<shd_warp_t *>::const_iterator &prioritized_iter) {
  scheduler_unit::do_on_warp_issued(warp_id, num_issued, prioritized_iter);
  if (SCHEDULER_PRIORITIZATION_LRR == m_inner_level_prioritization) {
    std::vector<shd_warp_t *> new_active;
    order_lrr(new_active, m_next_cycle_prioritized_warps, prioritized_iter,
              m_next_cycle_prioritized_warps.size());
    m_next_cycle_prioritized_warps = new_active;
  } else {
    fprintf(stderr, "Unimplemented m_inner_level_prioritization: %d\n",
            m_inner_level_prioritization);
    abort();
  }
}

void two_level_active_scheduler::order_warps() {
  // Move waiting warps to m_pending_warps
  unsigned num_demoted = 0;
  for (std::vector<shd_warp_t *>::iterator iter =
           m_next_cycle_prioritized_warps.begin();
       iter != m_next_cycle_prioritized_warps.end();) {
    bool waiting = (*iter)->waiting();
    for (int i = 0; i < MAX_INPUT_VALUES; i++) {
      const warp_inst_t *inst = (*iter)->ibuffer_next_inst();
      // Is the instruction waiting on a long operation?
      if (inst && inst->in[i] > 0 &&
          this->m_scoreboard->islongop((*iter)->get_warp_id(), inst->in[i])) {
        waiting = true;
      }
    }

    if (waiting) {
      m_pending_warps.push_back(*iter);
      iter = m_next_cycle_prioritized_warps.erase(iter);
      SCHED_DPRINTF("DEMOTED warp_id=%d, dynamic_warp_id=%d\n",
                    (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id());
      ++num_demoted;
    } else {
      ++iter;
    }
  }

  // If there is space in m_next_cycle_prioritized_warps, promote the next
  // m_pending_warps
  unsigned num_promoted = 0;
  if (SCHEDULER_PRIORITIZATION_SRR == m_outer_level_prioritization) {
    while (m_next_cycle_prioritized_warps.size() < m_max_active_warps) {
      m_next_cycle_prioritized_warps.push_back(m_pending_warps.front());
      m_pending_warps.pop_front();
      SCHED_DPRINTF(
          "PROMOTED warp_id=%d, dynamic_warp_id=%d\n",
          (m_next_cycle_prioritized_warps.back())->get_warp_id(),
          (m_next_cycle_prioritized_warps.back())->get_dynamic_warp_id());
      ++num_promoted;
    }
  } else {
    fprintf(stderr, "Unimplemented m_outer_level_prioritization: %d\n",
            m_outer_level_prioritization);
    abort();
  }
  assert(num_promoted == num_demoted);
}

swl_scheduler::swl_scheduler(shader_core_stats *stats, shader_core_ctx *shader,
                             Scoreboard *scoreboard, simt_stack **simt,
                             std::vector<shd_warp_t *> *warp,
                             register_set *sp_out, register_set *dp_out,
                             register_set *sfu_out, register_set *int_out,
                             register_set *tensor_core_out,
                             std::vector<register_set *> &spec_cores_out,
                             register_set *mem_out, int id, char *config_string)
    : scheduler_unit(stats, shader, scoreboard, simt, warp, sp_out, dp_out,
                     sfu_out, int_out, tensor_core_out, spec_cores_out, mem_out,
                     id) {
  unsigned m_prioritization_readin;
  int ret = sscanf(config_string, "warp_limiting:%d:%d",
                   &m_prioritization_readin, &m_num_warps_to_limit);
  assert(2 == ret);
  m_prioritization = (scheduler_prioritization_type)m_prioritization_readin;
  // Currently only GTO is implemented
  assert(m_prioritization == SCHEDULER_PRIORITIZATION_GTO);
  assert(m_num_warps_to_limit <= shader->get_config()->max_warps_per_shader);
}

void swl_scheduler::order_warps() {
  if (SCHEDULER_PRIORITIZATION_GTO == m_prioritization) {
    order_by_priority(m_next_cycle_prioritized_warps, m_supervised_warps,
                      m_last_supervised_issued,
                      MIN(m_num_warps_to_limit, m_supervised_warps.size()),
                      ORDERING_GREEDY_THEN_PRIORITY_FUNC,
                      scheduler_unit::sort_warps_by_oldest_dynamic_id);
  } else {
    fprintf(stderr, "swl_scheduler m_prioritization = %d\n", m_prioritization);
    abort();
  }
}

void shader_core_ctx::read_operands() {
  for (unsigned int i = 0; i < m_config->reg_file_port_throughput; ++i)
    m_operand_collector.step();
}

address_type coalesced_segment(address_type addr,
                               unsigned segment_size_lg2bytes) {
  return (addr >> segment_size_lg2bytes);
}

// Returns numbers of addresses in translated_addrs, each addr points to a 4B
// (32-bit) word
unsigned shader_core_ctx::translate_local_memaddr(
    address_type localaddr, unsigned tid, unsigned num_shader,
    unsigned datasize, new_addr_type *translated_addrs) {
  // During functional execution, each thread sees its own memory space for
  // local memory, but these need to be mapped to a shared address space for
  // timing simulation.  We do that mapping here.

  address_type thread_base = 0;
  unsigned max_concurrent_threads = 0;
  if (m_config->gpgpu_local_mem_map) {
    // Dnew = D*N + T%nTpC + nTpC*C
    // N = nTpC*nCpS*nS (max concurent threads)
    // C = nS*K + S (hw cta number per gpu)
    // K = T/nTpC   (hw cta number per core)
    // D = data index
    // T = thread
    // nTpC = number of threads per CTA
    // nCpS = number of CTA per shader
    //
    // for a given local memory address threads in a CTA map to contiguous
    // addresses, then distribute across memory space by CTAs from successive
    // shader cores first, then by successive CTA in same shader core
    thread_base =
        4 * (kernel_padded_threads_per_cta *
                 (m_sid + num_shader * (tid / kernel_padded_threads_per_cta)) +
             tid % kernel_padded_threads_per_cta);
    max_concurrent_threads =
        kernel_padded_threads_per_cta * kernel_max_cta_per_shader * num_shader;
  } else {
    // legacy mapping that maps the same address in the local memory space of
    // all threads to a single contiguous address region
    thread_base = 4 * (m_config->n_thread_per_shader * m_sid + tid);
    max_concurrent_threads = num_shader * m_config->n_thread_per_shader;
  }
  assert(thread_base < 4 /*word size*/ * max_concurrent_threads);

  // If requested datasize > 4B, split into multiple 4B accesses
  // otherwise do one sub-4 byte memory access
  unsigned num_accesses = 0;

  if (datasize >= 4) {
    // >4B access, split into 4B chunks
    assert(datasize % 4 == 0);  // Must be a multiple of 4B
    num_accesses = datasize / 4;
    assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD);  // max 32B
    assert(
        localaddr % 4 ==
        0);  // Address must be 4B aligned - required if accessing 4B per
             // request, otherwise access will overflow into next thread's space
    for (unsigned i = 0; i < num_accesses; i++) {
      address_type local_word = localaddr / 4 + i;
      address_type linear_address = local_word * max_concurrent_threads * 4 +
                                    thread_base + LOCAL_GENERIC_START;
      translated_addrs[i] = linear_address;
    }
  } else {
    // Sub-4B access, do only one access
    assert(datasize > 0);
    num_accesses = 1;
    address_type local_word = localaddr / 4;
    address_type local_word_offset = localaddr % 4;
    assert((localaddr + datasize - 1) / 4 ==
           local_word);  // Make sure access doesn't overflow into next 4B chunk
    address_type linear_address = local_word * max_concurrent_threads * 4 +
                                  local_word_offset + thread_base +
                                  LOCAL_GENERIC_START;
    translated_addrs[0] = linear_address;
  }
  return num_accesses;
}

/////////////////////////////////////////////////////////////////////////////////////////
int shader_core_ctx::test_res_bus(int latency) {
  for (unsigned i = 0; i < num_result_bus; i++) {
    if (!m_result_bus[i]->test(latency)) {
      return i;
    }
  }
  return -1;
}

void shader_core_ctx::execute() {
  for (unsigned i = 0; i < num_result_bus; i++) {
    *(m_result_bus[i]) >>= 1;
  }
  for (unsigned n = 0; n < m_num_function_units; n++) {
    unsigned multiplier = m_fu[n]->clock_multiplier();
    for (unsigned c = 0; c < multiplier; c++) m_fu[n]->cycle();
    m_fu[n]->active_lanes_in_pipeline();
    unsigned issue_port = m_issue_port[n];
    register_set &issue_inst = m_pipeline_reg[issue_port];
    unsigned reg_id;
    bool partition_issue =
        m_config->sub_core_model && m_fu[n]->is_issue_partitioned();
    if (partition_issue) {
      reg_id = m_fu[n]->get_issue_reg_id();
    }
    warp_inst_t **ready_reg = issue_inst.get_ready(partition_issue, reg_id);
    if (issue_inst.has_ready(partition_issue, reg_id) &&
        m_fu[n]->can_issue(**ready_reg)) {
      bool schedule_wb_now = !m_fu[n]->stallable();
      int resbus = -1;
      if (schedule_wb_now &&
          (resbus = test_res_bus((*ready_reg)->latency)) != -1) {
        assert((*ready_reg)->latency < MAX_ALU_LATENCY);
        m_result_bus[resbus]->set((*ready_reg)->latency);
        m_fu[n]->issue(issue_inst);
      } else if (!schedule_wb_now) {
        m_fu[n]->issue(issue_inst);
      } else {
        // stall issue (cannot reserve result bus)
      }
    }
  }

  // ITS: advance spill/fill state machines for all warps
  if (m_config->model == AWARE_RECONVERGENCE) {
    for (unsigned i = 0; i < m_warp_count; ++i) {
      AWARE_DPRINTF("Cycling SIMT tables for Shader %d: Warp %d...\n", m_sid, i);
      m_simt_tables[i]->cycle();
    }
  }
}

void ldst_unit::print_cache_stats(FILE *fp, unsigned &dl1_accesses,
                                  unsigned &dl1_misses) {
  if (m_L1D) {
    m_L1D->print(fp, dl1_accesses, dl1_misses);
  }
}

void ldst_unit::get_cache_stats(cache_stats &cs) {
  // Adds stats to 'cs' from each cache
  if (m_L1D) cs += m_L1D->get_stats();
  if (m_L1C) cs += m_L1C->get_stats();
  if (m_L1T) cs += m_L1T->get_stats();
}

void ldst_unit::get_L1D_sub_stats(struct cache_sub_stats &css) const {
  if (m_L1D) m_L1D->get_sub_stats(css);
}
void ldst_unit::get_L1C_sub_stats(struct cache_sub_stats &css) const {
  if (m_L1C) m_L1C->get_sub_stats(css);
}
void ldst_unit::get_L1T_sub_stats(struct cache_sub_stats &css) const {
  if (m_L1T) m_L1T->get_sub_stats(css);
}

// Add this function to unset depbar
void shader_core_ctx::unset_depbar(const warp_inst_t &inst) {
  bool done_flag = true;
  unsigned int end_group = m_warp[inst.warp_id()]->m_depbar_start_id == 0
                               ? m_warp[inst.warp_id()]->m_ldgdepbar_buf.size()
                               : (m_warp[inst.warp_id()]->m_depbar_start_id -
                                  m_warp[inst.warp_id()]->m_depbar_group + 1);

  if (inst.m_is_ldgsts) {
    for (int i = 0; i < m_warp[inst.warp_id()]->m_ldgdepbar_buf.size(); i++) {
      for (int j = 0; j < m_warp[inst.warp_id()]->m_ldgdepbar_buf[i].size();
           j++) {
        if (m_warp[inst.warp_id()]->m_ldgdepbar_buf[i][j].pc == inst.pc) {
          // Handle the case that same pc results in multiple LDGSTS
          // instructions
          if (m_warp[inst.warp_id()]->m_ldgdepbar_buf[i][j].get_addr(0) ==
              inst.get_addr(0)) {
            m_warp[inst.warp_id()]->m_ldgdepbar_buf[i][j].pc = -1;
            goto DoneWB;
          }
        }
      }
    }

  DoneWB:
    for (int i = 0; i < end_group; i++) {
      for (int j = 0; j < m_warp[inst.warp_id()]->m_ldgdepbar_buf[i].size();
           j++) {
        if (m_warp[inst.warp_id()]->m_ldgdepbar_buf[i][j].pc != -1) {
          done_flag = false;
          goto UpdateDEPBAR;
        }
      }
    }

  UpdateDEPBAR:
    if (done_flag) {
      if (m_warp[inst.warp_id()]->m_waiting_ldgsts) {
        m_warp[inst.warp_id()]->m_waiting_ldgsts = false;
      }
    }
  }
}

void shader_core_ctx::warp_inst_complete(const warp_inst_t &inst) {
#if 0
      printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu \n",
             inst.get_uid(), m_sid, inst.warp_id(), inst.pc,  m_gpu->gpu_tot_sim_cycle +  m_gpu->gpu_sim_cycle);
#endif
  if (inst.op_pipe == SP__OP)
    m_stats->m_num_sp_committed[m_sid]++;
  else if (inst.op_pipe == SFU__OP)
    m_stats->m_num_sfu_committed[m_sid]++;
  else if (inst.op_pipe == MEM__OP)
    m_stats->m_num_mem_committed[m_sid]++;

  if (m_config->gpgpu_clock_gated_lanes == false)
    m_stats->m_num_sim_insn[m_sid] += m_config->warp_size;
  else
    m_stats->m_num_sim_insn[m_sid] += inst.active_count();

  m_stats->m_num_sim_winsn[m_sid]++;
  m_gpu->gpu_sim_insn += inst.active_count();
  inst.completed(m_gpu->gpu_tot_sim_cycle + m_gpu->gpu_sim_cycle);
}

void shader_core_ctx::writeback() {
  unsigned max_committed_thread_instructions =
      m_config->warp_size *
      (m_config->pipe_widths[EX_WB]);  // from the functional units
  m_stats->m_pipeline_duty_cycle[m_sid] =
      ((float)(m_stats->m_num_sim_insn[m_sid] -
               m_stats->m_last_num_sim_insn[m_sid])) /
      max_committed_thread_instructions;

  m_stats->m_last_num_sim_insn[m_sid] = m_stats->m_num_sim_insn[m_sid];
  m_stats->m_last_num_sim_winsn[m_sid] = m_stats->m_num_sim_winsn[m_sid];

  warp_inst_t **preg = m_pipeline_reg[EX_WB].get_ready();
  warp_inst_t *pipe_reg = (preg == NULL) ? NULL : *preg;
  while (preg and !pipe_reg->empty()) {
    /*
     * Right now, the writeback stage drains all waiting instructions
     * assuming there are enough ports in the register file or the
     * conflicts are resolved at issue.
     */
    /*
     * The operand collector writeback can generally generate a stall
     * However, here, the pipelines should be un-stallable. This is
     * guaranteed because this is the first time the writeback function
     * is called after the operand collector's step function, which
     * resets the allocations. There is one case which could result in
     * the writeback function returning false (stall), which is when
     * an instruction tries to modify two registers (GPR and predicate)
     * To handle this case, we ignore the return value (thus allowing
     * no stalling).
     */

    m_operand_collector.writeback(*pipe_reg);
    unsigned warp_id = pipe_reg->warp_id();

    // Per-set writeback: release scoreboard and dec pipeline counter
    // for each unique warp participating in this composite instruction
    if (m_config->gpgpu_simd_partitioning && pipe_reg->has_simd_sets()) {
      std::set<unsigned> unique_warp_ids;
      const std::vector<simd_set_info> &sets = pipe_reg->get_simd_sets();
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid) continue;
        unsigned set_warp_id = sets[s].warp_id;
        unique_warp_ids.insert(set_warp_id);
      }
      // Release scoreboard for primary instruction (covers the outer warp_id)
      m_scoreboard->releaseRegisters(pipe_reg);
      // Release scoreboard for ALL co-issued sets (inter-warp AND intra-warp).
      // Dispatcher: mode 0 -> primary release; mode 1 -> mask-aware release
      // using the set's stamped source_mask. Idempotent; safe to call once
      // per set even when multiple sets share the same source_inst.
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        unsigned set_wid = sets[s].warp_id;
        unsigned set_slot =
            (sets[s].source_slot_id != (unsigned)-1) ? sets[s].source_slot_id
                                                     : 0u;
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          if (sets[s].source_inst->out[r] > 0) {
            m_scoreboard->releaseSetReg(set_wid,
                                        sets[s].source_inst->out[r],
                                        sets[s].source_mask,
                                        /*is_intra_legacy=*/false, set_slot);
          }
        }
      }
      // dec_inst_in_pipeline: once for the primary instruction, plus once
      // for each UNIQUE co-issued warp (not per-set — a co-issued warp may
      // span multiple sets but was only inc'd once at decode).
      // For intra-warp co-issue, the co-issued split shares the same warp_id
      // as the primary — dec once for primary + once for the co-issued split.
      m_warp[warp_id]->dec_inst_in_pipeline();
      std::set<unsigned> co_issued_warps_decremented;
      bool intra_warp_decremented = false;
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        if (sets[s].warp_id != warp_id) {
          // Inter-warp: dec once per unique co-issued warp
          if (co_issued_warps_decremented.find(sets[s].warp_id) ==
              co_issued_warps_decremented.end()) {
            m_warp[sets[s].warp_id]->dec_inst_in_pipeline();
            co_issued_warps_decremented.insert(sets[s].warp_id);
          }
        } else {
          // Intra-warp: release from secondary scoreboard (mode 0) or
          // mask-aware (mode 1) and dec once.
          if (!intra_warp_decremented) {
            unsigned set_slot =
                (sets[s].source_slot_id != (unsigned)-1)
                    ? sets[s].source_slot_id
                    : 1u;  // intra defaults to slot 1
            for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
              if (sets[s].source_inst->out[r] > 0) {
                m_scoreboard->releaseSetReg(
                    sets[s].warp_id, sets[s].source_inst->out[r],
                    sets[s].source_mask, /*is_intra_legacy=*/true, set_slot);
              }
            }
            m_warp[warp_id]->dec_inst_in_pipeline();
            intra_warp_decremented = true;
          }
        }
      }
    } else {
      m_scoreboard->releaseRegisters(pipe_reg);
      m_warp[warp_id]->dec_inst_in_pipeline();
    }

    warp_inst_complete(*pipe_reg);

    // Count additional co-issued instructions (not counted by warp_inst_complete)
    if (m_config->gpgpu_simd_partitioning && pipe_reg->has_simd_sets()) {
      const std::vector<simd_set_info> &sets = pipe_reg->get_simd_sets();
      // Accumulate active threads per unique co-issued warp/split
      std::map<unsigned, unsigned> inter_warp_threads;  // warp_id -> thread count
      unsigned intra_threads = 0;
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        if (sets[s].warp_id != warp_id) {
          inter_warp_threads[sets[s].warp_id] += sets[s].num_active_threads;
        } else {
          intra_threads += sets[s].num_active_threads;
        }
      }
      // Inter-warp: one warp-instruction per unique co-issued warp
      for (auto &entry : inter_warp_threads) {
        m_stats->m_num_sim_winsn[m_sid]++;
        m_gpu->gpu_sim_insn += entry.second;
        m_stats->m_num_sim_insn[m_sid] += entry.second;
      }
      // Intra-warp: one warp-instruction for the co-issued split
      if (intra_threads > 0) {
        m_stats->m_num_sim_winsn[m_sid]++;
        m_gpu->gpu_sim_insn += intra_threads;
        m_stats->m_num_sim_insn[m_sid] += intra_threads;
      }
    }

    m_gpu->gpu_sim_insn_last_update_sid = m_sid;
    m_gpu->gpu_sim_insn_last_update = m_gpu->gpu_sim_cycle;
    m_last_inst_gpu_sim_cycle = m_gpu->gpu_sim_cycle;
    m_last_inst_gpu_tot_sim_cycle = m_gpu->gpu_tot_sim_cycle;
    pipe_reg->clear();
    preg = m_pipeline_reg[EX_WB].get_ready();
    pipe_reg = (preg == NULL) ? NULL : *preg;
  }
}

bool ldst_unit::shared_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
                             mem_stage_access_type &fail_type) {
  // Mixed-space MEM co-issue (v2): relax the primary-space gate. Shared
  // work can live in (a) the primary's own cycles counter if primary is
  // shared, or (b) per-set `cycles` on any valid simd_set whose source
  // is a shared coissuer. Return immediately if the instruction has no
  // shared work at all.
  bool primary_is_shared = (inst.space.get_type() == shared_space ||
                            inst.space.get_type() == sstarr_space);
  bool has_shared_coissuer_set = false;
  if (inst.has_simd_sets()) {
    const std::vector<simd_set_info> &sets = inst.get_simd_sets();
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
      enum _memory_space_t src_sp = sets[s].source_inst->space.get_type();
      if (src_sp == shared_space || src_sp == sstarr_space) {
        has_shared_coissuer_set = true;
        break;
      }
    }
  }
  if (!primary_is_shared && !has_shared_coissuer_set) return true;

  if (inst.active_count() == 0) return true;

  bool any_stall = false;

  // MEMCO v3 Model B: when unified bank-conflict cycles are active for
  // this composite, replace the per-set max with a single unified counter
  // (computed at ldst_unit::issue across the union of all participating
  // lanes). Skip the primary + per-set walks below.
  if (inst.unified_shared_active()) {
    if (inst.has_dispatch_delay_unified()) {
      m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
    }
    if (inst.dispatch_delay_unified()) any_stall = true;
  } else {
    // Primary's own cycles (only counts if primary is shared).
    if (primary_is_shared) {
      if (inst.has_dispatch_delay()) {
        m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
      }
      if (inst.dispatch_delay()) any_stall = true;
    }

    // Per-set cycles for shared coissuer sets. Each set's cycles counts
    // down independently; we stall as long as ANY still has cycles > 0.
    if (inst.has_simd_sets()) {
      std::vector<simd_set_info> &sets = inst.get_simd_sets_mutable();
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        enum _memory_space_t src_sp = sets[s].source_inst->space.get_type();
        if (src_sp != shared_space && src_sp != sstarr_space) continue;
        if (sets[s].has_dispatch_delay()) {
          m_stats->gpgpu_n_shmem_bank_access[m_sid]++;
        }
        if (sets[s].dispatch_delay()) any_stall = true;
      }
    }
  }

  if (any_stall) {
    fail_type = S_MEM;
    rc_fail = BK_CONF;
    m_stats->gpgpu_n_shmem_bkconflict++;
  } else {
    rc_fail = NO_RC_FAIL;
  }
  return !any_stall;
}

mem_stage_stall_type ldst_unit::process_cache_access(
    cache_t *cache, new_addr_type address, warp_inst_t &inst,
    std::list<cache_event> &events, mem_fetch *mf,
    enum cache_request_status status) {
  mem_stage_stall_type result = NO_RC_FAIL;
  bool write_sent = was_write_sent(events);
  bool read_sent = was_read_sent(events);
  // Attribute store acks / load HIT register decrements to the access's
  // source warp + its source instruction (for MEM co-issue composites).
  unsigned src_wid_access = mf ? mf->get_wid() : inst.warp_id();
  unsigned src_split_access =
      mf ? mf->get_access_source_split_id() : (unsigned)-1;
  mem_src_t src = resolve_source(inst, src_wid_access, src_split_access);
  // "intra" here means the coissuer came from a secondary ibuffer slot
  // (scoreboard was reserved on the secondary map). It does NOT mean
  // same-warp-as-primary — try_utilization_max_coissue scans every warp's
  // secondary slots, so a coissuer from warp Y's secondary slot can pair
  // with warp X as primary. The correct discriminator is split_id != -1.
  bool src_is_intra_coissued = (src_split_access != (unsigned)-1);
  if (write_sent) {
    unsigned inc_ack = (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                           ? (mf->get_data_size() / SECTOR_SIZE)
                           : 1;

    for (unsigned i = 0; i < inc_ack; ++i)
      m_core->inc_store_req(src.wid);
  }
  if (status == HIT) {
    assert(!read_sent);
    inst.accessq_pop_back();
    if (inst.is_bru_st_fill_request() || inst.is_bru_rt_fill_request()) {
      release_virtual_entries(inst);
    } else if (mf && !mf->get_is_write()) {
      // Bug #2 fix: use per-access is_write (NOT composite primary op).
      // A STORE-primary composite can carry LOAD coissuer accesses; each
      // access's register-decrement or store-ack bookkeeping is selected
      // by the access's own type.
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
        if (src.out_inst->out[r] > 0) {
          if (src_is_intra_coissued)
            m_pending_writes_secondary[src.wid][src_split_access]
                                      [src.out_inst->out[r]]--;
          else
            m_pending_writes[src.wid][src.out_inst->out[r]]--;
          // Mode 1: per-mask decrement + release when the inst's own
          // accesses fully drain (independent of other in-flight insts'
          // contributions to the aggregate counter).
          dec_mask_pw_and_maybe_release(src.wid, src.out_inst->out[r],
                                         src.source_mask);
        }

      // release LDGSTS (primary-only: LDGSTS is excluded from MEM co-issue)
      if (inst.m_is_ldgsts) {
        m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)]--;
        if (m_pending_ldgsts[inst.warp_id()][inst.pc][inst.get_addr(0)] == 0) {
          m_core->unset_depbar(inst);
        }
      }
    }
    if (!write_sent) delete mf;
  } else if (status == RESERVATION_FAIL) {
    result = BK_CONF;
    assert(!read_sent);
    assert(!write_sent);
    delete mf;
  } else {
    assert(status == MISS || status == HIT_RESERVED);
    // inst.clear_active( access.get_warp_mask() ); // threads in mf writeback
    // when mf returns
    inst.accessq_pop_back();
  }
  if (!inst.accessq_empty() && result == NO_RC_FAIL) result = COAL_STALL;
  return result;
}

mem_stage_stall_type ldst_unit::process_memory_access_queue(cache_t *cache,
                                                            warp_inst_t &inst) {
  mem_stage_stall_type result = NO_RC_FAIL;
  if (inst.accessq_empty()) return result;

  if (!cache->data_port_free()) return DATA_PORT_STALL;

  // const mem_access_t &access = inst.accessq_back();
  mem_fetch *mf = m_mf_allocator->alloc(
      inst, inst.accessq_back(),
      m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle);
  std::list<cache_event> events;
  enum cache_request_status status = cache->access(
      mf->get_addr(), mf,
      m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle,
      events);
  return process_cache_access(cache, mf->get_addr(), inst, events, mf, status);
}

mem_stage_stall_type ldst_unit::process_memory_access_queue_l1cache(
    l1_cache *cache, warp_inst_t &inst) {
  mem_stage_stall_type result = NO_RC_FAIL;
  if (inst.accessq_empty()) return result;

  if (m_config->m_L1D_config.l1_latency > 0) {
    for (unsigned int j = 0; j < m_config->m_L1D_config.l1_banks;
         j++) {  // We can handle at max l1_banks reqs per cycle

      if (inst.accessq_empty()) return result;

      mem_fetch *mf =
          m_mf_allocator->alloc(inst, inst.accessq_back(),
                                m_core->get_gpu()->gpu_sim_cycle +
                                    m_core->get_gpu()->gpu_tot_sim_cycle);
      unsigned bank_id = m_config->m_L1D_config.set_bank(mf->get_addr());
      assert(bank_id < m_config->m_L1D_config.l1_banks);

      if ((l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1]) ==
          NULL) {
        l1_latency_queue[bank_id][m_config->m_L1D_config.l1_latency - 1] = mf;

        // Bug #2 fix: per-access write bit, NOT composite primary op.
        // A LOAD-primary composite can carry STORE coissuer accesses (and
        // vice versa). Each mf's own write bit determines whether it
        // needs an inc_store_req — the mf will be store_ack'd later when
        // WRITE_ACK returns, and that dec_store_req requires a matching
        // inc here.
        if (mf->get_is_write()) {
          unsigned inc_ack =
              (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                  ? (mf->get_data_size() / SECTOR_SIZE)
                  : 1;

          for (unsigned i = 0; i < inc_ack; ++i)
            m_core->inc_store_req(mf->get_wid());
        }

        inst.accessq_pop_back();
      } else {
        result = BK_CONF;
        m_stats->gpgpu_n_l1cache_bkconflict++;
        delete mf;
        break;  // do not try again, just break from the loop and try the next
                // cycle
      }
    }
    if (!inst.accessq_empty() && result != BK_CONF) result = COAL_STALL;

    return result;
  } else {
    mem_fetch *mf =
        m_mf_allocator->alloc(inst, inst.accessq_back(),
                              m_core->get_gpu()->gpu_sim_cycle +
                                  m_core->get_gpu()->gpu_tot_sim_cycle);
    std::list<cache_event> events;
    enum cache_request_status status = cache->access(
        mf->get_addr(), mf,
        m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle,
        events);
    return process_cache_access(cache, mf->get_addr(), inst, events, mf,
                                status);
  }
}

void ldst_unit::L1_latency_queue_cycle() {
  for (unsigned int j = 0; j < m_config->m_L1D_config.l1_banks; j++) {
    if ((l1_latency_queue[j][0]) != NULL) {
      mem_fetch *mf_next = l1_latency_queue[j][0];
      std::list<cache_event> events;
      enum cache_request_status status =
          m_L1D->access(mf_next->get_addr(), mf_next,
                        m_core->get_gpu()->gpu_sim_cycle +
                            m_core->get_gpu()->gpu_tot_sim_cycle,
                        events);

      bool write_sent = was_write_sent(events);
      bool read_sent = was_read_sent(events);

      if (status == HIT) {
        assert(!read_sent);
        l1_latency_queue[j][0] = NULL;
        // Bug #2 fix: per-access dispatch via mf's own write bit, not
        // composite primary op.
        if (!mf_next->get_is_write()) {
          // MEMCO v3: iterate mf's access source_list — a merged access has
          // multiple contributing sources, each with its own out[] regs to
          // decrement. Falls back to single-entry [(stamp_wid, stamp_split)]
          // when the access wasn't normalized (legacy / non-coissue path).
          std::vector<std::pair<unsigned, unsigned> > l1_srcs;
          const std::vector<std::pair<unsigned, unsigned> > &mf_srcs =
              mf_next->get_access_source_list();
          if (!mf_srcs.empty()) {
            l1_srcs = mf_srcs;
          } else {
            l1_srcs.push_back(std::make_pair(
                mf_next->get_wid(), mf_next->get_access_source_split_id()));
          }
          bool any_completed = false;
          for (unsigned si = 0; si < l1_srcs.size(); si++) {
            unsigned src_wid_i = l1_srcs[si].first;
            unsigned src_split_mf = l1_srcs[si].second;
            mem_src_t src = resolve_source(
                mf_next->get_inst(), src_wid_i, src_split_mf);
            // "intra" = coissuer came from a secondary ibuffer slot (secondary
            // scoreboard map). Not tied to warp equality with primary.
            bool src_is_intra = (src_split_mf != (unsigned)-1);
            for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
              if (src.out_inst->out[r] > 0) {
                unsigned reg = src.out_inst->out[r];
                if (src_is_intra) {
                  assert(m_pending_writes_secondary[src.wid][src_split_mf]
                                                   [reg] > 0);
                  unsigned still_pending = --m_pending_writes_secondary
                      [src.wid][src_split_mf][reg];
                  if (!still_pending) {
                    m_pending_writes_secondary[src.wid][src_split_mf].erase(
                        reg);
                    m_scoreboard->releaseSetReg(src.wid, reg, src.source_mask,
                                                /*is_intra_legacy=*/true,
                                                src.source_slot_id);
                    any_completed = true;
                  }
                } else {
                  assert(m_pending_writes[src.wid][reg] > 0);
                  unsigned still_pending = --m_pending_writes[src.wid][reg];
                  if (!still_pending) {
                    m_pending_writes[src.wid].erase(reg);
                    m_scoreboard->releaseSetReg(src.wid, reg, src.source_mask,
                                                /*is_intra_legacy=*/false,
                                                src.source_slot_id);
                    any_completed = true;
                  }
                }
                // Mode 1: per-mask release fires per inst, decoupled from
                // the aggregate counter above.
                dec_mask_pw_and_maybe_release(src.wid, reg, src.source_mask);
              }
          }
          if (any_completed) m_core->warp_inst_complete(mf_next->get_inst());

          // release LDGSTS (primary-only; LDGSTS excluded from MEM co-issue)
          if (mf_next->get_inst().m_is_ldgsts) {
            m_pending_ldgsts[mf_next->get_inst().warp_id()]
                            [mf_next->get_inst().pc]
                            [mf_next->get_inst().get_addr(0)]--;
            if (m_pending_ldgsts[mf_next->get_inst().warp_id()]
                                [mf_next->get_inst().pc]
                                [mf_next->get_inst().get_addr(0)] == 0) {
              m_core->unset_depbar(mf_next->get_inst());
            }
          }
        }

        // For write hit in WB policy (Bug #2 fix: per-access write bit)
        if (mf_next->get_is_write() && !write_sent) {
          unsigned dec_ack =
              (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                  ? (mf_next->get_data_size() / SECTOR_SIZE)
                  : 1;

          mf_next->set_reply();

          for (unsigned i = 0; i < dec_ack; ++i) m_core->store_ack(mf_next);
        }

        if (!write_sent) delete mf_next;

      } else if (status == RESERVATION_FAIL) {
        assert(!read_sent);
        assert(!write_sent);
      } else {
        assert(status == MISS || status == HIT_RESERVED);
        l1_latency_queue[j][0] = NULL;
        // Bug #2 fix: per-access write bit, not composite primary op.
        if (m_config->m_L1D_config.get_write_policy() != WRITE_THROUGH &&
            mf_next->get_is_write() &&
            (m_config->m_L1D_config.get_write_allocate_policy() ==
                 FETCH_ON_WRITE ||
             m_config->m_L1D_config.get_write_allocate_policy() ==
                 LAZY_FETCH_ON_READ) &&
            !was_writeallocate_sent(events)) {
          unsigned dec_ack =
              (m_config->m_L1D_config.get_mshr_type() == SECTOR_ASSOC)
                  ? (mf_next->get_data_size() / SECTOR_SIZE)
                  : 1;
          mf_next->set_reply();
          for (unsigned i = 0; i < dec_ack; ++i) m_core->store_ack(mf_next);
          if (!write_sent && !read_sent) delete mf_next;
        }
      }
    }

    for (unsigned stage = 0; stage < m_config->m_L1D_config.l1_latency - 1;
         ++stage)
      if (l1_latency_queue[j][stage] == NULL) {
        l1_latency_queue[j][stage] = l1_latency_queue[j][stage + 1];
        l1_latency_queue[j][stage + 1] = NULL;
      }
  }
}

bool ldst_unit::constant_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
                               mem_stage_access_type &fail_type) {
  if (inst.empty() || ((inst.space.get_type() != const_space) &&
                       (inst.space.get_type() != param_space_kernel)))
    return true;
  if (inst.active_count() == 0) return true;

  mem_stage_stall_type fail;
  if (m_config->perfect_inst_const_cache) {
    fail = NO_RC_FAIL;
    unsigned access_count = inst.accessq_count();
    while (inst.accessq_count() > 0) inst.accessq_pop_back();
    if (inst.is_load()) {
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
        if (inst.out[r] > 0)
          m_pending_writes[inst.warp_id()][inst.out[r]] -= access_count;
    }
  } else {
    fail = process_memory_access_queue(m_L1C, inst);
  }

  if (fail != NO_RC_FAIL) {
    rc_fail = fail;  // keep other fails if this didn't fail.
    fail_type = C_MEM;
    if (rc_fail == BK_CONF or rc_fail == COAL_STALL) {
      m_stats->gpgpu_n_cmem_portconflict++;  // coal stalls aren't really a bank
                                             // conflict, but this maintains
                                             // previous behavior.
    }
  }
  return inst.accessq_empty();  // done if empty.
}

bool ldst_unit::texture_cycle(warp_inst_t &inst, mem_stage_stall_type &rc_fail,
                              mem_stage_access_type &fail_type) {
  if (inst.empty() || inst.space.get_type() != tex_space) return true;
  if (inst.active_count() == 0) return true;
  mem_stage_stall_type fail = process_memory_access_queue(m_L1T, inst);
  if (fail != NO_RC_FAIL) {
    rc_fail = fail;  // keep other fails if this didn't fail.
    fail_type = T_MEM;
  }
  return inst.accessq_empty();  // done if empty.
}

bool ldst_unit::memory_cycle(warp_inst_t &inst,
                             mem_stage_stall_type &stall_reason,
                             mem_stage_access_type &access_type) {
  // Mixed-space MEM co-issue (v2): accessq is the source of truth for
  // global/local/param_local work. A shared-primary composite with a
  // global coissuer will have global accesses in its accessq (stamped
  // with source_wid per access), so we must enter this function even if
  // primary's space is shared. Accesses themselves carry their own
  // attribution; primary space only matters for the `else` stat-only
  // branch at the bottom of this function (access_type encoding).
  if (inst.empty()) return true;
  if (inst.active_count() == 0) return true;
  if (inst.accessq_empty()) return true;

  mem_stage_stall_type stall_cond = NO_RC_FAIL;
  const mem_access_t &access = inst.accessq_back();

  bool bypassL1D = false;
  if (CACHE_GLOBAL == inst.cache_op || (m_L1D == NULL)) {
    bypassL1D = true;
  } else if (inst.space.is_global()) {  // global memory access
    // skip L1 cache if the option is enabled
    if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op))
      bypassL1D = true;
  }
  if (bypassL1D) {
    // bypass L1 cache
    unsigned control_size =
        inst.is_store() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE;
    for (unsigned i = 0; i < m_config->m_L1D_config.l1_banks; i++) {
      if (inst.accessq_empty()) {
        break;
      }
      const mem_access_t &access = inst.accessq_back();
      unsigned size = access.get_size() + control_size;
      // printf("Interconnect:Addr: %x, size=%d\n",access.get_addr(),size);
      if (m_memory_config->SST_mode &&
          (static_cast<sst_memory_interface *>(m_icnt)->full(
              size, inst.is_store() || inst.isatomic(), access.get_type()))) {
        // SST need mf type here
        // Cast it to sst_memory_interface pointer first as this full() method
        // is not a virtual method in parent class
        stall_cond = ICNT_RC_FAIL;
        break;
      } else if (!m_memory_config->SST_mode &&
                 (m_icnt->full(size, inst.is_store() || inst.isatomic()))) {
        stall_cond = ICNT_RC_FAIL;
        break;
      } else {
        // Resolve source attribution for MEM co-issued accesses.
        unsigned src_wid = access.get_source_wid();
        if (src_wid == (unsigned)-1) src_wid = inst.warp_id();
        mem_fetch *mf =
            m_mf_allocator->alloc(inst, access,
                                  m_core->get_gpu()->gpu_sim_cycle +
                                      m_core->get_gpu()->gpu_tot_sim_cycle);
        m_icnt->push(mf);
        inst.accessq_pop_back();
        // inst.clear_active( access.get_warp_mask() );
        // Bug #2 fix: use per-access is_write, NOT composite primary op.
        // A STORE-primary composite can carry LOAD coissuer accesses, and
        // vice versa. Each access's type determines its own bookkeeping.
        if (!access.is_write()) {
          // For MEM-co-issue composites, sanity-check against the source
          // warp's pending_writes on its source out[] regs (if identifiable).
          const inst_t *src_out_inst = &inst;
          if (src_wid != inst.warp_id() && inst.has_simd_sets()) {
            const std::vector<simd_set_info> &sets = inst.get_simd_sets();
            for (unsigned s = 0; s < sets.size(); s++) {
              if (sets[s].valid && sets[s].source_inst != NULL &&
                  sets[s].warp_id == src_wid) {
                src_out_inst = sets[s].source_inst;
                break;
              }
            }
          }
          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++)
            if (src_out_inst->out[r] > 0)
              assert(m_pending_writes[src_wid][src_out_inst->out[r]] > 0);
        } else {
          m_core->inc_store_req(src_wid);
        }
      }
    }
  } else {
    assert(CACHE_UNDEFINED != inst.cache_op);
    stall_cond = process_memory_access_queue_l1cache(m_L1D, inst);
  }
  if (!inst.accessq_empty() && stall_cond == NO_RC_FAIL)
    stall_cond = COAL_STALL;
  if (stall_cond != NO_RC_FAIL) {
    stall_reason = stall_cond;
    bool iswrite = inst.is_store();
    if (inst.space.is_local())
      access_type = (iswrite) ? L_MEM_ST : L_MEM_LD;
    else
      access_type = (iswrite) ? G_MEM_ST : G_MEM_LD;
  }
  return inst.accessq_empty();
}

bool ldst_unit::response_buffer_full() const {
  return m_response_fifo.size() >= m_config->ldst_unit_response_queue_size;
}

void ldst_unit::fill(mem_fetch *mf) {
  mf->set_status(
      IN_SHADER_LDST_RESPONSE_FIFO,
      m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle);
  m_response_fifo.push_back(mf);
}

void ldst_unit::flush() {
  // Flush L1D cache
  m_L1D->flush();
}

void ldst_unit::invalidate() {
  // Flush L1D cache
  m_L1D->invalidate();
}

simd_function_unit::simd_function_unit(const shader_core_config *config) {
  m_config = config;
  m_dispatch_reg = new warp_inst_t(config);
}

void simd_function_unit::issue(register_set &source_reg) {
  bool partition_issue =
      m_config->sub_core_model && this->is_issue_partitioned();
  source_reg.move_out_to(partition_issue, this->get_issue_reg_id(),
                         m_dispatch_reg);
  occupied.set(m_dispatch_reg->latency);
}

sfu::sfu(register_set *result_port, const shader_core_config *config,
         shader_core_ctx *core, unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, config->max_sfu_latency, core,
                          issue_reg_id) {
  m_name = "SFU";
}

tensor_core::tensor_core(register_set *result_port,
                         const shader_core_config *config,
                         shader_core_ctx *core, unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, config->max_tensor_core_latency,
                          core, issue_reg_id) {
  m_name = "TENSOR_CORE";
}

void sfu::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));

  (*ready_reg)->op_pipe = SFU__OP;
  m_core->incsfu_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

void tensor_core::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));

  (*ready_reg)->op_pipe = TENSOR_CORE__OP;
  m_core->incsfu_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

unsigned pipelined_simd_unit::get_active_lanes_in_pipeline() {
  active_mask_t active_lanes;
  active_lanes.reset();
  if (m_core->get_gpu()->get_config().g_power_simulation_enabled) {
    for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++) {
      if (!m_pipeline_reg[stage]->empty())
        active_lanes |= m_pipeline_reg[stage]->get_active_mask();
    }
  }
  return active_lanes.count();
}

void ldst_unit::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incfumemactivelanes_stat(active_count);
}

void sp_unit::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incspactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}
void dp_unit::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  // m_core->incspactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}
void specialized_unit::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incspactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}

void int_unit::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incspactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}
void sfu::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incsfuactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}

void tensor_core::active_lanes_in_pipeline() {
  unsigned active_count = pipelined_simd_unit::get_active_lanes_in_pipeline();
  assert(active_count <= m_core->get_config()->warp_size);
  m_core->incsfuactivelanes_stat(active_count);
  m_core->incfuactivelanes_stat(active_count);
  m_core->incfumemactivelanes_stat(active_count);
}

sp_unit::sp_unit(register_set *result_port, const shader_core_config *config,
                 shader_core_ctx *core, unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, config->max_sp_latency, core,
                          issue_reg_id) {
  m_name = "SP ";
}

specialized_unit::specialized_unit(register_set *result_port,
                                   const shader_core_config *config,
                                   shader_core_ctx *core, int supported_op,
                                   char *unit_name, unsigned latency,
                                   unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, latency, core, issue_reg_id) {
  m_name = unit_name;
  m_supported_op = supported_op;
}

dp_unit::dp_unit(register_set *result_port, const shader_core_config *config,
                 shader_core_ctx *core, unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, config->max_dp_latency, core,
                          issue_reg_id) {
  m_name = "DP ";
}

int_unit::int_unit(register_set *result_port, const shader_core_config *config,
                   shader_core_ctx *core, unsigned issue_reg_id)
    : pipelined_simd_unit(result_port, config, config->max_int_latency, core,
                          issue_reg_id) {
  m_name = "INT ";
}

void sp_unit ::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));
  (*ready_reg)->op_pipe = SP__OP;
  m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

void dp_unit ::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));
  (*ready_reg)->op_pipe = DP__OP;
  m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

void specialized_unit ::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));
  (*ready_reg)->op_pipe = SPECIALIZED__OP;
  m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

void int_unit ::issue(register_set &source_reg) {
  warp_inst_t **ready_reg =
      source_reg.get_ready(m_config->sub_core_model, m_issue_reg_id);
  // m_core->incexecstat((*ready_reg));
  (*ready_reg)->op_pipe = INTP__OP;
  m_core->incsp_stat(m_core->get_config()->warp_size, (*ready_reg)->latency);
  pipelined_simd_unit::issue(source_reg);
}

pipelined_simd_unit::pipelined_simd_unit(register_set *result_port,
                                         const shader_core_config *config,
                                         unsigned max_latency,
                                         shader_core_ctx *core,
                                         unsigned issue_reg_id)
    : simd_function_unit(config) {
  m_result_port = result_port;
  m_pipeline_depth = max_latency;
  m_pipeline_reg = new warp_inst_t *[m_pipeline_depth];
  for (unsigned i = 0; i < m_pipeline_depth; i++)
    m_pipeline_reg[i] = new warp_inst_t(config);
  m_core = core;
  m_issue_reg_id = issue_reg_id;
  active_insts_in_pipeline = 0;
}

void pipelined_simd_unit::cycle() {
  if (!m_pipeline_reg[0]->empty()) {
    m_result_port->move_in(m_pipeline_reg[0]);
    assert(active_insts_in_pipeline > 0);
    active_insts_in_pipeline--;
  }
  if (active_insts_in_pipeline) {
    for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++)
      move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]);
  }
  if (!m_dispatch_reg->empty()) {
    if (!m_dispatch_reg->dispatch_delay()) {
      int start_stage =
          m_dispatch_reg->latency - m_dispatch_reg->initiation_interval;
      if (m_pipeline_reg[start_stage]->empty()) {
        move_warp(m_pipeline_reg[start_stage], m_dispatch_reg);
        active_insts_in_pipeline++;
      }
    }
  }
  occupied >>= 1;
}

void pipelined_simd_unit::issue(register_set &source_reg) {
  // move_warp(m_dispatch_reg,source_reg);
  bool partition_issue =
      m_config->sub_core_model && this->is_issue_partitioned();
  warp_inst_t **ready_reg =
      source_reg.get_ready(partition_issue, m_issue_reg_id);
  m_core->incexecstat((*ready_reg));
  // source_reg.move_out_to(m_dispatch_reg);
  simd_function_unit::issue(source_reg);
}

/*
    virtual void issue( register_set& source_reg )
    {
        //move_warp(m_dispatch_reg,source_reg);
        //source_reg.move_out_to(m_dispatch_reg);
        simd_function_unit::issue(source_reg);
    }
*/

void ldst_unit::init(mem_fetch_interface *icnt,
                     shader_core_mem_fetch_allocator *mf_allocator,
                     shader_core_ctx *core, opndcoll_rfu_t *operand_collector,
                     Scoreboard *scoreboard, const shader_core_config *config,
                     const memory_config *mem_config, shader_core_stats *stats,
                     unsigned sid, unsigned tpc) {
  m_memory_config = mem_config;
  m_icnt = icnt;
  m_mf_allocator = mf_allocator;
  m_core = core;
  m_operand_collector = operand_collector;
  m_scoreboard = scoreboard;
  m_stats = stats;
  m_sid = sid;
  m_tpc = tpc;
#define STRSIZE 1024
  char L1T_name[STRSIZE];
  char L1C_name[STRSIZE];
  snprintf(L1T_name, STRSIZE, "L1T_%03d", m_sid);
  snprintf(L1C_name, STRSIZE, "L1C_%03d", m_sid);
  m_L1T = new tex_cache(L1T_name, m_config->m_L1T_config, m_sid,
                        get_shader_texture_cache_id(), icnt, IN_L1T_MISS_QUEUE,
                        IN_SHADER_L1T_ROB);
  m_L1C = new read_only_cache(L1C_name, m_config->m_L1C_config, m_sid,
                              get_shader_constant_cache_id(), icnt,
                              IN_L1C_MISS_QUEUE, OTHER_GPU_CACHE, m_gpu);
  m_L1D = NULL;
  m_mem_rc = NO_RC_FAIL;
  m_num_writeback_clients =
      5;  // = shared memory, global/local (uncached), L1D, L1T, L1C
  m_writeback_arb = 0;
  m_next_global = NULL;
  m_next_wb_src_wid = (unsigned)-1;
  m_next_wb_src_split_id = (unsigned)-1;
  m_next_wb_src_list.clear();
  m_n_interset_merges = 0;
  m_n_interset_merged_sources = 0;
  m_last_inst_gpu_sim_cycle = 0;
  m_last_inst_gpu_tot_sim_cycle = 0;
}

ldst_unit::ldst_unit(mem_fetch_interface *icnt,
                     shader_core_mem_fetch_allocator *mf_allocator,
                     shader_core_ctx *core, opndcoll_rfu_t *operand_collector,
                     Scoreboard *scoreboard, const shader_core_config *config,
                     const memory_config *mem_config, shader_core_stats *stats,
                     unsigned sid, unsigned tpc, gpgpu_sim *gpu)
    : pipelined_simd_unit(NULL, config, config->smem_latency, core, 0),
      m_next_wb(config),
      m_gpu(gpu) {
  assert(config->smem_latency > 1);
  init(icnt, mf_allocator, core, operand_collector, scoreboard, config,
       mem_config, stats, sid, tpc);
  if (!m_config->m_L1D_config.disabled()) {
    char L1D_name[STRSIZE];
    snprintf(L1D_name, STRSIZE, "L1D_%03d", m_sid);
    m_L1D = new l1_cache(L1D_name, m_config->m_L1D_config, m_sid,
                         get_shader_normal_cache_id(), m_icnt, m_mf_allocator,
                         IN_L1D_MISS_QUEUE, core->get_gpu(), L1_GPU_CACHE);

    l1_latency_queue.resize(m_config->m_L1D_config.l1_banks);
    assert(m_config->m_L1D_config.l1_latency > 0);

    for (unsigned j = 0; j < m_config->m_L1D_config.l1_banks; j++)
      l1_latency_queue[j].resize(m_config->m_L1D_config.l1_latency,
                                 (mem_fetch *)NULL);
  }
  m_name = "MEM ";
}

ldst_unit::ldst_unit(mem_fetch_interface *icnt,
                     shader_core_mem_fetch_allocator *mf_allocator,
                     shader_core_ctx *core, opndcoll_rfu_t *operand_collector,
                     Scoreboard *scoreboard, const shader_core_config *config,
                     const memory_config *mem_config, shader_core_stats *stats,
                     unsigned sid, unsigned tpc, l1_cache *new_l1d_cache)
    : pipelined_simd_unit(NULL, config, 3, core, 0),
      m_L1D(new_l1d_cache),
      m_next_wb(config) {
  init(icnt, mf_allocator, core, operand_collector, scoreboard, config,
       mem_config, stats, sid, tpc);
}

void ldst_unit::dec_mask_pw_and_maybe_release(unsigned wid, unsigned reg,
                                              const active_mask_t &mask) {
  if (m_core->get_config()->gpgpu_scoreboard_mode != 1) return;
  unsigned long mk = mask.to_ulong();
  auto wit = m_pending_writes_mask.find(wid);
  if (wit == m_pending_writes_mask.end()) return;
  auto rit = wit->second.find(reg);
  if (rit == wit->second.end()) return;
  auto mit = rit->second.find(mk);
  if (mit == rit->second.end()) return;
  assert(mit->second > 0);
  if (--(mit->second) == 0) {
    rit->second.erase(mit);
    if (rit->second.empty()) {
      wit->second.erase(rit);
      if (wit->second.empty()) m_pending_writes_mask.erase(wit);
    }
    m_scoreboard->releaseRegisterMask(wid, reg, mask);
  }
}

ldst_unit::mem_src_t ldst_unit::resolve_source(
    const warp_inst_t &inst, unsigned access_src_wid,
    unsigned access_src_split_id) const {
  mem_src_t r;
  r.wid = (access_src_wid == (unsigned)-1) ? inst.warp_id() : access_src_wid;
  r.out_inst = &inst;
  r.source_mask.reset();
  r.source_slot_id = 0;  // primary's own slot
  // Primary access: (wid == primary AND split_id == -1). Return early with
  // out_inst = composite (primary's out[]). Note: split_id is the
  // discriminator — a coissuer can have the same warp_id as primary when
  // it came from the primary warp's own secondary ibuffer slot.
  if (r.wid == inst.warp_id() && access_src_split_id == (unsigned)-1) {
    // Primary's own mask is on the composite itself.
    r.source_mask = inst.get_active_mask();
    r.source_slot_id = inst.get_ibuffer_half_id();
    if (r.source_slot_id == (unsigned)-1) r.source_slot_id = 0;
    return r;
  }
  if (!inst.has_simd_sets()) return r;
  // Coissued access: search sets for an exact (wid, split_id) match.
  const std::vector<simd_set_info> &sets = inst.get_simd_sets();
  for (unsigned s = 0; s < sets.size(); s++) {
    if (!sets[s].valid || sets[s].source_inst == NULL) continue;
    if (sets[s].warp_id == r.wid &&
        sets[s].split_id == access_src_split_id) {
      r.out_inst = sets[s].source_inst;
      r.source_mask = sets[s].source_mask;
      if (sets[s].source_slot_id != (unsigned)-1) {
        r.source_slot_id = sets[s].source_slot_id;
      }
      break;
    }
  }
  return r;
}

// MEMCO v3: inter-set coalescing pass. Single post-splice walk of the
// composite's accessq that:
//  (1) Normalizes each eligible access's source_list to contain the
//      access's originating (wid, split_id) (from its single stamp).
//  (2) Merges segment-equal accesses (same access_type + block_address +
//      is_write) by unioning their warp/byte/sector masks into the first
//      occurrence and appending the later source to the first's source_list,
//      then erasing the merged duplicate.
//
// Provenance: each merged access carries the list of contributing sources.
// Writeback (ldst_unit::writeback) and L1 HIT path (L1_latency_queue_cycle)
// iterate this list to decrement pending_writes for each source.
//
// Exclusions: atomics (per-thread semantic), LDGSTS (excluded from co-issue),
// access types other than GLOBAL/LOCAL (shared doesn't populate accessq;
// const/texture take other pipe stages).
//
// Env vars:
//   MEMCOV3_DISABLE_INTERSET_COALESCE=1 → skip pass entirely (Stage A behavior)
//   MEMCOV3_DISABLE_MERGE=1             → normalize but skip merge (Stage B)
//   MEMCOV3_TRACE_MERGES=1              → print one line per merge event
void ldst_unit::coalesce_accessq_across_sets(warp_inst_t &inst) {
  static const bool disable = (getenv("MEMCOV3_DISABLE_INTERSET_COALESCE") !=
                               NULL);
  static const bool disable_merge = (getenv("MEMCOV3_DISABLE_MERGE") != NULL);
  static const bool trace = (getenv("MEMCOV3_TRACE_MERGES") != NULL);
  if (disable) return;
  if (inst.isatomic()) return;
  if (inst.m_is_ldgsts) return;
  unsigned primary_wid = inst.warp_id();

  // Pass 1: normalize source_list on every eligible access.
  for (std::list<mem_access_t>::iterator it = inst.accessq_mut_begin();
       it != inst.accessq_mut_end(); ++it) {
    mem_access_type at = it->get_type();
    bool eligible = (at == GLOBAL_ACC_R || at == GLOBAL_ACC_W ||
                     at == LOCAL_ACC_R || at == LOCAL_ACC_W);
    if (!eligible) continue;
    if (it->has_source_list()) continue;
    unsigned w = it->get_source_wid();
    unsigned sp = it->get_source_split_id();
    if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; }
    it->add_source(w, sp);
  }

  if (disable_merge) return;

  // Pass 2: merge segment-equal accesses. Hash key = (type, addr, size, write).
  // Include size to avoid merging accesses of different sector sizes (e.g.
  // 32B atomic sector vs 128B line). Merging is safe only when every field
  // the downstream cache/coalescer inspects is identical.
  struct merge_key {
    mem_access_type type;
    new_addr_type addr;
    unsigned size;
    bool write;
    bool operator<(const merge_key &o) const {
      if (type != o.type) return type < o.type;
      if (addr != o.addr) return addr < o.addr;
      if (size != o.size) return size < o.size;
      return write < o.write;
    }
  };
  std::map<merge_key, std::list<mem_access_t>::iterator> first_occ;

  unsigned merges_fired = 0;
  std::list<mem_access_t>::iterator it = inst.accessq_mut_begin();
  while (it != inst.accessq_mut_end()) {
    mem_access_type at = it->get_type();
    bool eligible = (at == GLOBAL_ACC_R || at == GLOBAL_ACC_W ||
                     at == LOCAL_ACC_R || at == LOCAL_ACC_W);
    if (!eligible) { ++it; continue; }
    merge_key k;
    k.type = at;
    k.addr = it->get_addr();
    k.size = it->get_size();
    k.write = it->is_write();
    std::map<merge_key, std::list<mem_access_t>::iterator>::iterator
        fit = first_occ.find(k);
    if (fit == first_occ.end()) {
      first_occ[k] = it;
      ++it;
    } else {
      mem_access_t &existing = *(fit->second);
      const mem_access_t &dup = *it;
      // Union masks.
      active_mask_t merged_active = existing.get_warp_mask() |
                                    dup.get_warp_mask();
      mem_access_byte_mask_t merged_byte = existing.get_byte_mask() |
                                           dup.get_byte_mask();
      mem_access_sector_mask_t merged_sector =
          existing.get_sector_mask() | dup.get_sector_mask();
      // Mutate existing via a rebuilt mem_access_t (no in-place setters for
      // masks). Preserve source_list from existing + append dup's sources.
      std::vector<std::pair<unsigned, unsigned> > merged_srcs =
          existing.get_source_list();
      const std::vector<std::pair<unsigned, unsigned> > &dup_srcs =
          dup.get_source_list();
      for (unsigned i = 0; i < dup_srcs.size(); i++) {
        bool already = false;
        for (unsigned j = 0; j < merged_srcs.size(); j++) {
          if (merged_srcs[j] == dup_srcs[i]) { already = true; break; }
        }
        if (!already) merged_srcs.push_back(dup_srcs[i]);
      }
      mem_access_t rebuilt(k.type, k.addr, k.size, k.write, merged_active,
                           merged_byte, merged_sector,
                           m_core->get_gpu()->gpgpu_ctx);
      // Preserve single-stamp from existing (first-arrival source), so
      // downstream code that still reads the single stamp sees a consistent
      // first source; source_list carries the full truth for iteration.
      rebuilt.set_source_wid(existing.get_source_wid());
      rebuilt.set_source_split_id(existing.get_source_split_id());
      for (unsigned i = 0; i < merged_srcs.size(); i++) {
        rebuilt.add_source(merged_srcs[i].first, merged_srcs[i].second);
      }
      *(fit->second) = rebuilt;
      it = inst.accessq_mut_erase(it);
      merges_fired++;
      m_n_interset_merges++;
      m_n_interset_merged_sources += dup_srcs.size();
      extern unsigned long long g_memcov3_n_merges_global;
      extern unsigned long long g_memcov3_n_merged_sources_global;
      g_memcov3_n_merges_global++;
      g_memcov3_n_merged_sources_global += dup_srcs.size();
      if (trace) {
        fprintf(stdout,
                "[MEMCOV3 merge] sid=%u pc=0x%llx type=%d addr=0x%llx "
                "size=%u wr=%d sources_now=%u\n",
                m_sid, (unsigned long long)inst.pc, (int)k.type,
                (unsigned long long)k.addr, k.size, k.write ? 1 : 0,
                (unsigned)merged_srcs.size());
      }
    }
  }
  (void)merges_fired;
}

void ldst_unit::issue(register_set &reg_set) {
  warp_inst_t *inst = *(reg_set.get_ready());

  // record how many pending register writes/memory accesses there are for this
  // instruction
  assert(inst->empty() == false);
  // Bug #1 fix: a composite's PRIMARY op may be STORE or shared while one
  // or more coissuer SETS carry LOAD accesses in non-shared spaces. Don't
  // gate the whole pending_writes bookkeeping block on primary. Gate the
  // primary portion on primary's own is_load+non-shared; gate per-set
  // accumulation on EACH set's source_inst is_load+non-shared.
  unsigned primary_wid = inst->warp_id();
  bool primary_is_tracked_load = inst->is_load() &&
                                 inst->space.get_type() != shared_space;

  // MEMCO v3: for co-issue composites, run the inter-set coalescing /
  // provenance-normalization pass before pending_writes accounting. Stage B:
  // populate each access's source_list with [(wid, split)] derived from the
  // access's single-stamp — no merging yet, same access count as before.
  bool is_coissue_composite = inst->has_simd_sets() &&
                              m_config->gpgpu_simd_partitioning &&
                              m_config->gpgpu_mem_coissue;
  if (is_coissue_composite && !inst->accessq_empty()) {
    coalesce_accessq_across_sets(*inst);
  }

  // MEMCO v3 Model B: compute unified shared bank-conflict cycles across
  // all participating sets when MEMCOV3_SHARED_UNIFIED is set.
  if (is_coissue_composite) {
    inst->compute_unified_shared_cycles();
  }

  if (is_coissue_composite) {
    // MEM co-issue: accesses in the composite's queue may originate from
    // different source warps (inter-warp) or different splits of the same
    // warp (intra-warp). Group them by (source_wid, source_split_id) to
    // charge pending_writes to the correct source's destination registers.
    // Post-MEMCO-v3: each access has a source_list (one entry for unmerged,
    // multiple for merged). Iterate the list so merged accesses count
    // toward EACH contributing source.
    std::map<std::pair<unsigned, unsigned>, unsigned> n_acc_per_src;
    for (std::list<mem_access_t>::const_iterator it = inst->accessq_begin();
         it != inst->accessq_end(); ++it) {
      if (it->has_source_list()) {
        const std::vector<std::pair<unsigned, unsigned> > &srcs =
            it->get_source_list();
        for (unsigned i = 0; i < srcs.size(); i++) {
          n_acc_per_src[srcs[i]] += 1;
        }
      } else {
        unsigned w = it->get_source_wid();
        unsigned sp = it->get_source_split_id();
        if (w == (unsigned)-1) { w = primary_wid; sp = (unsigned)-1; }
        n_acc_per_src[std::make_pair(w, sp)] += 1;
      }
    }

    // Primary's own registers — gated on primary being a tracked load
    // (stores have no out[]; shared loads don't use pending_writes).
    if (primary_is_tracked_load) {
      unsigned n_primary =
          n_acc_per_src[std::make_pair(primary_wid, (unsigned)-1)];
      if (n_primary > 0) {
        const bool mode1 =
            (m_core->get_config()->gpgpu_scoreboard_mode == 1);
        unsigned long pri_mask_key =
            mode1 ? inst->get_active_mask().to_ulong() : 0;
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = inst->out[r];
          if (reg_id > 0) {
            m_pending_writes[primary_wid][reg_id] += n_primary;
            if (mode1) {
              m_pending_writes_mask[primary_wid][reg_id][pri_mask_key] +=
                  n_primary;
            }
          }
        }
      }
    }

    // Per co-issued set's registers (use set's source_inst->out[]).
    // Gate per-set on the SET's source being a tracked load, independent
    // of primary. A STORE-primary composite can carry LOAD coissuers;
    // their pending_writes must still be tracked so their mf returns
    // decrement correctly.
    const std::vector<simd_set_info> &sets = inst->get_simd_sets();
    std::set<std::pair<unsigned, unsigned>> accounted;
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
      if (!sets[s].source_inst->is_load()) continue;
      if (sets[s].source_inst->space.get_type() == shared_space) continue;
      std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
      if (accounted.count(key)) continue;
      accounted.insert(key);
      unsigned n_src = n_acc_per_src[key];
      if (n_src == 0) continue;
      // is_intra = "uses secondary scoreboard map" = split came from a
      // secondary ibuffer slot. Not tied to warp_id == primary_wid.
      bool is_intra = (sets[s].split_id != (unsigned)-1);
      const bool mode1 =
          (m_core->get_config()->gpgpu_scoreboard_mode == 1);
      unsigned long set_mask_key =
          mode1 ? sets[s].source_mask.to_ulong() : 0;
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
        unsigned reg_id = sets[s].source_inst->out[r];
        if (reg_id > 0) {
          if (is_intra) {
            m_pending_writes_secondary[sets[s].warp_id][sets[s].split_id]
                                      [reg_id] += n_src;
          } else {
            m_pending_writes[sets[s].warp_id][reg_id] += n_src;
          }
          if (mode1) {
            m_pending_writes_mask[sets[s].warp_id][reg_id][set_mask_key] +=
                n_src;
          }
        }
      }
    }
  } else if (primary_is_tracked_load) {
    // Legacy (non-co-issue) path: single-warp pending_writes.
    unsigned n_accesses = inst->accessq_count();
    const bool mode1 =
        (m_core->get_config()->gpgpu_scoreboard_mode == 1);
    unsigned long pri_mask_key =
        mode1 ? inst->get_active_mask().to_ulong() : 0;
    for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
      unsigned reg_id = inst->out[r];
      if (reg_id > 0) {
        m_pending_writes[primary_wid][reg_id] += n_accesses;
        if (mode1) {
          m_pending_writes_mask[primary_wid][reg_id][pri_mask_key] +=
              n_accesses;
        }
      }
    }
  }

  if (primary_is_tracked_load && inst->m_is_ldgsts) {
    m_pending_ldgsts[primary_wid][inst->pc][inst->get_addr(0)] +=
        inst->accessq_count();
  }

  inst->op_pipe = MEM__OP;
  // stat collection
  m_core->mem_instruction_stats(*inst);
  m_core->incmem_stat(m_core->get_config()->warp_size, 1);
  pipelined_simd_unit::issue(reg_set);
}

void ldst_unit::release_virtual_entries(warp_inst_t &inst) {
  if (inst.is_bru_st_fill_request()) {
    unsigned wid = inst.warp_id();
    address_type addr = inst.pc;
    unsigned entry = (addr - BRU_VIR_START -
                      (wid * m_config->warp_size) * MAX_BRU_VIR_PER_SPLIT) /
                     MAX_BRU_VIR_PER_SPLIT;
    bool done = m_core->push_to_st_response_fifo(wid, entry);
    if (done) {
      inst.clear();
      inst.clear_pending_mem_requests();
    }
  } else {
    assert(inst.is_bru_rt_fill_request());
    unsigned wid = inst.warp_id();
    address_type addr = inst.pc;
    unsigned entry = (addr - BRU_VIR_START -
                      (wid * m_config->warp_size) * MAX_BRU_VIR_PER_SPLIT -
                      MAX_BRU_VIR_PER_SPLIT / 2) /
                     MAX_BRU_VIR_PER_SPLIT;
    bool done = m_core->push_to_rt_response_fifo(wid, entry);
    if (done) {
      inst.clear();
      inst.clear_pending_mem_requests();
    }
  }
}

void ldst_unit::writeback() {
  // process next instruction that is going to writeback
  if (!m_next_wb.empty()) {
    if (m_next_wb.is_bru_st_fill_request() ||
        m_next_wb.is_bru_rt_fill_request()) {
      release_virtual_entries(m_next_wb);
    } else if (m_operand_collector->writeback(m_next_wb)) {
      {
      bool insn_completed = false;
      // Resolve (src_wid, src_out_inst) for this writeback.  For non-shared
      // spaces the source is stamped on the feeding mem_fetch.  For shared
      // the source is always the composite's primary here; co-issued sets'
      // releases are handled in a dedicated pass below.
      unsigned primary_wid = m_next_wb.warp_id();
      // Mixed-space MEM co-issue (v2): for MIXED composites with shared
      // primary, a global coissuer's mf return arrives here via case 3/4
      // with m_next_wb_src_wid stamped to coissuer_wid. The mf's origin
      // is GLOBAL — it should take the non-shared pending_writes path,
      // not the shared-primary release path. Use the mf stamp to
      // disambiguate: stamped = mf-driven (non-shared routing), unstamped
      // = case 0 shared retire (shared routing).
      bool is_shared = (m_next_wb.space.get_type() == shared_space &&
                        m_next_wb_src_wid == (unsigned)-1);
      unsigned src_wid = (!is_shared && m_next_wb_src_wid != (unsigned)-1)
                             ? m_next_wb_src_wid
                             : primary_wid;
      unsigned src_split = m_next_wb_src_split_id;
      // is_intra = secondary-slot coissuer (not warp equality with primary).
      bool is_intra_coissued = (src_split != (unsigned)-1);

      if (!is_shared) {
        // MEMCO v3: iterate m_next_wb_src_list — each contributing source
        // gets its out[] regs decremented independently. Falls back to
        // single-entry [(src_wid, src_split)] when the access wasn't
        // normalized (legacy / non-coissue path).
        std::vector<std::pair<unsigned, unsigned> > wb_srcs;
        if (!m_next_wb_src_list.empty()) {
          wb_srcs = m_next_wb_src_list;
        } else {
          wb_srcs.push_back(std::make_pair(src_wid, src_split));
        }
        for (unsigned si = 0; si < wb_srcs.size(); si++) {
          unsigned src_wid_i = wb_srcs[si].first;
          unsigned src_split_i = wb_srcs[si].second;
          bool is_intra_i = (src_split_i != (unsigned)-1);
          mem_src_t src_i = resolve_source(m_next_wb, src_wid_i, src_split_i);
          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
            if (src_i.out_inst->out[r] > 0) {
              unsigned reg = src_i.out_inst->out[r];
              if (is_intra_i) {
                assert(m_pending_writes_secondary[src_i.wid][src_split_i]
                                                 [reg] > 0);
                unsigned still_pending =
                    --m_pending_writes_secondary[src_i.wid][src_split_i][reg];
                if (!still_pending) {
                  m_pending_writes_secondary[src_i.wid][src_split_i]
                      .erase(reg);
                  m_scoreboard->releaseSetReg(src_i.wid, reg,
                                              src_i.source_mask,
                                              /*is_intra_legacy=*/true,
                                              src_i.source_slot_id);
                  insn_completed = true;
                }
              } else {
                assert(m_pending_writes[src_i.wid][reg] > 0);
                unsigned still_pending =
                    --m_pending_writes[src_i.wid][reg];
                if (!still_pending) {
                  m_pending_writes[src_i.wid].erase(reg);
                  m_scoreboard->releaseSetReg(src_i.wid, reg,
                                              src_i.source_mask,
                                              /*is_intra_legacy=*/false,
                                              src_i.source_slot_id);
                  insn_completed = true;
                }
              }
              // Mode 1: per-mask release fires per inst.
              dec_mask_pw_and_maybe_release(src_i.wid, reg, src_i.source_mask);
            } else if (m_next_wb.m_is_ldgsts) {
              // LDGSTS excluded from co-issue; source_list always size<=1
              m_pending_ldgsts[primary_wid][m_next_wb.pc]
                              [m_next_wb.get_addr(0)]--;
              if (m_pending_ldgsts[primary_wid][m_next_wb.pc]
                                  [m_next_wb.get_addr(0)] == 0) {
                insn_completed = true;
              }
              break;
            }
          }
        }
      } else {
        // shared — release primary scoreboard only here; composite per-set
        // releases happen below.
        mem_src_t src = resolve_source(m_next_wb, src_wid, src_split);
        (void)is_intra_coissued;  // unused on shared path
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          if (src.out_inst->out[r] > 0) {
            m_scoreboard->releaseSetReg(src.wid, src.out_inst->out[r],
                                        src.source_mask,
                                        /*is_intra_legacy=*/false,
                                        src.source_slot_id);
            insn_completed = true;
          }
        }
      }

      // Shared-memory composite: release each co-issued set's scoreboard
      // regs in one pass (shared writeback is single-shot, not per-mf).
      if (is_shared && m_next_wb.has_simd_sets() &&
          m_config->gpgpu_simd_partitioning &&
          m_config->gpgpu_mem_coissue) {
        const std::vector<simd_set_info> &sets = m_next_wb.get_simd_sets();
        std::set<std::pair<unsigned, unsigned>> released;
        for (unsigned s = 0; s < sets.size(); s++) {
          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
          std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                             sets[s].split_id);
          if (released.count(key)) continue;
          released.insert(key);
          // Secondary slot coissuer → secondary scoreboard map (mode 0).
          bool coissued_intra = (sets[s].split_id != (unsigned)-1);
          unsigned set_slot =
              (sets[s].source_slot_id != (unsigned)-1)
                  ? sets[s].source_slot_id
                  : (coissued_intra ? 1u : 0u);
          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
            if (sets[s].source_inst->out[r] > 0) {
              m_scoreboard->releaseSetReg(
                  sets[s].warp_id, sets[s].source_inst->out[r],
                  sets[s].source_mask, /*is_intra_legacy=*/coissued_intra,
                  set_slot);
              insn_completed = true;
            }
          }
        }
      }

      if (insn_completed) {
        m_core->warp_inst_complete(m_next_wb);
        if (m_next_wb.m_is_ldgsts) {
          m_core->unset_depbar(m_next_wb);
        }
      }

      m_next_wb.clear();
      m_next_wb_src_wid = (unsigned)-1;
      m_next_wb_src_split_id = (unsigned)-1;
      m_next_wb_src_list.clear();
      m_last_inst_gpu_sim_cycle = m_core->get_gpu()->gpu_sim_cycle;
      m_last_inst_gpu_tot_sim_cycle = m_core->get_gpu()->gpu_tot_sim_cycle;
      }  // close inner block opened at line ~4176
    }  // close else
  }  // close if (!m_next_wb.empty())

  unsigned serviced_client = -1;
  for (unsigned c = 0; m_next_wb.empty() && (c < m_num_writeback_clients);
       c++) {
    unsigned next_client = (c + m_writeback_arb) % m_num_writeback_clients;
    switch (next_client) {
      case 0:  // shared memory
        if (!m_pipeline_reg[0]->empty()) {
          m_next_wb = *m_pipeline_reg[0];
          // Shared-mem writeback bypasses mem_fetch: there is no
          // per-transaction source. Treat this as primary-only. For a MEM
          // co-issue composite, enumerate the composite's simd_sets and
          // release each source's scoreboard/pipeline in a dedicated pass
          // below; here we just stamp to "primary" so the default path
          // handles the primary portion.
          m_next_wb_src_wid = (unsigned)-1;
          m_next_wb_src_split_id = (unsigned)-1;
          m_next_wb_src_list.clear();
          if (m_next_wb.isatomic()) {
            m_next_wb.do_atomic();
            m_core->decrement_atomic_count(m_next_wb.warp_id(),
                                           m_next_wb.active_count());
          }
          // dec_inst_in_pipeline for each unique (source wid, split_id)
          // participating in the composite (mirrors SP writeback at
          // shader.cc~3019-3044). Non-composite MEM inst still dec's once.
          unsigned primary_wid = m_pipeline_reg[0]->warp_id();
          m_core->dec_inst_in_pipeline(primary_wid);
          if (m_next_wb.has_simd_sets() &&
              m_config->gpgpu_simd_partitioning &&
              m_config->gpgpu_mem_coissue) {
            std::set<unsigned> inter_dec;
            bool intra_dec = false;
            const std::vector<simd_set_info> &sets =
                m_next_wb.get_simd_sets();
            for (unsigned s = 0; s < sets.size(); s++) {
              if (!sets[s].valid || sets[s].source_inst == NULL) continue;
              if (sets[s].warp_id != primary_wid) {
                if (inter_dec.insert(sets[s].warp_id).second)
                  m_core->dec_inst_in_pipeline(sets[s].warp_id);
              } else if (!intra_dec) {
                m_core->dec_inst_in_pipeline(primary_wid);
                intra_dec = true;
              }
            }
          }
          m_pipeline_reg[0]->clear();
          serviced_client = next_client;
        }
        break;
      case 1:  // texture response
        if (m_L1T->access_ready()) {
          mem_fetch *mf = m_L1T->next_access();
          m_next_wb = mf->get_inst();
          m_next_wb_src_wid = mf->get_wid();
          m_next_wb_src_split_id = mf->get_access_source_split_id();
          m_next_wb_src_list = mf->get_access_source_list();
          delete mf;
          serviced_client = next_client;
        }
        break;
      case 2:  // const cache response
        if (m_L1C->access_ready()) {
          mem_fetch *mf = m_L1C->next_access();
          m_next_wb = mf->get_inst();
          m_next_wb_src_wid = mf->get_wid();
          m_next_wb_src_split_id = mf->get_access_source_split_id();
          m_next_wb_src_list = mf->get_access_source_list();
          delete mf;
          serviced_client = next_client;
        }
        break;
      case 3:  // global/local
        if (m_next_global) {
          m_next_wb = m_next_global->get_inst();
          m_next_wb_src_wid = m_next_global->get_wid();
          m_next_wb_src_split_id =
              m_next_global->get_access_source_split_id();
          m_next_wb_src_list = m_next_global->get_access_source_list();
          if (m_next_global->isatomic()) {
            m_core->decrement_atomic_count(
                m_next_global->get_wid(),
                m_next_global->get_access_warp_mask().count());
          }
          delete m_next_global;
          m_next_global = NULL;
          serviced_client = next_client;
        }
        break;
      case 4:
        if (m_L1D && m_L1D->access_ready()) {
          mem_fetch *mf = m_L1D->next_access();
          m_next_wb = mf->get_inst();
          m_next_wb_src_wid = mf->get_wid();
          m_next_wb_src_split_id = mf->get_access_source_split_id();
          m_next_wb_src_list = mf->get_access_source_list();
          delete mf;
          serviced_client = next_client;
        }
        break;
      default:
        abort();
    }
  }
  // update arbitration priority only if:
  // 1. the writeback buffer was available
  // 2. a client was serviced
  if (serviced_client != (unsigned)-1) {
    m_writeback_arb = (serviced_client + 1) % m_num_writeback_clients;
  }
}

unsigned ldst_unit::clock_multiplier() const {
  // to model multiple read port, we give multiple cycles for the memory units
  if (m_config->mem_unit_ports)
    return m_config->mem_unit_ports;
  else
    return m_config->mem_warp_parts;
}
/*
void ldst_unit::issue( register_set &reg_set )
{
        warp_inst_t* inst = *(reg_set.get_ready());
   // stat collection
   m_core->mem_instruction_stats(*inst);

   // record how many pending register writes/memory accesses there are for this
instruction assert(inst->empty() == false); if (inst->is_load() and
inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id();
      unsigned n_accesses = inst->accessq_count();
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
         unsigned reg_id = inst->out[r];
         if (reg_id > 0) {
            m_pending_writes[warp_id][reg_id] += n_accesses;
         }
      }
   }

   pipelined_simd_unit::issue(reg_set);
}
*/
void ldst_unit::cycle() {
  writeback();

  // MEMCOV2_STATE_DUMP: periodic state-counter dump to detect leaks.
  // Prints total pending_writes map sizes + sum of their values every
  // 4096 cycles. Used for Stage 5 mixed-retire leak investigation.
  static const bool mc_state_dump = (getenv("MEMCOV2_STATE_DUMP") != NULL);
  if (mc_state_dump) {
    unsigned long long cyc =
        m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
    if ((cyc & 0xFFF) == 0) {
      unsigned pw_keys = 0, pw_sum = 0;
      for (auto &kv : m_pending_writes)
        for (auto &rv : kv.second) { pw_keys++; pw_sum += rv.second; }
      unsigned sw_keys = 0, sw_sum = 0;
      for (auto &kv : m_pending_writes_secondary)
        for (auto &sp : kv.second)
          for (auto &rv : sp.second) { sw_keys++; sw_sum += rv.second; }
      fprintf(stderr,
              "[MC2 state@%llu] sid=%u pw_keys=%u pw_sum=%u "
              "sw_keys=%u sw_sum=%u\n",
              cyc, m_sid, pw_keys, pw_sum, sw_keys, sw_sum);
    }
  }

  for (unsigned stage = 0; (stage + 1) < m_pipeline_depth; stage++)
    if (m_pipeline_reg[stage]->empty() && !m_pipeline_reg[stage + 1]->empty())
      move_warp(m_pipeline_reg[stage], m_pipeline_reg[stage + 1]);

  if (!m_response_fifo.empty()) {
    mem_fetch *mf = m_response_fifo.front();
    if (mf->get_access_type() == TEXTURE_ACC_R) {
      if (m_L1T->fill_port_free()) {
        m_L1T->fill(mf, m_core->get_gpu()->gpu_sim_cycle +
                            m_core->get_gpu()->gpu_tot_sim_cycle);
        m_response_fifo.pop_front();
      }
    } else if (mf->get_access_type() == CONST_ACC_R) {
      if (m_L1C->fill_port_free()) {
        mf->set_status(IN_SHADER_FETCHED,
                       m_core->get_gpu()->gpu_sim_cycle +
                           m_core->get_gpu()->gpu_tot_sim_cycle);
        m_L1C->fill(mf, m_core->get_gpu()->gpu_sim_cycle +
                            m_core->get_gpu()->gpu_tot_sim_cycle);
        m_response_fifo.pop_front();
      }
    } else {
      if (mf->get_type() == WRITE_ACK ||
          ((m_config->gpgpu_perfect_mem || m_memory_config->SST_mode) &&
           mf->get_is_write())) {
        // SST memory is handled by SST mem hierarchy
        // Perfect mem
        m_core->store_ack(mf);
        m_response_fifo.pop_front();
        delete mf;
      } else {
        assert(!mf->get_is_write());  // L1 cache is write evict, allocate line
                                      // on load miss only

        bool bypassL1D = false;
        if (CACHE_GLOBAL == mf->get_inst().cache_op || (m_L1D == NULL)) {
          bypassL1D = true;
        } else if (mf->get_access_type() == GLOBAL_ACC_R ||
                   mf->get_access_type() ==
                       GLOBAL_ACC_W) {  // global memory access
          if (m_core->get_config()->gmem_skip_L1D) bypassL1D = true;
        }
        if (bypassL1D) {
          if (m_next_global == NULL) {
            mf->set_status(IN_SHADER_FETCHED,
                           m_core->get_gpu()->gpu_sim_cycle +
                               m_core->get_gpu()->gpu_tot_sim_cycle);
            m_response_fifo.pop_front();
            m_next_global = mf;
          }
        } else {
          if (m_L1D->fill_port_free()) {
            m_L1D->fill(mf, m_core->get_gpu()->gpu_sim_cycle +
                                m_core->get_gpu()->gpu_tot_sim_cycle);
            m_response_fifo.pop_front();
          }
        }
      }
    }
  }

  m_L1T->cycle();
  m_L1C->cycle();
  if (m_L1D) {
    m_L1D->cycle();
    if (m_config->m_L1D_config.l1_latency > 0) L1_latency_queue_cycle();
  }

  warp_inst_t &pipe_reg = *m_dispatch_reg;
  enum mem_stage_stall_type rc_fail = NO_RC_FAIL;
  mem_stage_access_type type;
  bool done = true;
  done &= shared_cycle(pipe_reg, rc_fail, type);
  done &= constant_cycle(pipe_reg, rc_fail, type);
  done &= texture_cycle(pipe_reg, rc_fail, type);
  done &= memory_cycle(pipe_reg, rc_fail, type);
  m_mem_rc = rc_fail;

  // Option A (hardware-faithful per-source scoreboard release) for MIXED
  // composites: as soon as the SHARED side's bank-conflict cycles drain,
  // release shared-source scoreboards (primary if shared + shared
  // coissuer sets) regardless of whether global mfs are still in flight.
  // Global-source scoreboards continue to release via the pending_writes
  // path in writeback() on their own timeline. This matches hardware
  // where the shared-mem bank arbiter signals completion to the
  // scoreboard independently of global-memory transactions. One-shot
  // guarded by warp_inst_t::shared_side_released() to avoid re-release
  // as shared_cycle keeps returning true on subsequent cycles.
  if (!pipe_reg.empty() && pipe_reg.has_simd_sets() &&
      m_config->gpgpu_simd_partitioning &&
      m_config->gpgpu_mem_coissue &&
      pipe_reg.is_mixed_space_composite() &&
      !pipe_reg.shared_side_released()) {
    bool shared_done = true;
    // Primary's own cycles count only if primary itself is shared.
    if (warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type()) &&
        pipe_reg.has_dispatch_delay()) {
      shared_done = false;
    }
    if (shared_done) {
      const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        enum _memory_space_t sp = sets[s].source_inst->space.get_type();
        if (warp_inst_t::is_shared_bucket_space(sp) &&
            sets[s].has_dispatch_delay()) {
          shared_done = false;
          break;
        }
      }
    }
    if (shared_done) {
      // Release shared-source scoreboards.
      const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
      // Primary scoreboard if primary is a shared LOAD.
      if (pipe_reg.is_load() &&
          warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type())) {
        m_scoreboard->releaseRegisters(m_dispatch_reg);
      }
      // Shared coissuer LOAD sets.
      std::set<std::pair<unsigned, unsigned>> released;
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        if (!sets[s].source_inst->is_load()) continue;
        enum _memory_space_t sp = sets[s].source_inst->space.get_type();
        if (!warp_inst_t::is_shared_bucket_space(sp)) continue;
        std::pair<unsigned, unsigned> key(sets[s].warp_id, sets[s].split_id);
        if (released.count(key)) continue;
        released.insert(key);
        bool is_intra = (sets[s].split_id != (unsigned)-1);
        unsigned set_slot =
            (sets[s].source_slot_id != (unsigned)-1)
                ? sets[s].source_slot_id
                : (is_intra ? 1u : 0u);
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg = sets[s].source_inst->out[r];
          if (reg == 0) continue;
          m_scoreboard->releaseSetReg(sets[s].warp_id, reg,
                                      sets[s].source_mask,
                                      /*is_intra_legacy=*/is_intra, set_slot);
        }
      }
      pipe_reg.set_shared_side_released(true);
      if (getenv("MEMCOV2_TRACE")) {
        unsigned long long cyc = m_core->get_gpu()->gpu_sim_cycle +
                                 m_core->get_gpu()->gpu_tot_sim_cycle;
        fprintf(stderr,
                "[MC2 early-rel@%llu] wid=%u primary_space=%d "
                "primary_is_load=%d sets=%u\n",
                cyc, pipe_reg.warp_id(), (int)pipe_reg.space.get_type(),
                (int)pipe_reg.is_load(), (unsigned)sets.size());
      }
    }
  }

  if (!done) {  // log stall types and return
    assert(rc_fail != NO_RC_FAIL);
    m_stats->gpgpu_n_stall_shd_mem++;
    m_stats->gpu_stall_shd_mem_breakdown[type][rc_fail]++;
    return;
  }

  // Mixed-space MEM co-issue (v2): new inline retirement path for
  // composites with BOTH shared and global valid sets. Fires only when
  // has_simd_sets + partitioning + mem_coissue enabled + actually mixed.
  // Pure-shared (only shared sets) and pure-global (only global sets)
  // composites fall through to the v1 load/store branches below
  // unchanged. We retire at dispatch_reg (skipping pipeline_reg smem
  // chain + case 0 writeback) because mixed composites need to wait for
  // BOTH shared-cycle completion and global pending_writes completion;
  // routing them through pipeline_reg[smem_latency-1] first would cause
  // premature primary-scoreboard release at case 0 while global mfs are
  // still in flight (see re-plan doc).
  if (!pipe_reg.empty() && pipe_reg.has_simd_sets() &&
      m_config->gpgpu_simd_partitioning &&
      m_config->gpgpu_mem_coissue &&
      pipe_reg.is_mixed_space_composite()) {
    unsigned warp_id = pipe_reg.warp_id();
    const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();

    // mixed_retire_ready: shared_cycle already returned true (done), and
    // memory_cycle returned accessq_empty. We still need to wait for
    // outstanding global-source mfs to drain via the pending_writes
    // path. A global LOAD coissuer's scoreboard is released by the
    // pending_writes decrement in writeback() when its last mf returns;
    // our inline retire waits for that BEFORE releasing primary or
    // shared coissuer scoreboards.
    bool pending_writes_outstanding = false;
    // Check primary's own pending_writes (only if primary is a global
    // load — shared loads don't use pending_writes).
    if (pipe_reg.is_load() &&
        warp_inst_t::is_global_bucket_space(pipe_reg.space.get_type())) {
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
        unsigned reg_id = pipe_reg.out[r];
        if (reg_id == 0) continue;
        auto it = m_pending_writes[warp_id].find(reg_id);
        if (it != m_pending_writes[warp_id].end()) {
          if (it->second > 0) {
            pending_writes_outstanding = true;
            break;
          } else {
            m_pending_writes[warp_id].erase(reg_id);
          }
        }
      }
    }
    // Per-coissuer check for global LOAD sources.
    if (!pending_writes_outstanding) {
      std::set<std::pair<unsigned, unsigned>> checked;
      for (unsigned s = 0; s < sets.size() && !pending_writes_outstanding;
           s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        if (!sets[s].source_inst->is_load()) continue;
        enum _memory_space_t src_sp =
            sets[s].source_inst->space.get_type();
        if (!warp_inst_t::is_global_bucket_space(src_sp)) continue;
        std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                           sets[s].split_id);
        if (checked.count(key)) continue;
        checked.insert(key);
        bool is_intra = (sets[s].split_id != (unsigned)-1);
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = sets[s].source_inst->out[r];
          if (reg_id == 0) continue;
          if (is_intra) {
            auto &m = m_pending_writes_secondary[sets[s].warp_id]
                                                [sets[s].split_id];
            auto it = m.find(reg_id);
            if (it != m.end() && it->second > 0) {
              pending_writes_outstanding = true; break;
            }
          } else {
            auto &m = m_pending_writes[sets[s].warp_id];
            auto it = m.find(reg_id);
            if (it != m.end() && it->second > 0) {
              pending_writes_outstanding = true; break;
            }
          }
        }
      }
    }
    static const bool mc_trace = (getenv("MEMCOV2_TRACE") != NULL);
    if (pending_writes_outstanding) {
      if (mc_trace) {
        unsigned long long cyc =
            m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
        fprintf(stderr,
                "[MC2 stall@%llu] wid=%u pending_writes_outstanding\n",
                cyc, warp_id);
      }
      return;
    }

    // RF write-port fidelity: stall if operand collector not ready.
    if (!m_operand_collector->writeback(*m_dispatch_reg)) {
      if (mc_trace) {
        unsigned long long cyc =
            m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
        if ((cyc & 0x3FFF) == 0)
          fprintf(stderr, "[MC2 stall@%llu] wid=%u opnd_coll not ready\n",
                  cyc, warp_id);
      }
      return;
    }
    if (mc_trace) {
      unsigned long long cyc =
          m_core->get_gpu()->gpu_sim_cycle + m_core->get_gpu()->gpu_tot_sim_cycle;
      fprintf(stderr,
              "[MC2 retire@%llu] wid=%u is_load=%d primary_space=%d sets=%u\n",
              cyc, warp_id, (int)pipe_reg.is_load(),
              (int)pipe_reg.space.get_type(), (unsigned)sets.size());
    }

    // Primary retirement: release primary scoreboard if LOAD; always
    // call warp_inst_complete and dec_inst_in_pipeline for primary.
    // Stage 6 (Option A): if primary is shared, its scoreboard was
    // already released in the early-release block above — skip here to
    // avoid a spurious release that could race with a subsequent
    // reservation on the same reg.
    bool primary_is_shared =
        warp_inst_t::is_shared_bucket_space(pipe_reg.space.get_type());
    if (pipe_reg.is_load() &&
        !(primary_is_shared && pipe_reg.shared_side_released())) {
      m_scoreboard->releaseRegisters(m_dispatch_reg);
    }
    m_core->warp_inst_complete(*m_dispatch_reg);

    // Release ONLY shared-source coissuer scoreboards that haven't been
    // released yet. Global-source coissuer scoreboards were already
    // released via the pending_writes decrement in writeback() when
    // their last mf returned; releasing again here would race with the
    // coissuer's next-instruction reservation. Shared-source coissuer
    // scoreboards are released here OR in the early-release block above
    // (Stage 6 Option A) — guarded by shared_side_released() to avoid
    // double-release.
    if (!pipe_reg.shared_side_released()) {
      std::set<std::pair<unsigned, unsigned>> released_shared;
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid || sets[s].source_inst == NULL) continue;
        if (!sets[s].source_inst->is_load()) continue;
        enum _memory_space_t src_sp =
            sets[s].source_inst->space.get_type();
        if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue;
        std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                           sets[s].split_id);
        if (released_shared.count(key)) continue;
        released_shared.insert(key);
        bool is_intra = (sets[s].split_id != (unsigned)-1);
        unsigned set_slot =
            (sets[s].source_slot_id != (unsigned)-1)
                ? sets[s].source_slot_id
                : (is_intra ? 1u : 0u);
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = sets[s].source_inst->out[r];
          if (reg_id == 0) continue;
          m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id,
                                      sets[s].source_mask,
                                      /*is_intra_legacy=*/is_intra, set_slot);
        }
      }
    }

    // Defensive: global-source coissuer pending_writes should be drained
    // by now (we waited on them above). Assert to catch bookkeeping drift.
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
      if (!sets[s].source_inst->is_load()) continue;
      enum _memory_space_t src_sp =
          sets[s].source_inst->space.get_type();
      if (!warp_inst_t::is_global_bucket_space(src_sp)) continue;
      bool is_intra = (sets[s].split_id != (unsigned)-1);
      for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
        unsigned reg_id = sets[s].source_inst->out[r];
        if (reg_id == 0) continue;
        if (is_intra) {
          auto &m = m_pending_writes_secondary[sets[s].warp_id]
                                              [sets[s].split_id];
          auto it = m.find(reg_id);
          assert(it == m.end() || it->second == 0);
        } else {
          auto &m = m_pending_writes[sets[s].warp_id];
          auto it = m.find(reg_id);
          assert(it == m.end() || it->second == 0);
        }
      }
    }

    // dec_inst_in_pipeline for primary + each unique coissuer. Mirrors
    // case 0 shared writeback dedup: inter-warp coissuer decs once per
    // warp; intra (secondary slot of primary warp) decs primary warp
    // a second time.
    m_core->dec_inst_in_pipeline(warp_id);
    std::set<unsigned> inter_dec;
    bool intra_dec = false;
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid || sets[s].source_inst == NULL) continue;
      if (sets[s].warp_id != warp_id) {
        if (inter_dec.insert(sets[s].warp_id).second)
          m_core->dec_inst_in_pipeline(sets[s].warp_id);
      } else if (!intra_dec) {
        m_core->dec_inst_in_pipeline(warp_id);
        intra_dec = true;
      }
    }

    m_dispatch_reg->clear();
    return;
  }

  if (!pipe_reg.empty()) {
    unsigned warp_id = pipe_reg.warp_id();
    if (pipe_reg.is_load()) {
      if (pipe_reg.space.get_type() == shared_space) {
        if (m_pipeline_reg[m_config->smem_latency - 1]->empty()) {
          // new shared memory request
          move_warp(m_pipeline_reg[m_config->smem_latency - 1], m_dispatch_reg);
          m_dispatch_reg->clear();
        }
      } else {
        // if( pipe_reg.active_count() > 0 ) {
        //    if( !m_operand_collector->writeback(pipe_reg) )
        //        return;
        //}

        bool pending_requests = false;
        for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
          unsigned reg_id = pipe_reg.out[r];
          if (reg_id > 0) {
            if (m_pending_writes[warp_id].find(reg_id) !=
                m_pending_writes[warp_id].end()) {
              if (m_pending_writes[warp_id][reg_id] > 0) {
                pending_requests = true;
                break;
              } else {
                // this instruction is done already
                m_pending_writes[warp_id].erase(reg_id);
              }
            }
          }
        }
        // MEM co-issue: also check per-source pending for co-issued sets.
        // Inter-warp: m_pending_writes[coissue_wid][source_inst.out[r]].
        // Intra-warp: m_pending_writes_secondary[wid][split][source_inst.out[r]].
        if (!pending_requests && pipe_reg.has_simd_sets() &&
            m_config->gpgpu_simd_partitioning &&
            m_config->gpgpu_mem_coissue) {
          const std::vector<simd_set_info> &sets = pipe_reg.get_simd_sets();
          std::set<std::pair<unsigned, unsigned>> checked;
          for (unsigned s = 0; s < sets.size() && !pending_requests; s++) {
            if (!sets[s].valid || sets[s].source_inst == NULL) continue;
            std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                               sets[s].split_id);
            if (checked.count(key)) continue;
            checked.insert(key);
            // Secondary slot → secondary pending_writes map.
            bool is_intra = (sets[s].split_id != (unsigned)-1);
            for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
              unsigned reg_id = sets[s].source_inst->out[r];
              if (reg_id == 0) continue;
              if (is_intra) {
                auto &m = m_pending_writes_secondary[sets[s].warp_id]
                                                    [sets[s].split_id];
                auto it = m.find(reg_id);
                if (it != m.end() && it->second > 0) {
                  pending_requests = true; break;
                }
              } else {
                auto &m = m_pending_writes[sets[s].warp_id];
                auto it = m.find(reg_id);
                if (it != m.end() && it->second > 0) {
                  pending_requests = true; break;
                }
              }
            }
          }
        }
        if (!pending_requests) {
          m_core->warp_inst_complete(*m_dispatch_reg);
          m_scoreboard->releaseRegisters(m_dispatch_reg);

          // MEM co-issue retirement: also release each co-issued set's
          // scoreboard + call warp_inst_complete for each unique source
          // warp/split (inter-warp gets its own warp_inst_complete bump;
          // intra-warp keeps sharing the primary's warp_id but gets a
          // separate stats increment).
          if (m_dispatch_reg->has_simd_sets() &&
              m_config->gpgpu_simd_partitioning &&
              m_config->gpgpu_mem_coissue) {
            const std::vector<simd_set_info> &sets =
                m_dispatch_reg->get_simd_sets();
            std::set<std::pair<unsigned, unsigned>> released;
            for (unsigned s = 0; s < sets.size(); s++) {
              if (!sets[s].valid || sets[s].source_inst == NULL) continue;
              std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                                 sets[s].split_id);
              if (released.count(key)) continue;
              released.insert(key);
              // Secondary slot → secondary scoreboard map (mode 0).
              bool is_intra = (sets[s].split_id != (unsigned)-1);
              unsigned set_slot =
                  (sets[s].source_slot_id != (unsigned)-1)
                      ? sets[s].source_slot_id
                      : (is_intra ? 1u : 0u);
              for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
                unsigned reg_id = sets[s].source_inst->out[r];
                if (reg_id == 0) continue;
                m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id,
                                            sets[s].source_mask,
                                            /*is_intra_legacy=*/is_intra,
                                            set_slot);
              }
            }
          }

          // release LDGSTS
          if (m_dispatch_reg->m_is_ldgsts) {
            // m_pending_ldgsts[m_dispatch_reg->warp_id()][m_dispatch_reg->pc][m_dispatch_reg->get_addr(0)]--;
            if (m_pending_ldgsts[m_dispatch_reg->warp_id()][m_dispatch_reg->pc]
                                [m_dispatch_reg->get_addr(0)] == 0) {
              m_core->unset_depbar(*m_dispatch_reg);
            }
          }
        }
        m_core->dec_inst_in_pipeline(warp_id);
        // MEM co-issue: each unique co-issued (warp, split) was incremented
        // at decode time, so decrement once per unique here. Mirrors the
        // SP/SFU/INT retirement at shader.cc~3019-3044.
        if (m_dispatch_reg->has_simd_sets() &&
            m_config->gpgpu_simd_partitioning &&
            m_config->gpgpu_mem_coissue) {
          const std::vector<simd_set_info> &sets =
              m_dispatch_reg->get_simd_sets();
          std::set<unsigned> inter_dec;
          bool intra_dec = false;
          for (unsigned s = 0; s < sets.size(); s++) {
            if (!sets[s].valid || sets[s].source_inst == NULL) continue;
            if (sets[s].warp_id != warp_id) {
              if (inter_dec.insert(sets[s].warp_id).second)
                m_core->dec_inst_in_pipeline(sets[s].warp_id);
            } else if (!intra_dec) {
              m_core->dec_inst_in_pipeline(warp_id);
              intra_dec = true;
            }
          }
        }
        m_dispatch_reg->clear();
      }
    } else {
      // stores exit pipeline here
      m_core->dec_inst_in_pipeline(warp_id);
      m_core->warp_inst_complete(*m_dispatch_reg);
      // MEM co-issue: stores retiring — dec pipe + complete per source too.
      if (m_dispatch_reg->has_simd_sets() &&
          m_config->gpgpu_simd_partitioning &&
          m_config->gpgpu_mem_coissue) {
        const std::vector<simd_set_info> &sets =
            m_dispatch_reg->get_simd_sets();
        std::set<unsigned> inter_dec;
        bool intra_dec = false;
        for (unsigned s = 0; s < sets.size(); s++) {
          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
          if (sets[s].warp_id != warp_id) {
            if (inter_dec.insert(sets[s].warp_id).second)
              m_core->dec_inst_in_pipeline(sets[s].warp_id);
          } else if (!intra_dec) {
            m_core->dec_inst_in_pipeline(warp_id);
            intra_dec = true;
          }
        }
        // Mixed-space MEM co-issue (v2): for a STORE-primary composite,
        // SHARED LOAD coissuers bypass all the normal scoreboard-release
        // machinery (no mf/pending_writes for shared, no case 0 writeback
        // since composite never enters pipeline_reg via is_load branch).
        // Release their scoreboards here explicitly, mirroring case 0
        // writeback's shared-coissue release. Global LOAD coissuers are
        // handled by the pending_writes decrement path in writeback() and
        // MUST NOT be released here (would double-release).
        std::set<std::pair<unsigned, unsigned>> released;
        for (unsigned s = 0; s < sets.size(); s++) {
          if (!sets[s].valid || sets[s].source_inst == NULL) continue;
          if (!sets[s].source_inst->is_load()) continue;
          enum _memory_space_t src_sp =
              sets[s].source_inst->space.get_type();
          if (!warp_inst_t::is_shared_bucket_space(src_sp)) continue;
          std::pair<unsigned, unsigned> key(sets[s].warp_id,
                                             sets[s].split_id);
          if (released.count(key)) continue;
          released.insert(key);
          bool is_intra = (sets[s].split_id != (unsigned)-1);
          unsigned set_slot =
              (sets[s].source_slot_id != (unsigned)-1)
                  ? sets[s].source_slot_id
                  : (is_intra ? 1u : 0u);
          for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) {
            unsigned reg_id = sets[s].source_inst->out[r];
            if (reg_id == 0) continue;
            m_scoreboard->releaseSetReg(sets[s].warp_id, reg_id,
                                        sets[s].source_mask,
                                        /*is_intra_legacy=*/is_intra,
                                        set_slot);
          }
        }
      }
      m_dispatch_reg->clear();
    }
  }
}

void shader_core_ctx::register_cta_thread_exit(unsigned cta_num,
                                               kernel_info_t *kernel) {
  assert(m_cta_status[cta_num] > 0);
  m_cta_status[cta_num]--;
  if (!m_cta_status[cta_num]) {
    // Increment the completed CTAs
    m_stats->ctas_completed++;
    m_gpu->inc_completed_cta();
    m_n_active_cta--;
    m_barriers.deallocate_barrier(cta_num);
    shader_CTA_count_unlog(m_sid, 1);

    SHADER_DPRINTF(
        LIVENESS,
        "GPGPU-Sim uArch: Finished CTA #%u (%lld,%lld), %u CTAs running\n",
        cta_num, m_gpu->gpu_sim_cycle, m_gpu->gpu_tot_sim_cycle,
        m_n_active_cta);

    if (m_n_active_cta == 0) {
      SHADER_DPRINTF(
          LIVENESS,
          "GPGPU-Sim uArch: Empty (last released kernel %u \'%s\').\n",
          kernel->get_uid(), kernel->name().c_str());
      fflush(stdout);

      // Shader can only be empty when no more cta are dispatched
      if (kernel != m_kernel) {
        assert(m_kernel == NULL || !m_gpu->kernel_more_cta_left(m_kernel));
      }
      m_kernel = NULL;
    }

    // Jin: for concurrent kernels on sm
    release_shader_resource_1block(cta_num, *kernel);
    kernel->dec_running();
    if (!m_gpu->kernel_more_cta_left(kernel)) {
      if (!kernel->running()) {
        SHADER_DPRINTF(LIVENESS,
                       "GPGPU-Sim uArch: GPU detected kernel %u \'%s\' "
                       "finished on shader %u.\n",
                       kernel->get_uid(), kernel->name().c_str(), m_sid);

        if (m_kernel == kernel) m_kernel = NULL;
        m_gpu->set_kernel_done(kernel);
      }
    }
  }
}

void gpgpu_sim::shader_print_runtime_stat(FILE *fout) {
  /*
 fprintf(fout, "SHD_INSN: ");
 for (unsigned i=0;i<m_n_shader;i++)
    fprintf(fout, "%u ",m_sc[i]->get_num_sim_insn());
 fprintf(fout, "\n");
 fprintf(fout, "SHD_THDS: ");
 for (unsigned i=0;i<m_n_shader;i++)
    fprintf(fout, "%u ",m_sc[i]->get_not_completed());
 fprintf(fout, "\n");
 fprintf(fout, "SHD_DIVG: ");
 for (unsigned i=0;i<m_n_shader;i++)
    fprintf(fout, "%u ",m_sc[i]->get_n_diverge());
 fprintf(fout, "\n");

 fprintf(fout, "THD_INSN: ");
 for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++)
    fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn(i) );
 fprintf(fout, "\n");
 */
}

void gpgpu_sim::shader_print_scheduler_stat(FILE *fout,
                                            bool print_dynamic_info) const {
  fprintf(fout, "ctas_completed %d, ", m_shader_stats->ctas_completed);
  // Print out the stats from the sampling shader core
  const unsigned scheduler_sampling_core =
      m_shader_config->gpgpu_warp_issue_shader;
#define STR_SIZE 55
  char name_buff[STR_SIZE];
  name_buff[STR_SIZE - 1] = '\0';
  const std::vector<unsigned> &distro =
      print_dynamic_info
          ? m_shader_stats->get_dynamic_warp_issue()[scheduler_sampling_core]
          : m_shader_stats->get_warp_slot_issue()[scheduler_sampling_core];
  if (print_dynamic_info) {
    snprintf(name_buff, STR_SIZE - 1, "dynamic_warp_id");
  } else {
    snprintf(name_buff, STR_SIZE - 1, "warp_id");
  }
  fprintf(fout, "Shader %d %s issue ditsribution:\n", scheduler_sampling_core,
          name_buff);
  const unsigned num_warp_ids = distro.size();
  // First print out the warp ids
  fprintf(fout, "%s:\n", name_buff);
  for (unsigned warp_id = 0; warp_id < num_warp_ids; ++warp_id) {
    fprintf(fout, "%d, ", warp_id);
  }

  fprintf(fout, "\ndistro:\n");
  // Then print out the distribution of instuctions issued
  for (std::vector<unsigned>::const_iterator iter = distro.begin();
       iter != distro.end(); iter++) {
    fprintf(fout, "%d, ", *iter);
  }
  fprintf(fout, "\n");
}

void gpgpu_sim::shader_print_cache_stats(FILE *fout) const {
  // L1I
  struct cache_sub_stats total_css;
  struct cache_sub_stats css;

  if (!m_shader_config->m_L1I_config.disabled()) {
    total_css.clear();
    css.clear();
    fprintf(fout, "\n========= Core cache stats =========\n");
    fprintf(fout, "L1I_cache:\n");
    for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) {
      m_cluster[i]->get_L1I_sub_stats(css);
      total_css += css;
    }
    fprintf(fout, "\tL1I_total_cache_accesses = %llu\n", total_css.accesses);
    fprintf(fout, "\tL1I_total_cache_misses = %llu\n", total_css.misses);
    if (total_css.accesses > 0) {
      fprintf(fout, "\tL1I_total_cache_miss_rate = %.4lf\n",
              (double)total_css.misses / (double)total_css.accesses);
    }
    fprintf(fout, "\tL1I_total_cache_pending_hits = %llu\n",
            total_css.pending_hits);
    fprintf(fout, "\tL1I_total_cache_reservation_fails = %llu\n",
            total_css.res_fails);
  }

  // L1D
  if (!m_shader_config->m_L1D_config.disabled()) {
    total_css.clear();
    css.clear();
    fprintf(fout, "L1D_cache:\n");
    for (unsigned i = 0; i < m_shader_config->n_simt_clusters; i++) {
      m_cluster[i]->get_L1D_sub_stats(css);

      fprintf(stdout,
              "\tL1D_cache_core[%d]: Access = %llu, Miss = %llu, Miss_rate = "
              "%.3lf, Pending_hits = %llu, Reservation_fails = %llu\n",
              i, css.accesses, css.misses,
              (double)css.misses / (double)css.accesses, css.pending_hits,
              css.res_fails);

      total_css += css;
    }
    fprintf(fout, "\tL1D_total_cache_accesses = %llu\n", total_css.accesses);
    fprintf(fout, "\tL1D_total_cache_misses = %llu\n", total_css.misses);
    if (total_css.accesses > 0) {
      fprintf(fout, "\tL1D_total_cache_miss_rate = %.4lf\n",
              (double)total_css.misses / (double)total_css.accesses);
    }
    fprintf(fout, "\tL1D_total_cache_pending_hits = %llu\n",
            total_css.pending_hits);
    fprintf(fout, "\tL1D_total_cache_reservation_fails = %llu\n",
            total_css.res_fails);
    total_css.print_port_stats(fout, "\tL1D_cache");
  }

  // L1C
  if (!m_shader_config->m_L1C_config.disabled()) {
    total_css.clear();
    css.clear();
    fprintf(fout, "L1C_cache:\n");
    for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) {
      m_cluster[i]->get_L1C_sub_stats(css);
      total_css += css;
    }
    fprintf(fout, "\tL1C_total_cache_accesses = %llu\n", total_css.accesses);
    fprintf(fout, "\tL1C_total_cache_misses = %llu\n", total_css.misses);
    if (total_css.accesses > 0) {
      fprintf(fout, "\tL1C_total_cache_miss_rate = %.4lf\n",
              (double)total_css.misses / (double)total_css.accesses);
    }
    fprintf(fout, "\tL1C_total_cache_pending_hits = %llu\n",
            total_css.pending_hits);
    fprintf(fout, "\tL1C_total_cache_reservation_fails = %llu\n",
            total_css.res_fails);
  }

  // L1T
  if (!m_shader_config->m_L1T_config.disabled()) {
    total_css.clear();
    css.clear();
    fprintf(fout, "L1T_cache:\n");
    for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) {
      m_cluster[i]->get_L1T_sub_stats(css);
      total_css += css;
    }
    fprintf(fout, "\tL1T_total_cache_accesses = %llu\n", total_css.accesses);
    fprintf(fout, "\tL1T_total_cache_misses = %llu\n", total_css.misses);
    if (total_css.accesses > 0) {
      fprintf(fout, "\tL1T_total_cache_miss_rate = %.4lf\n",
              (double)total_css.misses / (double)total_css.accesses);
    }
    fprintf(fout, "\tL1T_total_cache_pending_hits = %llu\n",
            total_css.pending_hits);
    fprintf(fout, "\tL1T_total_cache_reservation_fails = %llu\n",
            total_css.res_fails);
  }
}

void gpgpu_sim::shader_print_l1_miss_stat(FILE *fout) const {
  unsigned total_d1_misses = 0, total_d1_accesses = 0;
  for (unsigned i = 0; i < m_shader_config->n_simt_clusters; ++i) {
    unsigned custer_d1_misses = 0, cluster_d1_accesses = 0;
    m_cluster[i]->print_cache_stats(fout, cluster_d1_accesses,
                                    custer_d1_misses);
    total_d1_misses += custer_d1_misses;
    total_d1_accesses += cluster_d1_accesses;
  }
  fprintf(fout, "total_dl1_misses=%d\n", total_d1_misses);
  fprintf(fout, "total_dl1_accesses=%d\n", total_d1_accesses);
  fprintf(fout, "total_dl1_miss_rate= %f\n",
          (float)total_d1_misses / (float)total_d1_accesses);
  /*
  fprintf(fout, "THD_INSN_AC: ");
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++)
     fprintf(fout, "%d ", m_sc[0]->get_thread_n_insn_ac(i));
  fprintf(fout, "\n");
  fprintf(fout, "T_L1_Mss: "); //l1 miss rate per thread
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++)
     fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i));
  fprintf(fout, "\n");
  fprintf(fout, "T_L1_Mgs: "); //l1 merged miss rate per thread
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++)
     fprintf(fout, "%d ", m_sc[0]->get_thread_n_l1_mis_ac(i) -
  m_sc[0]->get_thread_n_l1_mrghit_ac(i)); fprintf(fout, "\n"); fprintf(fout,
  "T_L1_Acc: "); //l1 access per thread for (unsigned i=0;
  i<m_shader_config->n_thread_per_shader; i++) fprintf(fout, "%d ",
  m_sc[0]->get_thread_n_l1_access_ac(i)); fprintf(fout, "\n");

  //per warp
  int temp =0;
  fprintf(fout, "W_L1_Mss: "); //l1 miss rate per warp
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) {
     temp += m_sc[0]->get_thread_n_l1_mis_ac(i);
     if (i%m_shader_config->warp_size ==
  (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp =
  0;
     }
  }
  fprintf(fout, "\n");
  temp=0;
  fprintf(fout, "W_L1_Mgs: "); //l1 merged miss rate per warp
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) {
     temp += (m_sc[0]->get_thread_n_l1_mis_ac(i) -
  m_sc[0]->get_thread_n_l1_mrghit_ac(i) ); if (i%m_shader_config->warp_size ==
  (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp =
  0;
     }
  }
  fprintf(fout, "\n");
  temp =0;
  fprintf(fout, "W_L1_Acc: "); //l1 access per warp
  for (unsigned i=0; i<m_shader_config->n_thread_per_shader; i++) {
     temp += m_sc[0]->get_thread_n_l1_access_ac(i);
     if (i%m_shader_config->warp_size ==
  (unsigned)(m_shader_config->warp_size-1)) { fprintf(fout, "%d ", temp); temp =
  0;
     }
  }
  fprintf(fout, "\n");
  */
}

void warp_inst_t::print(FILE *fout) const {
  if (empty()) {
    fprintf(fout, "bubble\n");
    return;
  } else
    fprintf(fout, "0x%04llx ", pc);
  fprintf(fout, "w%02d[", m_warp_id);
  for (unsigned j = 0; j < m_config->warp_size; j++)
    fprintf(fout, "%c", (active(j) ? '1' : '0'));
  fprintf(fout, "]: ");
  m_config->gpgpu_ctx->func_sim->ptx_print_insn(pc, fout);
  fprintf(fout, "\n");
}
void shader_core_ctx::incexecstat(warp_inst_t *&inst) {
  // Latency numbers for next operations are used to scale the power values
  // for special operations, according observations from microbenchmarking
  // TODO: put these numbers in the xml configuration
  if (get_gpu()->get_config().g_power_simulation_enabled) {
    switch (inst->sp_op) {
      case INT__OP:
        incialu_stat(inst->active_count(), scaling_coeffs->int_coeff);
        break;
      case INT_MUL_OP:
        incimul_stat(inst->active_count(), scaling_coeffs->int_mul_coeff);
        break;
      case INT_MUL24_OP:
        incimul24_stat(inst->active_count(), scaling_coeffs->int_mul24_coeff);
        break;
      case INT_MUL32_OP:
        incimul32_stat(inst->active_count(), scaling_coeffs->int_mul32_coeff);
        break;
      case INT_DIV_OP:
        incidiv_stat(inst->active_count(), scaling_coeffs->int_div_coeff);
        break;
      case FP__OP:
        incfpalu_stat(inst->active_count(), scaling_coeffs->fp_coeff);
        break;
      case FP_MUL_OP:
        incfpmul_stat(inst->active_count(), scaling_coeffs->fp_mul_coeff);
        break;
      case FP_DIV_OP:
        incfpdiv_stat(inst->active_count(), scaling_coeffs->fp_div_coeff);
        break;
      case DP___OP:
        incdpalu_stat(inst->active_count(), scaling_coeffs->dp_coeff);
        break;
      case DP_MUL_OP:
        incdpmul_stat(inst->active_count(), scaling_coeffs->dp_mul_coeff);
        break;
      case DP_DIV_OP:
        incdpdiv_stat(inst->active_count(), scaling_coeffs->dp_div_coeff);
        break;
      case FP_SQRT_OP:
        incsqrt_stat(inst->active_count(), scaling_coeffs->sqrt_coeff);
        break;
      case FP_LG_OP:
        inclog_stat(inst->active_count(), scaling_coeffs->log_coeff);
        break;
      case FP_SIN_OP:
        incsin_stat(inst->active_count(), scaling_coeffs->sin_coeff);
        break;
      case FP_EXP_OP:
        incexp_stat(inst->active_count(), scaling_coeffs->exp_coeff);
        break;
      case TENSOR__OP:
        inctensor_stat(inst->active_count(), scaling_coeffs->tensor_coeff);
        break;
      case TEX__OP:
        inctex_stat(inst->active_count(), scaling_coeffs->tex_coeff);
        break;
      default:
        break;
    }
    if (inst->const_cache_operand)  // warp has const address space load as one
                                    // operand
      inc_const_accesses(1);
  }
}
void shader_core_ctx::print_stage(unsigned int stage, FILE *fout) const {
  m_pipeline_reg[stage].print(fout);
  // m_pipeline_reg[stage].print(fout);
}

void shader_core_ctx::display_simt_state(FILE *fout, int mask) const {
  if ((mask & 4) && m_config->model == POST_DOMINATOR) {
    fprintf(fout, "per warp SIMT control-flow state:\n");
    unsigned n = m_config->n_thread_per_shader / m_config->warp_size;
    for (unsigned i = 0; i < n; i++) {
      unsigned nactive = 0;
      for (unsigned j = 0; j < m_config->warp_size; j++) {
        unsigned tid = i * m_config->warp_size + j;
        int done = ptx_thread_done(tid);
        nactive += (ptx_thread_done(tid) ? 0 : 1);
        if (done && (mask & 8)) {
          unsigned done_cycle = m_thread[tid]->donecycle();
          if (done_cycle) {
            printf("\n w%02u:t%03u: done @ cycle %u", i, tid, done_cycle);
          }
        }
      }
      if (nactive == 0) {
        continue;
      }
      if (m_config->model == POST_DOMINATOR)
        m_simt_stack[i]->print(fout);
      else
        m_simt_tables[i]->print(fout);
    }
    fprintf(fout, "\n");
  }
}

void ldst_unit::print(FILE *fout) const {
  fprintf(fout, "LD/ST unit  = ");
  m_dispatch_reg->print(fout);
  if (m_mem_rc != NO_RC_FAIL) {
    fprintf(fout, "              LD/ST stall condition: ");
    switch (m_mem_rc) {
      case BK_CONF:
        fprintf(fout, "BK_CONF");
        break;
      case MSHR_RC_FAIL:
        fprintf(fout, "MSHR_RC_FAIL");
        break;
      case ICNT_RC_FAIL:
        fprintf(fout, "ICNT_RC_FAIL");
        break;
      case COAL_STALL:
        fprintf(fout, "COAL_STALL");
        break;
      case WB_ICNT_RC_FAIL:
        fprintf(fout, "WB_ICNT_RC_FAIL");
        break;
      case WB_CACHE_RSRV_FAIL:
        fprintf(fout, "WB_CACHE_RSRV_FAIL");
        break;
      case N_MEM_STAGE_STALL_TYPE:
        fprintf(fout, "N_MEM_STAGE_STALL_TYPE");
        break;
      default:
        abort();
    }
    fprintf(fout, "\n");
  }
  fprintf(fout, "LD/ST wb    = ");
  m_next_wb.print(fout);
  fprintf(
      fout,
      "Last LD/ST writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n",
      m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle);
  fprintf(fout, "Pending register writes:\n");
  std::map<unsigned /*warp_id*/,
           std::map<unsigned /*regnum*/, unsigned /*count*/> >::const_iterator
      w;
  for (w = m_pending_writes.begin(); w != m_pending_writes.end(); w++) {
    unsigned warp_id = w->first;
    const std::map<unsigned /*regnum*/, unsigned /*count*/> &warp_info =
        w->second;
    if (warp_info.empty()) continue;
    fprintf(fout, "  w%2u : ", warp_id);
    std::map<unsigned /*regnum*/, unsigned /*count*/>::const_iterator r;
    for (r = warp_info.begin(); r != warp_info.end(); ++r) {
      fprintf(fout, "  %u(%u)", r->first, r->second);
    }
    fprintf(fout, "\n");
  }
  m_L1C->display_state(fout);
  m_L1T->display_state(fout);
  if (!m_config->m_L1D_config.disabled()) m_L1D->display_state(fout);
  fprintf(fout, "LD/ST response FIFO (occupancy = %zu):\n",
          m_response_fifo.size());
  for (std::list<mem_fetch *>::const_iterator i = m_response_fifo.begin();
       i != m_response_fifo.end(); i++) {
    const mem_fetch *mf = *i;
    mf->print(fout);
  }
}

void shader_core_ctx::display_pipeline(FILE *fout, int print_mem,
                                       int mask) const {
  fprintf(fout, "=================================================\n");
  fprintf(fout, "shader %u at cycle %Lu+%Lu (%u threads running)\n", m_sid,
          m_gpu->gpu_tot_sim_cycle, m_gpu->gpu_sim_cycle, m_not_completed);
  fprintf(fout, "=================================================\n");

  dump_warp_state(fout);
  fprintf(fout, "\n");

  m_L1I->display_state(fout);

  fprintf(fout, "IF/ID       = ");
  if (!m_inst_fetch_buffer.m_valid)
    fprintf(fout, "bubble\n");
  else {
    fprintf(fout, "w%2u : pc = 0x%llx, nbytes = %u\n",
            m_inst_fetch_buffer.m_warp_id, m_inst_fetch_buffer.m_pc,
            m_inst_fetch_buffer.m_nbytes);
  }
  fprintf(fout, "\nibuffer status:\n");
  for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) {
    if (!m_warp[i]->ibuffer_empty()) m_warp[i]->print_ibuffer(fout);
  }
  fprintf(fout, "\n");
  display_simt_state(fout, mask);
  fprintf(fout, "-------------------------- Scoreboard\n");
  m_scoreboard->printContents();
  /*
     fprintf(fout,"ID/OC (SP)  = ");
     print_stage(ID_OC_SP, fout);
     fprintf(fout,"ID/OC (SFU) = ");
     print_stage(ID_OC_SFU, fout);
     fprintf(fout,"ID/OC (MEM) = ");
     print_stage(ID_OC_MEM, fout);
  */
  fprintf(fout, "-------------------------- OP COL\n");
  m_operand_collector.dump(fout);
  /* fprintf(fout, "OC/EX (SP)  = ");
     print_stage(OC_EX_SP, fout);
     fprintf(fout, "OC/EX (SFU) = ");
     print_stage(OC_EX_SFU, fout);
     fprintf(fout, "OC/EX (MEM) = ");
     print_stage(OC_EX_MEM, fout);
  */
  fprintf(fout, "-------------------------- Pipe Regs\n");

  for (unsigned i = 0; i < N_PIPELINE_STAGES; i++) {
    fprintf(fout, "--- %s ---\n", pipeline_stage_name_decode[i]);
    print_stage(i, fout);
    fprintf(fout, "\n");
  }

  fprintf(fout, "-------------------------- Fu\n");
  for (unsigned n = 0; n < m_num_function_units; n++) {
    m_fu[n]->print(fout);
    fprintf(fout, "---------------\n");
  }
  fprintf(fout, "-------------------------- other:\n");

  for (unsigned i = 0; i < num_result_bus; i++) {
    std::string bits = m_result_bus[i]->to_string();
    fprintf(fout, "EX/WB sched[%d]= %s\n", i, bits.c_str());
  }
  fprintf(fout, "EX/WB      = ");
  print_stage(EX_WB, fout);
  fprintf(fout, "\n");
  fprintf(
      fout,
      "Last EX/WB writeback @ %llu + %llu (gpu_sim_cycle+gpu_tot_sim_cycle)\n",
      m_last_inst_gpu_sim_cycle, m_last_inst_gpu_tot_sim_cycle);

  if (m_active_threads.count() <= 2 * m_config->warp_size) {
    fprintf(fout, "Active Threads : ");
    unsigned last_warp_id = -1;
    for (unsigned tid = 0; tid < m_active_threads.size(); tid++) {
      unsigned warp_id = tid / m_config->warp_size;
      if (m_active_threads.test(tid)) {
        if (warp_id != last_warp_id) {
          fprintf(fout, "\n  warp %u : ", warp_id);
          last_warp_id = warp_id;
        }
        fprintf(fout, "%u ", tid);
      }
    }
  }
}

unsigned int shader_core_config::max_cta(const kernel_info_t &k) const {
  unsigned threads_per_cta = k.threads_per_cta();
  const class function_info *kernel = k.entry();
  unsigned int padded_cta_size = threads_per_cta;
  if (padded_cta_size % warp_size)
    padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size);

  // Limit by n_threads/shader
  unsigned int result_thread = n_thread_per_shader / padded_cta_size;

  const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel);

  // Limit by shmem/shader
  unsigned int result_shmem = (unsigned)-1;
  if (kernel_info->smem > 0)
    result_shmem = gpgpu_shmem_size / kernel_info->smem;

  // Limit by register count, rounded up to multiple of 4.
  unsigned int result_regs = (unsigned)-1;
  if (kernel_info->regs > 0)
    result_regs = gpgpu_shader_registers /
                  (padded_cta_size * ((kernel_info->regs + 3) & ~3));

  // Limit by CTA
  unsigned int result_cta = max_cta_per_core;

  unsigned result = result_thread;
  result = gs_min2(result, result_shmem);
  result = gs_min2(result, result_regs);
  result = gs_min2(result, result_cta);

  static const struct gpgpu_ptx_sim_info *last_kinfo = NULL;
  if (last_kinfo !=
      kernel_info) {  // Only print out stats if kernel_info struct changes
    last_kinfo = kernel_info;
    printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result);
    if (result == result_thread) printf(" threads");
    if (result == result_shmem) printf(" shmem");
    if (result == result_regs) printf(" regs");
    if (result == result_cta) printf(" cta_limit");
    printf("\n");
  }

  // gpu_max_cta_per_shader is limited by number of CTAs if not enough to keep
  // all cores busy
  if (k.num_blocks() < result * num_shader()) {
    result = k.num_blocks() / num_shader();
    if (k.num_blocks() % num_shader()) result++;
  }

  assert(result <= MAX_CTA_PER_SHADER);
  if (result < 1) {
    printf(
        "GPGPU-Sim uArch: ERROR ** Kernel requires more resources than shader "
        "has.\n");
    if (gpgpu_ignore_resources_limitation) {
      printf(
          "GPGPU-Sim uArch: gpgpu_ignore_resources_limitation is set, ignore "
          "the ERROR!\n");
      return 1;
    }
    abort();
  }

  if (adaptive_cache_config && !k.cache_config_set) {
    // For more info about adaptive cache, see
    // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
    unsigned total_shmem = kernel_info->smem * result;
    assert(total_shmem >= 0 && total_shmem <= shmem_opt_list.back());

    // Unified cache config is in KB. Converting to B
    unsigned total_unified = m_L1D_config.m_unified_cache_size * 1024;

    bool l1d_configured = false;
    unsigned max_assoc = m_L1D_config.get_max_assoc();

    for (std::vector<unsigned>::const_iterator it = shmem_opt_list.begin();
         it < shmem_opt_list.end(); it++) {
      if (total_shmem <= *it) {
        float l1_ratio = 1 - ((float)*(it) / total_unified);
        // make sure the ratio is between 0 and 1
        assert(0 <= l1_ratio && l1_ratio <= 1);
        // round to nearest instead of round down
        m_L1D_config.set_assoc(max_assoc * l1_ratio + 0.5f);
        l1d_configured = true;
        break;
      }
    }

    assert(l1d_configured && "no shared memory option found");

    if (m_L1D_config.is_streaming()) {
      // for streaming cache, if the whole memory is allocated
      // to the L1 cache, then make the allocation to be on_MISS
      // otherwise, make it ON_FILL to eliminate line allocation fails
      // i.e. MSHR throughput is the same, independent on the L1 cache
      // size/associativity
      if (total_shmem == 0) {
        m_L1D_config.set_allocation_policy(ON_MISS);
        printf("GPGPU-Sim: Reconfigure L1 allocation to ON_MISS\n");
      } else {
        m_L1D_config.set_allocation_policy(ON_FILL);
        printf("GPGPU-Sim: Reconfigure L1 allocation to ON_FILL\n");
      }
    }
    printf("GPGPU-Sim: Reconfigure L1 cache to %uKB\n",
           m_L1D_config.get_total_size_inKB());

    k.cache_config_set = true;
  }

  return result;
}

void shader_core_config::set_pipeline_latency() {
  // calculate the max latency  based on the input

  std::array<unsigned, 6> int_latency{};
  std::array<unsigned, 5> fp_latency{};
  std::array<unsigned, 5> dp_latency{};
  unsigned sfu_latency = 0;
  unsigned tensor_latency = 0;

  /*
   * [0] ADD,SUB
   * [1] MAX,Min
   * [2] MUL
   * [3] MAD
   * [4] DIV
   * [5] SHFL
   */
  sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u,%u",
         &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3],
         &int_latency[4], &int_latency[5]);
  sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u",
         &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3],
         &fp_latency[4]);
  sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u",
         &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3],
         &dp_latency[4]);
  sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency);
  sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency);

  // all div operation are executed on sfu
  // assume that the max latency are dp div or normal sfu_latency
  max_sfu_latency = std::max(dp_latency[4], sfu_latency);
  // assume that the max operation has the max latency
  max_sp_latency = fp_latency[1];
  max_int_latency = std::max(int_latency[1], int_latency[5]);
  max_dp_latency = dp_latency[1];
  max_tensor_core_latency = tensor_latency;
}

void shader_core_ctx::cycle() {
  if (!isactive() && get_not_completed() == 0) return;

  if (m_config->model == AWARE_RECONVERGENCE && m_config->rec_time_out > 0) {
    unsigned long long total_cycles =
        m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle;
    if (total_cycles % 10000 == 0) check_time_out();
  }

  m_stats->shader_cycles[m_sid]++;
  writeback();
  execute();
  read_operands();
  issue();
  for (unsigned int i = 0; i < m_config->inst_fetch_throughput; ++i) {
    decode();
    fetch();
  }
}

// Flushes all content of the cache to memory

void shader_core_ctx::cache_flush() { m_ldst_unit->flush(); }

void shader_core_ctx::cache_invalidate() { m_ldst_unit->invalidate(); }

// modifiers
std::list<opndcoll_rfu_t::op_t> opndcoll_rfu_t::arbiter_t::allocate_reads() {
  std::list<op_t>
      result;  // a list of registers that (a) are in different register banks,
               // (b) do not go to the same operand collector

  int input;
  int output;
  int _inputs = m_num_banks;
  int _outputs = m_num_collectors;
  int _square = (_inputs > _outputs) ? _inputs : _outputs;
  assert(_square > 0);
  int _pri = (int)m_last_cu;

  // Clear matching
  for (int i = 0; i < _inputs; ++i) _inmatch[i] = -1;
  for (int j = 0; j < _outputs; ++j) _outmatch[j] = -1;

  for (unsigned i = 0; i < m_num_banks; i++) {
    for (unsigned j = 0; j < m_num_collectors; j++) {
      assert(i < (unsigned)_inputs);
      assert(j < (unsigned)_outputs);
      _request[i][j] = 0;
    }
    if (!m_queue[i].empty()) {
      const op_t &op = m_queue[i].front();
      int oc_id = op.get_oc_id();
      assert(i < (unsigned)_inputs);
      assert(oc_id < _outputs);
      _request[i][oc_id] = 1;
    }
    if (m_allocated_bank[i].is_write()) {
      assert(i < (unsigned)_inputs);
      _inmatch[i] = 0;  // write gets priority
    }
  }

  ///// wavefront allocator from booksim... --->

  // Loop through diagonals of request matrix
  // printf("####\n");

  for (int p = 0; p < _square; ++p) {
    output = (_pri + p) % _outputs;

    // Step through the current diagonal
    for (input = 0; input < _inputs; ++input) {
      assert(input < _inputs);
      assert(output < _outputs);
      if ((output < _outputs) && (_inmatch[input] == -1) &&
          //( _outmatch[output] == -1 ) &&   //allow OC to read multiple reg
          // banks at the same cycle
          (_request[input][output] /*.label != -1*/)) {
        // Grant!
        _inmatch[input] = output;
        _outmatch[output] = input;
        // printf("Register File: granting bank %d to OC %d, schedid %d, warpid
        // %d, Regid %d\n", input, output, (m_queue[input].front()).get_sid(),
        // (m_queue[input].front()).get_wid(),
        // (m_queue[input].front()).get_reg());
      }

      output = (output + 1) % _outputs;
    }
  }

  // Round-robin the priority diagonal
  _pri = (_pri + 1) % _outputs;

  /// <--- end code from booksim

  m_last_cu = _pri;
  for (unsigned i = 0; i < m_num_banks; i++) {
    if (_inmatch[i] != -1) {
      if (!m_allocated_bank[i].is_write()) {
        unsigned bank = (unsigned)i;
        op_t &op = m_queue[bank].front();
        result.push_back(op);
        m_queue[bank].pop_front();
      }
    }
  }

  return result;
}

barrier_set_t::barrier_set_t(shader_core_ctx *shader,
                             unsigned max_warps_per_core,
                             unsigned max_cta_per_core,
                             unsigned max_barriers_per_cta,
                             unsigned warp_size) {
  m_max_warps_per_core = max_warps_per_core;
  m_max_cta_per_core = max_cta_per_core;
  m_max_barriers_per_cta = max_barriers_per_cta;
  m_warp_size = warp_size;
  m_shader = shader;
  if (max_warps_per_core > WARP_PER_CTA_MAX) {
    printf(
        "ERROR ** increase WARP_PER_CTA_MAX in shader.h from %u to >= %u or "
        "warps per cta in gpgpusim.config\n",
        WARP_PER_CTA_MAX, max_warps_per_core);
    exit(1);
  }
  if (max_barriers_per_cta > MAX_BARRIERS_PER_CTA) {
    printf(
        "ERROR ** increase MAX_BARRIERS_PER_CTA in abstract_hardware_model.h "
        "from %u to >= %u or barriers per cta in gpgpusim.config\n",
        MAX_BARRIERS_PER_CTA, max_barriers_per_cta);
    exit(1);
  }
  m_warp_active.reset();
  m_warp_at_barrier.reset();
  for (unsigned i = 0; i < max_barriers_per_cta; i++) {
    m_bar_id_to_warps[i].reset();
  }
}

// during cta allocation
void barrier_set_t::allocate_barrier(unsigned cta_id, warp_set_t warps) {
  assert(cta_id < m_max_cta_per_core);
  cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id);
  assert(w == m_cta_to_warps.end());  // cta should not already be active or
                                      // allocated barrier resources
  m_cta_to_warps[cta_id] = warps;
  assert(m_cta_to_warps.size() <=
         m_max_cta_per_core);  // catch cta's that were not properly deallocated

  m_warp_active |= warps;
  m_warp_at_barrier &= ~warps;
  for (unsigned i = 0; i < m_max_barriers_per_cta; i++) {
    m_bar_id_to_warps[i] &= ~warps;
  }
}

// during cta deallocation
void barrier_set_t::deallocate_barrier(unsigned cta_id) {
  cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id);
  if (w == m_cta_to_warps.end()) return;
  warp_set_t warps = w->second;
  warp_set_t at_barrier = warps & m_warp_at_barrier;
  assert(at_barrier.any() == false);  // no warps stuck at barrier
  warp_set_t active = warps & m_warp_active;
  assert(active.any() == false);  // no warps in CTA still running
  m_warp_active &= ~warps;
  m_warp_at_barrier &= ~warps;

  for (unsigned i = 0; i < m_max_barriers_per_cta; i++) {
    warp_set_t at_a_specific_barrier = warps & m_bar_id_to_warps[i];
    assert(at_a_specific_barrier.any() == false);  // no warps stuck at barrier
    m_bar_id_to_warps[i] &= ~warps;
  }
  m_cta_to_warps.erase(w);
}

// individual warp hits barrier
void barrier_set_t::warp_reaches_barrier(unsigned cta_id, unsigned warp_id,
                                         warp_inst_t *inst) {
  barrier_type bar_type = inst->bar_type;
  unsigned bar_id = inst->bar_id;
  unsigned bar_count = inst->bar_count;
  assert(bar_id != (unsigned)-1);
  cta_to_warp_t::iterator w = m_cta_to_warps.find(cta_id);

  if (w == m_cta_to_warps.end()) {  // cta is active
    printf(
        "ERROR ** cta_id %u not found in barrier set on cycle %llu+%llu...\n",
        cta_id, m_shader->get_gpu()->gpu_tot_sim_cycle,
        m_shader->get_gpu()->gpu_sim_cycle);
    dump();
    abort();
  }
  assert(w->second.test(warp_id) == true);  // warp is in cta

  m_bar_id_to_warps[bar_id].set(warp_id);
  if (bar_type == SYNC || bar_type == RED) {
    m_warp_at_barrier.set(warp_id);
  }
  warp_set_t warps_in_cta = w->second;
  warp_set_t at_barrier = warps_in_cta & m_bar_id_to_warps[bar_id];
  warp_set_t active = warps_in_cta & m_warp_active;
  if (bar_count == (unsigned)-1) {
    if (at_barrier == active) {
      // all warps have reached barrier, so release waiting warps...
      m_bar_id_to_warps[bar_id] &= ~at_barrier;
      m_warp_at_barrier &= ~at_barrier;
      if (bar_type == RED) {
        m_shader->broadcast_barrier_reduction(cta_id, bar_id, at_barrier);
      }
    }
  } else {
    // TODO: check on the hardware if the count should include warp that exited
    if ((at_barrier.count() * m_warp_size) == bar_count) {
      // required number of warps have reached barrier, so release waiting
      // warps...
      m_bar_id_to_warps[bar_id] &= ~at_barrier;
      m_warp_at_barrier &= ~at_barrier;
      if (bar_type == RED) {
        m_shader->broadcast_barrier_reduction(cta_id, bar_id, at_barrier);
      }
    }
  }
}

// warp reaches exit
void barrier_set_t::warp_exit(unsigned warp_id) {
  // caller needs to verify all threads in warp are done, e.g., by checking PDOM
  // stack to see it has only one entry during exit_impl()
  m_warp_active.reset(warp_id);

  // test for barrier release
  cta_to_warp_t::iterator w = m_cta_to_warps.begin();
  for (; w != m_cta_to_warps.end(); ++w) {
    if (w->second.test(warp_id) == true) break;
  }
  warp_set_t warps_in_cta = w->second;
  warp_set_t active = warps_in_cta & m_warp_active;

  for (unsigned i = 0; i < m_max_barriers_per_cta; i++) {
    warp_set_t at_a_specific_barrier = warps_in_cta & m_bar_id_to_warps[i];
    if (at_a_specific_barrier == active) {
      // all warps have reached barrier, so release waiting warps...
      m_bar_id_to_warps[i] &= ~at_a_specific_barrier;
      m_warp_at_barrier &= ~at_a_specific_barrier;
    }
  }
}

// assertions
bool barrier_set_t::warp_waiting_at_barrier(unsigned warp_id) const {
  return m_warp_at_barrier.test(warp_id);
}

void barrier_set_t::dump() {
  printf("barrier set information\n");
  printf("  m_max_cta_per_core = %u\n", m_max_cta_per_core);
  printf("  m_max_warps_per_core = %u\n", m_max_warps_per_core);
  printf(" m_max_barriers_per_cta =%u\n", m_max_barriers_per_cta);
  printf("  cta_to_warps:\n");

  cta_to_warp_t::const_iterator i;
  for (i = m_cta_to_warps.begin(); i != m_cta_to_warps.end(); i++) {
    unsigned cta_id = i->first;
    warp_set_t warps = i->second;
    printf("    cta_id %u : %s\n", cta_id, warps.to_string().c_str());
  }
  printf("  warp_active: %s\n", m_warp_active.to_string().c_str());
  printf("  warp_at_barrier: %s\n", m_warp_at_barrier.to_string().c_str());
  for (unsigned i = 0; i < m_max_barriers_per_cta; i++) {
    warp_set_t warps_reached_barrier = m_bar_id_to_warps[i];
    printf("  warp_at_barrier %u: %s\n", i,
           warps_reached_barrier.to_string().c_str());
  }
  fflush(stdout);
}

void shader_core_ctx::warp_exit(unsigned warp_id) {
  bool done = true;
  for (unsigned i = warp_id * get_config()->warp_size;
       i < (warp_id + 1) * get_config()->warp_size; i++) {
    //		if(this->m_thread[i]->m_functional_model_thread_state &&
    // this->m_thread[i].m_functional_model_thread_state->donecycle()==0) {
    // done = false;
    //		}

    if (m_thread[i] && !m_thread[i]->is_done()) done = false;
  }
  // if (m_warp[warp_id].get_n_completed() == get_config()->warp_size)
  // if (this->m_simt_stack[warp_id]->get_num_entries() == 0)
  if (done) m_barriers.warp_exit(warp_id);
}

bool shader_core_ctx::check_if_non_released_reduction_barrier(
    warp_inst_t &inst) {
  unsigned warp_id = inst.warp_id();
  bool bar_red_op = (inst.op == BARRIER_OP) && (inst.bar_type == RED);
  bool non_released_barrier_reduction = false;
  bool warp_stucked_at_barrier = warp_waiting_at_barrier(warp_id);
  bool single_inst_in_pipeline =
      (m_warp[warp_id]->num_issued_inst_in_pipeline() == 1);
  non_released_barrier_reduction =
      single_inst_in_pipeline and warp_stucked_at_barrier and bar_red_op;
  printf("non_released_barrier_reduction=%u\n", non_released_barrier_reduction);
  return non_released_barrier_reduction;
}

bool shader_core_ctx::warp_waiting_at_barrier(unsigned warp_id) const {
  return m_barriers.warp_waiting_at_barrier(warp_id);
}

bool shader_core_ctx::warp_waiting_at_mem_barrier(unsigned warp_id) {
  if (!m_warp[warp_id]->get_membar()) return false;
  if (!m_scoreboard->pendingWrites(warp_id)) {
    m_warp[warp_id]->clear_membar();
    if (m_gpu->get_config().flush_l1()) {
      // Mahmoud fixed this on Nov 2019
      // Invalidate L1 cache
      // Based on Nvidia Doc, at MEM barrier, we have to
      //(1) wait for all pending writes till they are acked
      //(2) invalidate L1 cache to ensure coherence and avoid reading stall data
      cache_invalidate();
      // TO DO: you need to stall the SM for 5k cycles.
    }
    return false;
  }
  return true;
}

void shader_core_ctx::set_max_cta(const kernel_info_t &kernel) {
  // calculate the max cta count and cta size for local memory address mapping
  kernel_max_cta_per_shader = m_config->max_cta(kernel);
  unsigned int gpu_cta_size = kernel.threads_per_cta();
  kernel_padded_threads_per_cta =
      (gpu_cta_size % m_config->warp_size)
          ? m_config->warp_size * ((gpu_cta_size / m_config->warp_size) + 1)
          : gpu_cta_size;
}

void shader_core_ctx::decrement_atomic_count(unsigned wid, unsigned n) {
  static const bool dbg =
      (getenv("MEMCO_DBG_DEADLOCK") != NULL);
  if (dbg) {
    fprintf(stdout,
            "[ATOMIC_DEC] sid=%u wid=%u dec=%u prev=%u new=%u\n",
            m_sid, wid, n, m_warp[wid]->get_n_atomic(),
            m_warp[wid]->get_n_atomic() - n);
    fflush(stdout);
  }
  assert(m_warp[wid]->get_n_atomic() >= n);
  m_warp[wid]->dec_n_atomic(n);
}

void shader_core_ctx::broadcast_barrier_reduction(unsigned cta_id,
                                                  unsigned bar_id,
                                                  warp_set_t warps) {
  for (unsigned i = 0; i < m_config->max_warps_per_shader; i++) {
    if (warps.test(i)) {
      const warp_inst_t *inst =
          m_warp[i]->restore_info_of_last_inst_at_barrier();
      const_cast<warp_inst_t *>(inst)->broadcast_barrier_reduction(
          inst->get_active_mask());
    }
  }
}

bool shader_core_ctx::fetch_unit_response_buffer_full() const { return false; }

void shader_core_ctx::accept_fetch_response(mem_fetch *mf) {
  mf->set_status(IN_SHADER_FETCHED,
                 m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
  m_L1I->fill(mf, m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
}

bool shader_core_ctx::ldst_unit_response_buffer_full() const {
  return m_ldst_unit->response_buffer_full();
}

void shader_core_ctx::accept_ldst_unit_response(mem_fetch *mf) {
  m_ldst_unit->fill(mf);
}

void shader_core_ctx::store_ack(class mem_fetch *mf) {
  assert(mf->get_type() == WRITE_ACK ||
         ((m_config->gpgpu_perfect_mem || m_memory_config->SST_mode) &&
          mf->get_is_write()));
  unsigned warp_id = mf->get_wid();
  m_warp[warp_id]->dec_store_req();
}

void shader_core_ctx::print_cache_stats(FILE *fp, unsigned &dl1_accesses,
                                        unsigned &dl1_misses) {
  m_ldst_unit->print_cache_stats(fp, dl1_accesses, dl1_misses);
}

void shader_core_ctx::print_scoreboard_accounting(FILE *fp) const {
  if (m_scoreboard) m_scoreboard->dumpAccounting(fp);
}

void shader_core_ctx::get_cache_stats(cache_stats &cs) {
  // Adds stats from each cache to 'cs'
  cs += m_L1I->get_stats();          // Get L1I stats
  m_ldst_unit->get_cache_stats(cs);  // Get L1D, L1C, L1T stats
}

void shader_core_ctx::get_L1I_sub_stats(struct cache_sub_stats &css) const {
  if (m_L1I) m_L1I->get_sub_stats(css);
}
void shader_core_ctx::get_L1D_sub_stats(struct cache_sub_stats &css) const {
  m_ldst_unit->get_L1D_sub_stats(css);
}
void shader_core_ctx::get_L1C_sub_stats(struct cache_sub_stats &css) const {
  m_ldst_unit->get_L1C_sub_stats(css);
}
void shader_core_ctx::get_L1T_sub_stats(struct cache_sub_stats &css) const {
  m_ldst_unit->get_L1T_sub_stats(css);
}

void shader_core_ctx::get_icnt_power_stats(long &n_simt_to_mem,
                                           long &n_mem_to_simt) const {
  n_simt_to_mem += m_stats->n_simt_to_mem[m_sid];
  n_mem_to_simt += m_stats->n_mem_to_simt[m_sid];
}

kernel_info_t *shd_warp_t::get_kernel_info() const {
  return m_shader->get_kernel_info();
}

bool shd_warp_t::functional_done() const {
  return get_n_completed() == m_warp_size;
}

bool shd_warp_t::hardware_done() const {
  return functional_done() && stores_done() && !inst_in_pipeline();
}

bool shd_warp_t::waiting() {
  if (functional_done()) {
    // waiting to be initialized with a kernel
    return true;
  } else if (m_shader->warp_waiting_at_barrier(m_warp_id)) {
    // waiting for other warps in CTA to reach barrier
    return true;
  } else if (m_shader->warp_waiting_at_mem_barrier(m_warp_id)) {
    // waiting for memory barrier
    return true;
  } else if (m_n_atomic > 0) {
    // waiting for atomic operation to complete at memory:
    // this stall is not required for accurate timing model, but rather we
    // stall here since if a call/return instruction occurs in the meantime
    // the functional execution of the atomic when it hits DRAM can cause
    // the wrong register to be read.
    return true;
  } else if (m_waiting_ldgsts) {  // Waiting for LDGSTS to finish
    return true;
  }
  return false;
}

// ITS (AWARE Reconvergence) warp eligibility helpers
bool shd_warp_t::virtualized() {
  return m_shader->is_virtualized(m_warp_id);
}

bool shd_warp_t::pending_reconvergence() {
  return m_shader->pending_reconvergence(m_warp_id);
}

bool shd_warp_t::blocked() {
  return m_shader->warp_blocked(m_warp_id);
}

bool shd_warp_t::valid() {
  return m_shader->warp_valid(m_warp_id);
}

// ITS (AWARE Reconvergence) shader_core_ctx methods
void shader_core_ctx::updateSIMTDivergenceStructuresInitialization() {
  unsigned n = m_config->n_thread_per_shader / m_config->warp_size;
  for (unsigned i = 0; i < n; i++) {
    m_simt_tables[i]->set_shader(this);
  }
}

bool shader_core_ctx::push_to_st_response_fifo(unsigned wid, unsigned entry) {
  return m_simt_tables[wid]->push_to_st_response_fifo(entry);
}

bool shader_core_ctx::push_to_rt_response_fifo(unsigned wid, unsigned entry) {
  return m_simt_tables[wid]->push_to_rt_response_fifo(entry);
}

bool shader_core_ctx::is_virtualized(unsigned wid) {
  return m_simt_tables[wid]->is_virtualized();
}

bool shader_core_ctx::pending_reconvergence(unsigned wid) {
  return m_simt_tables[wid]->is_pending_reconvergence();
}

bool shader_core_ctx::warp_blocked(unsigned wid) {
  return m_simt_tables[wid]->blocked();
}

bool shader_core_ctx::warp_valid(unsigned wid) {
  return m_simt_tables[wid]->valid();
}

bool shader_core_ctx::has_secondary_split(unsigned wid) {
  return m_simt_tables[wid]->has_secondary_split();
}

bool shader_core_ctx::get_secondary_split_info(unsigned wid, unsigned *pc,
                                               unsigned *rpc,
                                               unsigned *split_id,
                                               simt_mask_t *mask) {
  return m_simt_tables[wid]->get_secondary_split_info(pc, rpc, split_id, mask);
}

bool shader_core_ctx::is_split_valid(unsigned wid, unsigned split_id) {
  return m_simt_tables[wid]->is_split_valid(split_id);
}

void shader_core_ctx::get_split_info(unsigned wid, unsigned split_id,
                                     unsigned *pc, simt_mask_t *mask) {
  m_simt_tables[wid]->get_split_info(split_id, pc, mask);
}

bool shader_core_ctx::move_split_to_front(unsigned wid, unsigned split_id) {
  return m_simt_tables[wid]->move_split_to_front(split_id);
}

void shader_core_ctx::update_st_size(unsigned n) {
  // Stats not tracked in gpgpu-sim base; no-op.
  (void)n;
}

void shader_core_ctx::update_rt_size(unsigned n) {
  // Stats not tracked in gpgpu-sim base; no-op.
  (void)n;
}

bool shader_core_ctx::branch_unit_avail(unsigned wid) {
  return m_simt_tables[wid]->branch_unit_avail();
}

void shader_core_ctx::check_time_out() {
  for (unsigned w = 0; w < m_config->max_warps_per_shader; w++) {
    m_simt_tables[w]->check_time_out();
  }
}

bool shader_core_ctx::memory_cycle(warp_inst_t &inst,
                                   mem_stage_stall_type &rc_fail,
                                   mem_stage_access_type &fail_type) {
  return m_ldst_unit->memory_cycle(inst, rc_fail, fail_type);
}

void shd_warp_t::print(FILE *fout) const {
  if (!done_exit()) {
    fprintf(fout,
            "w%02u npc: 0x%04llx, done:%c%c%c%c:%2u i:%u s:%u a:%u (done: ",
            m_warp_id, m_next_pc, (functional_done() ? 'f' : ' '),
            (stores_done() ? 's' : ' '), (inst_in_pipeline() ? ' ' : 'i'),
            (done_exit() ? 'e' : ' '), n_completed, m_inst_in_pipeline,
            m_stores_outstanding, m_n_atomic);
    for (unsigned i = m_warp_id * m_warp_size;
         i < (m_warp_id + 1) * m_warp_size; i++) {
      if (m_shader->ptx_thread_done(i))
        fprintf(fout, "1");
      else
        fprintf(fout, "0");
      if ((((i + 1) % 4) == 0) && (i + 1) < (m_warp_id + 1) * m_warp_size)
        fprintf(fout, ",");
    }
    fprintf(fout, ") ");
    fprintf(fout, " active=%s", m_active_threads.to_string().c_str());
    fprintf(fout, " last fetched @ %5llu", m_last_fetch);
    if (m_imiss_pending) fprintf(fout, " i-miss pending");
    fprintf(fout, "\n");
  }
}

void shd_warp_t::print_ibuffer(FILE *fout) const {
  fprintf(fout, "  ibuffer[%2u] : ", m_warp_id);
  for (unsigned i = 0; i < IBUFFER_SIZE; i++) {
    const inst_t *inst = m_ibuffer[i].m_inst;
    if (inst)
      inst->print_insn(fout);
    else if (m_ibuffer[i].m_valid)
      fprintf(fout, " <invalid instruction> ");
    else
      fprintf(fout, " <empty> ");
  }
  fprintf(fout, "\n");
}

void opndcoll_rfu_t::add_cu_set(unsigned set_id, unsigned num_cu,
                                unsigned num_dispatch) {
  m_cus[set_id].reserve(num_cu);  // this is necessary to stop pointers in m_cu
                                  // from being invalid do to a resize;
  for (unsigned i = 0; i < num_cu; i++) {
    m_cus[set_id].push_back(collector_unit_t());
    m_cu.push_back(&m_cus[set_id].back());
  }
  // for now each collector set gets dedicated dispatch units.
  for (unsigned i = 0; i < num_dispatch; i++) {
    m_dispatch_units.push_back(dispatch_unit_t(&m_cus[set_id]));
  }
}

void opndcoll_rfu_t::add_port(port_vector_t &input, port_vector_t &output,
                              uint_vector_t cu_sets) {
  // m_num_ports++;
  // m_num_collectors += num_collector_units;
  // m_input.resize(m_num_ports);
  // m_output.resize(m_num_ports);
  // m_num_collector_units.resize(m_num_ports);
  // m_input[m_num_ports-1]=input_port;
  // m_output[m_num_ports-1]=output_port;
  // m_num_collector_units[m_num_ports-1]=num_collector_units;
  m_in_ports.push_back(input_port_t(input, output, cu_sets));
}

void opndcoll_rfu_t::init(unsigned num_banks, shader_core_ctx *shader) {
  m_shader = shader;
  m_arbiter.init(m_cu.size(), num_banks);
  // for( unsigned n=0; n<m_num_ports;n++ )
  //    m_dispatch_units[m_output[n]].init( m_num_collector_units[n] );
  m_num_banks = num_banks;
  m_warp_size = shader->get_config()->warp_size;

  sub_core_model = shader->get_config()->sub_core_model;
  m_num_warp_scheds = shader->get_config()->gpgpu_num_sched_per_core;
  unsigned reg_id = 0;
  if (sub_core_model) {
    assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0);
    assert(m_num_warp_scheds <= m_cu.size() &&
           m_cu.size() % m_num_warp_scheds == 0);
  }
  m_num_banks_per_sched =
      num_banks / shader->get_config()->gpgpu_num_sched_per_core;

  for (unsigned j = 0; j < m_cu.size(); j++) {
    if (sub_core_model) {
      unsigned cusPerSched = m_cu.size() / m_num_warp_scheds;
      reg_id = j / cusPerSched;
    }
    m_cu[j]->init(j, num_banks, shader->get_config(), this, sub_core_model,
                  reg_id, m_num_banks_per_sched);
  }
  for (unsigned j = 0; j < m_dispatch_units.size(); j++) {
    m_dispatch_units[j].init(sub_core_model, m_num_warp_scheds);
  }
  m_initialized = true;
}

unsigned register_bank(int regnum, int wid, unsigned num_banks,
                       bool sub_core_model, unsigned banks_per_sched,
                       unsigned sched_id) {
  int bank = regnum;
  bank += wid;
  if (sub_core_model) {
    unsigned bank_num = (bank % banks_per_sched) + (sched_id * banks_per_sched);
    assert(bank_num < num_banks);
    return bank_num;
  } else
    return bank % num_banks;
}

bool opndcoll_rfu_t::writeback(warp_inst_t &inst) {
  assert(!inst.empty());

  unsigned sched_id = inst.get_schd_id();

  // Phase 1: Primary warp's destination registers
  std::list<unsigned> regs = m_shader->get_regs_written(inst);
  for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) {
    int reg_num = inst.arch_reg.dst[op];
    if (reg_num >= 0) {
      unsigned bank =
          register_bank(reg_num, inst.warp_id(), m_num_banks, sub_core_model,
                        m_num_banks_per_sched, sched_id);
      if (m_arbiter.bank_idle(bank)) {
        m_arbiter.allocate_bank_for_write(
            bank, op_t(&inst, reg_num, m_num_banks, sub_core_model,
                       m_num_banks_per_sched, sched_id));
        inst.arch_reg.dst[op] = -1;
      } else {
        return false;
      }
    }
  }

  // Phase 2: Co-issued sets' destination registers (different instructions).
  // Phase 1 has already stamped dst=-1 on entries it wrote. If Phase 2 hits
  // a busy bank (can happen when primary.reg and coissued.reg hash to the
  // same bank under sub-core_model — e.g., (reg+wid_A)%banks_per_sched ==
  // (reg+wid_B)%banks_per_sched), SKIP that op instead of returning false.
  // Returning false would be swallowed by shader_core_ctx::writeback for
  // EX_WB composites (see comment at ~shader.cc:3096), but ldst_unit's
  // writeback uses the return value to decide whether to clear m_next_wb,
  // so a persistent false = deadlock.  Bank-busy bookkeeping doesn't affect
  // functional correctness — thread register values are committed during
  // func_exec_inst, not by this bank model.
  if (inst.has_simd_sets()) {
    const std::vector<simd_set_info> &sets = inst.get_simd_sets();
    for (unsigned s = 0; s < sets.size(); s++) {
      if (!sets[s].valid) continue;
      if (sets[s].source_inst == NULL) continue;  // primary sets have NULL source_inst

      for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) {
        int reg_num = sets[s].source_inst->arch_reg.dst[op];
        if (reg_num >= 0) {
          unsigned bank = register_bank(reg_num, sets[s].warp_id, m_num_banks,
                                        sub_core_model, m_num_banks_per_sched,
                                        sched_id);
          if (m_arbiter.bank_idle(bank)) {
            m_arbiter.allocate_bank_for_write(
                bank, op_t(sets[s].warp_id, reg_num, m_num_banks,
                           sub_core_model, m_num_banks_per_sched, sched_id));
          }
          // else: bank busy — skip the bookkeeping for this op. The
          // composite still completes; we just don't model this particular
          // bank-write's energy for this cycle.
        }
      }
    }
  }

  for (unsigned i = 0; i < (unsigned)regs.size(); i++) {
    if (m_shader->get_config()->gpgpu_clock_gated_reg_file) {
      unsigned active_count = 0;
      for (unsigned i = 0; i < m_shader->get_config()->warp_size;
           i = i + m_shader->get_config()->n_regfile_gating_group) {
        for (unsigned j = 0; j < m_shader->get_config()->n_regfile_gating_group;
             j++) {
          if (inst.get_active_mask().test(i + j)) {
            active_count += m_shader->get_config()->n_regfile_gating_group;
            break;
          }
        }
      }
      m_shader->incregfile_writes(active_count);
    } else {
      m_shader->incregfile_writes(
          m_shader->get_config()->warp_size);
    }
  }
  return true;
}

void opndcoll_rfu_t::dispatch_ready_cu() {
  for (unsigned p = 0; p < m_dispatch_units.size(); ++p) {
    dispatch_unit_t &du = m_dispatch_units[p];
    collector_unit_t *cu = du.find_ready();
    if (cu) {
      for (unsigned i = 0; i < (cu->get_num_operands() - cu->get_num_regs());
           i++) {
        if (m_shader->get_config()->gpgpu_clock_gated_reg_file) {
          unsigned active_count = 0;
          for (unsigned i = 0; i < m_shader->get_config()->warp_size;
               i = i + m_shader->get_config()->n_regfile_gating_group) {
            for (unsigned j = 0;
                 j < m_shader->get_config()->n_regfile_gating_group; j++) {
              if (cu->get_active_mask().test(i + j)) {
                active_count += m_shader->get_config()->n_regfile_gating_group;
                break;
              }
            }
          }
          m_shader->incnon_rf_operands(active_count);
        } else {
          m_shader->incnon_rf_operands(
              m_shader->get_config()->warp_size);  // cu->get_active_count());
        }
      }
      cu->dispatch();
    }
  }
}

void opndcoll_rfu_t::allocate_cu(unsigned port_num) {
  input_port_t &inp = m_in_ports[port_num];
  for (unsigned i = 0; i < inp.m_in.size(); i++) {
    if ((*inp.m_in[i]).has_ready()) {
      // find a free cu
      for (unsigned j = 0; j < inp.m_cu_sets.size(); j++) {
        std::vector<collector_unit_t> &cu_set = m_cus[inp.m_cu_sets[j]];
        bool allocated = false;
        unsigned cuLowerBound = 0;
        unsigned cuUpperBound = cu_set.size();
        unsigned schd_id;
        if (sub_core_model) {
          // Sub core model only allocates on the subset of CUs assigned to the
          // scheduler that issued
          unsigned reg_id = (*inp.m_in[i]).get_ready_reg_id();
          schd_id = (*inp.m_in[i]).get_schd_id(reg_id);
          assert(cu_set.size() % m_num_warp_scheds == 0 &&
                 cu_set.size() >= m_num_warp_scheds);
          unsigned cusPerSched = cu_set.size() / m_num_warp_scheds;
          cuLowerBound = schd_id * cusPerSched;
          cuUpperBound = cuLowerBound + cusPerSched;
          assert(0 <= cuLowerBound && cuUpperBound <= cu_set.size());
        }
        for (unsigned k = cuLowerBound; k < cuUpperBound; k++) {
          if (cu_set[k].is_free()) {
            collector_unit_t *cu = &cu_set[k];
            allocated = cu->allocate(inp.m_in[i], inp.m_out[i]);
            m_arbiter.add_read_requests(cu);
            break;
          }
        }
        if (allocated) break;  // cu has been allocated, no need to search more.
      }
      // break;  // can only service a single input, if it failed it will fail
      // for
      // others.
    }
  }
}

void opndcoll_rfu_t::deliver_operand(op_t &op) {
  unsigned cu = op.get_oc_id();
  unsigned operand = op.get_operand();
  m_cu[cu]->collect_operand(operand);
  if (m_shader->get_config()->gpgpu_clock_gated_reg_file) {
    unsigned active_count = 0;
    for (unsigned i = 0; i < m_shader->get_config()->warp_size;
         i = i + m_shader->get_config()->n_regfile_gating_group) {
      for (unsigned j = 0;
           j < m_shader->get_config()->n_regfile_gating_group; j++) {
        if (op.get_active_mask().test(i + j)) {
          active_count += m_shader->get_config()->n_regfile_gating_group;
          break;
        }
      }
    }
    m_shader->incregfile_reads(active_count);
  } else {
    m_shader->incregfile_reads(
        m_shader->get_config()->warp_size);  // op.get_active_count();
  }
}

void opndcoll_rfu_t::drain_pending_operands() {
  unsigned long long now = m_shader->get_gpu()->gpu_sim_cycle +
                           m_shader->get_gpu()->gpu_tot_sim_cycle;
  while (!m_pending_operands.empty() &&
         m_pending_operands.front().cycle_ready <= now) {
    deliver_operand(m_pending_operands.front().op);
    m_pending_operands.pop_front();
  }
}

void opndcoll_rfu_t::allocate_reads() {
  // process read requests that do not have conflicts
  std::list<op_t> allocated = m_arbiter.allocate_reads();
  std::map<unsigned, op_t> read_ops;
  for (std::list<op_t>::iterator r = allocated.begin(); r != allocated.end();
       r++) {
    const op_t &rr = *r;
    unsigned reg = rr.get_reg();
    unsigned wid = rr.get_wid();
    unsigned bank = register_bank(reg, wid, m_num_banks, sub_core_model,
                                  m_num_banks_per_sched, rr.get_sid());
    m_arbiter.allocate_for_read(bank, rr);
    read_ops[bank] = rr;
  }

  const unsigned latency =
      m_shader->get_config()->gpgpu_opndcoll_read_latency;
  if (latency == 0) {
    // Current behavior: deliver immediately, no queue. Bit-identical
    // to the pre-Change-3 tip.
    for (std::map<unsigned, op_t>::iterator r = read_ops.begin();
         r != read_ops.end(); ++r) {
      deliver_operand(r->second);
    }
  } else {
    // Enqueue; drain will deliver on `now + latency` or later.
    unsigned long long now = m_shader->get_gpu()->gpu_sim_cycle +
                             m_shader->get_gpu()->gpu_tot_sim_cycle;
    for (std::map<unsigned, op_t>::iterator r = read_ops.begin();
         r != read_ops.end(); ++r) {
      pending_operand_t pend;
      pend.op = r->second;
      pend.cycle_ready = now + latency;
      m_pending_operands.push_back(pend);
    }
  }
}

bool opndcoll_rfu_t::collector_unit_t::ready() const {
  return (!m_free) && m_not_ready.none() &&
         (*m_output_register).has_free(m_sub_core_model, m_reg_id);
}

void opndcoll_rfu_t::collector_unit_t::dump(
    FILE *fp, const shader_core_ctx *shader) const {
  if (m_free) {
    fprintf(fp, "    <free>\n");
  } else {
    m_warp->print(fp);
    for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) {
      if (m_not_ready.test(i)) {
        std::string r = m_src_op[i].get_reg_string();
        fprintf(fp, "    '%s' not ready\n", r.c_str());
      }
    }
  }
}

void opndcoll_rfu_t::collector_unit_t::init(unsigned n, unsigned num_banks,
                                            const core_config *config,
                                            opndcoll_rfu_t *rfu,
                                            bool sub_core_model,
                                            unsigned reg_id,
                                            unsigned banks_per_sched) {
  m_rfu = rfu;
  m_cuid = n;
  m_num_banks = num_banks;
  assert(m_warp == NULL);
  m_warp = new warp_inst_t(config);
  m_sub_core_model = sub_core_model;
  m_reg_id = reg_id;
  m_num_banks_per_sched = banks_per_sched;
}

bool opndcoll_rfu_t::collector_unit_t::allocate(register_set *pipeline_reg_set,
                                                register_set *output_reg_set) {
  assert(m_free);
  assert(m_not_ready.none());
  m_free = false;
  m_output_register = output_reg_set;
  warp_inst_t **pipeline_reg = pipeline_reg_set->get_ready();
  if ((pipeline_reg) and !((*pipeline_reg)->empty())) {
    m_warp_id = (*pipeline_reg)->warp_id();
    m_has_simd_sets = (*pipeline_reg)->has_simd_sets();
    unsigned sched_id = (*pipeline_reg)->get_schd_id();
    unsigned op_idx = 0;

    // Phase 1: Primary warp's source registers
    std::vector<int> prev_regs;
    for (unsigned op = 0; op < MAX_REG_OPERANDS; op++) {
      int reg_num = (*pipeline_reg)->arch_reg.src[op];
      bool new_reg = true;
      for (auto r : prev_regs) {
        if (r == reg_num) new_reg = false;
      }
      if (reg_num >= 0 && new_reg) {
        prev_regs.push_back(reg_num);
        m_src_op[op_idx] =
            op_t(this, op_idx, reg_num, m_num_banks, m_sub_core_model,
                 m_num_banks_per_sched, sched_id);
        m_not_ready.set(op_idx);
      } else {
        m_src_op[op_idx] = op_t();
      }
      op_idx++;
    }

    // Phase 2: Co-issued sets' source registers (different instructions)
    if (m_has_simd_sets) {
      const std::vector<simd_set_info> &sets =
          (*pipeline_reg)->get_simd_sets();
      for (unsigned s = 0; s < sets.size(); s++) {
        if (!sets[s].valid) continue;
        if (sets[s].source_inst == NULL) continue;  // primary sets have NULL source_inst

        std::vector<int> set_prev_regs;
        for (unsigned op = 0; op < MAX_REG_OPERANDS && op_idx < MAX_REG_OPERANDS * 2; op++) {
          int reg_num = sets[s].source_inst->arch_reg.src[op];
          bool new_reg = true;
          for (auto r : set_prev_regs) {
            if (r == reg_num) new_reg = false;
          }
          if (reg_num >= 0 && new_reg) {
            set_prev_regs.push_back(reg_num);
            m_src_op[op_idx] =
                op_t(this, op_idx, reg_num, m_num_banks, m_sub_core_model,
                     m_num_banks_per_sched, sched_id, sets[s].warp_id);
            m_not_ready.set(op_idx);
            op_idx++;
          }
        }
      }
    }

    // Clear remaining slots
    for (; op_idx < MAX_REG_OPERANDS * 2; op_idx++) {
      m_src_op[op_idx] = op_t();
    }

    pipeline_reg_set->move_out_to(m_warp);
    return true;
  }
  return false;
}

void opndcoll_rfu_t::collector_unit_t::dispatch() {
  assert(m_not_ready.none());
  m_output_register->move_in(m_sub_core_model, m_reg_id, m_warp);
  m_free = true;
  m_output_register = NULL;
  for (unsigned i = 0; i < MAX_REG_OPERANDS * 2; i++) m_src_op[i].reset();
}

void exec_simt_core_cluster::create_shader_core_ctx() {
  m_core = new shader_core_ctx *[m_config->n_simt_cores_per_cluster];
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) {
    unsigned sid = m_config->cid_to_sid(i, m_cluster_id);
    m_core[i] = new exec_shader_core_ctx(m_gpu, this, sid, m_cluster_id,
                                         m_config, m_mem_config, m_stats);
    m_core_sim_order.push_back(i);
  }
}

simt_core_cluster::simt_core_cluster(class gpgpu_sim *gpu, unsigned cluster_id,
                                     const shader_core_config *config,
                                     const memory_config *mem_config,
                                     shader_core_stats *stats,
                                     class memory_stats_t *mstats) {
  m_config = config;
  m_cta_issue_next_core = m_config->n_simt_cores_per_cluster -
                          1;  // this causes first launch to use hw cta 0
  m_cluster_id = cluster_id;
  m_gpu = gpu;
  m_stats = stats;
  m_memory_stats = mstats;
  m_mem_config = mem_config;
}

void simt_core_cluster::core_cycle() {
  for (std::list<unsigned>::iterator it = m_core_sim_order.begin();
       it != m_core_sim_order.end(); ++it) {
    m_core[*it]->cycle();
  }

  if (m_config->simt_core_sim_order == 1) {
    m_core_sim_order.splice(m_core_sim_order.end(), m_core_sim_order,
                            m_core_sim_order.begin());
  }
}

void simt_core_cluster::reinit() {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    m_core[i]->reinit(0, m_config->n_thread_per_shader, true);
}

unsigned simt_core_cluster::max_cta(const kernel_info_t &kernel) {
  return m_config->n_simt_cores_per_cluster * m_config->max_cta(kernel);
}

unsigned simt_core_cluster::get_not_completed() const {
  unsigned not_completed = 0;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    not_completed += m_core[i]->get_not_completed();
  return not_completed;
}

void simt_core_cluster::print_not_completed(FILE *fp) const {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) {
    unsigned not_completed = m_core[i]->get_not_completed();
    unsigned sid = m_config->cid_to_sid(i, m_cluster_id);
    fprintf(fp, "%u(%u) ", sid, not_completed);
  }
}

// Read-only diagnostic — dispatches a per-warp state dump for every core
// in this cluster that still has un-completed threads. Called from
// gpu-sim.cc::deadlock_check() under MEMCO_DBG_DEADLOCK env var.
void simt_core_cluster::dump_warps_at_deadlock(FILE *fp) {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) {
    if (m_core[i]->get_not_completed() == 0) continue;
    m_core[i]->dump_warps_at_deadlock(fp);
  }
}

// Read-only diagnostic — for each warp on this core, prints flags,
// ibuffer slots, simt-table state, and scoreboard reservations. Helps
// pinpoint which warp is stuck and why when the deadlock detector fires.
// No state mutation: only invokes accessors.
void shader_core_ctx::dump_warps_at_deadlock(FILE *fout) {
  fprintf(fout, "[DEADLOCK_DUMP] === core sid=%u not_completed=%u ===\n",
          m_sid, get_not_completed());
  for (unsigned wid = 0; wid < m_warp.size(); wid++) {
    if (m_warp[wid] == NULL) continue;
    shd_warp_t *w = m_warp[wid];
    bool valid_f = w->valid();
    bool done = w->done_exit();
    bool waiting = w->waiting();
    bool blocked = w->blocked();
    bool prec = w->pending_reconvergence();
    bool virt = w->virtualized();
    bool ib_empty = w->ibuffer_empty();
    // Skip warps with nothing interesting (done or fully empty + no
    // scoreboard activity). The stuck warp will have ib_empty=false OR
    // active scoreboard regs — keep those prominent.
    if (done && ib_empty) continue;
    fprintf(fout,
            "[DEADLOCK_DUMP] wid=%u valid=%d done_exit=%d waiting=%d "
            "blocked=%d pending_rec=%d virt=%d ibuf_empty=%d active_split=%u\n",
            wid, (int)valid_f, (int)done, (int)waiting, (int)blocked,
            (int)prec, (int)virt, (int)ib_empty,
            (m_simt_tables && m_simt_tables[wid])
                ? m_simt_tables[wid]->get_active_split_id()
                : 0);
    // Why is waiting() true? Show each branch's contribution.
    if (waiting) {
      bool fdone = w->functional_done();
      bool wb = warp_waiting_at_barrier(wid);
      bool wmb = warp_waiting_at_mem_barrier(wid);
      unsigned natomic = w->get_n_atomic();
      fprintf(fout,
              "[DEADLOCK_DUMP]   waiting_reason: functional_done=%d "
              "barrier=%d mem_barrier=%d n_atomic=%u\n",
              (int)fdone, (int)wb, (int)wmb, natomic);
    }
    // ibuffer slots 0..3 (IBUFFER_SIZE=4, hardcoded since the constant is
    // private to shd_warp_t)
    for (unsigned slot = 0; slot < 4; slot++) {
      if (!w->ibuffer_slot_valid(slot)) {
        fprintf(fout, "[DEADLOCK_DUMP]   ibuffer[%u]: <empty>\n", slot);
        continue;
      }
      const warp_inst_t *pI = w->ibuffer_slot_inst(slot);
      unsigned split_id = w->ibuffer_slot_split_id(slot);
      const active_mask_t &mask = w->ibuffer_slot_split_mask(slot);
      fprintf(fout,
              "[DEADLOCK_DUMP]   ibuffer[%u]: pc=0x%lx op=%d "
              "split_id=%d mask_count=%u\n",
              slot, pI ? (unsigned long)pI->pc : 0ul,
              pI ? (int)pI->op : -1, (int)split_id,
              (unsigned)mask.count());
    }
    // simt_tables summary — use safe accessors only (full print() can
    // segfault on corrupted state, which is likely with a stuck warp).
    if (m_simt_tables && m_simt_tables[wid]) {
      simt_tables *st = m_simt_tables[wid];
      unsigned active_pc = 0, active_rpc = 0;
      st->get_pdom_active_split_info(&active_pc, &active_rpc);
      fprintf(fout,
              "[DEADLOCK_DUMP]   simt: active_split_id=%u active_pc=0x%lx "
              "active_rpc=0x%lx ST_size=%u RT_size=%u blocked=%d "
              "pending_recvg=%d virt=%d valid=%d\n",
              st->get_active_split_id(),
              (unsigned long)active_pc, (unsigned long)active_rpc,
              st->getSTsize(), st->getRTsize(),
              (int)st->blocked(), (int)st->is_pending_reconvergence(),
              (int)st->is_virtualized(), (int)st->valid());
    }
    // scoreboard
    if (m_scoreboard) m_scoreboard->dump_warp_state(fout, wid);
  }
  fflush(fout);
}

float simt_core_cluster::get_current_occupancy(
    unsigned long long &active, unsigned long long &total) const {
  float aggregate = 0.f;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) {
    aggregate += m_core[i]->get_current_occupancy(active, total);
  }
  return aggregate / m_config->n_simt_cores_per_cluster;
}

unsigned simt_core_cluster::get_n_active_cta() const {
  unsigned n = 0;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    n += m_core[i]->get_n_active_cta();
  return n;
}

unsigned simt_core_cluster::get_n_active_sms() const {
  unsigned n = 0;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    n += m_core[i]->isactive();
  return n;
}

unsigned simt_core_cluster::issue_block2core() {
  unsigned num_blocks_issued = 0;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++) {
    unsigned core =
        (i + m_cta_issue_next_core + 1) % m_config->n_simt_cores_per_cluster;

    kernel_info_t *kernel;
    // Jin: fetch kernel according to concurrent kernel setting
    if (m_config->gpgpu_concurrent_kernel_sm) {  // concurrent kernel on sm
      // always select latest issued kernel
      kernel_info_t *k = m_gpu->select_kernel();
      kernel = k;
    } else {
      // first select core kernel, if no more cta, get a new kernel
      // only when core completes
      kernel = m_core[core]->get_kernel();
      if (!m_gpu->kernel_more_cta_left(kernel)) {
        // wait till current kernel finishes
        if (m_core[core]->get_not_completed() == 0) {
          kernel_info_t *k = m_gpu->select_kernel();
          if (k) m_core[core]->set_kernel(k);
          kernel = k;
        }
      }
    }

    if (m_gpu->kernel_more_cta_left(kernel) &&
        //            (m_core[core]->get_n_active_cta() <
        //            m_config->max_cta(*kernel)) ) {
        m_core[core]->can_issue_1block(*kernel)) {
      m_core[core]->issue_block2core(*kernel);
      num_blocks_issued++;
      m_cta_issue_next_core = core;
      break;
    }
  }
  return num_blocks_issued;
}

void simt_core_cluster::cache_flush() {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    m_core[i]->cache_flush();
}

void simt_core_cluster::cache_invalidate() {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; i++)
    m_core[i]->cache_invalidate();
}

bool simt_core_cluster::icnt_injection_buffer_full(unsigned size, bool write) {
  unsigned request_size = size;
  if (!write) request_size = READ_PACKET_SIZE;
  return !::icnt_has_buffer(m_cluster_id, request_size);
}

bool sst_simt_core_cluster::SST_injection_buffer_full(unsigned size, bool write,
                                                      mem_access_type type) {
  switch (type) {
    case CONST_ACC_R:
    case INST_ACC_R: {
      return response_queue_full();
      break;
    }
    default: {
      return ::is_SST_buffer_full(m_cluster_id);
      break;
    }
  }
}

void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf) {
  // Update stats based on mf type
  update_icnt_stats(mf);

  // The packet size varies depending on the type of request:
  // - For write request and atomic request, the packet contains the data
  // - For read request (i.e. not write nor atomic), the packet only has control
  // metadata
  unsigned int packet_size = mf->size();
  if (!mf->get_is_write() && !mf->isatomic()) {
    packet_size = mf->get_ctrl_size();
  }
  m_stats->m_outgoing_traffic_stats->record_traffic(mf, packet_size);
  unsigned destination = mf->get_sub_partition_id();
  mf->set_status(IN_ICNT_TO_MEM,
                 m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
  if (!mf->get_is_write() && !mf->isatomic())
    ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void *)mf,
                mf->get_ctrl_size());
  else
    ::icnt_push(m_cluster_id, m_config->mem2device(destination), (void *)mf,
                mf->size());
}

void simt_core_cluster::update_icnt_stats(class mem_fetch *mf) {
  // stats
  if (mf->get_is_write())
    m_stats->made_write_mfs++;
  else
    m_stats->made_read_mfs++;
  switch (mf->get_access_type()) {
    case CONST_ACC_R:
      m_stats->gpgpu_n_mem_const++;
      break;
    case TEXTURE_ACC_R:
      m_stats->gpgpu_n_mem_texture++;
      break;
    case GLOBAL_ACC_R:
      m_stats->gpgpu_n_mem_read_global++;
      break;
    // case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++;
    // printf("read_global%d\n",m_stats->gpgpu_n_mem_read_global); break;
    case GLOBAL_ACC_W:
      m_stats->gpgpu_n_mem_write_global++;
      break;
    case LOCAL_ACC_R:
      m_stats->gpgpu_n_mem_read_local++;
      break;
    case LOCAL_ACC_W:
      m_stats->gpgpu_n_mem_write_local++;
      break;
    case INST_ACC_R:
      m_stats->gpgpu_n_mem_read_inst++;
      break;
    case L1_WRBK_ACC:
      m_stats->gpgpu_n_mem_write_global++;
      break;
    case L2_WRBK_ACC:
      m_stats->gpgpu_n_mem_l2_writeback++;
      break;
    case L1_WR_ALLOC_R:
      m_stats->gpgpu_n_mem_l1_write_allocate++;
      break;
    case L2_WR_ALLOC_R:
      m_stats->gpgpu_n_mem_l2_write_allocate++;
      break;
    default:
      assert(0);
  }
}

void sst_simt_core_cluster::icnt_inject_request_packet_to_SST(
    class mem_fetch *mf) {
  // Update stats
  update_icnt_stats(mf);

  // The packet size varies depending on the type of request:
  // - For write request and atomic request, the packet contains the data
  // - For read request (i.e. not write nor atomic), the packet only has control
  // metadata
  unsigned int packet_size = mf->size();
  if (!mf->get_is_write() && !mf->isatomic()) {
    packet_size = mf->get_ctrl_size();
  }
  m_stats->m_outgoing_traffic_stats->record_traffic(mf, packet_size);
  mf->set_status(IN_ICNT_TO_MEM,
                 m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
  switch (mf->get_access_type()) {
    case CONST_ACC_R:
    case INST_ACC_R: {
      push_response_fifo(mf);
      break;
    }
    default: {
      if (!mf->get_is_write() && !mf->isatomic())
        ::send_read_request_SST(m_cluster_id, mf->get_addr(),
                                mf->get_data_size(), (void *)mf);
      else
        ::send_write_request_SST(m_cluster_id, mf->get_addr(),
                                 mf->get_data_size(), (void *)mf);

      break;
    }
  }
}

void simt_core_cluster::icnt_cycle() {
  if (!m_response_fifo.empty()) {
    mem_fetch *mf = m_response_fifo.front();
    unsigned cid = m_config->sid_to_cid(mf->get_sid());
    if (mf->get_access_type() == INST_ACC_R) {
      // instruction fetch response
      if (!m_core[cid]->fetch_unit_response_buffer_full()) {
        m_response_fifo.pop_front();
        m_core[cid]->accept_fetch_response(mf);
      }
    } else {
      // data response
      if (!m_core[cid]->ldst_unit_response_buffer_full()) {
        m_response_fifo.pop_front();
        m_memory_stats->memlatstat_read_done(mf);
        m_core[cid]->accept_ldst_unit_response(mf);
      }
    }
  }
  if (m_response_fifo.size() < m_config->n_simt_ejection_buffer_size) {
    mem_fetch *mf = (mem_fetch *)::icnt_pop(m_cluster_id);
    if (!mf) return;
    assert(mf->get_tpc() == m_cluster_id);
    assert(mf->get_type() == READ_REPLY || mf->get_type() == WRITE_ACK);

    // The packet size varies depending on the type of request:
    // - For read request and atomic request, the packet contains the data
    // - For write-ack, the packet only has control metadata
    unsigned int packet_size =
        (mf->get_is_write()) ? mf->get_ctrl_size() : mf->size();
    m_stats->m_incoming_traffic_stats->record_traffic(mf, packet_size);
    mf->set_status(IN_CLUSTER_TO_SHADER_QUEUE,
                   m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
    // m_memory_stats->memlatstat_read_done(mf,m_shader_config->max_warps_per_shader);
    m_response_fifo.push_back(mf);
    m_stats->n_mem_to_simt[m_cluster_id] += mf->get_num_flits(false);
  }
}

void sst_simt_core_cluster::icnt_cycle_SST() {
  if (!m_response_fifo.empty()) {
    mem_fetch *mf = m_response_fifo.front();
    unsigned cid = m_config->sid_to_cid(mf->get_sid());
    if (mf->get_access_type() == INST_ACC_R) {
      // instruction fetch response
      if (!m_core[cid]->fetch_unit_response_buffer_full()) {
        m_response_fifo.pop_front();
        m_core[cid]->accept_fetch_response(mf);
      }
    } else {
      // data response
      if (!m_core[cid]->ldst_unit_response_buffer_full()) {
        m_response_fifo.pop_front();
        m_memory_stats->memlatstat_read_done(mf);
        m_core[cid]->accept_ldst_unit_response(mf);
      }
    }
  }

  // pop from SST buffers
  if (m_response_fifo.size() < m_config->n_simt_ejection_buffer_size) {
    mem_fetch *mf = (mem_fetch *)(static_cast<sst_gpgpu_sim *>(get_gpu())
                                      ->SST_pop_mem_reply(m_cluster_id));
    if (!mf) return;
    assert(mf->get_tpc() == m_cluster_id);

    // do atomic here
    // For now, we execute atomic when the mem reply comes back
    // This needs to be validated
    if (mf && mf->isatomic()) mf->do_atomic();

    unsigned int packet_size =
        (mf->get_is_write()) ? mf->get_ctrl_size() : mf->size();
    m_stats->m_incoming_traffic_stats->record_traffic(mf, packet_size);
    mf->set_status(IN_CLUSTER_TO_SHADER_QUEUE,
                   m_gpu->gpu_sim_cycle + m_gpu->gpu_tot_sim_cycle);
    // m_memory_stats->memlatstat_read_done(mf,m_shader_config->max_warps_per_shader);
    m_response_fifo.push_back(mf);
    m_stats->n_mem_to_simt[m_cluster_id] += mf->get_num_flits(false);
  }
}

void simt_core_cluster::get_pdom_stack_top_info(unsigned sid, unsigned tid,
                                                unsigned *pc,
                                                unsigned *rpc) const {
  unsigned cid = m_config->sid_to_cid(sid);
  m_core[cid]->get_pdom_stack_top_info(tid, pc, rpc);
}

void simt_core_cluster::display_pipeline(unsigned sid, FILE *fout,
                                         int print_mem, int mask) {
  m_core[m_config->sid_to_cid(sid)]->display_pipeline(fout, print_mem, mask);

  fprintf(fout, "\n");
  fprintf(fout, "Cluster %u pipeline state\n", m_cluster_id);
  fprintf(fout, "Response FIFO (occupancy = %zu):\n", m_response_fifo.size());
  for (std::list<mem_fetch *>::const_iterator i = m_response_fifo.begin();
       i != m_response_fifo.end(); i++) {
    const mem_fetch *mf = *i;
    mf->print(fout);
  }
}

void simt_core_cluster::print_cache_stats(FILE *fp, unsigned &dl1_accesses,
                                          unsigned &dl1_misses) const {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->print_cache_stats(fp, dl1_accesses, dl1_misses);
  }
}

void simt_core_cluster::print_scoreboard_accounting(FILE *fp) const {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->print_scoreboard_accounting(fp);
  }
}

void simt_core_cluster::get_icnt_stats(long &n_simt_to_mem,
                                       long &n_mem_to_simt) const {
  long simt_to_mem = 0;
  long mem_to_simt = 0;
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_icnt_power_stats(simt_to_mem, mem_to_simt);
  }
  n_simt_to_mem = simt_to_mem;
  n_mem_to_simt = mem_to_simt;
}

void simt_core_cluster::get_cache_stats(cache_stats &cs) const {
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_cache_stats(cs);
  }
}

void simt_core_cluster::get_L1I_sub_stats(struct cache_sub_stats &css) const {
  struct cache_sub_stats temp_css;
  struct cache_sub_stats total_css;
  temp_css.clear();
  total_css.clear();
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_L1I_sub_stats(temp_css);
    total_css += temp_css;
  }
  css = total_css;
}
void simt_core_cluster::get_L1D_sub_stats(struct cache_sub_stats &css) const {
  struct cache_sub_stats temp_css;
  struct cache_sub_stats total_css;
  temp_css.clear();
  total_css.clear();
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_L1D_sub_stats(temp_css);
    total_css += temp_css;
  }
  css = total_css;
}
void simt_core_cluster::get_L1C_sub_stats(struct cache_sub_stats &css) const {
  struct cache_sub_stats temp_css;
  struct cache_sub_stats total_css;
  temp_css.clear();
  total_css.clear();
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_L1C_sub_stats(temp_css);
    total_css += temp_css;
  }
  css = total_css;
}
void simt_core_cluster::get_L1T_sub_stats(struct cache_sub_stats &css) const {
  struct cache_sub_stats temp_css;
  struct cache_sub_stats total_css;
  temp_css.clear();
  total_css.clear();
  for (unsigned i = 0; i < m_config->n_simt_cores_per_cluster; ++i) {
    m_core[i]->get_L1T_sub_stats(temp_css);
    total_css += temp_css;
  }
  css = total_css;
}

void exec_shader_core_ctx::checkExecutionStatusAndUpdate(warp_inst_t &inst,
                                                         unsigned t,
                                                         unsigned tid) {
  if (inst.isatomic()) {
    static const bool dbg =
        (getenv("MEMCO_DBG_DEADLOCK") != NULL);
    unsigned wid = inst.warp_id();
    if (dbg) {
      fprintf(stdout,
              "[ATOMIC_INC] sid=%u wid=%u pc=0x%lx tid=%u prev=%u new=%u "
              "dbg_path=%u space=%d\n",
              m_sid, wid, (unsigned long)inst.pc, tid,
              m_warp[wid]->get_n_atomic(),
              m_warp[wid]->get_n_atomic() + 1,
              inst.get_dbg_path(),
              (int)inst.space.get_type());
      fflush(stdout);
    }
    m_warp[wid]->inc_n_atomic();
  }
  if (inst.space.is_local() && (inst.is_load() || inst.is_store())) {
    new_addr_type localaddrs[MAX_ACCESSES_PER_INSN_PER_THREAD];
    unsigned num_addrs;
    num_addrs = translate_local_memaddr(
        inst.get_addr(t), tid,
        m_config->n_simt_clusters * m_config->n_simt_cores_per_cluster,
        inst.data_size, (new_addr_type *)localaddrs);
    inst.set_addr(t, (new_addr_type *)localaddrs, num_addrs);
  }
  if (ptx_thread_done(tid)) {
    m_warp[inst.warp_id()]->set_completed(t);
    m_warp[inst.warp_id()]->ibuffer_flush();
    m_scoreboard->clearSecondary(inst.warp_id());
  }

  // PC-Histogram Update
  unsigned warp_id = inst.warp_id();
  unsigned pc = inst.pc;
  for (unsigned t = 0; t < m_config->warp_size; t++) {
    if (inst.active(t)) {
      int tid = warp_id * m_config->warp_size + t;
      cflog_update_thread_pc(m_sid, tid, pc);
    }
  }
}