Added shfl instruction

10 files changed, 153 insertions, 2 deletions

diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 09e9a81..8bf4ec8 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -769,6 +769,10 @@ void ptx_instruction::set_opcode_and_latency()
 	  initiation_interval = dp_init[2];
       op = SFU_OP;
       break;
+   case SHFL_OP:
+	   latency = 32;
+	   initiation_interval = 15;
+	   break;
    default: 
        break;
    }
@@ -845,8 +849,10 @@ void ptx_instruction::pre_decode()
 
    switch ( get_opcode() ) {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break;
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break;
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
    default:
       printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() );
       break;
@@ -1240,8 +1246,10 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id)
       }
       switch ( pI->get_opcode() ) {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break;
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break;
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
       default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break;
       }
       delete pJ;
@@ -1408,6 +1416,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
 
    static std::map<unsigned,memory_space*> shared_memory_lookup;
    static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup;
+   static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup;
    static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup;
 
    if ( *thread_info != NULL ) {
@@ -1486,7 +1495,16 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel,
       kernel.increment_thread_id();
       new_tid += tid;
       ptx_thread_info *thd = new ptx_thread_info(kernel);
-   
+
+      ptx_warp_info *warp_info = NULL;
+      if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) {
+    	  warp_info = new ptx_warp_info(); // num_threads should be threads in the warp
+    	  ptx_warp_lookup[hw_warp_id] = warp_info;
+      } else {
+    	  warp_info = ptx_warp_lookup[hw_warp_id];
+      }
+      thd->m_warp_info = warp_info;
+
       memory_space *local_mem = NULL;
       std::map<unsigned,memory_space*>::iterator l = local_mem_lookup.find(new_tid);
       if ( l != local_mem_lookup.end() ) {
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index 7b0f4fa..05ba78f 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -47,8 +47,10 @@ unsigned ptx_instruction::g_num_ptx_inst_uid=0;
 
 const char *g_opcode_string[NUM_OPCODES] = {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR,
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR,
 #include "opcodes.def"
 #undef OP_DEF
+#undef OP_W_DEF
 };
 
 void inst_not_implemented( const ptx_instruction * pI ) ;
@@ -3516,6 +3518,79 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread )
 
 }
 
+void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst )
+{
+	unsigned i_type = pI->get_type();
+	int tid = inst.warp_id() * core->get_warp_size();
+	ptx_thread_info *thread = core->get_thread_info()[tid];
+	ptx_warp_info *warp_info = thread->m_warp_info;
+	int lane = warp_info->get_done_threads();
+	thread = core->get_thread_info()[tid + lane];
+
+	const operand_info &dst = pI->dst();
+	const operand_info &src1 = pI->src1();
+	const operand_info &src2 = pI->src2();
+	const operand_info &src3 = pI->src3();
+	int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32;
+	int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32;
+	int mask = cval >> 8;
+	cval &= 0x1F;
+
+	int maxLane = (lane & mask) | (cval & ~mask);
+	int minLane = lane & mask;
+
+	int src_idx;
+	int p;
+	switch(pI->shfl_op()) {
+	case UP_OPTION:
+		src_idx = lane - bval;
+		p = (src_idx >= maxLane);
+		break;
+	case DOWN_OPTION:
+		src_idx = lane + bval;
+		p = (src_idx <= maxLane);
+		break;
+	case BFLY_OPTION:
+		src_idx = lane ^ bval;
+		p = (src_idx <= maxLane);
+		break;
+	case IDX_OPTION:
+		src_idx = minLane | (bval & ~mask);
+		p = (src_idx <= maxLane);
+		break;
+	default:
+		printf("GPGPU-Sim PTX: ERROR: Unrecognized shfl option\n");
+		assert(0);
+		break;
+	}
+	// copy from own lane
+	if (!p) src_idx = lane;
+	// copy input from lane src_idx
+	ptx_reg_t data;
+	/*if (inst.active(src_idx) && i_type == PRED_TYPE) {
+		ptx_thread_info *source = core->get_thread_info()[tid + src_idx];
+		data = source->get_operand_value(src1, dst, i_type, source, 1);
+		data.pred = p;
+	} else {
+		printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive/predicated-off threads in a warp\n");
+		data.u32 = 0;
+	}*/
+	if (inst.active(src_idx)) {
+		ptx_thread_info *source = core->get_thread_info()[tid + src_idx];
+		data = source->get_operand_value(src1, dst, i_type, source, 1);
+	}
+	if (i_type == PRED_TYPE) {
+		data.pred = p;
+	}
+	thread->set_operand_value(dst, data, i_type, thread, pI);
+
+	// keep track of the number of threads that have executed in the warp
+	warp_info->inc_done_threads();
+	if (warp_info->get_done_threads() == inst.active_count()) {
+		warp_info->reset_done_threads();
+	}
+}
+
 void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) 
 {
    ptx_reg_t a, b, d;
diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def
index 2ee6976..e1b1422 100644
--- a/src/cuda-sim/opcodes.def
+++ b/src/cuda-sim/opcodes.def
@@ -98,6 +98,7 @@ OP_DEF(SAD_OP,sad_impl,"sad",1,1)
 OP_DEF(SELP_OP,selp_impl,"selp",1,1)
 OP_DEF(SETP_OP,setp_impl,"setp",1,1)
 OP_DEF(SET_OP,set_impl,"set",1,1)
+OP_W_DEF(SHFL_OP,shfl_impl,"shfl",1,10)
 OP_DEF(SHL_OP,shl_impl,"shl",1,1)
 OP_DEF(SHR_OP,shr_impl,"shr",1,1)
 OP_DEF(SIN_OP,sin_impl,"sin",1,4)
diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h
index 871091c..aa133da 100644
--- a/src/cuda-sim/opcodes.h
+++ b/src/cuda-sim/opcodes.h
@@ -30,9 +30,11 @@
 
 enum opcode_t {
 #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP,
+#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP,
 #include "opcodes.def"
-   NUM_OPCODES
+	NUM_OPCODES
 #undef OP_DEF
+#undef OP_W_DEF
 };
 
 enum special_regs {
diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l
index 88ccf6a..8fac4ac 100644
--- a/src/cuda-sim/ptx.l
+++ b/src/cuda-sim/ptx.l
@@ -115,6 +115,7 @@ sad     TC; ptx_lval.int_value = SAD_OP; return OPCODE;
 selp	TC; ptx_lval.int_value = SELP_OP; return OPCODE;
 setp    TC; ptx_lval.int_value = SETP_OP; return OPCODE;
 set	TC; ptx_lval.int_value = SET_OP; return OPCODE;
+shfl	TC; ptx_lval.int_value = SHFL_OP; return OPCODE;
 shl     TC; ptx_lval.int_value = SHL_OP; return OPCODE;
 shr     TC; ptx_lval.int_value = SHR_OP; return OPCODE;
 sin	TC; ptx_lval.int_value = SIN_OP; return OPCODE;
@@ -329,6 +330,11 @@ breakaddr  TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE;
 
 \.nc	TC; return NC_OPTION;
 
+\.up	TC; return UP_OPTION;
+\.down	TC; return DOWN_OPTION;
+\.bfly	TC; return BFLY_OPTION;
+\.idx	TC; return IDX_OPTION;
+
 \.popc	TC; return ATOMIC_POPC;
 \.and	TC; return ATOMIC_AND;
 \.or	TC; return ATOMIC_OR;
diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y
index 4de39d1..166b15d 100644
--- a/src/cuda-sim/ptx.y
+++ b/src/cuda-sim/ptx.y
@@ -194,6 +194,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 %token  WB_OPTION;
 %token  WT_OPTION;
 %token	NC_OPTION;
+%token	UP_OPTION;
+%token	DOWN_OPTION;
+%token	BFLY_OPTION;
+%token	IDX_OPTION;
 
 %type <int_value> function_decl_header
 %type <ptr_value> function_decl
@@ -451,6 +455,10 @@ option: type_spec
 	| WB_OPTION { add_option(WB_OPTION); }
 	| WT_OPTION { add_option(WT_OPTION); }
 	| NC_OPTION { add_option(NC_OPTION); }
+	| UP_OPTION { add_option(UP_OPTION); }
+	| DOWN_OPTION { add_option(DOWN_OPTION); }
+	| BFLY_OPTION { add_option(BFLY_OPTION); }
+	| IDX_OPTION { add_option(IDX_OPTION); }
 	;
 
 atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } 
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index 2eccabc..4cfe1b9 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -1171,6 +1171,12 @@ ptx_instruction::ptx_instruction( int opcode,
 		 break;
 	  case NC_OPTION:
 		 break;
+	  case UP_OPTION:
+	  case DOWN_OPTION:
+	  case BFLY_OPTION:
+	  case IDX_OPTION:
+		  m_shfl_op = last_ptx_inst_option;
+		  break;
       default:
          assert(0);
          break;
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 601a13d..0abbc83 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -993,6 +993,7 @@ public:
    unsigned saturation_mode() const { return m_saturation_mode;}
    unsigned dimension() const { return m_geom_spec;}
    unsigned barrier_op() const {return m_barrier_op;}
+   unsigned shfl_op() const {return m_shfl_op;}
    enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot };
    enum vote_mode_t vote_mode() const { return m_vote_mode; }
 
@@ -1058,6 +1059,7 @@ private:
    unsigned            m_compare_op;
    unsigned            m_saturation_mode;
    unsigned 		   m_barrier_op;
+   unsigned			   m_shfl_op;
 
    std::list<int>          m_scalar_type;
    memory_space_t m_space_spec;
diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc
index 09844ae..a3e43aa 100644
--- a/src/cuda-sim/ptx_sim.cc
+++ b/src/cuda-sim/ptx_sim.cc
@@ -128,6 +128,26 @@ unsigned ptx_cta_info::get_sm_idx() const
    return m_sm_idx;
 }
 
+ptx_warp_info::ptx_warp_info()
+{
+	reset_done_threads();
+}
+
+unsigned ptx_warp_info::get_done_threads() const
+{
+	return m_done_threads;
+}
+
+void ptx_warp_info::inc_done_threads()
+{
+	m_done_threads++;
+}
+
+void ptx_warp_info::reset_done_threads()
+{
+	m_done_threads = 0;
+}
+
 unsigned g_ptx_thread_info_uid_next=1;
 unsigned g_ptx_thread_info_delete_count=0;
 
@@ -153,6 +173,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel )
    m_last_memory_space = undefined_space; 
    m_branch_taken = 0;
    m_shared_mem = NULL;
+   m_warp_info = NULL;
    m_cta_info = NULL;
    m_local_mem = NULL;
    m_symbol_table = NULL;
diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h
index f926e6d..449511f 100644
--- a/src/cuda-sim/ptx_sim.h
+++ b/src/cuda-sim/ptx_sim.h
@@ -167,6 +167,17 @@ private:
    std::set<ptx_thread_info*>  m_dangling_pointers;
 };
 
+class ptx_warp_info {
+public:
+	ptx_warp_info();
+	unsigned get_done_threads() const;
+	void inc_done_threads();
+	void reset_done_threads();
+
+private:
+	unsigned m_done_threads;
+};
+
 class symbol;
 
 struct stack_entry {
@@ -425,6 +436,7 @@ public:
    dram_callback_t   m_last_dram_callback; 
    memory_space   *m_shared_mem;
    memory_space   *m_local_mem;
+   ptx_warp_info  *m_warp_info;
    ptx_cta_info   *m_cta_info;
    ptx_reg_t m_last_set_operand_value;