Merge pull request #1 from tgrogers/copy-kernel

Copy kernel

7 files changed, 99 insertions, 13 deletions

diff --git a/src/gpgpu-sim/gpu-cache.cc b/src/gpgpu-sim/gpu-cache.cc
index 32c2bb1..e1e41a5 100644
--- a/src/gpgpu-sim/gpu-cache.cc
+++ b/src/gpgpu-sim/gpu-cache.cc
@@ -190,11 +190,17 @@ void tag_array::init( int core_id, int type_id )
     m_type_id = type_id;
 }
 
+
 enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_fetch* mf) const {
+    mem_access_sector_mask_t mask = mf->get_access_sector_mask();
+    return probe(addr, idx, mask);
+}
+
+
+enum cache_request_status tag_array::probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask) const {
     //assert( m_config.m_write_policy == READ_ONLY );
     unsigned set_index = m_config.set_index(addr);
     new_addr_type tag = m_config.tag(addr);
-    mem_access_sector_mask_t mask = mf->get_access_sector_mask();
 
     unsigned invalid_line = (unsigned)-1;
     unsigned valid_line = (unsigned)-1;
@@ -310,18 +316,23 @@ enum cache_request_status tag_array::access( new_addr_type addr, unsigned time,
 
 void tag_array::fill( new_addr_type addr, unsigned time, mem_fetch* mf)
 {
-    assert( m_config.m_alloc_policy == ON_FILL );
+    fill(addr, time, mf->get_access_sector_mask());
+}
+
+void tag_array::fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask )
+{
+    //assert( m_config.m_alloc_policy == ON_FILL );
     unsigned idx;
-    enum cache_request_status status = probe(addr,idx,mf);
-    assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request
+    enum cache_request_status status = probe(addr,idx,mask);
+    //assert(status==MISS||status==SECTOR_MISS); // MSHR should have prevented redundant memory request
     if(status==MISS)
-    	m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mf->get_access_sector_mask() );
+    	m_lines[idx]->allocate( m_config.tag(addr), m_config.block_addr(addr), time, mask );
     else if (status==SECTOR_MISS) {
     	assert(m_config.m_cache_type == SECTOR);
-    	((sector_cache_block*)m_lines[idx])->allocate_sector( time, mf->get_access_sector_mask() );
+    	((sector_cache_block*)m_lines[idx])->allocate_sector( time, mask );
     }
 
-    m_lines[idx]->fill(time, mf->get_access_sector_mask());
+    m_lines[idx]->fill(time, mask);
 }
 
 void tag_array::fill( unsigned index, unsigned time, mem_fetch* mf)
diff --git a/src/gpgpu-sim/gpu-cache.h b/src/gpgpu-sim/gpu-cache.h
index 3e1691a..6a84a3a 100644
--- a/src/gpgpu-sim/gpu-cache.h
+++ b/src/gpgpu-sim/gpu-cache.h
@@ -153,8 +153,8 @@ struct line_cache_block: public cache_block_t  {
 	    }
 		void fill( unsigned time, mem_access_sector_mask_t sector_mask )
 	    {
-	    	if(!m_ignore_on_fill_status)
-	    		assert( m_status == RESERVED );
+	    	//if(!m_ignore_on_fill_status)
+	    	//	assert( m_status == RESERVED );
 
 	    	m_status = m_set_modified_on_fill? MODIFIED : VALID;
 
@@ -292,8 +292,8 @@ struct sector_cache_block : public cache_block_t {
     {
     	unsigned sidx = get_sector_index(sector_mask);
 
-    	if(!m_ignore_on_fill_status[sidx])
-    	         assert( m_status[sidx] == RESERVED );
+    //	if(!m_ignore_on_fill_status[sidx])
+    //	         assert( m_status[sidx] == RESERVED );
 
     	m_status[sidx] = m_set_modified_on_fill[sidx]? MODIFIED : VALID;
 
@@ -690,11 +690,13 @@ public:
     ~tag_array();
 
     enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_fetch* mf ) const;
+    enum cache_request_status probe( new_addr_type addr, unsigned &idx, mem_access_sector_mask_t mask ) const;
     enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, mem_fetch* mf );
     enum cache_request_status access( new_addr_type addr, unsigned time, unsigned &idx, bool &wb, evicted_block_info &evicted, mem_fetch* mf );
 
     void fill( new_addr_type addr, unsigned time, mem_fetch* mf );
     void fill( unsigned idx, unsigned time, mem_fetch* mf );
+    void fill( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask );
 
     unsigned size() const { return m_config.get_num_lines();}
     cache_block_t* get_block(unsigned idx) { return m_lines[idx];}
@@ -969,6 +971,15 @@ public:
     bool data_port_free() const { return m_bandwidth_management.data_port_free(); } 
     bool fill_port_free() const { return m_bandwidth_management.fill_port_free(); } 
 
+    // This is a gapping hole we are poking in the system to quickly handle
+    // filling the cache on cudamemcopies. We don't care about anything other than
+    // L2 state after the memcopy - so just force the tag array to act as though
+    // something is read or written without doing anything else.
+    void force_tag_access( new_addr_type addr, unsigned time, mem_access_sector_mask_t mask )
+    {
+        m_tag_array->fill( addr, time, mask );
+    }
+
 protected:
     // Constructor that can be used by derived classes with custom tag arrays
     baseline_cache( const char *name,
diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc
index 11ac5df..17f1714 100644
--- a/src/gpgpu-sim/gpu-sim.cc
+++ b/src/gpgpu-sim/gpu-sim.cc
@@ -140,6 +140,8 @@ void power_config::reg_options(class OptionParser * opp)
 
 void memory_config::reg_options(class OptionParser * opp)
 {
+    option_parser_register(opp, "-perf_sim_memcpy", OPT_BOOL, &m_perf_sim_memcpy, 
+                                "Fill the L2 cache on memcpy", "1");
     option_parser_register(opp, "-gpgpu_dram_scheduler", OPT_INT32, &scheduler_type, 
                                 "0 = fifo, 1 = FR-FCFS (defaul)", "1");
     option_parser_register(opp, "-gpgpu_dram_partition_queues", OPT_CSTR, &gpgpu_L2_queue_config, 
@@ -1595,6 +1597,24 @@ void shader_core_ctx::dump_warp_state( FILE *fout ) const
        m_warp[w].print(fout);
 }
 
+
+void gpgpu_sim::perf_memcpy_to_gpu( size_t dst_start_addr, size_t count )
+{
+    if (m_memory_config->m_perf_sim_memcpy) {
+       assert (dst_start_addr % 32 == 0);
+
+       for ( unsigned counter = 0; counter < count; counter += 32 ) {
+           const unsigned wr_addr = dst_start_addr + counter;
+           addrdec_t raw_addr;
+           mem_access_sector_mask_t mask;
+           mask.set(wr_addr % 128 / 32);
+           m_memory_config->m_address_mapping.addrdec_tlx( wr_addr, &raw_addr );
+           const unsigned partition_id = raw_addr.sub_partition / m_memory_config->m_n_sub_partition_per_memory_channel;
+           m_memory_partition_unit[ partition_id ]->handle_memcpy_to_gpu( wr_addr, raw_addr.sub_partition, mask );
+       }
+    }
+}
+
 void gpgpu_sim::dump_pipeline( int mask, int s, int m ) const
 {
 /*
diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h
index c04648c..1778008 100644
--- a/src/gpgpu-sim/gpu-sim.h
+++ b/src/gpgpu-sim/gpu-sim.h
@@ -290,6 +290,7 @@ struct memory_config {
    unsigned gpgpu_frfcfs_dram_write_queue_size;
    unsigned write_high_watermark;
    unsigned write_low_watermark;
+   bool m_perf_sim_memcpy;
 };
 
 // global counters and flags (please try not to add to this list!!!)
@@ -425,6 +426,8 @@ public:
    void gpu_print_stat();
    void dump_pipeline( int mask, int s, int m ) const;
 
+    void perf_memcpy_to_gpu( size_t dst_start_addr, size_t count );
+
    //The next three functions added to be used by the functional simulation function
    
    //! Get shader core configuration
diff --git a/src/gpgpu-sim/l2cache.cc b/src/gpgpu-sim/l2cache.cc
index 8fbf448..b1465a8 100644
--- a/src/gpgpu-sim/l2cache.cc
+++ b/src/gpgpu-sim/l2cache.cc
@@ -74,6 +74,15 @@ memory_partition_unit::memory_partition_unit( unsigned partition_id,
     }
 }
 
+void memory_partition_unit::handle_memcpy_to_gpu( size_t addr, unsigned global_subpart_id, mem_access_sector_mask_t mask )
+{
+    unsigned p = global_sub_partition_id_to_local_id(global_subpart_id);
+    std::string mystring =
+        mask.to_string<char,std::string::traits_type,std::string::allocator_type>();
+    MEMPART_DPRINTF("Copy Engine Request Received For Address=%llx, local_subpart=%u, global_subpart=%u, sector_mask=%s \n", addr, p, global_subpart_id, mystring.c_str()); 
+    m_sub_partition[p]->force_l2_tag_update(addr,gpu_sim_cycle+gpu_tot_sim_cycle, mask);
+}
+
 memory_partition_unit::~memory_partition_unit() 
 {
     delete m_dram; 
@@ -306,6 +315,7 @@ memory_sub_partition::memory_sub_partition( unsigned sub_partition_id,
     m_id = sub_partition_id;
     m_config=config;
     m_stats=stats;
+    m_memcpy_cycle_offset = 0;
 
     assert(m_id < m_config->m_n_mem_sub_partition); 
 
@@ -369,7 +379,7 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
         if ( !m_config->m_L2_config.disabled() && m_L2cache->waiting_for_fill(mf) ) {
             if (m_L2cache->fill_port_free()) {
                 mf->set_status(IN_PARTITION_L2_FILL_QUEUE,gpu_sim_cycle+gpu_tot_sim_cycle);
-                m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle);
+                m_L2cache->fill(mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset);
                 m_dram_L2_queue->pop();
             }
         } else if ( !m_L2_icnt_queue->full() ) {
@@ -395,9 +405,10 @@ void memory_sub_partition::cache_cycle( unsigned cycle )
             bool port_free = m_L2cache->data_port_free(); 
             if ( !output_full && port_free ) {
                 std::list<cache_event> events;
-                enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle,events);
+                enum cache_request_status status = m_L2cache->access(mf->get_addr(),mf,gpu_sim_cycle+gpu_tot_sim_cycle+m_memcpy_cycle_offset,events);
                 bool write_sent = was_write_sent(events);
                 bool read_sent = was_read_sent(events);
+                MEM_SUBPART_DPRINTF("Probing L2 cache Address=%llx, status=%u\n", mf->get_addr(), status); 
 
                 if ( status == HIT ) {
                     if( !write_sent ) {
diff --git a/src/gpgpu-sim/l2cache.h b/src/gpgpu-sim/l2cache.h
index 2cc0e76..2d13918 100644
--- a/src/gpgpu-sim/l2cache.h
+++ b/src/gpgpu-sim/l2cache.h
@@ -72,6 +72,7 @@ public:
    void print_stat( FILE *fp ) { m_dram->print_stat(fp); }
    void visualize() const { m_dram->visualize(); }
    void print( FILE *fp ) const;
+   void handle_memcpy_to_gpu( size_t dst_start_addr, unsigned subpart_id, mem_access_sector_mask_t mask );
 
    class memory_sub_partition * get_sub_partition(int sub_partition_id) 
    {
@@ -178,6 +179,12 @@ public:
    void accumulate_L2cache_stats(class cache_stats &l2_stats) const;
    void get_L2cache_sub_stats(struct cache_sub_stats &css) const;
 
+   void force_l2_tag_update(new_addr_type addr, unsigned time, mem_access_sector_mask_t mask)
+   {
+        m_L2cache->force_tag_access( addr, m_memcpy_cycle_offset + time, mask );
+        m_memcpy_cycle_offset += 1;
+   }
+
 private:
 // data
    unsigned m_id;  //< the global sub partition ID
@@ -210,6 +217,13 @@ private:
    friend class L2interface;
 
    std::vector<mem_fetch*> breakdown_request_to_sector_requests(mem_fetch* mf);
+
+   // This is a cycle offset that has to be applied to the l2 accesses to account for
+   // the cudamemcpy read/writes. We want GPGPU-Sim to only count cycles for kernel execution
+   // but we want cudamemcpy to go through the L2. Everytime an access is made from cudamemcpy
+   // this counter is incremented, and when the l2 is accessed (in both cudamemcpyies and otherwise)
+   // this value is added to the gpgpu-sim cycle counters.
+   unsigned m_memcpy_cycle_offset;
 };
 
 class L2interface : public mem_fetch_interface {
diff --git a/src/gpgpu-sim/l2cache_trace.h b/src/gpgpu-sim/l2cache_trace.h
index 3dac87d..2235cdc 100644
--- a/src/gpgpu-sim/l2cache_trace.h
+++ b/src/gpgpu-sim/l2cache_trace.h
@@ -34,6 +34,9 @@
 #define MEMPART_PRINT_STR SIM_PRINT_STR " %d - "
 #define MEMPART_DTRACE(x)  ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)get_mpid()) )
 
+#define MEM_SUBPART_PRINT_STR SIM_PRINT_STR " %d - "
+#define MEM_SUBPART_DTRACE(x)  ( DTRACE(x) && (Trace::sampling_memory_partition == -1 || Trace::sampling_memory_partition == (int)m_id) )
+
 // Intended to be called from inside components of a memory partition
 // Depends on a get_mpid() function
 #define MEMPART_DPRINTF(...) do {\
@@ -46,10 +49,23 @@
     }\
 } while (0)
 
+#define MEM_SUBPART_DPRINTF(...) do {\
+    if (MEM_SUBPART_DTRACE(MEMORY_PARTITION_UNIT)) {\
+        printf( MEM_SUBPART_PRINT_STR,\
+                gpu_sim_cycle + gpu_tot_sim_cycle,\
+                Trace::trace_streams_str[Trace::MEMORY_SUBPARTITION_UNIT],\
+                m_id );\
+        printf(__VA_ARGS__);\
+    }\
+} while (0)
+
 #else
 
 #define MEMPART_DTRACE(x)  (false)
 #define MEMPART_DPRINTF(x, ...) do {} while (0)
 
+#define MEM_SUBPART_DTRACE(x)  (false)
+#define MEM_SUBPART_DPRINTF(x, ...) do {} while (0)
+
 #endif