BUG: kernels should return to stream if not pushed to concurrent kernel pool

4 files changed, 41 insertions, 10 deletions

diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc
index be87f6a..c53ea04 100644
--- a/src/cuda-sim/cuda_device_runtime.cc
+++ b/src/cuda-sim/cuda_device_runtime.cc
@@ -122,7 +122,7 @@ void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info *
         addr_t from_addr = actual_param_op.get_symbol()->get_address();
 
         if(arg == 0) {//paramter buffer for child kernel (in global memory)
-            //get parameter_buffer from the cudaDeviceLaunchV2_param0
+            //get parameter_buffer from the cudaLaunchDeviceV2_param0
 			assert(size == sizeof(void *));
             thread->m_local_mem->read(from_addr, size, &parameter_buffer);
             assert((size_t)parameter_buffer >= GLOBAL_HEAP_START);
@@ -152,11 +152,13 @@ void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info *
             kernel_info_t & parent_kernel = thread->get_kernel();
             if(child_stream == 0) { //default stream on device for current CTA
                 child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); 
-                DEV_RUNTIME_REPORT("launching child kernel to default stream of the cta " << child_stream->get_uid() << ": " << child_stream);
+                DEV_RUNTIME_REPORT("launching child kernel " << child_grid->get_uid() << 
+                    " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream);
             }
             else {
                assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); 
-                DEV_RUNTIME_REPORT("launching child kernel to stream " << child_stream->get_uid() << ": " << child_stream);
+                DEV_RUNTIME_REPORT("launching child kernel " << child_grid->get_uid() << 
+                " to stream " << child_stream->get_uid() << ": " << child_stream);
             }
         }
         
diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc
index ff2fac7..de5bcf6 100644
--- a/src/gpgpu-sim/shader.cc
+++ b/src/gpgpu-sim/shader.cc
@@ -1934,7 +1934,8 @@ void shader_core_ctx::register_cta_thread_exit( unsigned cta_num )
                  m_kernel->name().c_str() );
           if( !m_gpu->kernel_more_cta_left(m_kernel) ) {
               if( !m_kernel->running() ) {
-                  printf("GPGPU-Sim uArch: GPU detected kernel \'%s\' finished on shader %u.\n", m_kernel->name().c_str(), m_sid );
+                  printf("GPGPU-Sim uArch: GPU detected kernel %u \'%s\' finished on shader %u.\n", m_kernel->get_uid(),
+                    m_kernel->name().c_str(), m_sid );
                   m_gpu->set_kernel_done( m_kernel );
               }
           }
diff --git a/src/stream_manager.cc b/src/stream_manager.cc
index 687d544..546ab3b 100644
--- a/src/stream_manager.cc
+++ b/src/stream_manager.cc
@@ -95,6 +95,15 @@ stream_operation CUstream_st::next()
     return result;
 }
 
+void CUstream_st::cancel_front()
+{
+    pthread_mutex_lock(&m_lock);
+    assert(m_pending);
+    m_pending = false;
+    pthread_mutex_unlock(&m_lock);
+
+}
+
 void CUstream_st::print(FILE *fp)
 {
     pthread_mutex_lock(&m_lock);
@@ -111,10 +120,10 @@ void CUstream_st::print(FILE *fp)
 }
 
 
-void stream_operation::do_operation( gpgpu_sim *gpu )
+bool stream_operation::do_operation( gpgpu_sim *gpu )
 {
     if( is_noop() ) 
-        return;
+        return true;
 
     assert(!m_done && m_stream);
     if(g_debug_execution >= 3)
@@ -153,16 +162,23 @@ void stream_operation::do_operation( gpgpu_sim *gpu )
     case stream_kernel_launch:
         if( gpu->can_start_kernel() ) {
         	gpu->set_cache_config(m_kernel->name());
-        	printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() );
+            if(g_debug_execution >= 3)
+        	    printf("kernel %d: \'%s\' transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() );
             m_kernel->print_parent_info();
             if( m_sim_mode )
                 gpu->functional_launch( m_kernel );
             else
                 gpu->launch( m_kernel );
         }
+        else {
+            if(g_debug_execution >= 3)
+        	    printf("kernel %d: \'%s\' not ready to transfer to GPU hardware scheduler\n", m_kernel->get_uid(), m_kernel->name().c_str() );
+            return false;    
+        }
         break;
     case stream_event: {
-        printf("event update\n");
+        if(g_debug_execution >= 3)
+            printf("event update\n");
         time_t wallclock = time((time_t *)NULL);
         m_event->update( gpu_tot_sim_cycle, wallclock );
         m_stream->record_next_done();
@@ -173,6 +189,7 @@ void stream_operation::do_operation( gpgpu_sim *gpu )
     }
     m_done=true;
     fflush(stdout);
+    return true;
 }
 
 void stream_operation::print( FILE *fp ) const
@@ -204,7 +221,16 @@ bool stream_manager::operation( bool * sim)
     bool check=check_finished_kernel();
     if(check)m_gpu->print_stats();
     stream_operation op =front();
-    op.do_operation( m_gpu );
+    if(!op.do_operation( m_gpu )) //not ready to execute
+    {
+        //cancel operation
+        if( op.is_kernel() ) {
+            unsigned grid_uid = op.get_kernel()->get_uid();
+            m_grid_id_to_stream.erase(grid_uid);
+        }
+        op.get_stream()->cancel_front();
+
+    }
     pthread_mutex_unlock(&m_lock);
     //pthread_mutex_lock(&m_lock);
     // simulate a clock cycle on the GPU
@@ -228,6 +254,7 @@ bool stream_manager::register_finished_kernel(unsigned grid_uid)
 
         //Jin: should check children kernels for CDP
         if(kernel->is_finished()) {
+            printf("kernel %d finishes, retires from stream %d\n", grid_uid, stream->get_uid());
             stream->record_next_done();
             m_grid_id_to_stream.erase(grid_uid);
             kernel->notify_parent_finished();
diff --git a/src/stream_manager.h b/src/stream_manager.h
index 701b33c..222a1b2 100644
--- a/src/stream_manager.h
+++ b/src/stream_manager.h
@@ -150,7 +150,7 @@ public:
     bool is_noop() const { return m_type == stream_no_op; }
     bool is_done() const { return m_done; }
     kernel_info_t *get_kernel() { return m_kernel; }
-    void do_operation( gpgpu_sim *gpu );
+    bool do_operation( gpgpu_sim *gpu );
     void print( FILE *fp ) const;
     struct CUstream_st *get_stream() { return m_stream; }
     void set_stream( CUstream_st *stream ) { m_stream = stream; }
@@ -218,6 +218,7 @@ public:
     void push( const stream_operation &op );
     void record_next_done();
     stream_operation next();
+    void cancel_front(); //front operation fails, cancle the pending status
     stream_operation &front() { return m_operations.front(); }
     void print( FILE *fp );
     unsigned get_uid() const { return m_uid; }