Review: 46001. Fixing a source of non-determinism in GPGPU-Sim (Bug 174). Not sure if this is the only issue, but this was definitely a source of non-determinism (checked with many of the <rodinia>-FT benchmarks). In gpgpu_sim_thread_concurrent() (gpgpusim_entrypoint.cc), the inner do{...}while(active); only breaks when the gpu is not active. As a result, the gpu is only initialized when the gpu becomes momentarily inactive. If one kernel completes while another kernel is currently pending in the stream's queue, the next kernel will start running immediately causing the GPU to not be reinitialized (which includes resetting per-kernel stats). The fix simply recognizes that a kernel has completed and breaks from the loop prior to starting the next operation.

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 16463]

3 files changed, 27 insertions, 18 deletions

diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 9999bb1..20fb88e 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -115,14 +115,23 @@ void *gpgpu_sim_thread_concurrent(void*)
         g_the_gpu->init();
         do {
             // check if a kernel has completed
-			// launch operation on device if one is pending and can be run
-			g_stream_manager->operation(&sim_cycles);
-		    if( g_the_gpu->active() ) {
-		        g_the_gpu->cycle();
-		        sim_cycles = true;
-		        g_the_gpu->deadlock_check();
-		    }
-		    active=g_the_gpu->active() || !g_stream_manager->empty_protected();
+            // launch operation on device if one is pending and can be run
+
+            // Need to break this loop when a kernel completes. This was a
+            // source of non-deterministic behaviour in GPGPU-Sim (bug 147).
+            // If another stream operation is available, g_the_gpu remains active,
+            // causing this loop to not break. If the next operation happens to be
+            // another kernel, the gpu is not re-initialized and the inter-kernel
+            // behaviour may be incorrect.
+            if(g_stream_manager->operation(&sim_cycles))
+                break;
+
+            if( g_the_gpu->active() ) {
+                g_the_gpu->cycle();
+                sim_cycles = true;
+                g_the_gpu->deadlock_check();
+            }
+            active=g_the_gpu->active() || !g_stream_manager->empty_protected();
         } while( active );
         if(g_debug_execution >= 3) {
            printf("GPGPU-Sim: ** STOP simulation thread (no work) **\n");
diff --git a/src/stream_manager.cc b/src/stream_manager.cc
index 3b8f57f..07cd44a 100644
--- a/src/stream_manager.cc
+++ b/src/stream_manager.cc
@@ -197,17 +197,17 @@ stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking )
     pthread_mutex_init(&m_lock,NULL);
 }
 
-void stream_manager::operation( bool * sim)
+bool stream_manager::operation( bool * sim)
 {
-	pthread_mutex_lock(&m_lock);
-	bool check=check_finished_kernel();
-	if(check) m_gpu->print_stats();
-	stream_operation op =front();
-	op.do_operation( m_gpu );
-	pthread_mutex_unlock(&m_lock);
-	//pthread_mutex_lock(&m_lock);
+    pthread_mutex_lock(&m_lock);
+    bool check=check_finished_kernel();
+    if(check)m_gpu->print_stats();
+    stream_operation op =front();
+    op.do_operation( m_gpu );
+    pthread_mutex_unlock(&m_lock);
+    //pthread_mutex_lock(&m_lock);
     // simulate a clock cycle on the GPU
-
+    return check;
 }
 
 bool stream_manager::check_finished_kernel()
diff --git a/src/stream_manager.h b/src/stream_manager.h
index a30e21b..7275402 100644
--- a/src/stream_manager.h
+++ b/src/stream_manager.h
@@ -245,7 +245,7 @@ public:
     bool empty();
     void print( FILE *fp);
     void push( stream_operation op );
-    void operation(bool * sim);
+    bool operation(bool * sim);
 private:
     void print_impl( FILE *fp);