- Checkpointing new support for concurrent kernel execution (CUDA only, not OpenCL)

  This changelist adds full support for streams supported by a new class,
  stream_manager and enables concurrent execution of kernels from different
  streams.

- fast_regression.sh fails for simpleMultiCopy, simpleStreams (other tests
  passing)

** Known issues **
- Kernel parameter passing is not done correctly for concurrent kernel execution
  (somehow concurrentKernels is not affected by this): the parameters are
  stored inside function_info, which is shared among parallel kernel launches
  so that the values passed into the launch are likely to get overwritten if
  multiple grids are launched in parallel streams.

- Statistics are printed out whenever the simulation thread runs out of
  cuda commands (doesn't make sense to print out when a kernel ends during
  concurrent kernel execution).  This will probably require further tweaking
  so as to be more compatible with data collection scripts.

[git-p4: depot-paths = "//depot/gpgpu_sim_research/fermi/distribution/": change = 8302]

1 files changed, 383 insertions, 0 deletions

diff --git a/src/stream_manager.cc b/src/stream_manager.cc
new file mode 100644
index 0000000..5d1cfe2
--- /dev/null
+++ b/src/stream_manager.cc
@@ -0,0 +1,383 @@
+/*
+ * stream_manager.cc
+ *
+ * Copyright © 2009 by Tor M. Aamodt, Wilson W. L. Fung, Ali Bakhoda, 
+ * George L. Yuan and the University of British Columbia, Vancouver, 
+ * BC V6T 1Z4, All Rights Reserved.
+ * 
+ * THIS IS A LEGAL DOCUMENT BY DOWNLOADING GPGPU-SIM, YOU ARE AGREEING TO THESE
+ * TERMS AND CONDITIONS.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * 
+ * NOTE: The files libcuda/cuda_runtime_api.c and src/cuda-sim/cuda-math.h
+ * are derived from the CUDA Toolset available from http://www.nvidia.com/cuda
+ * (property of NVIDIA).  The files benchmarks/BlackScholes/ and 
+ * benchmarks/template/ are derived from the CUDA SDK available from 
+ * http://www.nvidia.com/cuda (also property of NVIDIA).  The files from 
+ * src/intersim/ are derived from Booksim (a simulator provided with the 
+ * textbook "Principles and Practices of Interconnection Networks" available 
+ * from http://cva.stanford.edu/books/ppin/). As such, those files are bound by 
+ * the corresponding legal terms and conditions set forth separately (original 
+ * copyright notices are left in files from these sources and where we have 
+ * modified a file our copyright notice appears before the original copyright 
+ * notice).  
+ * 
+ * Using this version of GPGPU-Sim requires a complete installation of CUDA 
+ * which is distributed seperately by NVIDIA under separate terms and 
+ * conditions.  To use this version of GPGPU-Sim with OpenCL requires a
+ * recent version of NVIDIA's drivers which support OpenCL.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 
+ * 3. Neither the name of the University of British Columbia nor the names of
+ * its contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ * 
+ * 4. This version of GPGPU-SIM is distributed freely for non-commercial use only.  
+ *  
+ * 5. No nonprofit user may place any restrictions on the use of this software,
+ * including as modified by the user, by any other authorized user.
+ * 
+ * 6. GPGPU-SIM was developed primarily by Tor M. Aamodt, Wilson W. L. Fung, 
+ * Ali Bakhoda, George L. Yuan, at the University of British Columbia, 
+ * Vancouver, BC V6T 1Z4
+ */
+
+#include "stream_manager.h"
+#include "gpgpusim_entrypoint.h"
+#include "cuda-sim/cuda-sim.h"
+#include "gpgpu-sim/gpu-sim.h"
+
+unsigned CUstream_st::sm_next_stream_uid = 0;
+
+CUstream_st::CUstream_st() 
+{
+    m_pending = false;
+    m_uid = sm_next_stream_uid++;
+    pthread_mutex_init(&m_lock,NULL);
+}
+
+bool CUstream_st::empty()
+{
+    pthread_mutex_lock(&m_lock);
+    bool empty = m_operations.empty();
+    pthread_mutex_unlock(&m_lock);
+    return empty;
+}
+
+bool CUstream_st::busy()
+{
+    pthread_mutex_lock(&m_lock);
+    bool pending = m_pending;
+    pthread_mutex_unlock(&m_lock);
+    return pending;
+}
+
+void CUstream_st::synchronize() 
+{
+    // called by host thread
+    bool done=false;
+    do{
+        pthread_mutex_lock(&m_lock);
+        done = m_operations.empty();
+        pthread_mutex_unlock(&m_lock);
+    } while ( !done );
+}
+
+void CUstream_st::push( const stream_operation &op )
+{
+    // called by host thread
+    pthread_mutex_lock(&m_lock);
+    m_operations.push_back( op );
+    pthread_mutex_unlock(&m_lock);
+}
+
+void CUstream_st::record_next_done()
+{
+    // called by gpu thread
+    pthread_mutex_lock(&m_lock);
+    assert(m_pending);
+    m_operations.pop_front();
+    m_pending=false;
+    pthread_mutex_unlock(&m_lock);
+}
+
+
+stream_operation CUstream_st::next()
+{
+    // called by gpu thread
+    pthread_mutex_lock(&m_lock);
+    m_pending = true;
+    stream_operation result = m_operations.front();
+    pthread_mutex_unlock(&m_lock);
+    return result;
+}
+
+void CUstream_st::print(FILE *fp)
+{
+    pthread_mutex_lock(&m_lock);
+    fprintf(fp,"GPGPU-Sim API:    stream %u has %zu operations\n", m_uid, m_operations.size() );
+    std::list<stream_operation>::iterator i;
+    unsigned n=0;
+    for( i=m_operations.begin(); i!=m_operations.end(); i++ ) {
+        stream_operation &op = *i;
+        fprintf(fp,"GPGPU-Sim API:       %u : ", n++);
+        op.print(fp);
+        fprintf(fp,"\n");
+    }
+    pthread_mutex_unlock(&m_lock);
+}
+
+void stream_operation::do_operation( gpgpu_sim *gpu )
+{
+    if( is_noop() ) 
+        return;
+
+    assert(!m_done && m_stream);
+    printf("GPGPU-Sim API: stream %u performing ", m_stream->get_uid() );
+    switch( m_type ) {
+    case stream_memcpy_host_to_device:
+        printf("memcpy host-to-device\n");
+        gpu->memcpy_to_gpu(m_device_address_dst,m_host_address_src,m_cnt);
+        m_stream->record_next_done();
+        break;
+    case stream_memcpy_device_to_host: 
+        printf("memcpy device-to-host\n");
+        gpu->memcpy_from_gpu(m_host_address_dst,m_device_address_src,m_cnt);
+        m_stream->record_next_done();
+        break;
+    case stream_memcpy_device_to_device:
+        printf("memcpy device-to-device\n");
+        gpu->memcpy_gpu_to_gpu(m_device_address_dst,m_device_address_src,m_cnt); 
+        m_stream->record_next_done();
+        break;
+    case stream_memcpy_to_symbol:
+        printf("memcpy to symbol\n");
+        gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_src,m_cnt,m_offset,1,gpu);
+        m_stream->record_next_done();
+        break;
+    case stream_memcpy_from_symbol:
+        printf("memcpy from symbol\n");
+        gpgpu_ptx_sim_memcpy_symbol(m_symbol,m_host_address_dst,m_cnt,m_offset,0,gpu);
+        m_stream->record_next_done();
+        break;
+    case stream_kernel_launch:
+        if( gpu->can_start_kernel() ) {
+            printf("kernel \'%s\' transfer to GPU hardware scheduler\n", m_kernel.name().c_str() );
+            if( m_sim_mode )
+                gpgpu_cuda_ptx_sim_main_func( m_kernel );
+            else
+                gpu->launch( m_kernel );
+        }
+        break;
+    case stream_event: {
+        printf("event update\n");
+        time_t wallclock = time((time_t *)NULL);
+        m_event->update( gpu_tot_sim_cycle, wallclock );
+        m_stream->record_next_done();
+        } 
+        break;
+    default:
+        abort();
+    }
+    m_done=true;
+    fflush(stdout);
+}
+
+void stream_operation::print( FILE *fp ) const
+{
+    fprintf(fp," stream operation " );
+    switch( m_type ) {
+    case stream_event: fprintf(fp,"event"); break;
+    case stream_kernel_launch: fprintf(fp,"kernel"); break;
+    case stream_memcpy_device_to_device: fprintf(fp,"memcpy device-to-device"); break;
+    case stream_memcpy_device_to_host: fprintf(fp,"memcpy device-to-host"); break;
+    case stream_memcpy_host_to_device: fprintf(fp,"memcpy host-to-device"); break;
+    case stream_memcpy_to_symbol: fprintf(fp,"memcpy to symbol"); break;
+    case stream_memcpy_from_symbol: fprintf(fp,"memcpy from symbol"); break;
+    case stream_no_op: fprintf(fp,"no-op"); break;
+    }
+}
+
+stream_manager::stream_manager( gpgpu_sim *gpu, bool cuda_launch_blocking ) 
+{
+    m_gpu = gpu;
+    m_service_stream_zero = false;
+    m_cuda_launch_blocking = cuda_launch_blocking;
+    pthread_mutex_init(&m_lock,NULL);
+}
+
+void stream_manager::register_finished_kernel( unsigned grid_uid ) 
+{
+    // called by gpu simulation thread
+    pthread_mutex_lock(&m_lock);
+    CUstream_st *stream = m_grid_id_to_stream[grid_uid];
+    assert( grid_uid == stream->front().get_kernel().get_uid() );
+    stream->record_next_done();
+    m_grid_id_to_stream.erase(grid_uid);
+    pthread_mutex_unlock(&m_lock);
+}
+
+stream_operation stream_manager::front() 
+{
+    // called by gpu simulation thread
+    stream_operation result;
+    pthread_mutex_lock(&m_lock);
+    if( concurrent_streams_empty() )
+        m_service_stream_zero = true;
+    if( m_service_stream_zero ) {
+        if( !m_stream_zero.empty() ) {
+            if( !m_stream_zero.busy() ) {
+                result = m_stream_zero.next();
+                if( result.is_kernel() ) {
+                    unsigned grid_id = result.get_kernel().get_uid();
+                    m_grid_id_to_stream[grid_id] = &m_stream_zero;
+                }
+            }
+        } else {
+            m_service_stream_zero = false;
+        }
+    } else {
+        std::list<struct CUstream_st*>::iterator s;
+        for( s=m_streams.begin(); s != m_streams.end(); s++) {
+            CUstream_st *stream = *s;
+            if( !stream->busy() && !stream->empty() ) {
+                result = stream->next();
+//                stream_barrier *b = result.get_barrier();
+//                if( b && b->value() > 1 ) {
+//                    b->dec();
+//                    result = stream_operation();
+//                } else {
+//                    assert( b->value() == 1 );
+//                    delete b;
+                    if( result.is_kernel() ) {
+                        unsigned grid_id = result.get_kernel().get_uid();
+                        m_grid_id_to_stream[grid_id] = stream;
+                    }
+//                }
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&m_lock);
+    return result;
+}
+
+void stream_manager::add_stream( struct CUstream_st *stream )
+{
+    // called by host thread
+    pthread_mutex_lock(&m_lock);
+    m_streams.push_back(stream);
+    pthread_mutex_unlock(&m_lock);
+}
+
+void stream_manager::destroy_stream( CUstream_st *stream )
+{
+    // called by host thread
+    pthread_mutex_lock(&m_lock);
+    while( !stream->empty() )
+        ; 
+    std::list<CUstream_st *>::iterator s;
+    for( s=m_streams.begin(); s != m_streams.end(); s++ ) {
+        if( *s == stream ) {
+            m_streams.erase(s);
+            break;
+        }
+    }
+    delete stream; 
+    pthread_mutex_unlock(&m_lock);
+}
+
+bool stream_manager::concurrent_streams_empty()
+{
+    bool result = true;
+    // called by gpu simulation thread
+    std::list<struct CUstream_st *>::iterator s;
+    for( s=m_streams.begin(); s!=m_streams.end();++s ) {
+        struct CUstream_st *stream = *s;
+        if( !stream->empty() ) {
+            //stream->print(stdout);
+            result = false;
+        }
+    }
+    return result;
+}
+
+bool stream_manager::empty()
+{
+    bool result = true;
+    pthread_mutex_lock(&m_lock);
+    if( !concurrent_streams_empty() ) 
+        result = false;
+    if( !m_stream_zero.empty() ) 
+        result = false;
+    pthread_mutex_unlock(&m_lock);
+    return result;
+}
+
+void stream_manager::print( FILE *fp)
+{
+    pthread_mutex_lock(&m_lock);
+    print_impl(fp);
+    pthread_mutex_unlock(&m_lock);
+}
+
+void stream_manager::print_impl( FILE *fp)
+{
+    fprintf(fp,"GPGPU-Sim API: Stream Manager State\n");
+    std::list<struct CUstream_st *>::iterator s;
+    for( s=m_streams.begin(); s!=m_streams.end();++s ) {
+        struct CUstream_st *stream = *s;
+        if( !stream->empty() ) 
+            stream->print(fp);
+    }
+    if( !m_stream_zero.empty() ) 
+        m_stream_zero.print(fp);
+}
+
+void stream_manager::push( stream_operation op )
+{
+    struct CUstream_st *stream = op.get_stream();
+
+    // block if stream 0 (or concurrency disabled) and pending concurrent operations exist
+    bool block= !stream || m_cuda_launch_blocking;
+    while(block) {
+        pthread_mutex_lock(&m_lock);
+        block = !concurrent_streams_empty();
+        pthread_mutex_unlock(&m_lock);
+    };
+
+    pthread_mutex_lock(&m_lock);
+    if( stream && !m_cuda_launch_blocking ) {
+        stream->push(op);
+    } else {
+        op.set_stream(&m_stream_zero);
+        m_stream_zero.push(op);
+    }
+    print_impl(stdout);
+    pthread_mutex_unlock(&m_lock);
+    if( m_cuda_launch_blocking || stream == NULL ) {
+        while( !empty() )
+            ; 
+    }
+}
+