Add SST integration into gpgpusim (#44)

* Add accommodations to run gpgpusim with SST simulation framework through balar

* Output setup_environment options when sourcing

* Add SST directive check when creating sim thread

* Add sst side test for jenkins

* sst-integration: update Jenkinsfile with offical sst-elements repo and fix bugs in pipeline script

* sst-integration: direct jenkins to rebuild gpgpusim before testing for sst

* sst-integration: fix bugs in sst repos config

* sst-integration: let Jenkins rebuilds simulator

Since the simulator needs to be configured with both normal mode and sst mode, need to rebuild make target to clean prior runs.

* sst-integration: Update Jenkinsfile to source env vars when running balar test

* sst-integration: refactor code to remove __SST__ flag

* sst-integration: fix a bug that init cluster twice for sst

* sst-integration: fix a bug of not sending mem packets to SST

* sst-integration: remove sst flags from makefiles and setup_env

* sst-integration: add comments to SST changes

* sst-integration: remove rebuilding simulator in jenkins when testing for SST

* sst-integration: revert simulator build script

* Add a function to support querying function argument info for SST

* sst-integration: add version detection for vanadis binary

* Automated Format

* add version detection support for gcc 10+

* sst-integration: add cudaMallocHost for SST

* sst-integration: fix a compilation bug

* sst-integration: add sst balar unittest CI

* sst-integration: specify GPU_ARCH for CI test

* sst-integration: use bash for github actions

* sst-integration: use https links for sst repos

* sst-integration: add SST dependencies to CI config

* sst-integration: remove sudo

* sst-integration: default to yes for apt install

* sst-integration: add manual trigger for github action

* sst-integration: remove wrong on event

* sst-integration: limit CPU usage for compilation

* sst-integration: fix wrong path

* sst-integration: use personal repo for testing

* sst-integration: remove sst-core source in CI to free space

* sst-integration: SST_Cycle use print stats with stream id

* Automated Format

* sst-integration: check for diskspace and try to clean it

* sst-integration: move out of docker image

* sst-integration: testing for ci path

* sst-integration: fix syntax

* sst-integration: pass env vars

* sst-integration: set env properly

* sst-integration: merge LLVM build and test into same job

* sst-integration: fix step order

* sst-integration: checkout correct branch for env-setup

* sst-integration: remove resourcing gpu apps

* sst-integration: revert back to docker github action

* sst-integration: enable debug trace for sst testing

* sst-integration: resourcing gpu app for env vars

* sst-integration: use GPUAPPS_ROOT for path for gpu app

* sst-integration: use GPUAPPS_ROOT for path for gpu app

* sst-integration: enable parallel ci tests and fix not returning with cudaMallocHostSST

* sst-integration: using debug flag for CI run

* sst-integration: revert debug ci run

* sst-integration: CI skips cuda sdk download and launch multiple jobs

* sst-integration: reenable parallel tests

* sst-integration: reduce concurrent test thread count

* sst-integration: skip long test for github runner

* sst-integration: try running CI with single core

* sst-integrtion: add callback to SST to check thread sync is done in SST_Cycle()

* sst-integration: ignore lookup if already found and add callbacks to SST

* Automated Format

* sst-integration: add support for indirect texture access

* Automated Format

* sste-integration: fix up for PR

* Automated Format

---------

Co-authored-by: purdue-jenkins <[email protected]>

1 files changed, 129 insertions, 9 deletions

diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc
index 42c6981..839fef6 100644
--- a/src/gpgpusim_entrypoint.cc
+++ b/src/gpgpusim_entrypoint.cc
@@ -43,6 +43,20 @@
 static int sg_argc = 3;
 static const char *sg_argv[] = {"", "-config", "gpgpusim.config"};
 
+// Help funcs to avoid multiple '->' for SST
+GPGPUsim_ctx *GPGPUsim_ctx_ptr() { return GPGPU_Context()->the_gpgpusim; }
+
+class sst_gpgpu_sim *g_the_gpu() {
+  return static_cast<sst_gpgpu_sim *>(GPGPUsim_ctx_ptr()->g_the_gpu);
+}
+
+class stream_manager *g_stream_manager() {
+  return GPGPUsim_ctx_ptr()->g_stream_manager;
+}
+
+// SST callback
+extern void SST_callback_cudaThreadSynchronize_done();
+
 void *gpgpu_sim_thread_sequential(void *ctx_ptr) {
   gpgpu_context *ctx = (gpgpu_context *)ctx_ptr;
   // at most one kernel running at a time
@@ -169,6 +183,75 @@ void *gpgpu_sim_thread_concurrent(void *ctx_ptr) {
   return NULL;
 }
 
+bool sst_sim_cycles = false;
+
+bool SST_Cycle() {
+  // Check if Synchronize is done when SST previously requested
+  // cudaThreadSynchronize
+  if (GPGPU_Context()->requested_synchronize &&
+      ((g_stream_manager()->empty() && !GPGPUsim_ctx_ptr()->g_sim_active) ||
+       GPGPUsim_ctx_ptr()->g_sim_done)) {
+    SST_callback_cudaThreadSynchronize_done();
+    GPGPU_Context()->requested_synchronize = false;
+  }
+
+  if (g_stream_manager()->empty_protected() &&
+      !GPGPUsim_ctx_ptr()->g_sim_done && !g_the_gpu()->active()) {
+    GPGPUsim_ctx_ptr()->g_sim_active = false;
+    // printf("stream is empty %d \n",  g_stream_manager->empty());
+    return false;
+  }
+
+  if (g_stream_manager()->operation(&sst_sim_cycles) &&
+      !g_the_gpu()->active()) {
+    if (sst_sim_cycles) {
+      sst_sim_cycles = false;
+    }
+    return false;
+  }
+
+  // printf("GPGPU-Sim: Give GPU Cycle\n");
+  GPGPUsim_ctx_ptr()->g_sim_active = true;
+
+  // functional simulation
+  if (g_the_gpu()->is_functional_sim()) {
+    kernel_info_t *kernel = g_the_gpu()->get_functional_kernel();
+    assert(kernel);
+    GPGPUsim_ctx_ptr()->gpgpu_ctx->func_sim->gpgpu_cuda_ptx_sim_main_func(
+        *kernel);
+    g_the_gpu()->finish_functional_sim(kernel);
+  }
+
+  // performance simulation
+  if (g_the_gpu()->active()) {
+    g_the_gpu()->SST_cycle();
+    sst_sim_cycles = true;
+    g_the_gpu()->deadlock_check();
+  } else {
+    if (g_the_gpu()->cycle_insn_cta_max_hit()) {
+      g_stream_manager()->stop_all_running_kernels();
+      GPGPUsim_ctx_ptr()->g_sim_done = true;
+      GPGPUsim_ctx_ptr()->g_sim_active = false;
+      GPGPUsim_ctx_ptr()->break_limit = true;
+    }
+  }
+
+  if (!g_the_gpu()->active()) {
+    g_the_gpu()->print_stats(GPGPUsim_ctx_ptr()->g_the_gpu->last_streamID);
+    g_the_gpu()->update_stats();
+    GPGPU_Context()->print_simulation_time();
+  }
+
+  if (GPGPUsim_ctx_ptr()->break_limit) {
+    printf(
+        "GPGPU-Sim: ** break due to reaching the maximum cycles (or "
+        "instructions) **\n");
+    return true;
+  }
+
+  return false;
+}
+
 void gpgpu_context::synchronize() {
   printf("GPGPU-Sim: synchronize waiting for inactive GPU simulation\n");
   the_gpgpusim->g_stream_manager->print(stdout);
@@ -187,6 +270,27 @@ void gpgpu_context::synchronize() {
   //    sem_post(&g_sim_signal_start);
 }
 
+bool gpgpu_context::synchronize_check() {
+  // printf("GPGPU-Sim: synchronize checking for inactive GPU simulation\n");
+  requested_synchronize = true;
+  the_gpgpusim->g_stream_manager->print(stdout);
+  fflush(stdout);
+  //    sem_wait(&g_sim_signal_finish);
+  bool done = false;
+  pthread_mutex_lock(&(the_gpgpusim->g_sim_lock));
+  done = (the_gpgpusim->g_stream_manager->empty() &&
+          !the_gpgpusim->g_sim_active) ||
+         the_gpgpusim->g_sim_done;
+  pthread_mutex_unlock(&(the_gpgpusim->g_sim_lock));
+  if (done) {
+    printf(
+        "GPGPU-Sim: synchronize checking: detected inactive GPU simulation "
+        "thread\n");
+  }
+  fflush(stdout);
+  return done;
+}
+
 void gpgpu_context::exit_simulation() {
   the_gpgpusim->g_sim_done = true;
   printf("GPGPU-Sim: exit_simulation called\n");
@@ -220,8 +324,14 @@ gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() {
   assert(setlocale(LC_NUMERIC, "C"));
   the_gpgpusim->g_the_gpu_config->init();
 
-  the_gpgpusim->g_the_gpu =
-      new exec_gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this);
+  if (the_gpgpusim->g_the_gpu_config->is_SST_mode()) {
+    // Create SST specific GPGPUSim
+    the_gpgpusim->g_the_gpu =
+        new sst_gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this);
+  } else {
+    the_gpgpusim->g_the_gpu =
+        new exec_gpgpu_sim(*(the_gpgpusim->g_the_gpu_config), this);
+  }
   the_gpgpusim->g_stream_manager = new stream_manager(
       (the_gpgpusim->g_the_gpu), func_sim->g_cuda_launch_blocking);
 
@@ -237,12 +347,17 @@ gpgpu_sim *gpgpu_context::gpgpu_ptx_sim_init_perf() {
 void gpgpu_context::start_sim_thread(int api) {
   if (the_gpgpusim->g_sim_done) {
     the_gpgpusim->g_sim_done = false;
-    if (api == 1) {
-      pthread_create(&(the_gpgpusim->g_simulation_thread), NULL,
-                     gpgpu_sim_thread_concurrent, (void *)this);
+    if (the_gpgpusim->g_the_gpu_config->is_SST_mode()) {
+      // Do not create concurrent thread in SST mode
+      g_the_gpu()->init();
     } else {
-      pthread_create(&(the_gpgpusim->g_simulation_thread), NULL,
-                     gpgpu_sim_thread_sequential, (void *)this);
+      if (api == 1) {
+        pthread_create(&(the_gpgpusim->g_simulation_thread), NULL,
+                       gpgpu_sim_thread_concurrent, (void *)this);
+      } else {
+        pthread_create(&(the_gpgpusim->g_simulation_thread), NULL,
+                       gpgpu_sim_thread_sequential, (void *)this);
+      }
     }
   }
 }
@@ -266,8 +381,13 @@ void gpgpu_context::print_simulation_time() {
   const unsigned cycles_per_sec =
       (unsigned)(the_gpgpusim->g_the_gpu->gpu_tot_sim_cycle / difference);
   printf("gpgpu_simulation_rate = %u (cycle/sec)\n", cycles_per_sec);
-  printf("gpgpu_silicon_slowdown = %ux\n",
-         the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec);
+
+  if (cycles_per_sec == 0) {
+    printf("gpgpu_silicon_slowdown = Nan\n");
+  } else {
+    printf("gpgpu_silicon_slowdown = %ux\n",
+           the_gpgpusim->g_the_gpu->shader_clock() * 1000 / cycles_per_sec);
+  }
   fflush(stdout);
 }