WIP adding support for PTX JIT and dumping params to cudaLaunches

6 files changed, 252 insertions, 9 deletions

diff --git a/libcuda/cuda_runtime_api.cc b/libcuda/cuda_runtime_api.cc
index 08ee413..300fa28 100644
--- a/libcuda/cuda_runtime_api.cc
+++ b/libcuda/cuda_runtime_api.cc
@@ -126,6 +126,9 @@
 #include "host_defines.h"
 #include "builtin_types.h"
 #include "driver_types.h"
+#if (CUDART_VERSION >= 8000)
+#include "cuda.h"
+#endif
 #if (CUDART_VERSION < 8000)
 #include "__cudaFatFormat.h"
 #endif
@@ -287,6 +290,34 @@ struct CUctx_st {
 		}
 	}
 
+    void register_hostFun_function( const char*hostFun, function_info* f){
+        m_kernel_lookup[hostFun] = f;
+    }
+
+	dim3 get_blockdim(const char *hostFun)
+	{
+		std::map<const char*,dim3>::iterator i=m_hostFun_blockdim.find(hostFun);
+		assert( i != m_hostFun_blockdim.end() );
+        return i->second;
+	}
+
+	dim3 get_griddim(const char *hostFun)
+	{
+		std::map<const char*,dim3>::iterator i=m_hostFun_griddim.find(hostFun);
+		assert( i != m_hostFun_griddim.end() );
+        return i->second;
+	}
+
+	void set_blockdim(const char *hostFun, dim3 dims)
+	{
+		m_hostFun_blockdim[hostFun] = dims;
+	}
+
+	void set_griddim(const char *hostFun, dim3 dims)
+	{
+		m_hostFun_griddim[hostFun] = dims;
+	}
+
 	function_info *get_kernel(const char *hostFun)
 	{
 		std::map<const void*,function_info*>::iterator i=m_kernel_lookup.find(hostFun);
@@ -298,6 +329,8 @@ private:
 	_cuda_device_id *m_gpu; // selected gpu
 	std::map<unsigned,symbol_table*> m_code; // fat binary handle => global symbol table
 	unsigned m_last_fat_cubin_handle;
+    std::map<const char*, dim3> m_hostFun_blockdim;
+    std::map<const char*, dim3> m_hostFun_griddim;
 	std::map<const void*,function_info*> m_kernel_lookup; // unique id (CUDA app function address) => kernel entry point
 	struct gpgpu_ptx_sim_info m_binary_info;
 
@@ -1300,16 +1333,15 @@ int CUDARTAPI __cudaSynchronizeThreads(void**, void*)
  *                                                                              *
  *******************************************************************************/
 
-#if (CUDART_VERSION >= 3010)
+#if (CUDART_VERSION >= 3010 && CUDART_VERSION < 8000)
 
 typedef struct CUuuid_st {                                /**< CUDA definition of UUID */
     char bytes[16];
 } CUuuid;
 
-/**
- * CUDA UUID types
- */
-// typedef __device_builtin__ struct CUuuid_st cudaUUID_t;
+#endif
+
+#if (CUDART_VERSION >= 3010)
 
 __host__ cudaError_t CUDARTAPI cudaGetExportTable(const void **ppExportTable, const cudaUUID_t *pExportTableId)
 {
@@ -1958,10 +1990,10 @@ void cuobjdumpParseBinary(unsigned int handle){
           symtab = gpgpu_ptx_sim_load_ptx_from_filename( ptx_filename.c_str() );
       }
    }
-	name_symtab[fname] = symtab;
-	context->add_binary(symtab, handle);
-	load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
-	load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
+   name_symtab[fname] = symtab;
+   context->add_binary(symtab, handle);
+   load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
+   load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
    return;
 #endif
 
@@ -2602,5 +2634,123 @@ kernel_info_t *gpgpu_cuda_ptx_sim_init_grid( const char *hostFun,
 	g_ptx_kernel_count++;
 	fflush(stdout);
 
+	if(g_debug_execution >= 3){
+        entry->debug_param();
+    }
+
 	return result;
 }
+
+CUresult CUDAAPI cuLinkCreate(unsigned int numOptions, CUjit_option *options, void **optionValues, CUlinkState *stateOut)
+{
+    //currently do not support options or multiple CUlinkStates
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuLinkAddData(CUlinkState state, CUjitInputType type, void *data, size_t size, const char *name,
+    unsigned int numOptions, CUjit_option *options, void **optionValues)
+{
+    assert(type==CU_JIT_INPUT_PTX);
+	cuda_not_implemented(__my_func__,__LINE__);
+	return CUDA_ERROR_UNKNOWN;
+}
+
+CUresult CUDAAPI cuLinkAddFile(CUlinkState state, CUjitInputType type, const char *path,
+    unsigned int numOptions, CUjit_option *options, void **optionValues)
+{
+    static bool addedFile = false;
+    if (addedFile){
+        printf("GPGPU-Sim PTX: ERROR: cuLinkAddFile does not support multiple file");
+        abort();
+    }
+
+    //blocking
+    assert(type==CU_JIT_INPUT_PTX);
+	CUctx_st *context = GPGPUSim_Context();
+    char *file = getenv("PTX_JIT_PATH");
+    if(file==NULL){
+        printf("GPGPU-Sim PTX: ERROR: PTX_JIT_PATH has not been set");
+        abort();
+    }
+    strcat(file,path);
+	symbol_table *symtab = gpgpu_ptx_sim_load_ptx_from_filename( file );
+    std::string fname(path);
+    name_symtab[fname] = symtab;
+    context->add_binary(symtab, 1);
+    load_static_globals(symtab,STATIC_ALLOC_LIMIT,0xFFFFFFFF,context->get_device()->get_gpgpu());
+    load_constants(symtab,STATIC_ALLOC_LIMIT,context->get_device()->get_gpgpu());
+    addedFile = true;
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuLinkComplete(CUlinkState state, void **cubinOut, size_t *sizeOut)
+{
+    //all cuLink* function are implemented to block until completion so nothing to do here
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuLinkDestroy(CUlinkState state)
+{
+    //currently do not support options or multiple CUlinkStates
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuModuleLoadData(CUmodule *module, const void *image)
+{
+    //Currently do not support multiple modules
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuModuleGetFunction(CUfunction *hfunc, CUmodule hmod, const char *name)
+{
+	CUctx_st* context = GPGPUSim_Context();
+    std::string key(name);
+    //only support one file
+    assert(name_symtab.size()==1);
+    symbol_table* symtab = name_symtab.begin()->second;
+    function_info* f = symtab->lookup_function( std::string(name) );
+    //just need to add given pointer to map for cudaLaunch
+    context->register_hostFun_function( (const char*) hfunc, f);
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuModuleUnload(CUmodule hmod)
+{
+    //Currently do not support multiple modules
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z)
+{
+	CUctx_st* context = GPGPUSim_Context();
+    dim3 dims(x,y,z);
+    context->set_blockdim((const char *)hfunc, dims);
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuParamSetSize(CUfunction hfunc, unsigned int numbytes)
+{
+    //Nothing to do
+	return CUDA_SUCCESS;
+}
+
+CUresult CUDAAPI cuParamSetv(CUfunction hfunc, int offset, void *ptr, unsigned int numbytes)
+{
+	cuda_not_implemented(__my_func__,__LINE__);
+	return CUDA_ERROR_UNKNOWN;
+}
+
+CUresult CUDAAPI cuLaunchGrid(CUfunction f, int grid_width, int grid_height)
+{
+	cuda_not_implemented(__my_func__,__LINE__);
+	return CUDA_ERROR_UNKNOWN;
+
+	CUctx_st* context = GPGPUSim_Context();
+    const char *hostFun = (const char*) f;
+    dim3 dims(grid_width,grid_height,1);
+    context->set_griddim((const char *)f, dims);
+    cudaConfigureCall(context->get_griddim(hostFun), context->get_blockdim(hostFun), 0, NULL);
+
+    cudaLaunch(hostFun);
+	return CUDA_SUCCESS;
+}
diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc
index 34368ce..6875edd 100644
--- a/src/cuda-sim/cuda-sim.cc
+++ b/src/cuda-sim/cuda-sim.cc
@@ -1226,6 +1226,87 @@ void function_info::list_param( FILE *fout ) const
    fflush(fout);
 }
 
+void function_info::debug_param( ) const
+{
+   char filename[] = "params.txt";
+   char buff[1024];
+   snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename);
+   system(buff);
+   FILE *fp = fopen(filename, "r");
+   fgets(buff, 1024, fp);
+   fclose(fp);
+
+   std::string fn(buff);
+   size_t pos1, pos2;
+   pos1 = fn.find("(");
+   pos2 = fn.find(")");
+   assert(pos2>pos1&&pos1>0);
+   strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str());
+   printf("params: %s\n", buff);
+   char *tok;
+   std::vector<std::string> params;
+   tok = strtok(buff, ",");
+   while(tok!=NULL){
+       std::string param(tok);
+       param.erase(0, param.find_first_not_of(" "));
+       param.erase(param.find_last_not_of(" ")+1);
+       params.push_back(param);
+       tok = strtok(NULL, ",");
+   }
+   for (auto const& it : params){
+      std::cout<<it<<std::endl;
+   }
+
+   FILE *fout  = fopen (filename, "w");
+   fprintf(fout, "Name of function:%s\n", fn.c_str());
+
+   for( std::map<unsigned,param_info>::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) {
+      const param_info &p = i->second;
+      std::string name = p.get_name();
+      param_t param_value = p.get_value();
+      if(params[i->first].find("const")!=std::string::npos){
+         fprintf(fout, "Input: ");
+      } else {
+         fprintf(fout, "Input/output: ");
+      }
+
+      symbol *param = m_symtab->lookup(name.c_str());
+      addr_t param_addr = param->get_address();
+      fprintf(fout, "%s: %#08x, ", name.c_str(), param_addr);
+
+      if(params[i->first].find("int")!=std::string::npos){
+         size_t len = param_value.size/sizeof(int);
+         int val[len];
+         memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size);
+         fprintf(fout, "val (int) = ");
+         for (unsigned i = 0; i<len; i++){
+             fprintf(fout, "%d ", val[i]);
+         }
+         fprintf(fout, "\n");
+      } else if(params[i->first].find("float")!=std::string::npos){
+         size_t len = param_value.size/sizeof(float);
+         float val[len];
+         memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size);
+         fprintf(fout, "val (float) = ");
+         for (unsigned i = 0; i<len; i++){
+             fprintf(fout, "%f ", val[i]);
+         }
+         fprintf(fout, "\n");
+      }else{
+         size_t len = param_value.size/sizeof(char);
+         char val[len];
+         memcpy((void*) val, param_value.pdata+param_value.offset, param_value.size);
+         fprintf(fout, "val (char) = ");
+         for (unsigned i = 0; i<len; i++){
+             fprintf(fout, "%c ", val[i]);
+         }
+         fprintf(fout, "\n");
+      }
+   }
+   fflush(fout);
+   fclose(fout);
+}
+
 template<int activate_level> 
 bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc)
 {
diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h
index 958daba..9049a84 100644
--- a/src/cuda-sim/cuda-sim.h
+++ b/src/cuda-sim/cuda-sim.h
@@ -32,6 +32,7 @@
 #include"../gpgpu-sim/shader.h"
 #include <stdlib.h>
 #include <map>
+#include <vector>
 #include <string>
 #include"ptx_sim.h"
 
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc
index c77e4da..034a7b9 100644
--- a/src/cuda-sim/instructions.cc
+++ b/src/cuda-sim/instructions.cc
@@ -2753,6 +2753,7 @@ void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread )
    const operand_info &dst  = pI->dst();
    const operand_info &src1 = pI->src1();
    unsigned i_type = pI->get_type();
+   assert( src1.is_param_local() == 0 );
 
    if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) {
       // pack or unpack operation
diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc
index 016c600..482b9e0 100644
--- a/src/cuda-sim/ptx_ir.cc
+++ b/src/cuda-sim/ptx_ir.cc
@@ -257,6 +257,14 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio
    return prior_decl;
 }
 
+function_info *symbol_table::lookup_function( std::string name )
+{
+   std::string key = std::string(name);
+   std::map<std::string,function_info*>::iterator it = m_function_info_lookup.find(key);
+   assert ( it != m_function_info_lookup.end() );
+   return it->second;
+}
+
 type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec )
 {
    if( space_spec == param_space_unclassified ) 
diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h
index 5b68fcf..341f9b7 100644
--- a/src/cuda-sim/ptx_ir.h
+++ b/src/cuda-sim/ptx_ir.h
@@ -313,6 +313,7 @@ public:
    symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line );
    void add_function( function_info *func, const char *filename, unsigned linenumber );
    bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table );
+   function_info *lookup_function(std::string name);
    type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec );
    type_info *add_type( function_info *func );
    type_info *get_array_type( type_info *base_type, unsigned array_dim ); 
@@ -1256,6 +1257,7 @@ public:
    void finalize( memory_space *param_mem );
    void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); 
    void list_param( FILE *fout ) const;
+   void debug_param() const;
 
    const struct gpgpu_ptx_sim_info* get_kernel_info () const
    {