diff options
Diffstat (limited to 'src')
32 files changed, 9539 insertions, 497 deletions
diff --git a/src/abstract_hardware_model.cc b/src/abstract_hardware_model.cc index 39ed81c..307df40 100644 --- a/src/abstract_hardware_model.cc +++ b/src/abstract_hardware_model.cc @@ -35,10 +35,70 @@ #include "gpgpu-sim/gpu-sim.h" #include "option_parser.h" #include <algorithm> +#include <sys/stat.h> +#include <sstream> +#include <iostream> unsigned mem_access_t::sm_next_access_uid = 0; unsigned warp_inst_t::sm_next_uid = 0; +checkpoint::checkpoint() +{ + + struct stat st = {0}; + + if (stat("checkpoint_files", &st) == -1) { + mkdir("checkpoint_files", 0777); + } + +} +void checkpoint::load_global_mem(class memory_space *temp_mem, char * f1name) +{ + + FILE * fp2 = fopen(f1name, "r"); + assert(fp2!=NULL); + char line [ 128 ]; /* or other suitable maximum line size */ + unsigned int offset ; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + unsigned int index; + char * pch; + pch = strtok (line," "); + if (pch[0]=='g' || pch[0]=='s' || pch[0]=='l') + { + + pch = strtok (NULL, " "); + + std::stringstream ss; + ss << std::hex << pch; + ss >> index; + + offset=0; + } + else { + unsigned int data; + std::stringstream ss; + ss << std::hex << pch; + ss >> data; + temp_mem->write_only(offset,index, 4,&data); + offset= offset+4; + } + //fputs ( line, stdout ); /* write the line */ + } + fclose ( fp2 ); +} + + + +void checkpoint::store_global_mem(class memory_space * mem, char *fname, char * format) +{ + + FILE * fp3 = fopen(fname, "w"); + assert(fp3!=NULL); + mem->print(format,fp3); + fclose(fp3); +} + void move_warp( warp_inst_t *&dst, warp_inst_t *&src ) { assert( dst->empty() ); @@ -64,6 +124,31 @@ void gpgpu_functional_sim_config::reg_options(class OptionParser * opp) &m_experimental_lib_support, "Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]", "0"); + option_parser_register(opp, "-checkpoint_option", OPT_INT32, &checkpoint_option, + " checkpointing flag (0 = no checkpoint)", + "0"); + option_parser_register(opp, "-checkpoint_kernel", OPT_INT32, &checkpoint_kernel, + " checkpointing during execution of which kernel (1- 1st kernel)", + "1"); + option_parser_register(opp, "-checkpoint_CTA", OPT_INT32, &checkpoint_CTA, + " checkpointing after # of CTA (< less than total CTA)", + "0"); + option_parser_register(opp, "-resume_option", OPT_INT32, &resume_option, + " resume flag (0 = no resume)", + "0"); + option_parser_register(opp, "-resume_kernel", OPT_INT32, &resume_kernel, + " Resume from which kernel (1= 1st kernel)", + "0"); + option_parser_register(opp, "-resume_CTA", OPT_INT32, &resume_CTA, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_CTA_t", OPT_INT32, &checkpoint_CTA_t, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-checkpoint_insn_Y", OPT_INT32, &checkpoint_insn_Y, + " resume from which CTA ", + "0"); + option_parser_register(opp, "-gpgpu_ptx_convert_to_ptxplus", OPT_BOOL, &m_ptx_convert_to_ptxplus, "Convert SASS (native ISA) to ptxplus and run ptxplus", @@ -93,10 +178,20 @@ gpgpu_t::gpgpu_t( const gpgpu_functional_sim_config &config ) : m_function_model_config(config) { m_global_mem = new memory_space_impl<8192>("global",64*1024); + m_tex_mem = new memory_space_impl<8192>("tex",64*1024); m_surf_mem = new memory_space_impl<8192>("surf",64*1024); m_dev_malloc=GLOBAL_HEAP_START; + checkpoint_option = m_function_model_config.get_checkpoint_option(); + checkpoint_kernel = m_function_model_config.get_checkpoint_kernel(); + checkpoint_CTA = m_function_model_config.get_checkpoint_CTA(); + resume_option = m_function_model_config.get_resume_option(); + resume_kernel = m_function_model_config.get_resume_kernel(); + resume_CTA = m_function_model_config.get_resume_CTA(); + checkpoint_CTA_t = m_function_model_config.get_checkpoint_CTA_t(); + checkpoint_insn_Y = m_function_model_config.get_checkpoint_insn_Y(); + if(m_function_model_config.get_ptx_inst_debug_to_file() != 0) ptx_inst_debug_file = fopen(m_function_model_config.get_ptx_inst_debug_file(), "w"); @@ -184,7 +279,7 @@ void warp_inst_t::generate_mem_accesses() { if( empty() || op == MEMORY_BARRIER_OP || m_mem_accesses_created ) return; - if ( !((op == LOAD_OP) || (op == STORE_OP)) ) + if (!((op == LOAD_OP) || (op==TENSOR_CORE_LOAD_OP) || (op == STORE_OP)||(op==TENSOR_CORE_STORE_OP))) return; if( m_warp_active_mask.count() == 0 ) return; // predicated off @@ -213,6 +308,7 @@ void warp_inst_t::generate_mem_accesses() access_type = is_write? LOCAL_ACC_W: LOCAL_ACC_R; break; case shared_space: break; + case sstarr_space: break; default: assert(0); break; } @@ -220,7 +316,8 @@ void warp_inst_t::generate_mem_accesses() new_addr_type cache_block_size = 0; // in bytes switch( space.get_type() ) { - case shared_space: { + case shared_space: + case sstarr_space: { unsigned subwarp_size = m_config->warp_size / m_config->mem_warp_parts; unsigned total_accesses=0; for( unsigned subwarp=0; subwarp < m_config->mem_warp_parts; subwarp++ ) { @@ -314,7 +411,7 @@ void warp_inst_t::generate_mem_accesses() break; case global_space: case local_space: case param_space_local: - if( m_config->gpgpu_coalesce_arch >= 13 && m_config->gpgpu_coalesce_arch <= 62) { + if( m_config->gpgpu_coalesce_arch >= 13) { if(isatomic()) memory_coalescing_arch_atomic(is_write, access_type); else @@ -369,7 +466,7 @@ void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_ } else if(m_config->gpgpu_coalesce_arch >= 40) { - //Maxwell and Pascal, L1 and L2 are sectors + //Maxwell, Pascal and Volta, L1 and L2 are sectors //all requests should be 32 bytes sector_segment_size = true; } @@ -404,7 +501,8 @@ void warp_inst_t::memory_coalescing_arch( bool is_write, mem_access_type access_ assert(num_accesses <= MAX_ACCESSES_PER_INSN_PER_THREAD); - for(unsigned access=0; access<num_accesses; access++) { +// for(unsigned access=0; access<num_accesses; access++) { + for(unsigned access=0; (access<MAX_ACCESSES_PER_INSN_PER_THREAD)&&(m_per_scalar_thread[thread].memreqaddr[access]!=0); access++) { new_addr_type addr = m_per_scalar_thread[thread].memreqaddr[access]; unsigned block_address = line_size_based_tag_func(addr,segment_size); unsigned chunk = (addr&127)/32; // which 32-byte chunk within in a 128-byte chunk does this thread access? @@ -440,11 +538,28 @@ void warp_inst_t::memory_coalescing_arch_atomic( bool is_write, mem_access_type // see the CUDA manual where it discusses coalescing rules before reading this unsigned segment_size = 0; - unsigned warp_parts = 2; + unsigned warp_parts = m_config->mem_warp_parts; + bool sector_segment_size = false; + + if(m_config->gpgpu_coalesce_arch >= 20 && m_config->gpgpu_coalesce_arch < 39) + { + //Fermi and Kepler, L1 is normal and L2 is sector + if(m_config->gmem_skip_L1D || cache_op == CACHE_GLOBAL) + sector_segment_size = true; + else + sector_segment_size = false; + } + else if(m_config->gpgpu_coalesce_arch >= 40) + { + //Maxwell, Pascal and Volta, L1 and L2 are sectors + //all requests should be 32 bytes + sector_segment_size = true; + } + switch( data_size ) { case 1: segment_size = 32; break; - case 2: segment_size = 64; break; - case 4: case 8: case 16: segment_size = 128; break; + case 2: segment_size = sector_segment_size? 32 : 64; break; + case 4: case 8: case 16: segment_size = sector_segment_size? 32 : 128; break; } unsigned subwarp_size = m_config->warp_size / warp_parts; @@ -730,6 +845,51 @@ void simt_stack::launch( address_type start_pc, const simt_mask_t &active_mask ) m_stack.push_back(new_stack_entry); } +void simt_stack::resume( char * fname ) +{ + reset(); + + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + + char line [ 200 ]; /* or other suitable maximum line size */ + + while ( fgets ( line, sizeof line, fp2 ) != NULL ) /* read a line */ + { + simt_stack_entry new_stack_entry; + char * pch; + pch = strtok (line," "); + for (unsigned j=0; j<m_warp_size; j++) + { + if (pch[0]=='1') + new_stack_entry.m_active_mask.set(j); + else + new_stack_entry.m_active_mask.reset(j); + pch = strtok (NULL," "); + + } + + new_stack_entry.m_pc=atoi(pch); + pch = strtok (NULL," "); + new_stack_entry.m_calldepth=atoi(pch); + pch = strtok (NULL," "); + new_stack_entry.m_recvg_pc=atoi(pch); + pch = strtok (NULL," "); + new_stack_entry.m_branch_div_cycle=atoi(pch); + pch = strtok (NULL," "); + if(pch[0]=='0') + new_stack_entry.m_type= STACK_ENTRY_TYPE_NORMAL; + else + new_stack_entry.m_type= STACK_ENTRY_TYPE_CALL; + m_stack.push_back(new_stack_entry); + } + fclose ( fp2 ); + + +} + const simt_mask_t &simt_stack::get_active_mask() const { assert(m_stack.size() > 0); @@ -774,6 +934,20 @@ void simt_stack::print (FILE *fout) const ptx_print_insn( stack_entry.m_pc, fout ); fprintf(fout,"\n"); } + +} + +void simt_stack::print_checkpoint (FILE *fout) const +{ + for ( unsigned k=0; k < m_stack.size(); k++ ) { + simt_stack_entry stack_entry = m_stack[k]; + + for (unsigned j=0; j<m_warp_size; j++) + fprintf(fout, "%c ", (stack_entry.m_active_mask.test(j)?'1':'0') ); + fprintf(fout, "%d %d %d %lld %d ", stack_entry.m_pc,stack_entry.m_calldepth,stack_entry.m_recvg_pc,stack_entry.m_branch_div_cycle,stack_entry.m_type ); + fprintf(fout, "%d %d\n",m_warp_id, m_warp_size ); + + } } void simt_stack::update( simt_mask_t &thread_done, addr_vector_t &next_pc, address_type recvg_pc, op_type next_inst_op,unsigned next_inst_size, address_type next_inst_pc ) diff --git a/src/abstract_hardware_model.h b/src/abstract_hardware_model.h index 8333247..08aa88c 100644 --- a/src/abstract_hardware_model.h +++ b/src/abstract_hardware_model.h @@ -36,11 +36,16 @@ class kernel_info_t; #define MAX_CTA_PER_SHADER 32 #define MAX_BARRIERS_PER_CTA 16 +//After expanding the vector input and output operands +#define MAX_INPUT_VALUES 24 +#define MAX_OUTPUT_VALUES 8 + enum _memory_space_t { undefined_space=0, reg_space, local_space, shared_space, + sstarr_space, param_space_unclassified, param_space_kernel, /* global to all threads in a kernel : read-only */ param_space_local, /* local to a thread : read-writable */ @@ -65,6 +70,7 @@ enum FuncCache #include <string.h> #include <stdio.h> +#include <set> typedef unsigned long long new_addr_type; typedef unsigned address_type; @@ -76,9 +82,14 @@ enum uarch_op_t { NO_OP=-1, ALU_OP=1, SFU_OP, + TENSOR_CORE_OP, DP_OP, + SP_OP, + INTP_OP, ALU_SFU_OP, LOAD_OP, + TENSOR_CORE_LOAD_OP, + TENSOR_CORE_STORE_OP, STORE_OP, BRANCH_OP, BARRIER_OP, @@ -133,7 +144,9 @@ enum operation_pipeline_t { UNKOWN_OP, SP__OP, DP__OP, + INTP__OP, SFU__OP, + TENSOR_CORE__OP, MEM__OP }; typedef enum operation_pipeline_t operation_pipeline; @@ -184,6 +197,9 @@ void increment_x_then_y_then_z( dim3 &i, const dim3 &bound); class stream_manager; struct CUstream_st; extern stream_manager * g_stream_manager; +//support for pinned memories added +extern std::map<void *,void **> pinned_memory; +extern std::map<void *, size_t> pinned_memory_size; class kernel_info_t { public: @@ -233,6 +249,10 @@ public: m_next_tid.z=0; } dim3 get_next_cta_id() const { return m_next_cta; } + unsigned get_next_cta_id_single() const + { + return m_next_cta.x + m_grid_dim.x*m_next_cta.y + m_grid_dim.x*m_grid_dim.y*m_next_cta.z; + } bool no_more_ctas_to_run() const { return (m_next_cta.x >= m_grid_dim.x || m_next_cta.y >= m_grid_dim.y || m_next_cta.z >= m_grid_dim.z ); @@ -336,6 +356,7 @@ struct core_config { unsigned gpgpu_shmem_sizeDefault; unsigned gpgpu_shmem_sizePrefL1; unsigned gpgpu_shmem_sizePrefShared; + unsigned mem_unit_ports; // texture and constant cache line sizes (used to determine number of memory accesses) unsigned gpgpu_cache_texl1_linesize; @@ -363,7 +384,9 @@ public: const simt_mask_t &get_active_mask() const; void get_pdom_stack_top_info( unsigned *pc, unsigned *rpc ) const; unsigned get_rp() const; - void print(FILE*fp) const; + void print(FILE *fp) const; + void resume(char * fname) ; + void print_checkpoint (FILE *fout) const; protected: unsigned m_warp_id; @@ -483,14 +506,28 @@ public: const char* get_ptx_inst_debug_file() const { return g_ptx_inst_debug_file; } int get_ptx_inst_debug_thread_uid() const { return g_ptx_inst_debug_thread_uid; } unsigned get_texcache_linesize() const { return m_texcache_linesize; } - + int get_checkpoint_option() const {return checkpoint_option; } + int get_checkpoint_kernel() const {return checkpoint_kernel; } + int get_checkpoint_CTA() const {return checkpoint_CTA; } + int get_resume_option() const {return resume_option; } + int get_resume_kernel() const {return resume_kernel; } + int get_resume_CTA() const {return resume_CTA; } + int get_checkpoint_CTA_t() const {return checkpoint_CTA_t; } + int get_checkpoint_insn_Y() const {return checkpoint_insn_Y; } private: // PTX options int m_ptx_convert_to_ptxplus; int m_ptx_use_cuobjdump; int m_experimental_lib_support; unsigned m_ptx_force_max_capability; - + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; int g_ptx_inst_debug_to_file; char* g_ptx_inst_debug_file; int g_ptx_inst_debug_thread_uid; @@ -502,6 +539,14 @@ private: class gpgpu_t { public: gpgpu_t( const gpgpu_functional_sim_config &config ); + int checkpoint_option; + int checkpoint_kernel; + int checkpoint_CTA; + int resume_option; + int resume_kernel; + int resume_CTA; + int checkpoint_CTA_t; + int checkpoint_insn_Y; void* gpu_malloc( size_t size ); void* gpu_mallocarray( size_t count ); void gpu_memset( size_t dst_start_addr, int c, size_t count ); @@ -515,38 +560,34 @@ public: void gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array); void gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext); + void gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref); const char* gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref); - const struct textureReference* get_texref(const std::string &texname) const + const struct textureReference* get_texref( const std::string &texname ) const { - std::map<std::string, const struct textureReference*>::const_iterator t=m_NameToTextureRef.find(texname); - if( t == m_NameToTextureRef.end() ) { - // search for :: prefixed names - std::string temp("::" + texname); - t=m_NameToTextureRef.find(temp); - } - - assert(t != m_NameToTextureRef.end()); - - return t->second; + std::map<std::string, std::set<const struct textureReference*> >::const_iterator t=m_NameToTextureRef.find(texname); + assert( t != m_NameToTextureRef.end() ); + return *(t->second.begin()); } - const struct cudaArray* get_texarray( const struct textureReference *texref ) const + + const struct cudaArray* get_texarray( const std::string &texname ) const { - std::map<const struct textureReference*,const struct cudaArray*>::const_iterator t=m_TextureRefToCudaArray.find(texref); - assert(t != m_TextureRefToCudaArray.end()); + std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); + assert(t != m_NameToCudaArray.end()); return t->second; } - const struct textureInfo* get_texinfo( const struct textureReference *texref ) const + + const struct textureInfo* get_texinfo( const std::string &texname ) const { - std::map<const struct textureReference*, const struct textureInfo*>::const_iterator t=m_TextureRefToTexureInfo.find(texref); - assert(t != m_TextureRefToTexureInfo.end()); + std::map<std::string, const struct textureInfo*>::const_iterator t=m_NameToTexureInfo.find(texname); + assert(t != m_NameToTexureInfo.end()); return t->second; } - const struct textureReferenceAttr* get_texattr( const struct textureReference *texref ) const + const struct textureReferenceAttr* get_texattr( const std::string &texname ) const { - std::map<const struct textureReference*, const struct textureReferenceAttr*>::const_iterator t=m_TextureRefToAttribute.find(texref); - assert(t != m_TextureRefToAttribute.end()); + std::map<std::string, const struct textureReferenceAttr*>::const_iterator t=m_NameToAttribute.find(texname); + assert(t != m_NameToAttribute.end()); return t->second; } @@ -563,10 +604,11 @@ protected: unsigned long long m_dev_malloc; - std::map<std::string, const struct textureReference*> m_NameToTextureRef; - std::map<const struct textureReference*,const struct cudaArray*> m_TextureRefToCudaArray; - std::map<const struct textureReference*, const struct textureInfo*> m_TextureRefToTexureInfo; - std::map<const struct textureReference*, const struct textureReferenceAttr*> m_TextureRefToAttribute; + std::map<std::string, std::set<const struct textureReference*> > m_NameToTextureRef; + std::map<const struct textureReference*, std::string> m_TextureRefToName; + std::map<std::string, const struct cudaArray*> m_NameToCudaArray; + std::map<std::string, const struct textureInfo*> m_NameToTexureInfo; + std::map<std::string, const struct textureReferenceAttr*> m_NameToAttribute; }; struct gpgpu_ptx_sim_info @@ -616,6 +658,7 @@ public: return false; } enum _memory_space_t get_type() const { return m_type; } + void set_type( enum _memory_space_t t ) { m_type = t; } unsigned get_bank() const { return m_bank; } void set_bank( unsigned b ) { m_bank = b; } bool is_const() const { return (m_type == const_space) || (m_type == param_space_kernel); } @@ -769,7 +812,7 @@ public: }; // the maximum number of destination, source, or address uarch operands in a instruction -#define MAX_REG_OPERANDS 8 +#define MAX_REG_OPERANDS 32 struct dram_callback_t { dram_callback_t() { function=NULL; instruction=NULL; thread=NULL; } @@ -816,8 +859,8 @@ public: { fprintf(fp," [inst @ pc=0x%04x] ", pc ); } - bool is_load() const { return (op == LOAD_OP || memory_op == memory_load); } - bool is_store() const { return (op == STORE_OP || memory_op == memory_store); } + bool is_load() const { return (op == LOAD_OP ||op==TENSOR_CORE_LOAD_OP || memory_op == memory_load); } + bool is_store() const { return (op == STORE_OP ||op==TENSOR_CORE_STORE_OP || memory_op == memory_store); } unsigned get_num_operands() const {return num_operands;} unsigned get_num_regs() const {return num_regs;} void set_num_regs(unsigned num) {num_regs=num;} @@ -844,8 +887,10 @@ public: address_type reconvergence_pc; // -1 => not a branch, -2 => use function return address - unsigned out[4]; - unsigned in[4]; + unsigned out[8]; + unsigned outcount; + unsigned in[24]; + unsigned incount; unsigned char is_vectorin; unsigned char is_vectorout; int pred; // predicate register number @@ -856,7 +901,7 @@ public: int src[MAX_REG_OPERANDS]; } arch_reg; //int arch_reg[MAX_REG_OPERANDS]; // register number for bank conflict evaluation - unsigned latency; // operation latency + unsigned latency; // operation latency unsigned initiation_interval; unsigned data_size; // what is the size of the word being operated on? @@ -908,7 +953,7 @@ public: { m_empty=true; } - void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id ) + void issue( const active_mask_t &mask, unsigned warp_id, unsigned long long cycle, int dynamic_warp_id, int sch_id ) { m_warp_active_mask = mask; m_warp_issued_mask = mask; @@ -919,6 +964,7 @@ public: cycles = initiation_interval; m_cache_hit=false; m_empty=false; + m_scheduler_id=sch_id; } const active_mask_t & get_active_mask() const { @@ -944,7 +990,18 @@ public: for(unsigned i=0; i<num_addrs; i++) m_per_scalar_thread[n].memreqaddr[i] = addr[i]; } - + void print_m_accessq(){ + + if(accessq_empty()) + return; + else{ + printf("Printing mem access generated\n"); + std::list<mem_access_t>::iterator it; + for (it = m_accessq.begin(); it != m_accessq.end(); ++it){ + printf("MEM_TXN_GEN:%s:%x, Size:%d \n",mem_access_type_str(it->get_type()), it->get_addr(),it->get_size()); + } + } + } struct transaction_info { std::bitset<4> chunks; // bitmask: 32-byte chunks accessed mem_access_byte_mask_t bytes; @@ -999,6 +1056,10 @@ public: assert( !m_empty ); return m_warp_id; } + unsigned warp_id_func() const // to be used in functional simulations only + { + return m_warp_id; + } unsigned dynamic_warp_id() const { assert( !m_empty ); @@ -1037,6 +1098,7 @@ public: void print( FILE *fout ) const; unsigned get_uid() const { return m_uid; } + unsigned get_schd_id() const { return m_scheduler_id; } protected: @@ -1069,6 +1131,8 @@ protected: static unsigned sm_next_uid; + unsigned m_scheduler_id; //the scheduler that issues this inst + //Jin: cdp support public: int m_is_cdp; @@ -1078,7 +1142,21 @@ public: void move_warp( warp_inst_t *&dst, warp_inst_t *&src ); size_t get_kernel_code_size( class function_info *entry ); +class checkpoint +{ +public: + + checkpoint(); + ~checkpoint(){ + printf("clasfsfss destructed\n"); + } + void load_global_mem(class memory_space *temp_mem, char * f1name); + void store_global_mem(class memory_space *mem, char * fname , char * format); + unsigned radnom; + + +}; /* * This abstract class used as a base for functional and performance and simulation, it has basic functional simulation * data structures and procedures. @@ -1161,6 +1239,14 @@ public: } return false; } + bool has_free(bool sub_core_model, unsigned reg_id){ + //in subcore model, each sched has a one specific reg to use (based on sched id) + if(!sub_core_model) + return has_free(); + + assert(reg_id < regs.size()); + return regs[reg_id]->empty(); + } bool has_ready(){ for( unsigned i = 0; i < regs.size(); i++ ) { if( not regs[i]->empty() ) { @@ -1216,6 +1302,23 @@ public: return NULL; } + warp_inst_t ** get_free(bool sub_core_model, unsigned reg_id){ + //in subcore model, each sched has a one specific reg to use (based on sched id) + if(!sub_core_model) + return get_free(); + + assert(reg_id < regs.size()); + if( regs[reg_id]->empty() ) { + return ®s[reg_id]; + } + assert(0 && "No free register found"); + return NULL; + } + + unsigned get_size(){ + return regs.size(); + } + private: std::vector<warp_inst_t*> regs; const char* m_name; diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index afac330..a5db337 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -64,6 +64,7 @@ * the above Disclaimer and U.S. Government End Users Notice. */ + #ifndef CUDA_MATH #define CUDA_MATH @@ -360,7 +361,7 @@ int __signbitd(double d) #ifdef __APPLE__ int isnanf(float a) { - return (isnan(a)); + return (std::isnan(a)); } #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 99e054d..3773f6f 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -33,7 +33,7 @@ #include "ptx.tab.h" #include "ptx_sim.h" #include <stdio.h> - +#include <sstream> #include "opcodes.h" #include "../statwrapper.h" #include <set> @@ -58,12 +58,14 @@ int g_debug_execution = 0; int g_debug_thread_uid = 0; addr_t g_debug_pc = 0xBEEF1518; // Output debug information to file options +int cp_count; +int cp_cta_resume; unsigned g_ptx_sim_num_insn = 0; unsigned gpgpu_param_num_shaders = 0; -char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu; -char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu; +char *opcode_latency_int, *opcode_latency_fp, *opcode_latency_dp,*opcode_latency_sfu,*opcode_latency_tensor; +char *opcode_initiation_int, *opcode_initiation_fp, *opcode_initiation_dp,*opcode_initiation_sfu,*opcode_initiation_tensor; char *cdp_latency_str; unsigned cdp_latency[5]; @@ -84,6 +86,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) { "Opcode latencies for SFU instructions" "Default 8", "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, "Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV>" "Default 1,1,4,4,32", @@ -100,6 +106,10 @@ void ptx_opcocde_latency_options (option_parser_t opp) { "Opcode initiation intervals for sfu instructions" "Default 8", "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, "CDP API latency <cudaStreamCreateWithFlags, \ cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, \ @@ -113,22 +123,35 @@ static address_type get_converge_point(address_type pc); void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) { std::string texname(name); - m_NameToTextureRef[texname] = texref; + if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ + m_NameToTextureRef[texname] = std::set<const struct textureReference*>(); + }else{ + const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); + assert(tr!=NULL); + //asserts that all texrefs in set have same fields + assert(tr->normalized==texref->normalized&& + tr->filterMode==texref->filterMode&& + tr->addressMode[0]==texref->addressMode[0]&& + tr->addressMode[1]==texref->addressMode[1]&& + tr->addressMode[2]==texref->addressMode[2]&& + tr->channelDesc.x==texref->channelDesc.x&& + tr->channelDesc.y==texref->channelDesc.y&& + tr->channelDesc.z==texref->channelDesc.z&& + tr->channelDesc.w==texref->channelDesc.w&& + tr->channelDesc.f==texref->channelDesc.f + ); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_TextureRefToAttribute[texref] = texAttr; + m_NameToAttribute[texname] = texAttr; } const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) { - std::map<std::string, const struct textureReference*>::iterator itr = m_NameToTextureRef.begin(); - while (itr != m_NameToTextureRef.end()) { - if ((*itr).second == texref) { - const char *p = ((*itr).first).c_str(); - return p; - } - itr++; - } - return NULL; + std::map<const struct textureReference*, std::string>::const_iterator t=m_TextureRefToName.find(texref); + assert( t != m_TextureRefToName.end() ); + return t->second.c_str(); } unsigned int intLOGB2( unsigned int v ) { @@ -148,7 +171,14 @@ unsigned int intLOGB2( unsigned int v ) { void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) { - m_TextureRefToCudaArray[texref] = array; + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + + std::map<std::string,const struct cudaArray*>::const_iterator t=m_NameToCudaArray.find(texname); + //check that there's nothing there first + if(t != m_NameToCudaArray.end()){ + printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; unsigned int texel_size = texel_size_bits/8; unsigned int Tx, Ty; @@ -188,7 +218,15 @@ void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* te texInfo->Ty_numbits = intLOGB2(Ty); texInfo->texel_size = texel_size; texInfo->texel_size_numbits = intLOGB2(texel_size); - m_TextureRefToTexureInfo[texref] = texInfo; + m_NameToTexureInfo[texname] = texInfo; +} + +void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) +{ + //assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTexureInfo.erase(texname); } unsigned g_assemble_code_next_pc=0; @@ -220,7 +258,9 @@ void function_info::ptx_assemble() m_start_PC = PC; addr_t n=0; // offset in m_instr_mem - s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); + //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. + //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); + s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { ptx_instruction *pI = *i; if ( pI->is_label() ) { @@ -258,11 +298,13 @@ void function_info::ptx_assemble() target.set_type(label_t); } } - + m_n = n; printf(" done.\n"); fflush(stdout); - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); + //disable pdom analysis here and do it at runtime +#if 0 + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); connect_basic_blocks(); bool modified = false; @@ -296,6 +338,7 @@ void function_info::ptx_assemble() fflush(stdout); m_assembled = true; +#endif } addr_t shared_to_generic( unsigned smid, addr_t addr ) @@ -536,7 +579,7 @@ void ptx_instruction::set_mul_div_or_other_archop(){ sp_op=FP_EXP_OP; break; default: - if(op==ALU_OP) + if((op==ALU_OP)||(op==TENSOR_CORE_OP)) sp_op=FP__OP; break; @@ -558,7 +601,7 @@ void ptx_instruction::set_mul_div_or_other_archop(){ sp_op=INT_DIV_OP; break; default: - if(op==ALU_OP) + if((op==ALU_OP)) sp_op=INT__OP; break; } @@ -597,6 +640,9 @@ void ptx_instruction::set_bar_type() abort(); } } + else if(m_opcode==SST_OP) { + bar_type = SYNC; + } } @@ -606,10 +652,12 @@ void ptx_instruction::set_opcode_and_latency() unsigned fp_latency[5]; unsigned dp_latency[5]; unsigned sfu_latency; + unsigned tensor_latency; unsigned int_init[5]; unsigned fp_init[5]; unsigned dp_init[5]; unsigned sfu_init; + unsigned tensor_init; /* * [0] ADD,SUB * [1] MAX,Min @@ -628,6 +676,8 @@ void ptx_instruction::set_opcode_and_latency() &dp_latency[3],&dp_latency[4]); sscanf(opcode_latency_sfu, "%u", &sfu_latency); + sscanf(opcode_latency_tensor, "%u", + &tensor_latency); sscanf(opcode_initiation_int, "%u,%u,%u,%u,%u", &int_init[0],&int_init[1],&int_init[2], &int_init[3],&int_init[4]); @@ -639,6 +689,8 @@ void ptx_instruction::set_opcode_and_latency() &dp_init[3],&dp_init[4]); sscanf(opcode_initiation_sfu, "%u", &sfu_init); + sscanf(opcode_initiation_tensor, "%u", + &tensor_init); sscanf(cdp_latency_str, "%u,%u,%u,%u,%u", &cdp_latency[0],&cdp_latency[1],&cdp_latency[2], &cdp_latency[3],&cdp_latency[4]); @@ -662,13 +714,16 @@ void ptx_instruction::set_opcode_and_latency() if ( has_memory_write() ) op = STORE_OP; break; case LD_OP: op = LOAD_OP; break; + case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; case LDU_OP: op = LOAD_OP; break; case ST_OP: op = STORE_OP; break; + case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; case BRA_OP: op = BRANCH_OP; break; case BREAKADDR_OP: op = BRANCH_OP; break; case TEX_OP: op = LOAD_OP; mem_op=TEX; break; case ATOM_OP: op = LOAD_OP; break; case BAR_OP: op = BARRIER_OP; break; + case SST_OP: op = BARRIER_OP; break; case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; case CALL_OP: { @@ -695,6 +750,7 @@ void ptx_instruction::set_opcode_and_latency() case F32_TYPE: latency = fp_latency[0]; initiation_interval = fp_init[0]; + op = SP_OP; break; case F64_TYPE: case FF64_TYPE: @@ -708,6 +764,7 @@ void ptx_instruction::set_opcode_and_latency() default: //Use int settings for default latency = int_latency[0]; initiation_interval = int_init[0]; + op = INTP_OP; break; } break; @@ -717,6 +774,7 @@ void ptx_instruction::set_opcode_and_latency() case F32_TYPE: latency = fp_latency[1]; initiation_interval = fp_init[1]; + op = SP_OP; break; case F64_TYPE: case FF64_TYPE: @@ -730,6 +788,7 @@ void ptx_instruction::set_opcode_and_latency() default: //Use int settings for default latency = int_latency[1]; initiation_interval = int_init[1]; + op = INTP_OP; break; } break; @@ -739,6 +798,7 @@ void ptx_instruction::set_opcode_and_latency() case F32_TYPE: latency = fp_latency[2]; initiation_interval = fp_init[2]; + op = SP_OP; break; case F64_TYPE: case FF64_TYPE: @@ -752,7 +812,7 @@ void ptx_instruction::set_opcode_and_latency() default: //Use int settings for default latency = int_latency[2]; initiation_interval = int_init[2]; - op = SFU_OP; + op = INTP_OP; break; } break; @@ -762,6 +822,7 @@ void ptx_instruction::set_opcode_and_latency() case F32_TYPE: latency = fp_latency[3]; initiation_interval = fp_init[3]; + op = SP_OP; break; case F64_TYPE: case FF64_TYPE: @@ -775,7 +836,7 @@ void ptx_instruction::set_opcode_and_latency() default: //Use int settings for default latency = int_latency[3]; initiation_interval = int_init[3]; - op = SFU_OP; + op = INTP_OP; break; } break; @@ -802,11 +863,15 @@ void ptx_instruction::set_opcode_and_latency() } break; case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: - //Using double to approximate those latency = sfu_latency; initiation_interval = sfu_init; op = SFU_OP; break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op=TENSOR_CORE_OP; + break; case SHFL_OP: latency = 32; initiation_interval = 4; @@ -863,10 +928,14 @@ void ptx_instruction::pre_decode() { pc = m_PC; isize = m_inst_size; - for( unsigned i=0; i<4; i++) { + for(unsigned i=0; i<MAX_OUTPUT_VALUES; i++) { out[i] = 0; + } + for(unsigned i=0; i<MAX_INPUT_VALUES; i++) { in[i] = 0; } + incount=0; + outcount=0; is_vectorin = 0; is_vectorout = 0; std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); @@ -906,9 +975,11 @@ void ptx_instruction::pre_decode() case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; default: - if( m_opcode == LD_OP || m_opcode == LDU_OP ) + //if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) cache_op = CACHE_ALL; - else if( m_opcode == ST_OP ) + //else if( m_opcode == ST_OP ) + else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) cache_op = CACHE_WRITE_BACK; else if( m_opcode == ATOM_OP ) cache_op = CACHE_GLOBAL; @@ -934,6 +1005,10 @@ void ptx_instruction::pre_decode() if( num_elem >= 2 ) out[1] = o.reg2_num(); if( num_elem >= 3 ) out[2] = o.reg3_num(); if( num_elem >= 4 ) out[3] = o.reg4_num(); + if( num_elem >= 5 ) out[4] = o.reg5_num(); + if( num_elem >= 6 ) out[5] = o.reg6_num(); + if( num_elem >= 7 ) out[6] = o.reg7_num(); + if( num_elem >= 8 ) out[7] = o.reg8_num(); for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } @@ -952,16 +1027,29 @@ void ptx_instruction::pre_decode() //assert(m == 0); //only support 1 vector operand (for textures) right now is_vectorout = 1; unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[0] = o.reg1_num(); - if( num_elem >= 2 ) in[1] = o.reg2_num(); - if( num_elem >= 3 ) in[2] = o.reg3_num(); - if( num_elem >= 4 ) in[3] = o.reg4_num(); + if( num_elem >= 1 ) in[m+0] = o.reg1_num(); + if( num_elem >= 2 ) in[m+1] = o.reg2_num(); + if( num_elem >= 3 ) in[m+2] = o.reg3_num(); + if( num_elem >= 4 ) in[m+3] = o.reg4_num(); + if( num_elem >= 5 ) in[m+4] = o.reg5_num(); + if( num_elem >= 6 ) in[m+5] = o.reg6_num(); + if( num_elem >= 7 ) in[m+6] = o.reg7_num(); + if( num_elem >= 8 ) in[m+7] = o.reg8_num(); for (int i = 0; i < num_elem; i++) - arch_reg.src[i] = o.arch_reg_num(i); - m+=4; + arch_reg.src[m+i] = o.arch_reg_num(i); + m+=num_elem; } } } + + //Setting number of input and output operands which is required for scoreboard check + for(int i=0;i<MAX_OUTPUT_VALUES;i++) + if(out[i]>0) + outcount++; + + for(int i=0;i<MAX_INPUT_VALUES;i++) + if(in[i]>0) + incount++; // Get predicate if(has_pred()) { @@ -1159,14 +1247,19 @@ void function_info::finalize( memory_space *param_mem ) } // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over //Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned + const size_t word_size = 4; - //param_address = (param_address + size - 1) / size * size; //aligned with size TODO: align not correct + //param_address = (param_address + size - 1) / size * size; //aligned with size for (size_t idx = 0; idx < size; idx += word_size) { const char *pdata = reinterpret_cast<const char*>(param_value.pdata) + idx; // cast to char * for ptr arithmetic param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); } unsigned offset = p.get_offset(); - //assert(offset == param_address); + assert(offset == param_address); param->set_address(param_address); param_address += size; } @@ -1213,6 +1306,165 @@ void function_info::list_param( FILE *fout ) const fflush(fout); } +void function_info::ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) +{ + static unsigned long counter = 0; + std::vector< std::pair<size_t, unsigned char*> > param_data; + std::vector<unsigned> offsets; + std::vector<bool> paramIsPointer; + + char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path!=NULL); + char * wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path!=NULL); + std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); + + //initialize paramList + char buff[1024]; + std::string filename_c(filename+"_c"); + snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); + system(buff); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2>pos1&&pos1>0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while(tok!=NULL){ + std::string param(tok); + if(param.find("<")!=std::string::npos){ + assert(param.find(">")==std::string::npos); + assert(param.find("*")==std::string::npos); + tmp = param; + } else { + if (tmp.length()>0){ + tmp = ""; + assert(param.find(">")!=std::string::npos); + assert(param.find("<")==std::string::npos); + assert(param.find("*")==std::string::npos); + } + printf("%s\n", param.c_str()); + if(param.find("*")!=std::string::npos){ + paramIsPointer.push_back(true); + }else{ + paramIsPointer.push_back(false); + } + } + tok = strtok(NULL, ","); + } + + + for( std::map<unsigned,param_info>::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); + + if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ + //is pointer + assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; + if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ + array_size = mallocPtr_Size[param_pointer]; + }else{ + for( std::map<unsigned long long, size_t>::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { + if(param_pointer>j->first&¶m_pointer<j->first + j->second){ + array_size = j->first + j->second - param_pointer; + break; + } + } + assert(array_size>0&&"pointer was not previously malloc'd"); + } + + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + unsigned char* array_val = (unsigned char*) malloc(array_size); + gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); + param_data.push_back(std::pair<size_t, unsigned char*>(array_size,array_val)); + paramIsPointer.push_back(true); + }else{ + unsigned char* val = (unsigned char*) malloc(param_value.size); + param_mem->read(param_addr,param_value.size,(void*)val); + param_data.push_back(std::pair<size_t, unsigned char*>(param_value.size,val)); + paramIsPointer.push_back(false); + } + } + + FILE *fout = fopen (filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for( std::vector< std::pair<size_t,unsigned char*> >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { + if (paramIsPointer[index]){ + fprintf(fout, "*"); + } + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j<i->first; j++){ + fprintf(fout, " %u", i->second[j]); + } + fprintf(fout, " : %u", offsets[index]); + free (i->second); + fprintf(fout, "\n"); + index++; + } + fflush(fout); + fclose(fout); + + //ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); + snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff)!=0){ + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin!=NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout!=NULL); + for (unsigned i = 0; i<line_number; i++){ + fgets(buff, 1024, fin); + assert(!feof(fin)); + } + fprintf(fout, "\n\n"); + do{ + fprintf(fout, "%s", buff); + fgets(buff, 1024, fin); + if(feof(fin)){ + break; + } + } while(strstr(buff, "entry")==NULL); + + fclose(fin); + fflush(fout); + fclose(fout); + counter++; +} + template<int activate_level> bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { @@ -1255,13 +1507,19 @@ static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *th std::string texname = src1.name(); gpgpu_t *gpu = thread->get_gpu(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct textureInfo* texInfo = gpu->get_texinfo(texref); + const struct textureInfo* texInfo = gpu->get_texinfo(texname); unsigned data_size = texInfo->texel_size; return data_size; } +int tensorcore_op(int inst_opcode){ + + if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) + return 1; + else + return 0; +} void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) { @@ -1270,6 +1528,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) addr_t pc = next_instr(); assert( pc == inst.pc ); // make sure timing model and functional model are in sync const ptx_instruction *pI = m_func_info->get_instruction(pc); + set_npc( pc + pI->inst_size() ); @@ -1302,6 +1561,7 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } } + int inst_opcode=pI->get_opcode(); if( skip ) { inst.set_not_active(lane_id); @@ -1313,13 +1573,25 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) *((warp_inst_t*)pJ) = inst; // copy active mask information pI = pJ; } - switch ( pI->get_opcode() ) { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; -#include "opcodes.def" -#undef OP_DEF -#undef OP_W_DEF - default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; + + if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ + if(inst.active_count()!=MAX_WARP_SIZE) + { + printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); + assert(0); + } + } + + //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation + if(!tensorcore_op(inst_opcode)||(tensorcore_op(inst_opcode))&&(lane_id==0)){ + switch ( inst_opcode ) { + #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; + #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; + #include "opcodes.def" + #undef OP_DEF + #undef OP_W_DEF + default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; + } } delete pJ; pI = pI_saved; @@ -1362,13 +1634,16 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) _memory_op_t insn_memory_op = no_memory_op; unsigned insn_data_size = 0; if ( (pI->has_memory_read() || pI->has_memory_write()) ) { - insn_memaddr = last_eaddr(); - insn_space = last_space(); - unsigned to_type = pI->get_type(); - insn_data_size = datatype2size(to_type); - insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { + insn_memaddr = last_eaddr(); + insn_space = last_space(); + unsigned to_type = pI->get_type(); + insn_data_size = datatype2size(to_type); + insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + } } - + if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); } @@ -1440,12 +1715,15 @@ void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) // "Return values" if(!skip) { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert( inst.memory_op == insn_memory_op ); - } - + if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) + { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert( inst.memory_op == insn_memory_op ); + } + } + } catch ( int x ) { printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); @@ -1484,6 +1762,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, std::list<ptx_thread_info *> &active_threads = kernel.active_threads(); static std::map<unsigned,memory_space*> shared_memory_lookup; + static std::map<unsigned,memory_space*> sstarr_memory_lookup; static std::map<unsigned,ptx_cta_info*> ptx_cta_lookup; static std::map<unsigned,ptx_warp_info*> ptx_warp_lookup; static std::map<unsigned,std::map<unsigned,memory_space*> > local_memory_lookup; @@ -1528,6 +1807,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, //initializing new CTA ptx_cta_info *cta_info = NULL; memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; unsigned cta_size = kernel.threads_per_cta(); unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 @@ -1545,6 +1825,9 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, snprintf(buf,512,"shared_%u", sid); shared_mem = new memory_space_impl<16*1024>(buf,4); shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf,512,"sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16*1024>(buf,4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; cta_info = new ptx_cta_info(sm_idx); ptx_cta_lookup[sm_idx] = cta_info; } else { @@ -1553,6 +1836,7 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, sm_idx, sid, max_cta_per_sm ); } shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; cta_info = ptx_cta_lookup[sm_idx]; cta_info->check_cta_thread_status_and_reset(); } @@ -1565,7 +1849,6 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, kernel.increment_thread_id(); new_tid += tid; ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { warp_info = new ptx_warp_info(); @@ -1594,9 +1877,11 @@ unsigned ptx_sim_init_thread( kernel_info_t &kernel, thd->cpy_tid_to_reg(tid3d); thd->set_valid(); thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; function_info *finfo = thd->func_info(); symbol_table *st = finfo->get_symtab(); thd->func_info()->param_to_shared(thd->m_shared_mem,st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); thd->m_cta_info = cta_info; cta_info->add_thread(thd); thd->m_local_mem = local_mem; @@ -1814,6 +2099,38 @@ ptx_cta_info *g_func_cta_info = NULL; #define MAX(a,b) (((a)>(b))?(a):(b)) +unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) +{ + + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size%warp_size) + padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; + + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); + printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); + //Limit by CTA + unsigned int result_cta = max_cta_per_core; + + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf (" threads"); + if (result == result_shmem) printf (" shmem"); + if (result == result_regs) printf (" regs"); + if (result == result_cta) printf (" cta_limit"); + printf ("\n"); + + return result; +} /*! This function simulates the CUDA code functionally, it takes a kernel_info_t parameter which holds the data for the CUDA kernel to be executed @@ -1824,20 +2141,71 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here extern gpgpu_sim *g_the_gpu; + //before we execute, we should do PDOM analysis for functional simulation scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + + unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), g_the_gpu->getShaderCoreConfig()->warp_size, g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, g_the_gpu->getShaderCoreConfig()->max_cta_per_core); + printf("Max CTA : %d\n",max_cta_tot); + + + + + + int inst_count=50; + int cp_op= g_the_gpu->checkpoint_option; + int cp_CTA = g_the_gpu->checkpoint_CTA; + int cp_kernel= g_the_gpu->checkpoint_kernel; + cp_count= g_the_gpu->checkpoint_insn_Y; + cp_cta_resume= g_the_gpu->checkpoint_CTA_t; + int cta_launched =0; //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise while(!kernel.no_more_ctas_to_run()){ - functionalCoreSim cta( - &kernel, - g_the_gpu, - g_the_gpu->getShaderCoreConfig()->warp_size - ); - cta.execute(); + unsigned temp=kernel.get_next_cta_id_single(); + -#if (CUDART_VERSION >= 5000) - launch_all_device_kernels(); -#endif + if(cp_op==0 || (cp_op==1 && cta_launched<cp_cta_resume && kernel.get_uid()==cp_kernel) || kernel.get_uid()< cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, + g_the_gpu, + g_the_gpu->getShaderCoreConfig()->warp_size + ); + cta.execute(cp_count,temp); + + #if (CUDART_VERSION >= 5000) + launch_all_device_kernels(); + #endif + } + else + { + kernel.increment_cta_id(); + } + cta_launched++; } + + + + if(cp_op==1) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); + g_checkpoint->store_global_mem(g_the_gpu->get_global_memory(), f1name , "%08x"); + } + + + //registering this kernel as done @@ -1875,9 +2243,10 @@ void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) fflush(stdout); } -void functionalCoreSim::initializeCTA() +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { int ctaLiveThreads=0; + symbol_table * symtab= m_kernel->entry()->get_symtab(); for(int i=0; i< m_warp_count; i++){ m_warpAtBarrier[i]=false; @@ -1890,9 +2259,13 @@ void functionalCoreSim::initializeCTA() for(unsigned i=0; i<m_kernel->threads_per_cta();i++) { ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); + if(cp_cta_resume==1) + m_thread[i]->resume_reg_thread(fname,symtab); ctaLiveThreads++; } - + for(int k=0;k<m_warp_count;k++) createWarp(k); } @@ -1909,27 +2282,91 @@ void functionalCoreSim::createWarp(unsigned warpId) assert(m_thread[warpId*m_warp_size]!=NULL); m_simt_stack[warpId]->launch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); + + if(cp_cta_resume==1) + { + unsigned pc,rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); + for(int i=warpId*m_warp_size; i<warpId*m_warp_size+m_warp_size;i++){ + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } + + } m_liveThreadCount[warpId]= liveThreadsCount; } -void functionalCoreSim::execute() +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { - initializeCTA(); + cp_count= m_gpu->checkpoint_insn_Y; + cp_cta_resume= m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); - //start executing the CTA + int count=0; while(true){ bool someOneLive= false; bool allAtBarrier = true; for(unsigned i=0;i<m_warp_count;i++){ executeWarp(i,allAtBarrier,someOneLive); + count++; } + + if(inst_count>0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option==1) + { + someOneLive=false; + break; + } if(!someOneLive) break; if(allAtBarrier){ for(unsigned i=0;i<m_warp_count;i++) m_warpAtBarrier[i]=false; } } - } + + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + symbol * sym; + ptx_reg_t regval; + regval.u64= 123; + symbol_table * symtab= m_kernel->entry()->get_symtab(); + + + unsigned ctaid =m_kernel->get_next_cta_id_single(); + if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cp<m_gpu->checkpoint_CTA_t)) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , "%08x"); + for(int i=0; i<32*m_warp_count;i++) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , "%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); + } + + for(int i=0;i<m_warp_count;i++) + { + + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i,ctaid-1 ); + FILE * fp = fopen(fname,"w"); + assert(fp!=NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); + } + } + +} void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) { diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 958daba..abd32f9 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -32,6 +32,7 @@ #include"../gpgpu-sim/shader.h" #include <stdlib.h> #include <map> +#include <vector> #include <string> #include"ptx_sim.h" @@ -97,7 +98,7 @@ public: delete[] m_warpAtBarrier; } //! executes all warps till completion - void execute(); + void execute(int inst_count, unsigned ctaid_cp); virtual void warp_exit( unsigned warp_id ); virtual bool warp_waiting_at_barrier( unsigned warp_id ) const { @@ -107,7 +108,7 @@ public: private: void executeWarp(unsigned, bool &, bool &); //initializes threads in the CTA block which we are executing - void initializeCTA(); + void initializeCTA(unsigned ctaid_cp); virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) { if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 4a8ffe5..b399133 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -177,6 +177,20 @@ void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * //device_grid = op.grid; device_kernel_entry = config.entry; DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); + + //PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() >0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } //copy data in parameter_buffer to device kernel param memory unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h new file mode 100644 index 0000000..8f1a8eb --- /dev/null +++ b/src/cuda-sim/half.h @@ -0,0 +1,3067 @@ +// half - IEEE 754-based half-precision floating point library.
+//
+// Copyright (c) 2012-2017 Christian Rau <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// Version 1.12.0
+
+/// \file
+/// Main header file for half precision functionality.
+
+#ifndef HALF_HALF_HPP
+#define HALF_HALF_HPP
+
+/// Combined gcc version number.
+#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
+
+//check C++11 language features
+#if defined(__clang__) //clang
+ #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+/*#elif defined(__INTEL_COMPILER) //Intel C++
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif*/
+#elif defined(__GNUC__) //gcc
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #endif
+#elif defined(_MSC_VER) //Visual C++
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #define HALF_POP_WARNINGS 1
+ #pragma warning(push)
+ #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
+#endif
+
+//check C++11 library features
+#include <utility>
+#if defined(_LIBCPP_VERSION) //libc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+#elif defined(__GLIBCXX__) //libstdc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifdef __clang__
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #else
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #endif
+#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
+ #if _CPPLIB_VER >= 520
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #if _CPPLIB_VER >= 610
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #endif
+#endif
+#undef HALF_GNUC_VERSION
+
+//support constexpr
+#if HALF_ENABLE_CPP11_CONSTEXPR
+ #define HALF_CONSTEXPR constexpr
+ #define HALF_CONSTEXPR_CONST constexpr
+#else
+ #define HALF_CONSTEXPR
+ #define HALF_CONSTEXPR_CONST const
+#endif
+
+//support noexcept
+#if HALF_ENABLE_CPP11_NOEXCEPT
+ #define HALF_NOEXCEPT noexcept
+ #define HALF_NOTHROW noexcept
+#else
+ #define HALF_NOEXCEPT
+ #define HALF_NOTHROW throw()
+#endif
+
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#if HALF_ENABLE_CPP11_TYPE_TRAITS
+ #include <type_traits>
+#endif
+#if HALF_ENABLE_CPP11_CSTDINT
+ #include <cstdint>
+#endif
+#if HALF_ENABLE_CPP11_HASH
+ #include <functional>
+#endif
+
+
+/// Default rounding mode.
+/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
+/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
+/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
+///
+/// `std::float_round_style` | value | rounding
+/// ---------------------------------|-------|-------------------------
+/// `std::round_indeterminate` | -1 | fastest (default)
+/// `std::round_toward_zero` | 0 | toward zero
+/// `std::round_to_nearest` | 1 | to nearest
+/// `std::round_toward_infinity` | 2 | toward positive infinity
+/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
+///
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
+/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
+/// to synchronize the rounding mode with that of the underlying single-precision implementation.
+#ifndef HALF_ROUND_STYLE
+ #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+#endif
+
+/// Tie-breaking behaviour for round to nearest.
+/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
+/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
+/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
+/// behaviour is needed.
+#ifndef HALF_ROUND_TIES_TO_EVEN
+ #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+#endif
+
+/// Value signaling overflow.
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
+/// operation, in particular it just evaluates to positive infinity.
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+
+/// Fast half-precision fma function.
+/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
+/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
+/// arithmetic operations, this is in fact always the case.
+#define FP_FAST_FMAH 1
+
+#ifndef FP_ILOGB0
+ #define FP_ILOGB0 INT_MIN
+#endif
+#ifndef FP_ILOGBNAN
+ #define FP_ILOGBNAN INT_MAX
+#endif
+#ifndef FP_SUBNORMAL
+ #define FP_SUBNORMAL 0
+#endif
+#ifndef FP_ZERO
+ #define FP_ZERO 1
+#endif
+#ifndef FP_NAN
+ #define FP_NAN 2
+#endif
+#ifndef FP_INFINITE
+ #define FP_INFINITE 3
+#endif
+#ifndef FP_NORMAL
+ #define FP_NORMAL 4
+#endif
+
+
+/// Main namespace for half precision functionality.
+/// This namespace contains all the functionality provided by the library.
+namespace half_float
+{
+ class half;
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ /// Library-defined half-precision literals.
+ /// Import this namespace to enable half-precision floating point literals:
+ /// ~~~~{.cpp}
+ /// using namespace half_float::literal;
+ /// half_float::half = 4.2_h;
+ /// ~~~~
+ namespace literal
+ {
+ half operator"" _h(long double);
+ }
+#endif
+
+ /// \internal
+ /// \brief Implementation details.
+ namespace detail
+ {
+ #if HALF_ENABLE_CPP11_TYPE_TRAITS
+ /// Conditional type.
+ template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
+
+ /// Helper for tag dispatching.
+ template<bool B> struct bool_type : std::integral_constant<bool,B> {};
+ using std::true_type;
+ using std::false_type;
+
+ /// Type traits for floating point types.
+ template<typename T> struct is_float : std::is_floating_point<T> {};
+ #else
+ /// Conditional type.
+ template<bool,typename T,typename> struct conditional { typedef T type; };
+ template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
+
+ /// Helper for tag dispatching.
+ template<bool> struct bool_type {};
+ typedef bool_type<true> true_type;
+ typedef bool_type<false> false_type;
+
+ /// Type traits for floating point types.
+ template<typename> struct is_float : false_type {};
+ template<typename T> struct is_float<const T> : is_float<T> {};
+ template<typename T> struct is_float<volatile T> : is_float<T> {};
+ template<typename T> struct is_float<const volatile T> : is_float<T> {};
+ template<> struct is_float<float> : true_type {};
+ template<> struct is_float<double> : true_type {};
+ template<> struct is_float<long double> : true_type {};
+ #endif
+
+ /// Type traits for floating point bits.
+ template<typename T> struct bits { typedef unsigned char type; };
+ template<typename T> struct bits<const T> : bits<T> {};
+ template<typename T> struct bits<volatile T> : bits<T> {};
+ template<typename T> struct bits<const volatile T> : bits<T> {};
+
+ #if HALF_ENABLE_CPP11_CSTDINT
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef std::uint_least16_t uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> { typedef std::uint_least32_t type; };
+
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef std::uint_least64_t type; };
+ #else
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef unsigned short uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
+ #else
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef unsigned long type; };
+ #endif
+ #endif
+
+ /// Tag type for binary construction.
+ struct binary_t {};
+
+ /// Tag for binary construction.
+ HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+
+ /// Temporary half-precision expression.
+ /// This class represents a half-precision expression which just stores a single-precision value internally.
+ struct expr
+ {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+ };
+
+ /// SFINAE helper for generic half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member equivalent to \a T.
+ /// \tparam T type to return
+ template<typename T,typename,typename=void,typename=void> struct enable {};
+ template<typename T> struct enable<T,half,void,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,void,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,void> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,half> { typedef T type; };
+ template<typename T> struct enable<T,half,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
+
+ /// Return type for specialized generic 2-argument half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member denoting the appropriate return type.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct result : enable<expr,T,U> {};
+ template<> struct result<half,half> { typedef half type; };
+
+ /// \name Classification helpers
+ /// \{
+
+ /// Check for infinity.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if infinity
+ /// \retval false else
+ template<typename T> bool builtin_isinf(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+ #elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
+ #else
+ return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
+ #endif
+ }
+
+ /// Check for NaN.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if not a number
+ /// \retval false else
+ template<typename T> bool builtin_isnan(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+ #elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+ #else
+ return arg != arg;
+ #endif
+ }
+
+ /// Check sign.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if signbit set
+ /// \retval false else
+ template<typename T> bool builtin_signbit(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+ #else
+ return arg < T() || (arg == T() && T(1)/arg < T());
+ #endif
+ }
+
+ /// \}
+ /// \name Conversion
+ /// \{
+
+ /// Convert IEEE single-precision to half-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value single-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+/* uint16 hbits = (bits>>16) & 0x8000;
+ bits &= 0x7FFFFFFF;
+ int exp = bits >> 23;
+ if(exp == 255)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0));
+ if(exp > 142)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s;
+ if(exp > 112)
+ {
+ g = (bits>>12) & 1;
+ s = (bits&0xFFF) != 0;
+ hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
+ }
+ else if(exp > 101)
+ {
+ int i = 125 - exp;
+ bits = (bits&0x7FFFFF) | 0x800000;
+ g = (bits>>i) & 1;
+ s = (bits&((1L<<i)-1)) != 0;
+ hbits |= bits >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s = bits != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+*/ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
+ 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
+ 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
+ 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
+ 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
+ uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
+ if(R == std::round_to_nearest)
+ hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
+ #endif
+ ;
+ else if(R == std::round_toward_zero)
+ hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
+ else if(R == std::round_toward_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
+ ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
+ else if(R == std::round_toward_neg_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
+ ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
+ return hbits;
+ }
+
+ /// Convert IEEE double-precision to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value double-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi>>16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if(exp == 2047)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
+ if(exp > 1038)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if(exp > 1008)
+ {
+ g = (hi>>9) & 1;
+ s |= (hi&0x1FF) != 0;
+ hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
+ }
+ else if(exp > 997)
+ {
+ int i = 1018 - exp;
+ hi = (hi&0xFFFFF) | 0x100000;
+ g = (hi>>i) & 1;
+ s |= (hi&((1L<<i)-1)) != 0;
+ hbits |= hi >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s |= hi != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ return hbits;
+ }
+
+ /// Convert non-IEEE floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
+ {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if(value == T())
+ return hbits;
+ if(builtin_isnan(value))
+ return hbits | 0x7FFF;
+ if(builtin_isinf(value))
+ return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if(exp > 16)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ else if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ if(exp < -13)
+ value = std::ldexp(value, 24);
+ else
+ {
+ value = std::ldexp(value, 11-exp);
+ hbits |= ((exp+13)<<10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if(R == std::round_to_nearest)
+ {
+ frac = std::abs(frac);
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
+ #else
+ hbits += frac >= T(0.5);
+ #endif
+ }
+ else if(R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if(R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+ }
+
+ /// Convert floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half(T value)
+ {
+ return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam S `true` if value negative, `false` else
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value non-negative integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
+ #endif
+ if(S)
+ value = -value;
+ uint16 bits = S << 15;
+ if(value > 0xFFFF)
+ {
+ if(R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if(R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R!=std::round_toward_zero);
+ }
+ else if(value)
+ {
+ unsigned int m = value, exp = 24;
+ for(; m<0x400; m<<=1,--exp) ;
+ for(; m>0x7FF; m>>=1,++exp) ;
+ bits |= (exp<<10) + m;
+ if(exp > 24)
+ {
+ if(R == std::round_to_nearest)
+ bits += (value>>(exp-25)) & 1
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((1<<(exp-25))-1)&value)!=0)|bits)
+ #endif
+ ;
+ else if(R == std::round_toward_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & !S;
+ else if(R == std::round_toward_neg_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & S;
+ }
+ }
+ return bits;
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 int2half(T value)
+ {
+ return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
+ }
+
+ /// Convert half-precision to IEEE single-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \param value binary representation of half-precision value
+ /// \return single-precision value
+ inline float half2float_impl(uint16 value, float, true_type)
+ {
+ typedef bits<float>::type uint32;
+/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ bits |= 0x38000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,bits-=0x800000) ;
+ bits += static_cast<uint32>(abs) << 13;
+ }
+*/ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
+ 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
+ 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
+ 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
+ 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
+ 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
+ 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
+ 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
+ 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
+ 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
+ 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
+ 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
+ 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
+ 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
+ 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
+ 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
+ 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
+ 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
+ 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
+ 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
+ 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
+ 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
+ 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
+ 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
+ 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
+ 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
+ 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
+ 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
+ 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
+ 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
+ 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
+ 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
+ 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
+ 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
+ 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
+ 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
+ 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
+ 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
+ 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
+ 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
+ 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
+ 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
+ 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
+ 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
+ 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
+ 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
+ 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
+ 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
+ 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
+ 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
+ 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
+ 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
+ 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
+ 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
+ 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
+ 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
+ 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
+ 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
+ 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
+ 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
+ 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
+ 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
+ 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
+ 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
+ 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
+ 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
+ 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
+ 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
+ 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
+ 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
+ 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
+ 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
+ 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
+ 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
+ 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
+ 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
+ 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
+ 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
+ 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
+ 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
+ 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
+ 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
+ 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
+ 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
+ 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
+ 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
+ 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
+ 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
+ uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
+// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+ }
+
+ /// Convert half-precision to IEEE double-precision.
+ /// \param value binary representation of half-precision value
+ /// \return double-precision value
+ inline double half2float_impl(uint16 value, double, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,hi-=0x100000) ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+ }
+
+ /// Convert half-precision to non-IEEE floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float_impl(uint16 value, T, ...)
+ {
+ T out;
+ int abs = value & 0x7FFF;
+ if(abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
+ else if(abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
+ else if(abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value&0x8000) ? -out : out;
+ }
+
+ /// Convert half-precision to floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float(uint16 value)
+ {
+ return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
+ #endif
+ unsigned int e = value & 0x7FFF;
+ if(e >= 0x7C00)
+ return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
+ if(e < 0x3800)
+ {
+ if(R == std::round_toward_infinity)
+ return T(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ return -T(value>0x8000);
+ return T();
+ }
+ unsigned int m = (value&0x3FF) | 0x400;
+ e >>= 10;
+ if(e < 25)
+ {
+ if(R == std::round_to_nearest)
+ m += (1<<(24-e)) - (~(m>>(25-e))&E);
+ else if(R == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(25-e))-1U);
+ else if(R == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(25-e))-1U);
+ m >>= 25 - e;
+ }
+ else
+ m <<= e - 25;
+ return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
+
+ /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
+ {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if(e < 0x3C00)
+ {
+ result &= 0x8000;
+ if(R == std::round_to_nearest)
+ result |= 0x3C00U & -(e>=(0x3800+E));
+ else if(R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value>0x8000);
+ }
+ else if(e < 0x6400)
+ {
+ e = 25 - (e>>10);
+ unsigned int mask = (1<<e) - 1;
+ if(R == std::round_to_nearest)
+ result += (1<<(e-1)) - (~(result>>e)&E);
+ else if(R == std::round_toward_infinity)
+ result += mask & ((value>>15)-1);
+ else if(R == std::round_toward_neg_infinity)
+ result += mask & -(value>>15);
+ result &= ~mask;
+ }
+ return result;
+ }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
+
+ /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
+ /// \}
+
+ struct functions;
+ template<typename> struct unary_specialized;
+ template<typename,typename> struct binary_specialized;
+ template<typename,typename,std::float_round_style> struct half_caster;
+ }
+
+ /// Half-precision floating point type.
+ /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
+ /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
+ /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
+ /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
+ /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
+ /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
+ ///
+ /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
+ /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
+ /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
+ /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
+ /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
+ /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
+ /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
+ /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
+ /// nearly any reasonable platform.
+ ///
+ /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
+ /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
+ class half
+ {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half,half>;
+ template<typename,typename,std::float_round_style> friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+ #if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+ #endif
+ #if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+ #endif
+
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
+ /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
+
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
+
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
+
+ /// Prefix increment.
+ /// \return incremented half value
+ half& operator++() { return *this += 1.0f; }
+
+ /// Prefix decrement.
+ /// \return decremented half value
+ half& operator--() { return *this -= 1.0f; }
+
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) { half out(*this); ++*this; return out; }
+
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) { half out(*this); --*this; return out; }
+
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
+
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
+
+ /// Internal binary representation
+ detail::uint16 data_;
+ };
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ namespace literal
+ {
+ /// Half literal.
+ /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
+ /// to rather involved conversions.
+ /// \param value literal value
+ /// \return half with given value (if representable)
+ inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
+ }
+#endif
+
+ namespace detail
+ {
+ /// Wrapper implementing unspecialized half-precision functions.
+ struct functions
+ {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x+y); }
+
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x-y); }
+
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x*y); }
+
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x/y); }
+
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
+
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
+ {
+ float f;
+ if(in >> f)
+ arg = f;
+ return in;
+ }
+
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay+ay);
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ if(ax >= y2)
+ ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+ #endif
+ }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f*ay);
+ int cquo = 0;
+ if(ax >= 4.0f * ay)
+ {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if(ax >= 2.0f * ay)
+ {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ ++cquo;
+ if(ax >= y2)
+ {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+ #endif
+ }
+
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+ #else
+ return expr((x<=y) ? 0.0f : (x-y));
+ #endif
+ }
+
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z)
+ {
+ #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+ #else
+ return expr(x*y+z);
+ #endif
+ }
+
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+ #else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
+ #endif
+ }
+
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+ #else
+ return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
+ #endif
+ }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
+
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+ #else
+ return expr(static_cast<float>(std::log(1.0+arg)));
+ #endif
+ }
+
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+ #else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
+ #endif
+ }
+
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+ #else
+ if(builtin_isnan(arg) || builtin_isinf(arg))
+ return expr(arg);
+ return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
+ static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
+ #endif
+ }
+
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+ #else
+ return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
+ static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
+ #endif
+ }
+
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
+
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
+
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
+
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
+
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
+
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
+
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
+
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+ #else
+ return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+ #else
+ return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+ #else
+ return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
+ #endif
+ }
+
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+ #else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+ #else
+ return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+ #else
+ if(builtin_isinf(arg))
+ return expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(1.1447298858494001741434273513531-
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+ #else
+ if(arg == 0.0f)
+ return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
+ return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
+ }
+ if(builtin_isinf(arg))
+ return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+ #endif
+ }
+
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
+
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
+
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
+ #endif
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp)
+ {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if(m >= 0x7C00 || !m)
+ return *exp = 0, arg;
+ for(; m<0x400; m<<=1,--e) ;
+ return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
+ }
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr)
+ {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if(e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
+ if(e < 0x3C00)
+ return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if(!m)
+ return half(binary, arg.data_&0x8000);
+ for(; m<0x400; m<<=1,--e) ;
+ return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
+ }
+
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp)
+ {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if(m >= 0x7C00 || !m)
+ return arg;
+ for(; m<0x400; m<<=1,--exp) ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if(exp > 30)
+ {
+ if(half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if(half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value>>15);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value>>15);
+ else
+ value |= 0x7C00;
+ }
+ else if(exp > 0)
+ value |= (exp<<10) | (m&0x3FF);
+ else if(exp > -11)
+ {
+ m = (m&0x3FF) | 0x400;
+ if(half::round_style == std::round_to_nearest)
+ {
+ m += 1 << -exp;
+ #if HALF_ROUND_TIES_TO_EVEN
+ m -= (m>>(1-exp)) & 1;
+ #endif
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(1-exp))-1U);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(1-exp))-1U);
+ value |= m >> (1-exp);
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ value -= (value>>15) - 1;
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return FP_ILOGB0;
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ return exp;
+ }
+ if(abs > 0x7C00)
+ return FP_ILOGBNAN;
+ return INT_MAX;
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return half(binary, 0xFC00);
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ uint16 bits = (exp<0) << 15;
+ if(exp)
+ {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for(; m<0x400; m<<=1,--e) ;
+ bits |= (e<<10) + m;
+ }
+ return half(binary, bits);
+ }
+ if(abs > 0x7C00)
+ return arg;
+ return half(binary, 0x7C00);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to)
+ {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if(fabs > 0x7C00)
+ return from;
+ if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
+ return to;
+ if(!fabs)
+ return half(binary, (to.data_&0x8000)+1);
+ bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
+ ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to)
+ {
+ if(isnan(from))
+ return from;
+ long double lfrom = static_cast<long double>(from);
+ if(builtin_isnan(to) || lfrom == to)
+ return half(static_cast<float>(to));
+ if(!(from.data_&0x7FFF))
+ return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
+ }
+
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg)
+ {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
+ }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
+
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00 || yabs > 0x7C00)
+ return false;
+ int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+
+ private:
+ static double erf(double arg)
+ {
+ if(builtin_isinf(arg))
+ return (arg<0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
+
+ static double lgamma(double arg)
+ {
+ double v = 1.0;
+ for(; arg<8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg*arg);
+ return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
+ -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
+ -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
+ -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
+ }
+ };
+
+ /// Wrapper for unary half-precision functions needing specialization for individual argument types.
+ /// \tparam T argument type
+ template<typename T> struct unary_specialized
+ {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
+
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
+ };
+ template<> struct unary_specialized<expr>
+ {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+ };
+
+ /// Wrapper for binary half-precision functions needing specialization for individual argument types.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct binary_specialized
+ {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::min(x, y));
+ #endif
+ }
+
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::max(x, y));
+ #endif
+ }
+ };
+ template<> struct binary_specialized<half,half>
+ {
+ static half fmin(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ static half fmax(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ };
+
+ /// Helper class for half casts.
+ /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
+ /// function and a corresponding `type` member denoting its return type.
+ /// \tparam T destination type
+ /// \tparam U source type
+ /// \tparam R rounding mode to use
+ template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
+ template<typename U,std::float_round_style R> struct half_caster<half,U,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
+ #endif
+
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+
+ private:
+ static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
+ static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,half,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<std::float_round_style R> struct half_caster<half,half,R>
+ {
+ static half cast(half arg) { return arg; }
+ };
+ template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
+
+ /// \name Comparison operators
+ /// \{
+
+ /// Comparison for equality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
+
+ /// Comparison for inequality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
+
+ /// \}
+ /// \name Arithmetic operators
+ /// \{
+
+ /// Add halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return sum of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
+
+ /// Subtract halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return difference of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
+
+ /// Multiply halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return product of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
+
+ /// Divide halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return quotient of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
+
+ /// Identity.
+ /// \param arg operand
+ /// \return uncahnged operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
+
+ /// Negation.
+ /// \param arg operand
+ /// \return negated operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
+
+ /// \}
+ /// \name Input and output
+ /// \{
+
+ /// Output operator.
+ /// \param out output stream to write into
+ /// \param arg half expression to write
+ /// \return reference to output stream
+ template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
+ operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
+
+ /// Input operator.
+ /// \param in input stream to read from
+ /// \param arg half to read into
+ /// \return reference to input stream
+ template<typename charT,typename traits> std::basic_istream<charT,traits>&
+ operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
+
+ /// \}
+ /// \name Basic mathematical operations
+ /// \{
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store some bits of quotient at
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+
+ /// Fused multiply add.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+
+ /// Maximum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+ inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
+ inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
+ inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
+ inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
+
+ /// Minimum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+ inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
+ inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
+ inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
+ inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
+
+ /// Positive difference.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+
+ /// Get NaN value.
+ /// \return quiet NaN
+ inline half nanh(const char*) { return functions::nanh(); }
+
+ /// \}
+ /// \name Exponential functions
+ /// \{
+
+ /// Exponential function.
+ /// \param arg function argument
+ /// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
+ inline expr exp(half arg) { return functions::exp(arg); }
+ inline expr exp(expr arg) { return functions::exp(arg); }
+
+ /// Exponential minus one.
+ /// \param arg function argument
+ /// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
+ inline expr expm1(half arg) { return functions::expm1(arg); }
+ inline expr expm1(expr arg) { return functions::expm1(arg); }
+
+ /// Binary exponential.
+ /// \param arg function argument
+ /// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
+ inline expr exp2(half arg) { return functions::exp2(arg); }
+ inline expr exp2(expr arg) { return functions::exp2(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
+ inline expr log(half arg) { return functions::log(arg); }
+ inline expr log(expr arg) { return functions::log(arg); }
+
+ /// Common logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
+ inline expr log10(half arg) { return functions::log10(arg); }
+ inline expr log10(expr arg) { return functions::log10(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
+ inline expr log1p(half arg) { return functions::log1p(arg); }
+ inline expr log1p(expr arg) { return functions::log1p(arg); }
+
+ /// Binary logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
+ inline expr log2(half arg) { return functions::log2(arg); }
+ inline expr log2(expr arg) { return functions::log2(arg); }
+
+ /// \}
+ /// \name Power functions
+ /// \{
+
+ /// Square root.
+ /// \param arg function argument
+ /// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
+ inline expr sqrt(half arg) { return functions::sqrt(arg); }
+ inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+
+ /// Cubic root.
+ /// \param arg function argument
+ /// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
+ inline expr cbrt(half arg) { return functions::cbrt(arg); }
+ inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+
+ /// Hypotenuse function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+
+ /// Power function.
+ /// \param base first argument
+ /// \param exp second argument
+ /// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+
+ /// \}
+ /// \name Trigonometric functions
+ /// \{
+
+ /// Sine function.
+ /// \param arg function argument
+ /// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
+ inline expr sin(half arg) { return functions::sin(arg); }
+ inline expr sin(expr arg) { return functions::sin(arg); }
+
+ /// Cosine function.
+ /// \param arg function argument
+ /// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
+ inline expr cos(half arg) { return functions::cos(arg); }
+ inline expr cos(expr arg) { return functions::cos(arg); }
+
+ /// Tangent function.
+ /// \param arg function argument
+ /// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
+ inline expr tan(half arg) { return functions::tan(arg); }
+ inline expr tan(expr arg) { return functions::tan(arg); }
+
+ /// Arc sine.
+ /// \param arg function argument
+ /// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
+ inline expr asin(half arg) { return functions::asin(arg); }
+ inline expr asin(expr arg) { return functions::asin(arg); }
+
+ /// Arc cosine function.
+ /// \param arg function argument
+ /// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
+ inline expr acos(half arg) { return functions::acos(arg); }
+ inline expr acos(expr arg) { return functions::acos(arg); }
+
+ /// Arc tangent function.
+ /// \param arg function argument
+ /// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
+ inline expr atan(half arg) { return functions::atan(arg); }
+ inline expr atan(expr arg) { return functions::atan(arg); }
+
+ /// Arc tangent function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+
+ /// \}
+ /// \name Hyperbolic functions
+ /// \{
+
+ /// Hyperbolic sine.
+ /// \param arg function argument
+ /// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
+ inline expr sinh(half arg) { return functions::sinh(arg); }
+ inline expr sinh(expr arg) { return functions::sinh(arg); }
+
+ /// Hyperbolic cosine.
+ /// \param arg function argument
+ /// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
+ inline expr cosh(half arg) { return functions::cosh(arg); }
+ inline expr cosh(expr arg) { return functions::cosh(arg); }
+
+ /// Hyperbolic tangent.
+ /// \param arg function argument
+ /// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
+ inline expr tanh(half arg) { return functions::tanh(arg); }
+ inline expr tanh(expr arg) { return functions::tanh(arg); }
+
+ /// Hyperbolic area sine.
+ /// \param arg function argument
+ /// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
+ inline expr asinh(half arg) { return functions::asinh(arg); }
+ inline expr asinh(expr arg) { return functions::asinh(arg); }
+
+ /// Hyperbolic area cosine.
+ /// \param arg function argument
+ /// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
+ inline expr acosh(half arg) { return functions::acosh(arg); }
+ inline expr acosh(expr arg) { return functions::acosh(arg); }
+
+ /// Hyperbolic area tangent.
+ /// \param arg function argument
+ /// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
+ inline expr atanh(half arg) { return functions::atanh(arg); }
+ inline expr atanh(expr arg) { return functions::atanh(arg); }
+
+ /// \}
+ /// \name Error and gamma functions
+ /// \{
+
+ /// Error function.
+ /// \param arg function argument
+ /// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
+ inline expr erf(half arg) { return functions::erf(arg); }
+ inline expr erf(expr arg) { return functions::erf(arg); }
+
+ /// Complementary error function.
+ /// \param arg function argument
+ /// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
+ inline expr erfc(half arg) { return functions::erfc(arg); }
+ inline expr erfc(expr arg) { return functions::erfc(arg); }
+
+ /// Natural logarithm of gamma function.
+ /// \param arg function argument
+ /// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
+ inline expr lgamma(half arg) { return functions::lgamma(arg); }
+ inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+
+ /// Gamma function.
+ /// \param arg function argument
+ /// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
+ inline expr tgamma(half arg) { return functions::tgamma(arg); }
+ inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+
+ /// \}
+ /// \name Rounding
+ /// \{
+
+ /// Nearest integer not less than half value.
+ /// \param arg half to round
+ /// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
+ inline half ceil(half arg) { return functions::ceil(arg); }
+ inline half ceil(expr arg) { return functions::ceil(arg); }
+
+ /// Nearest integer not greater than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
+ inline half floor(half arg) { return functions::floor(arg); }
+ inline half floor(expr arg) { return functions::floor(arg); }
+
+ /// Nearest integer not greater in magnitude than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
+ inline half trunc(half arg) { return functions::trunc(arg); }
+ inline half trunc(expr arg) { return functions::trunc(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
+ inline half round(half arg) { return functions::round(arg); }
+ inline half round(expr arg) { return functions::round(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
+ inline long lround(half arg) { return functions::lround(arg); }
+ inline long lround(expr arg) { return functions::lround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
+ inline half nearbyint(half arg) { return functions::rint(arg); }
+ inline half nearbyint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
+ inline half rint(half arg) { return functions::rint(arg); }
+ inline half rint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
+ inline long lrint(half arg) { return functions::lrint(arg); }
+ inline long lrint(expr arg) { return functions::lrint(arg); }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
+ inline long long llround(half arg) { return functions::llround(arg); }
+ inline long long llround(expr arg) { return functions::llround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
+ inline long long llrint(half arg) { return functions::llrint(arg); }
+ inline long long llrint(expr arg) { return functions::llrint(arg); }
+ #endif
+
+ /// \}
+ /// \name Floating point manipulation
+ /// \{
+
+ /// Decompress floating point number.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract integer and fractional parts.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+ /// \retval FP_ILOGB0 for zero
+ /// \retval FP_ILOGBNAN for NaN
+ /// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
+ inline int ilogb(half arg) { return functions::ilogb(arg); }
+ inline int ilogb(expr arg) { return functions::ilogb(arg); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
+ inline half logb(half arg) { return functions::logb(arg); }
+ inline half logb(expr arg) { return functions::logb(arg); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
+
+ /// Take sign.
+ /// \param x value to change sign for
+ /// \param y value to take sign from
+ /// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
+ inline half copysign(half x, half y) { return functions::copysign(x, y); }
+ inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+
+ /// \}
+ /// \name Floating point classification
+ /// \{
+
+
+ /// Classify floating point value.
+ /// \param arg number to classify
+ /// \retval FP_ZERO for positive and negative zero
+ /// \retval FP_SUBNORMAL for subnormal numbers
+ /// \retval FP_INFINITY for positive and negative infinity
+ /// \retval FP_NAN for NaNs
+ /// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+
+ /// Check if finite number.
+ /// \param arg number to check
+ /// \retval true if neither infinity nor NaN
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
+ inline bool isfinite(half arg) { return functions::isfinite(arg); }
+ inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+
+ /// Check for infinity.
+ /// \param arg number to check
+ /// \retval true for positive or negative infinity
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
+ inline bool isinf(half arg) { return functions::isinf(arg); }
+ inline bool isinf(expr arg) { return functions::isinf(arg); }
+
+ /// Check for NaN.
+ /// \param arg number to check
+ /// \retval true for NaNs
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
+ inline bool isnan(half arg) { return functions::isnan(arg); }
+ inline bool isnan(expr arg) { return functions::isnan(arg); }
+
+ /// Check if normal number.
+ /// \param arg number to check
+ /// \retval true if normal number
+ /// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
+ inline bool isnormal(half arg) { return functions::isnormal(arg); }
+ inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+
+ /// Check sign.
+ /// \param arg number to check
+ /// \retval true for negative number
+ /// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
+ inline bool signbit(half arg) { return functions::signbit(arg); }
+ inline bool signbit(expr arg) { return functions::signbit(arg); }
+
+ /// \}
+ /// \name Comparison
+ /// \{
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
+ inline bool isless(half x, half y) { return functions::isless(x, y); }
+ inline bool isless(half x, expr y) { return functions::isless(x, y); }
+ inline bool isless(expr x, half y) { return functions::isless(x, y); }
+ inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+
+ /// Comarison for less or greater.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either less or greater
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
+
+ /// Check if unordered.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if unordered (one or two NaN operands)
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+
+ /// \name Casting
+ /// \{
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ /// It uses the default rounding mode.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam R rounding mode to use.
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
+ /// \}
+ }
+
+ using detail::operator==;
+ using detail::operator!=;
+ using detail::operator<;
+ using detail::operator>;
+ using detail::operator<=;
+ using detail::operator>=;
+ using detail::operator+;
+ using detail::operator-;
+ using detail::operator*;
+ using detail::operator/;
+ using detail::operator<<;
+ using detail::operator>>;
+
+ using detail::abs;
+ using detail::fabs;
+ using detail::fmod;
+ using detail::remainder;
+ using detail::remquo;
+ using detail::fma;
+ using detail::fmax;
+ using detail::fmin;
+ using detail::fdim;
+ using detail::nanh;
+ using detail::exp;
+ using detail::expm1;
+ using detail::exp2;
+ using detail::log;
+ using detail::log10;
+ using detail::log1p;
+ using detail::log2;
+ using detail::sqrt;
+ using detail::cbrt;
+ using detail::hypot;
+ using detail::pow;
+ using detail::sin;
+ using detail::cos;
+ using detail::tan;
+ using detail::asin;
+ using detail::acos;
+ using detail::atan;
+ using detail::atan2;
+ using detail::sinh;
+ using detail::cosh;
+ using detail::tanh;
+ using detail::asinh;
+ using detail::acosh;
+ using detail::atanh;
+ using detail::erf;
+ using detail::erfc;
+ using detail::lgamma;
+ using detail::tgamma;
+ using detail::ceil;
+ using detail::floor;
+ using detail::trunc;
+ using detail::round;
+ using detail::lround;
+ using detail::nearbyint;
+ using detail::rint;
+ using detail::lrint;
+#if HALF_ENABLE_CPP11_LONG_LONG
+ using detail::llround;
+ using detail::llrint;
+#endif
+ using detail::frexp;
+ using detail::ldexp;
+ using detail::modf;
+ using detail::scalbn;
+ using detail::scalbln;
+ using detail::ilogb;
+ using detail::logb;
+ using detail::nextafter;
+ using detail::nexttoward;
+ using detail::copysign;
+ using detail::fpclassify;
+ using detail::isfinite;
+ using detail::isinf;
+ using detail::isnan;
+ using detail::isnormal;
+ using detail::signbit;
+ using detail::isgreater;
+ using detail::isgreaterequal;
+ using detail::isless;
+ using detail::islessequal;
+ using detail::islessgreater;
+ using detail::isunordered;
+
+ using detail::half_cast;
+}
+
+
+/// Extensions to the C++ standard library.
+namespace std
+{
+ /// Numeric limits for half-precision floats.
+ /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
+ /// `std::numeric_limits<float>`.
+ template<> class numeric_limits<half_float::half> : public numeric_limits<float>
+ {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
+
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
+
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
+
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
+
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
+ /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
+ /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
+ /// single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
+ half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
+
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
+
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
+
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
+
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
+
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
+
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
+
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
+
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
+
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
+
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
+ { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
+
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
+
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
+
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
+
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
+ };
+
+#if HALF_ENABLE_CPP11_HASH
+ /// Hash function for half-precision floats.
+ /// This is only defined if C++11 `std::hash` is supported and enabled.
+ template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+ {
+ /// Type of function argument.
+ typedef half_float::half argument_type;
+
+ /// Function return type.
+ typedef size_t result_type;
+
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const
+ { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
+ };
+#endif
+}
+
+
+#undef HALF_CONSTEXPR
+#undef HALF_CONSTEXPR_CONST
+#undef HALF_NOEXCEPT
+#undef HALF_NOTHROW
+#ifdef HALF_POP_WARNINGS
+ #pragma warning(pop)
+ #undef HALF_POP_WARNINGS
+#endif
+
+#endif
diff --git a/src/cuda-sim/half.hpp b/src/cuda-sim/half.hpp new file mode 100644 index 0000000..8f1a8eb --- /dev/null +++ b/src/cuda-sim/half.hpp @@ -0,0 +1,3067 @@ +// half - IEEE 754-based half-precision floating point library.
+//
+// Copyright (c) 2012-2017 Christian Rau <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// Version 1.12.0
+
+/// \file
+/// Main header file for half precision functionality.
+
+#ifndef HALF_HALF_HPP
+#define HALF_HALF_HPP
+
+/// Combined gcc version number.
+#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__)
+
+//check C++11 language features
+#if defined(__clang__) //clang
+ #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+/*#elif defined(__INTEL_COMPILER) //Intel C++
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ????????
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ????????
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ????????
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ????????
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif*/
+#elif defined(__GNUC__) //gcc
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #endif
+#elif defined(_MSC_VER) //Visual C++
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR)
+ #define HALF_ENABLE_CPP11_CONSTEXPR 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT)
+ #define HALF_ENABLE_CPP11_NOEXCEPT 1
+ #endif
+ #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS)
+ #define HALF_ENABLE_CPP11_USER_LITERALS 1
+ #endif
+ #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT)
+ #define HALF_ENABLE_CPP11_STATIC_ASSERT 1
+ #endif
+ #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG)
+ #define HALF_ENABLE_CPP11_LONG_LONG 1
+ #endif
+ #define HALF_POP_WARNINGS 1
+ #pragma warning(push)
+ #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned
+#endif
+
+//check C++11 library features
+#include <utility>
+#if defined(_LIBCPP_VERSION) //libc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+#elif defined(__GLIBCXX__) //libstdc++
+ #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103
+ #ifdef __clang__
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS)
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #else
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT)
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH)
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH)
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #endif
+#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++
+ #if _CPPLIB_VER >= 520
+ #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS
+ #define HALF_ENABLE_CPP11_TYPE_TRAITS 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_CSTDINT
+ #define HALF_ENABLE_CPP11_CSTDINT 1
+ #endif
+ #ifndef HALF_ENABLE_CPP11_HASH
+ #define HALF_ENABLE_CPP11_HASH 1
+ #endif
+ #endif
+ #if _CPPLIB_VER >= 610
+ #ifndef HALF_ENABLE_CPP11_CMATH
+ #define HALF_ENABLE_CPP11_CMATH 1
+ #endif
+ #endif
+#endif
+#undef HALF_GNUC_VERSION
+
+//support constexpr
+#if HALF_ENABLE_CPP11_CONSTEXPR
+ #define HALF_CONSTEXPR constexpr
+ #define HALF_CONSTEXPR_CONST constexpr
+#else
+ #define HALF_CONSTEXPR
+ #define HALF_CONSTEXPR_CONST const
+#endif
+
+//support noexcept
+#if HALF_ENABLE_CPP11_NOEXCEPT
+ #define HALF_NOEXCEPT noexcept
+ #define HALF_NOTHROW noexcept
+#else
+ #define HALF_NOEXCEPT
+ #define HALF_NOTHROW throw()
+#endif
+
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#if HALF_ENABLE_CPP11_TYPE_TRAITS
+ #include <type_traits>
+#endif
+#if HALF_ENABLE_CPP11_CSTDINT
+ #include <cstdint>
+#endif
+#if HALF_ENABLE_CPP11_HASH
+ #include <functional>
+#endif
+
+
+/// Default rounding mode.
+/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as
+/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one
+/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`:
+///
+/// `std::float_round_style` | value | rounding
+/// ---------------------------------|-------|-------------------------
+/// `std::round_indeterminate` | -1 | fastest (default)
+/// `std::round_toward_zero` | 0 | toward zero
+/// `std::round_to_nearest` | 1 | to nearest
+/// `std::round_toward_infinity` | 2 | toward positive infinity
+/// `std::round_toward_neg_infinity` | 3 | toward negative infinity
+///
+/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows
+/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits<float>::round_style`
+/// to synchronize the rounding mode with that of the underlying single-precision implementation.
+#ifndef HALF_ROUND_STYLE
+ #define HALF_ROUND_STYLE -1 // = std::round_indeterminate
+#endif
+
+/// Tie-breaking behaviour for round to nearest.
+/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is
+/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and
+/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant
+/// behaviour is needed.
+#ifndef HALF_ROUND_TIES_TO_EVEN
+ #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero
+#endif
+
+/// Value signaling overflow.
+/// In correspondence with `HUGE_VAL[F|L]` from `<cmath>` this symbol expands to a positive value signaling the overflow of an
+/// operation, in particular it just evaluates to positive infinity.
+#define HUGE_VALH std::numeric_limits<half_float::half>::infinity()
+
+/// Fast half-precision fma function.
+/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate
+/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all
+/// arithmetic operations, this is in fact always the case.
+#define FP_FAST_FMAH 1
+
+#ifndef FP_ILOGB0
+ #define FP_ILOGB0 INT_MIN
+#endif
+#ifndef FP_ILOGBNAN
+ #define FP_ILOGBNAN INT_MAX
+#endif
+#ifndef FP_SUBNORMAL
+ #define FP_SUBNORMAL 0
+#endif
+#ifndef FP_ZERO
+ #define FP_ZERO 1
+#endif
+#ifndef FP_NAN
+ #define FP_NAN 2
+#endif
+#ifndef FP_INFINITE
+ #define FP_INFINITE 3
+#endif
+#ifndef FP_NORMAL
+ #define FP_NORMAL 4
+#endif
+
+
+/// Main namespace for half precision functionality.
+/// This namespace contains all the functionality provided by the library.
+namespace half_float
+{
+ class half;
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ /// Library-defined half-precision literals.
+ /// Import this namespace to enable half-precision floating point literals:
+ /// ~~~~{.cpp}
+ /// using namespace half_float::literal;
+ /// half_float::half = 4.2_h;
+ /// ~~~~
+ namespace literal
+ {
+ half operator"" _h(long double);
+ }
+#endif
+
+ /// \internal
+ /// \brief Implementation details.
+ namespace detail
+ {
+ #if HALF_ENABLE_CPP11_TYPE_TRAITS
+ /// Conditional type.
+ template<bool B,typename T,typename F> struct conditional : std::conditional<B,T,F> {};
+
+ /// Helper for tag dispatching.
+ template<bool B> struct bool_type : std::integral_constant<bool,B> {};
+ using std::true_type;
+ using std::false_type;
+
+ /// Type traits for floating point types.
+ template<typename T> struct is_float : std::is_floating_point<T> {};
+ #else
+ /// Conditional type.
+ template<bool,typename T,typename> struct conditional { typedef T type; };
+ template<typename T,typename F> struct conditional<false,T,F> { typedef F type; };
+
+ /// Helper for tag dispatching.
+ template<bool> struct bool_type {};
+ typedef bool_type<true> true_type;
+ typedef bool_type<false> false_type;
+
+ /// Type traits for floating point types.
+ template<typename> struct is_float : false_type {};
+ template<typename T> struct is_float<const T> : is_float<T> {};
+ template<typename T> struct is_float<volatile T> : is_float<T> {};
+ template<typename T> struct is_float<const volatile T> : is_float<T> {};
+ template<> struct is_float<float> : true_type {};
+ template<> struct is_float<double> : true_type {};
+ template<> struct is_float<long double> : true_type {};
+ #endif
+
+ /// Type traits for floating point bits.
+ template<typename T> struct bits { typedef unsigned char type; };
+ template<typename T> struct bits<const T> : bits<T> {};
+ template<typename T> struct bits<volatile T> : bits<T> {};
+ template<typename T> struct bits<const volatile T> : bits<T> {};
+
+ #if HALF_ENABLE_CPP11_CSTDINT
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef std::uint_least16_t uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> { typedef std::uint_least32_t type; };
+
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef std::uint_least64_t type; };
+ #else
+ /// Unsigned integer of (at least) 16 bits width.
+ typedef unsigned short uint16;
+
+ /// Unsigned integer of (at least) 32 bits width.
+ template<> struct bits<float> : conditional<std::numeric_limits<unsigned int>::digits>=32,unsigned int,unsigned long> {};
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> : conditional<std::numeric_limits<unsigned long>::digits>=64,unsigned long,unsigned long long> {};
+ #else
+ /// Unsigned integer of (at least) 64 bits width.
+ template<> struct bits<double> { typedef unsigned long type; };
+ #endif
+ #endif
+
+ /// Tag type for binary construction.
+ struct binary_t {};
+
+ /// Tag for binary construction.
+ HALF_CONSTEXPR_CONST binary_t binary = binary_t();
+
+ /// Temporary half-precision expression.
+ /// This class represents a half-precision expression which just stores a single-precision value internally.
+ struct expr
+ {
+ /// Conversion constructor.
+ /// \param f single-precision value to convert
+ explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; }
+
+ private:
+ /// Internal expression value stored in single-precision.
+ float value_;
+ };
+
+ /// SFINAE helper for generic half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member equivalent to \a T.
+ /// \tparam T type to return
+ template<typename T,typename,typename=void,typename=void> struct enable {};
+ template<typename T> struct enable<T,half,void,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,void,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,void> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,void> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,void> { typedef T type; };
+ template<typename T> struct enable<T,half,half,half> { typedef T type; };
+ template<typename T> struct enable<T,half,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,half,expr,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,half,expr> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,half> { typedef T type; };
+ template<typename T> struct enable<T,expr,expr,expr> { typedef T type; };
+
+ /// Return type for specialized generic 2-argument half-precision functions.
+ /// This class template has to be specialized for each valid combination of argument types to provide a corresponding
+ /// `type` member denoting the appropriate return type.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct result : enable<expr,T,U> {};
+ template<> struct result<half,half> { typedef half type; };
+
+ /// \name Classification helpers
+ /// \{
+
+ /// Check for infinity.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if infinity
+ /// \retval false else
+ template<typename T> bool builtin_isinf(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isinf(arg);
+ #elif defined(_MSC_VER)
+ return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
+ #else
+ return arg == std::numeric_limits<T>::infinity() || arg == -std::numeric_limits<T>::infinity();
+ #endif
+ }
+
+ /// Check for NaN.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if not a number
+ /// \retval false else
+ template<typename T> bool builtin_isnan(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::isnan(arg);
+ #elif defined(_MSC_VER)
+ return ::_isnan(static_cast<double>(arg)) != 0;
+ #else
+ return arg != arg;
+ #endif
+ }
+
+ /// Check sign.
+ /// \tparam T argument type (builtin floating point type)
+ /// \param arg value to query
+ /// \retval true if signbit set
+ /// \retval false else
+ template<typename T> bool builtin_signbit(T arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return std::signbit(arg);
+ #else
+ return arg < T() || (arg == T() && T(1)/arg < T());
+ #endif
+ }
+
+ /// \}
+ /// \name Conversion
+ /// \{
+
+ /// Convert IEEE single-precision to half-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value single-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(float value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ uint32 bits;// = *reinterpret_cast<uint32*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(float));
+/* uint16 hbits = (bits>>16) & 0x8000;
+ bits &= 0x7FFFFFFF;
+ int exp = bits >> 23;
+ if(exp == 255)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0x7FFFFF)!=0));
+ if(exp > 142)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s;
+ if(exp > 112)
+ {
+ g = (bits>>12) & 1;
+ s = (bits&0xFFF) != 0;
+ hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF);
+ }
+ else if(exp > 101)
+ {
+ int i = 125 - exp;
+ bits = (bits&0x7FFFFF) | 0x800000;
+ g = (bits>>i) & 1;
+ s = (bits&((1L<<i)-1)) != 0;
+ hbits |= bits >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s = bits != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+*/ static const uint16 base_table[512] = {
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100,
+ 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00,
+ 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100,
+ 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00,
+ 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00,
+ 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 };
+ static const unsigned char shift_table[512] = {
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
+ 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 };
+ uint16 hbits = base_table[bits>>23] + static_cast<uint16>((bits&0x7FFFFF)>>shift_table[bits>>23]);
+ if(R == std::round_to_nearest)
+ hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00)
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((static_cast<uint32>(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits)
+ #endif
+ ;
+ else if(R == std::round_toward_zero)
+ hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23];
+ else if(R == std::round_toward_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)&
+ ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511));
+ else if(R == std::round_toward_neg_infinity)
+ hbits += ((((bits&0x7FFFFF&((static_cast<uint32>(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)&
+ ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255));
+ return hbits;
+ }
+
+ /// Convert IEEE double-precision to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value double-precision value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R> uint16 float2half_impl(double value, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint64 bits;// = *reinterpret_cast<uint64*>(&value); //violating strict aliasing!
+ std::memcpy(&bits, &value, sizeof(double));
+ uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF;
+ uint16 hbits = (hi>>16) & 0x8000;
+ hi &= 0x7FFFFFFF;
+ int exp = hi >> 20;
+ if(exp == 2047)
+ return hbits | 0x7C00 | (0x3FF&-static_cast<unsigned>((bits&0xFFFFFFFFFFFFF)!=0));
+ if(exp > 1038)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ int g, s = lo != 0;
+ if(exp > 1008)
+ {
+ g = (hi>>9) & 1;
+ s |= (hi&0x1FF) != 0;
+ hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF);
+ }
+ else if(exp > 997)
+ {
+ int i = 1018 - exp;
+ hi = (hi&0xFFFFF) | 0x100000;
+ g = (hi>>i) & 1;
+ s |= (hi&((1L<<i)-1)) != 0;
+ hbits |= hi >> (i+1);
+ }
+ else
+ {
+ g = 0;
+ s |= hi != 0;
+ }
+ if(R == std::round_to_nearest)
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += g & (s|hbits);
+ #else
+ hbits += g;
+ #endif
+ else if(R == std::round_toward_infinity)
+ hbits += ~(hbits>>15) & (s|g);
+ else if(R == std::round_toward_neg_infinity)
+ hbits += (hbits>>15) & (g|s);
+ return hbits;
+ }
+
+ /// Convert non-IEEE floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half_impl(T value, ...)
+ {
+ uint16 hbits = static_cast<unsigned>(builtin_signbit(value)) << 15;
+ if(value == T())
+ return hbits;
+ if(builtin_isnan(value))
+ return hbits | 0x7FFF;
+ if(builtin_isinf(value))
+ return hbits | 0x7C00;
+ int exp;
+ std::frexp(value, &exp);
+ if(exp > 16)
+ {
+ if(R == std::round_toward_infinity)
+ return hbits | 0x7C00 - (hbits>>15);
+ else if(R == std::round_toward_neg_infinity)
+ return hbits | 0x7BFF + (hbits>>15);
+ return hbits | 0x7BFF + (R!=std::round_toward_zero);
+ }
+ if(exp < -13)
+ value = std::ldexp(value, 24);
+ else
+ {
+ value = std::ldexp(value, 11-exp);
+ hbits |= ((exp+13)<<10);
+ }
+ T ival, frac = std::modf(value, &ival);
+ hbits += static_cast<uint16>(std::abs(static_cast<int>(ival)));
+ if(R == std::round_to_nearest)
+ {
+ frac = std::abs(frac);
+ #if HALF_ROUND_TIES_TO_EVEN
+ hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits);
+ #else
+ hbits += frac >= T(0.5);
+ #endif
+ }
+ else if(R == std::round_toward_infinity)
+ hbits += frac > T();
+ else if(R == std::round_toward_neg_infinity)
+ hbits += frac < T();
+ return hbits;
+ }
+
+ /// Convert floating point to half-precision.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T source type (builtin floating point type)
+ /// \param value floating point value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 float2half(T value)
+ {
+ return float2half_impl<R>(value, bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam S `true` if value negative, `false` else
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value non-negative integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,bool S,typename T> uint16 int2half_impl(T value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "int to half conversion only supports builtin integer types");
+ #endif
+ if(S)
+ value = -value;
+ uint16 bits = S << 15;
+ if(value > 0xFFFF)
+ {
+ if(R == std::round_toward_infinity)
+ bits |= 0x7C00 - S;
+ else if(R == std::round_toward_neg_infinity)
+ bits |= 0x7BFF + S;
+ else
+ bits |= 0x7BFF + (R!=std::round_toward_zero);
+ }
+ else if(value)
+ {
+ unsigned int m = value, exp = 24;
+ for(; m<0x400; m<<=1,--exp) ;
+ for(; m>0x7FF; m>>=1,++exp) ;
+ bits |= (exp<<10) + m;
+ if(exp > 24)
+ {
+ if(R == std::round_to_nearest)
+ bits += (value>>(exp-25)) & 1
+ #if HALF_ROUND_TIES_TO_EVEN
+ & (((((1<<(exp-25))-1)&value)!=0)|bits)
+ #endif
+ ;
+ else if(R == std::round_toward_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & !S;
+ else if(R == std::round_toward_neg_infinity)
+ bits += ((value&((1<<(exp-24))-1))!=0) & S;
+ }
+ }
+ return bits;
+ }
+
+ /// Convert integer to half-precision floating point.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert (builtin integer type)
+ /// \param value integral value
+ /// \return binary representation of half-precision value
+ template<std::float_round_style R,typename T> uint16 int2half(T value)
+ {
+ return (value<0) ? int2half_impl<R,true>(value) : int2half_impl<R,false>(value);
+ }
+
+ /// Convert half-precision to IEEE single-precision.
+ /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
+ /// \param value binary representation of half-precision value
+ /// \return single-precision value
+ inline float half2float_impl(uint16 value, float, true_type)
+ {
+ typedef bits<float>::type uint32;
+/* uint32 bits = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ bits |= 0x38000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,bits-=0x800000) ;
+ bits += static_cast<uint32>(abs) << 13;
+ }
+*/ static const uint32 mantissa_table[2048] = {
+ 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000,
+ 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000,
+ 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000,
+ 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000,
+ 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000,
+ 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000,
+ 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000,
+ 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000,
+ 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000,
+ 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000,
+ 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000,
+ 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000,
+ 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000,
+ 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000,
+ 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000,
+ 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000,
+ 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000,
+ 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000,
+ 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000,
+ 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000,
+ 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000,
+ 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000,
+ 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000,
+ 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000,
+ 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000,
+ 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000,
+ 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000,
+ 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000,
+ 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000,
+ 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000,
+ 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000,
+ 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000,
+ 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000,
+ 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000,
+ 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000,
+ 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000,
+ 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000,
+ 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000,
+ 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000,
+ 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000,
+ 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000,
+ 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000,
+ 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000,
+ 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000,
+ 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000,
+ 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000,
+ 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000,
+ 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000,
+ 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000,
+ 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000,
+ 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000,
+ 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000,
+ 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000,
+ 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000,
+ 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000,
+ 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000,
+ 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000,
+ 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000,
+ 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000,
+ 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000,
+ 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000,
+ 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000,
+ 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000,
+ 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000,
+ 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000,
+ 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000,
+ 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000,
+ 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000,
+ 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000,
+ 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000,
+ 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000,
+ 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000,
+ 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000,
+ 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000,
+ 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000,
+ 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000,
+ 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000,
+ 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000,
+ 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000,
+ 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000,
+ 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000,
+ 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000,
+ 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000,
+ 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000,
+ 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000,
+ 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000,
+ 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000,
+ 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000,
+ 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000,
+ 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000,
+ 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000,
+ 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000,
+ 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000,
+ 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000,
+ 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000,
+ 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000,
+ 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000,
+ 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000,
+ 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000,
+ 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000,
+ 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000,
+ 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000,
+ 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000,
+ 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000,
+ 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000,
+ 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000,
+ 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000,
+ 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000,
+ 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000,
+ 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000,
+ 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000,
+ 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000,
+ 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000,
+ 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000,
+ 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000,
+ 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000,
+ 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000,
+ 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000,
+ 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000,
+ 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000,
+ 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000,
+ 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000,
+ 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000,
+ 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000,
+ 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000,
+ 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000,
+ 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000,
+ 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 };
+ static const uint32 exponent_table[64] = {
+ 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000,
+ 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000,
+ 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000,
+ 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 };
+ static const unsigned short offset_table[64] = {
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
+ 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 };
+ uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];
+// return *reinterpret_cast<float*>(&bits); //violating strict aliasing!
+ float out;
+ std::memcpy(&out, &bits, sizeof(float));
+ return out;
+ }
+
+ /// Convert half-precision to IEEE double-precision.
+ /// \param value binary representation of half-precision value
+ /// \return double-precision value
+ inline double half2float_impl(uint16 value, double, true_type)
+ {
+ typedef bits<float>::type uint32;
+ typedef bits<double>::type uint64;
+ uint32 hi = static_cast<uint32>(value&0x8000) << 16;
+ int abs = value & 0x7FFF;
+ if(abs)
+ {
+ hi |= 0x3F000000 << static_cast<unsigned>(abs>=0x7C00);
+ for(; abs<0x400; abs<<=1,hi-=0x100000) ;
+ hi += static_cast<uint32>(abs) << 10;
+ }
+ uint64 bits = static_cast<uint64>(hi) << 32;
+// return *reinterpret_cast<double*>(&bits); //violating strict aliasing!
+ double out;
+ std::memcpy(&out, &bits, sizeof(double));
+ return out;
+ }
+
+ /// Convert half-precision to non-IEEE floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float_impl(uint16 value, T, ...)
+ {
+ T out;
+ int abs = value & 0x7FFF;
+ if(abs > 0x7C00)
+ out = std::numeric_limits<T>::has_quiet_NaN ? std::numeric_limits<T>::quiet_NaN() : T();
+ else if(abs == 0x7C00)
+ out = std::numeric_limits<T>::has_infinity ? std::numeric_limits<T>::infinity() : std::numeric_limits<T>::max();
+ else if(abs > 0x3FF)
+ out = std::ldexp(static_cast<T>((abs&0x3FF)|0x400), (abs>>10)-25);
+ else
+ out = std::ldexp(static_cast<T>(abs), -24);
+ return (value&0x8000) ? -out : out;
+ }
+
+ /// Convert half-precision to floating point.
+ /// \tparam T type to convert to (builtin integer type)
+ /// \param value binary representation of half-precision value
+ /// \return floating point value
+ template<typename T> T half2float(uint16 value)
+ {
+ return half2float_impl(value, T(), bool_type<std::numeric_limits<T>::is_iec559&&sizeof(typename bits<T>::type)==sizeof(T)>());
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,bool E,typename T> T half2int_impl(uint16 value)
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_integral<T>::value, "half to int conversion only supports builtin integer types");
+ #endif
+ unsigned int e = value & 0x7FFF;
+ if(e >= 0x7C00)
+ return (value&0x8000) ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
+ if(e < 0x3800)
+ {
+ if(R == std::round_toward_infinity)
+ return T(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ return -T(value>0x8000);
+ return T();
+ }
+ unsigned int m = (value&0x3FF) | 0x400;
+ e >>= 10;
+ if(e < 25)
+ {
+ if(R == std::round_to_nearest)
+ m += (1<<(24-e)) - (~(m>>(25-e))&E);
+ else if(R == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(25-e))-1U);
+ else if(R == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(25-e))-1U);
+ m >>= 25 - e;
+ }
+ else
+ m <<= e - 25;
+ return (value&0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+ }
+
+ /// Convert half-precision floating point to integer.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<std::float_round_style R,typename T> T half2int(uint16 value) { return half2int_impl<R,HALF_ROUND_TIES_TO_EVEN,T>(value); }
+
+ /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero.
+ /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits)
+ /// \param value binary representation of half-precision value
+ /// \return integral value
+ template<typename T> T half2int_up(uint16 value) { return half2int_impl<std::round_to_nearest,0,T>(value); }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \tparam E `true` for round to even, `false` for round away from zero
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R,bool E> uint16 round_half_impl(uint16 value)
+ {
+ unsigned int e = value & 0x7FFF;
+ uint16 result = value;
+ if(e < 0x3C00)
+ {
+ result &= 0x8000;
+ if(R == std::round_to_nearest)
+ result |= 0x3C00U & -(e>=(0x3800+E));
+ else if(R == std::round_toward_infinity)
+ result |= 0x3C00U & -(~(value>>15)&(e!=0));
+ else if(R == std::round_toward_neg_infinity)
+ result |= 0x3C00U & -(value>0x8000);
+ }
+ else if(e < 0x6400)
+ {
+ e = 25 - (e>>10);
+ unsigned int mask = (1<<e) - 1;
+ if(R == std::round_to_nearest)
+ result += (1<<(e-1)) - (~(result>>e)&E);
+ else if(R == std::round_toward_infinity)
+ result += mask & ((value>>15)-1);
+ else if(R == std::round_toward_neg_infinity)
+ result += mask & -(value>>15);
+ result &= ~mask;
+ }
+ return result;
+ }
+
+ /// Round half-precision number to nearest integer value.
+ /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ template<std::float_round_style R> uint16 round_half(uint16 value) { return round_half_impl<R,HALF_ROUND_TIES_TO_EVEN>(value); }
+
+ /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero.
+ /// \param value binary representation of half-precision value
+ /// \return half-precision bits for nearest integral value
+ inline uint16 round_half_up(uint16 value) { return round_half_impl<std::round_to_nearest,0>(value); }
+ /// \}
+
+ struct functions;
+ template<typename> struct unary_specialized;
+ template<typename,typename> struct binary_specialized;
+ template<typename,typename,std::float_round_style> struct half_caster;
+ }
+
+ /// Half-precision floating point type.
+ /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and
+ /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and
+ /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations
+ /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to
+ /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic
+ /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type).
+ ///
+ /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and
+ /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which
+ /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the
+ /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of
+ /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most
+ /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit
+ /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if
+ /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on
+ /// nearly any reasonable platform.
+ ///
+ /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable
+ /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation.
+ class half
+ {
+ friend struct detail::functions;
+ friend struct detail::unary_specialized<half>;
+ friend struct detail::binary_specialized<half,half>;
+ template<typename,typename,std::float_round_style> friend struct detail::half_caster;
+ friend class std::numeric_limits<half>;
+ #if HALF_ENABLE_CPP11_HASH
+ friend struct std::hash<half>;
+ #endif
+ #if HALF_ENABLE_CPP11_USER_LITERALS
+ friend half literal::operator"" _h(long double);
+ #endif
+
+ public:
+ /// Default constructor.
+ /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics
+ /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics.
+ HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {}
+
+ /// Copy constructor.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ half(detail::expr rhs) : data_(detail::float2half<round_style>(static_cast<float>(rhs))) {}
+
+ /// Conversion constructor.
+ /// \param rhs float to convert
+ explicit half(float rhs) : data_(detail::float2half<round_style>(rhs)) {}
+
+ /// Conversion to single-precision.
+ /// \return single precision value representing expression value
+ operator float() const { return detail::half2float<float>(data_); }
+
+ /// Assignment operator.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to copy from
+ /// \return reference to this half
+ half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to add
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator+=(T rhs) { return *this += static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to subtract
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator-=(T rhs) { return *this -= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to multiply with
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator*=(T rhs) { return *this *= static_cast<float>(rhs); }
+
+ /// Arithmetic assignment.
+ /// \tparam T type of concrete half expression
+ /// \param rhs half expression to divide by
+ /// \return reference to this half
+ template<typename T> typename detail::enable<half&,T>::type operator/=(T rhs) { return *this /= static_cast<float>(rhs); }
+
+ /// Assignment operator.
+ /// \param rhs single-precision value to copy from
+ /// \return reference to this half
+ half& operator=(float rhs) { data_ = detail::float2half<round_style>(rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to add
+ /// \return reference to this half
+ half& operator+=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)+rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to subtract
+ /// \return reference to this half
+ half& operator-=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)-rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to multiply with
+ /// \return reference to this half
+ half& operator*=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)*rhs); return *this; }
+
+ /// Arithmetic assignment.
+ /// \param rhs single-precision value to divide by
+ /// \return reference to this half
+ half& operator/=(float rhs) { data_ = detail::float2half<round_style>(detail::half2float<float>(data_)/rhs); return *this; }
+
+ /// Prefix increment.
+ /// \return incremented half value
+ half& operator++() { return *this += 1.0f; }
+
+ /// Prefix decrement.
+ /// \return decremented half value
+ half& operator--() { return *this -= 1.0f; }
+
+ /// Postfix increment.
+ /// \return non-incremented half value
+ half operator++(int) { half out(*this); ++*this; return out; }
+
+ /// Postfix decrement.
+ /// \return non-decremented half value
+ half operator--(int) { half out(*this); --*this; return out; }
+
+ private:
+ /// Rounding mode to use
+ static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE);
+
+ /// Constructor.
+ /// \param bits binary representation to set half to
+ HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {}
+
+ /// Internal binary representation
+ detail::uint16 data_;
+ };
+
+#if HALF_ENABLE_CPP11_USER_LITERALS
+ namespace literal
+ {
+ /// Half literal.
+ /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due
+ /// to rather involved conversions.
+ /// \param value literal value
+ /// \return half with given value (if representable)
+ inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half<half::round_style>(value)); }
+ }
+#endif
+
+ namespace detail
+ {
+ /// Wrapper implementing unspecialized half-precision functions.
+ struct functions
+ {
+ /// Addition implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision sum stored in single-precision
+ static expr plus(float x, float y) { return expr(x+y); }
+
+ /// Subtraction implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision difference stored in single-precision
+ static expr minus(float x, float y) { return expr(x-y); }
+
+ /// Multiplication implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision product stored in single-precision
+ static expr multiplies(float x, float y) { return expr(x*y); }
+
+ /// Division implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision quotient stored in single-precision
+ static expr divides(float x, float y) { return expr(x/y); }
+
+ /// Output implementation.
+ /// \param out stream to write to
+ /// \param arg value to write
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_ostream<charT,traits>& write(std::basic_ostream<charT,traits> &out, float arg) { return out << arg; }
+
+ /// Input implementation.
+ /// \param in stream to read from
+ /// \param arg half to read into
+ /// \return reference to stream
+ template<typename charT,typename traits> static std::basic_istream<charT,traits>& read(std::basic_istream<charT,traits> &in, half &arg)
+ {
+ float f;
+ if(in >> f)
+ arg = f;
+ return in;
+ }
+
+ /// Modulo implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr fmod(float x, float y) { return expr(std::fmod(x, y)); }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remainder(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remainder(x, y));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return expr(builtin_signbit(x) ? -0.0f : 0.0f);
+ ax = std::fmod(ax, ay+ay);
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ if(ax >= y2)
+ ax -= ay;
+ }
+ return expr(builtin_signbit(x) ? -ax : ax);
+ #endif
+ }
+
+ /// Remainder implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store quotient bits at
+ /// \return Half-precision division remainder stored in single-precision
+ static expr remquo(float x, float y, int *quo)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::remquo(x, y, quo));
+ #else
+ if(builtin_isnan(x) || builtin_isnan(y))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ bool sign = builtin_signbit(x), qsign = static_cast<bool>(sign^builtin_signbit(y));
+ float ax = std::fabs(x), ay = std::fabs(y);
+ if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24))
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ if(ay >= 65536.0f)
+ return expr(x);
+ if(ax == ay)
+ return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f);
+ ax = std::fmod(ax, 8.0f*ay);
+ int cquo = 0;
+ if(ax >= 4.0f * ay)
+ {
+ ax -= 4.0f * ay;
+ cquo += 4;
+ }
+ if(ax >= 2.0f * ay)
+ {
+ ax -= 2.0f * ay;
+ cquo += 2;
+ }
+ float y2 = 0.5f * ay;
+ if(ax > y2)
+ {
+ ax -= ay;
+ ++cquo;
+ if(ax >= y2)
+ {
+ ax -= ay;
+ ++cquo;
+ }
+ }
+ return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax);
+ #endif
+ }
+
+ /// Positive difference implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return Positive difference stored in single-precision
+ static expr fdim(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fdim(x, y));
+ #else
+ return expr((x<=y) ? 0.0f : (x-y));
+ #endif
+ }
+
+ /// Fused multiply-add implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return \a x * \a y + \a z stored in single-precision
+ static expr fma(float x, float y, float z)
+ {
+ #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF)
+ return expr(std::fma(x, y, z));
+ #else
+ return expr(x*y+z);
+ #endif
+ }
+
+ /// Get NaN.
+ /// \return Half-precision quiet NaN
+ static half nanh() { return half(binary, 0x7FFF); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp(float arg) { return expr(std::exp(arg)); }
+
+ /// Exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr expm1(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::expm1(arg));
+ #else
+ return expr(static_cast<float>(std::exp(static_cast<double>(arg))-1.0));
+ #endif
+ }
+
+ /// Binary exponential implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr exp2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::exp2(arg));
+ #else
+ return expr(static_cast<float>(std::exp(arg*0.69314718055994530941723212145818)));
+ #endif
+ }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log(float arg) { return expr(std::log(arg)); }
+
+ /// Common logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log10(float arg) { return expr(std::log10(arg)); }
+
+ /// Logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log1p(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log1p(arg));
+ #else
+ return expr(static_cast<float>(std::log(1.0+arg)));
+ #endif
+ }
+
+ /// Binary logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr log2(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::log2(arg));
+ #else
+ return expr(static_cast<float>(std::log(static_cast<double>(arg))*1.4426950408889634073599246810019));
+ #endif
+ }
+
+ /// Square root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sqrt(float arg) { return expr(std::sqrt(arg)); }
+
+ /// Cubic root implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cbrt(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::cbrt(arg));
+ #else
+ if(builtin_isnan(arg) || builtin_isinf(arg))
+ return expr(arg);
+ return expr(builtin_signbit(arg) ? -static_cast<float>(std::pow(-static_cast<double>(arg), 1.0/3.0)) :
+ static_cast<float>(std::pow(static_cast<double>(arg), 1.0/3.0)));
+ #endif
+ }
+
+ /// Hypotenuse implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr hypot(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::hypot(x, y));
+ #else
+ return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits<float>::infinity() :
+ static_cast<float>(std::sqrt(static_cast<double>(x)*x+static_cast<double>(y)*y)));
+ #endif
+ }
+
+ /// Power implementation.
+ /// \param base value to exponentiate
+ /// \param exp power to expontiate to
+ /// \return function value stored in single-preicision
+ static expr pow(float base, float exp) { return expr(std::pow(base, exp)); }
+
+ /// Sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sin(float arg) { return expr(std::sin(arg)); }
+
+ /// Cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cos(float arg) { return expr(std::cos(arg)); }
+
+ /// Tan implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tan(float arg) { return expr(std::tan(arg)); }
+
+ /// Arc sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asin(float arg) { return expr(std::asin(arg)); }
+
+ /// Arc cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acos(float arg) { return expr(std::acos(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atan(float arg) { return expr(std::atan(arg)); }
+
+ /// Arc tangent implementation.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return function value stored in single-preicision
+ static expr atan2(float x, float y) { return expr(std::atan2(x, y)); }
+
+ /// Hyperbolic sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr sinh(float arg) { return expr(std::sinh(arg)); }
+
+ /// Hyperbolic cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr cosh(float arg) { return expr(std::cosh(arg)); }
+
+ /// Hyperbolic tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tanh(float arg) { return expr(std::tanh(arg)); }
+
+ /// Hyperbolic area sine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr asinh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::asinh(arg));
+ #else
+ return expr((arg==-std::numeric_limits<float>::infinity()) ? arg : static_cast<float>(std::log(arg+std::sqrt(arg*arg+1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area cosine implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr acosh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::acosh(arg));
+ #else
+ return expr((arg<-1.0f) ? std::numeric_limits<float>::quiet_NaN() : static_cast<float>(std::log(arg+std::sqrt(arg*arg-1.0))));
+ #endif
+ }
+
+ /// Hyperbolic area tangent implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr atanh(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::atanh(arg));
+ #else
+ return expr(static_cast<float>(0.5*std::log((1.0+arg)/(1.0-arg))));
+ #endif
+ }
+
+ /// Error function implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erf(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erf(arg));
+ #else
+ return expr(static_cast<float>(erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Complementary implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr erfc(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::erfc(arg));
+ #else
+ return expr(static_cast<float>(1.0-erf(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma logarithm implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr lgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::lgamma(arg));
+ #else
+ if(builtin_isinf(arg))
+ return expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::infinity());
+ return expr(static_cast<float>(1.1447298858494001741434273513531-
+ std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg)));
+ }
+ return expr(static_cast<float>(lgamma(static_cast<double>(arg))));
+ #endif
+ }
+
+ /// Gamma implementation.
+ /// \param arg function argument
+ /// \return function value stored in single-preicision
+ static expr tgamma(float arg)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::tgamma(arg));
+ #else
+ if(arg == 0.0f)
+ return builtin_signbit(arg) ? expr(-std::numeric_limits<float>::infinity()) : expr(std::numeric_limits<float>::infinity());
+ if(arg < 0.0f)
+ {
+ float i, f = std::modf(-arg, &i);
+ if(f == 0.0f)
+ return expr(std::numeric_limits<float>::quiet_NaN());
+ double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg)));
+ return expr(static_cast<float>((std::fmod(i, 2.0f)==0.0f) ? -value : value));
+ }
+ if(builtin_isinf(arg))
+ return expr(arg);
+ return expr(static_cast<float>(std::exp(lgamma(static_cast<double>(arg)))));
+ #endif
+ }
+
+ /// Floor implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half floor(half arg) { return half(binary, round_half<std::round_toward_neg_infinity>(arg.data_)); }
+
+ /// Ceiling implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half ceil(half arg) { return half(binary, round_half<std::round_toward_infinity>(arg.data_)); }
+
+ /// Truncation implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half trunc(half arg) { return half(binary, round_half<std::round_toward_zero>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half round(half arg) { return half(binary, round_half_up(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lround(half arg) { return detail::half2int_up<long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static half rint(half arg) { return half(binary, round_half<half::round_style>(arg.data_)); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long lrint(half arg) { return detail::half2int<half::round_style,long>(arg.data_); }
+
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llround(half arg) { return detail::half2int_up<long long>(arg.data_); }
+
+ /// Nearest integer implementation.
+ /// \param arg value to round
+ /// \return rounded value
+ static long long llrint(half arg) { return detail::half2int<half::round_style,long long>(arg.data_); }
+ #endif
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return normalized significant
+ static half frexp(half arg, int *exp)
+ {
+ int m = arg.data_ & 0x7FFF, e = -14;
+ if(m >= 0x7C00 || !m)
+ return *exp = 0, arg;
+ for(; m<0x400; m<<=1,--e) ;
+ return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF));
+ }
+
+ /// Decompression implementation.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+ static half modf(half arg, half *iptr)
+ {
+ unsigned int e = arg.data_ & 0x7FFF;
+ if(e >= 0x6400)
+ return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00)));
+ if(e < 0x3C00)
+ return iptr->data_ = arg.data_ & 0x8000, arg;
+ e >>= 10;
+ unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask;
+ iptr->data_ = arg.data_ & ~mask;
+ if(!m)
+ return half(binary, arg.data_&0x8000);
+ for(; m<0x400; m<<=1,--e) ;
+ return half(binary, static_cast<uint16>((arg.data_&0x8000)|(e<<10)|(m&0x3FF)));
+ }
+
+ /// Scaling implementation.
+ /// \param arg number to scale
+ /// \param exp power of two to scale by
+ /// \return scaled number
+ static half scalbln(half arg, long exp)
+ {
+ unsigned int m = arg.data_ & 0x7FFF;
+ if(m >= 0x7C00 || !m)
+ return arg;
+ for(; m<0x400; m<<=1,--exp) ;
+ exp += m >> 10;
+ uint16 value = arg.data_ & 0x8000;
+ if(exp > 30)
+ {
+ if(half::round_style == std::round_toward_zero)
+ value |= 0x7BFF;
+ else if(half::round_style == std::round_toward_infinity)
+ value |= 0x7C00 - (value>>15);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value |= 0x7BFF + (value>>15);
+ else
+ value |= 0x7C00;
+ }
+ else if(exp > 0)
+ value |= (exp<<10) | (m&0x3FF);
+ else if(exp > -11)
+ {
+ m = (m&0x3FF) | 0x400;
+ if(half::round_style == std::round_to_nearest)
+ {
+ m += 1 << -exp;
+ #if HALF_ROUND_TIES_TO_EVEN
+ m -= (m>>(1-exp)) & 1;
+ #endif
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ m += ((value>>15)-1) & ((1<<(1-exp))-1U);
+ else if(half::round_style == std::round_toward_neg_infinity)
+ m += -(value>>15) & ((1<<(1-exp))-1U);
+ value |= m >> (1-exp);
+ }
+ else if(half::round_style == std::round_toward_infinity)
+ value -= (value>>15) - 1;
+ else if(half::round_style == std::round_toward_neg_infinity)
+ value += value >> 15;
+ return half(binary, value);
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static int ilogb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return FP_ILOGB0;
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ return exp;
+ }
+ if(abs > 0x7C00)
+ return FP_ILOGBNAN;
+ return INT_MAX;
+ }
+
+ /// Exponent implementation.
+ /// \param arg number to query
+ /// \return floating point exponent
+ static half logb(half arg)
+ {
+ int abs = arg.data_ & 0x7FFF;
+ if(!abs)
+ return half(binary, 0xFC00);
+ if(abs < 0x7C00)
+ {
+ int exp = (abs>>10) - 15;
+ if(abs < 0x400)
+ for(; abs<0x200; abs<<=1,--exp) ;
+ uint16 bits = (exp<0) << 15;
+ if(exp)
+ {
+ unsigned int m = std::abs(exp) << 6, e = 18;
+ for(; m<0x400; m<<=1,--e) ;
+ bits |= (e<<10) + m;
+ }
+ return half(binary, bits);
+ }
+ if(abs > 0x7C00)
+ return arg;
+ return half(binary, 0x7C00);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nextafter(half from, half to)
+ {
+ uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF;
+ if(fabs > 0x7C00)
+ return from;
+ if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs))
+ return to;
+ if(!fabs)
+ return half(binary, (to.data_&0x8000)+1);
+ bool lt = ((fabs==from.data_) ? static_cast<int>(fabs) : -static_cast<int>(fabs)) <
+ ((tabs==to.data_) ? static_cast<int>(tabs) : -static_cast<int>(tabs));
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lt))<<1)-1);
+ }
+
+ /// Enumeration implementation.
+ /// \param from number to increase/decrease
+ /// \param to direction to enumerate into
+ /// \return next representable number
+ static half nexttoward(half from, long double to)
+ {
+ if(isnan(from))
+ return from;
+ long double lfrom = static_cast<long double>(from);
+ if(builtin_isnan(to) || lfrom == to)
+ return half(static_cast<float>(to));
+ if(!(from.data_&0x7FFF))
+ return half(binary, (static_cast<detail::uint16>(builtin_signbit(to))<<15)+1);
+ return half(binary, from.data_+(((from.data_>>15)^static_cast<unsigned>(lfrom<to))<<1)-1);
+ }
+
+ /// Sign implementation
+ /// \param x first operand
+ /// \param y second operand
+ /// \return composed value
+ static half copysign(half x, half y) { return half(binary, x.data_^((x.data_^y.data_)&0x8000)); }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static int fpclassify(half arg)
+ {
+ unsigned int abs = arg.data_ & 0x7FFF;
+ return abs ? ((abs>0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO;
+ }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if finite number
+ /// \retval false else
+ static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if infinite number
+ /// \retval false else
+ static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if not a number
+ /// \retval false else
+ static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; }
+
+ /// Classification implementation.
+ /// \param arg value to classify
+ /// \retval true if normal number
+ /// \retval false else
+ static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); }
+
+ /// Sign bit implementation.
+ /// \param arg value to check
+ /// \retval true if signed
+ /// \retval false if unsigned
+ static bool signbit(half arg) { return (arg.data_&0x8000) != 0; }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x > \a y
+ /// \retval false else
+ static bool isgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x >= \a y
+ /// \retval false else
+ static bool isgreaterequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x < \a y
+ /// \retval false else
+ static bool isless(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x <= \a y
+ /// \retval false else
+ static bool islessequal(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs));
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either \a x > \a y nor \a x < \a y
+ /// \retval false else
+ static bool islessgreater(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00 || yabs > 0x7C00)
+ return false;
+ int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs;
+ return a < b || a > b;
+ }
+
+ /// Comparison implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operand unordered
+ /// \retval false else
+ static bool isunordered(half x, half y) { return isnan(x) || isnan(y); }
+
+ private:
+ static double erf(double arg)
+ {
+ if(builtin_isinf(arg))
+ return (arg<0.0) ? -1.0 : 1.0;
+ double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2)));
+ return builtin_signbit(arg) ? -value : value;
+ }
+
+ static double lgamma(double arg)
+ {
+ double v = 1.0;
+ for(; arg<8.0; ++arg) v *= arg;
+ double w = 1.0 / (arg*arg);
+ return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+
+ -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+
+ -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+
+ -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg +
+ 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg);
+ }
+ };
+
+ /// Wrapper for unary half-precision functions needing specialization for individual argument types.
+ /// \tparam T argument type
+ template<typename T> struct unary_specialized
+ {
+ /// Negation implementation.
+ /// \param arg value to negate
+ /// \return negated value
+ static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); }
+
+ /// Absolute value implementation.
+ /// \param arg function argument
+ /// \return absolute value
+ static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); }
+ };
+ template<> struct unary_specialized<expr>
+ {
+ static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); }
+ static expr fabs(float arg) { return expr(std::fabs(arg)); }
+ };
+
+ /// Wrapper for binary half-precision functions needing specialization for individual argument types.
+ /// \tparam T first argument type
+ /// \tparam U first argument type
+ template<typename T,typename U> struct binary_specialized
+ {
+ /// Minimum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum value
+ static expr fmin(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmin(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::min(x, y));
+ #endif
+ }
+
+ /// Maximum implementation.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum value
+ static expr fmax(float x, float y)
+ {
+ #if HALF_ENABLE_CPP11_CMATH
+ return expr(std::fmax(x, y));
+ #else
+ if(builtin_isnan(x))
+ return expr(y);
+ if(builtin_isnan(y))
+ return expr(x);
+ return expr(std::max(x, y));
+ #endif
+ }
+ };
+ template<> struct binary_specialized<half,half>
+ {
+ static half fmin(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ static half fmax(half x, half y)
+ {
+ int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF;
+ if(xabs > 0x7C00)
+ return y;
+ if(yabs > 0x7C00)
+ return x;
+ return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x;
+ }
+ };
+
+ /// Helper class for half casts.
+ /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member
+ /// function and a corresponding `type` member denoting its return type.
+ /// \tparam T destination type
+ /// \tparam U source type
+ /// \tparam R rounding mode to use
+ template<typename T,typename U,std::float_round_style R=(std::float_round_style)(HALF_ROUND_STYLE)> struct half_caster {};
+ template<typename U,std::float_round_style R> struct half_caster<half,U,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<U>::value, "half_cast from non-arithmetic type unsupported");
+ #endif
+
+ static half cast(U arg) { return cast_impl(arg, is_float<U>()); };
+
+ private:
+ static half cast_impl(U arg, true_type) { return half(binary, float2half<R>(arg)); }
+ static half cast_impl(U arg, false_type) { return half(binary, int2half<R>(arg)); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,half,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(half arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(half arg, true_type) { return half2float<T>(arg.data_); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<typename T,std::float_round_style R> struct half_caster<T,expr,R>
+ {
+ #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS
+ static_assert(std::is_arithmetic<T>::value, "half_cast to non-arithmetic type unsupported");
+ #endif
+
+ static T cast(expr arg) { return cast_impl(arg, is_float<T>()); }
+
+ private:
+ static T cast_impl(float arg, true_type) { return static_cast<T>(arg); }
+ static T cast_impl(half arg, false_type) { return half2int<R,T>(arg.data_); }
+ };
+ template<std::float_round_style R> struct half_caster<half,half,R>
+ {
+ static half cast(half arg) { return arg; }
+ };
+ template<std::float_round_style R> struct half_caster<half,expr,R> : half_caster<half,half,R> {};
+
+ /// \name Comparison operators
+ /// \{
+
+ /// Comparison for equality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator==(T x, U y) { return functions::isequal(x, y); }
+
+ /// Comparison for inequality.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if operands not equal
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator!=(T x, U y) { return functions::isnotequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<(T x, U y) { return functions::isless(x, y); }
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>(T x, U y) { return functions::isgreater(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator<=(T x, U y) { return functions::islessequal(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+ template<typename T,typename U> typename enable<bool,T,U>::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); }
+
+ /// \}
+ /// \name Arithmetic operators
+ /// \{
+
+ /// Add halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return sum of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator+(T x, U y) { return functions::plus(x, y); }
+
+ /// Subtract halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return difference of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator-(T x, U y) { return functions::minus(x, y); }
+
+ /// Multiply halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return product of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator*(T x, U y) { return functions::multiplies(x, y); }
+
+ /// Divide halfs.
+ /// \param x left operand
+ /// \param y right operand
+ /// \return quotient of half expressions
+ template<typename T,typename U> typename enable<expr,T,U>::type operator/(T x, U y) { return functions::divides(x, y); }
+
+ /// Identity.
+ /// \param arg operand
+ /// \return uncahnged operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator+(T arg) { return arg; }
+
+ /// Negation.
+ /// \param arg operand
+ /// \return negated operand
+ template<typename T> HALF_CONSTEXPR typename enable<T,T>::type operator-(T arg) { return unary_specialized<T>::negate(arg); }
+
+ /// \}
+ /// \name Input and output
+ /// \{
+
+ /// Output operator.
+ /// \param out output stream to write into
+ /// \param arg half expression to write
+ /// \return reference to output stream
+ template<typename T,typename charT,typename traits> typename enable<std::basic_ostream<charT,traits>&,T>::type
+ operator<<(std::basic_ostream<charT,traits> &out, T arg) { return functions::write(out, arg); }
+
+ /// Input operator.
+ /// \param in input stream to read from
+ /// \param arg half to read into
+ /// \return reference to input stream
+ template<typename charT,typename traits> std::basic_istream<charT,traits>&
+ operator>>(std::basic_istream<charT,traits> &in, half &arg) { return functions::read(in, arg); }
+
+ /// \}
+ /// \name Basic mathematical operations
+ /// \{
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type abs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half abs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr abs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Absolute value.
+ /// \param arg operand
+ /// \return absolute value of \a arg
+// template<typename T> typename enable<T,T>::type fabs(T arg) { return unary_specialized<T>::fabs(arg); }
+ inline half fabs(half arg) { return unary_specialized<half>::fabs(arg); }
+ inline expr fabs(expr arg) { return unary_specialized<expr>::fabs(arg); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type fmod(T x, U y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(half x, expr y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, half y) { return functions::fmod(x, y); }
+ inline expr fmod(expr x, expr y) { return functions::fmod(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remainder(T x, U y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(half x, expr y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, half y) { return functions::remainder(x, y); }
+ inline expr remainder(expr x, expr y) { return functions::remainder(x, y); }
+
+ /// Remainder of division.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param quo address to store some bits of quotient at
+ /// \return remainder of floating point division.
+// template<typename T,typename U> typename enable<expr,T,U>::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); }
+ inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); }
+
+ /// Fused multiply add.
+ /// \param x first operand
+ /// \param y second operand
+ /// \param z third operand
+ /// \return ( \a x * \a y ) + \a z rounded as one operation.
+// template<typename T,typename U,typename V> typename enable<expr,T,U,V>::type fma(T x, U y, V z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); }
+ inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); }
+
+ /// Maximum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return maximum of operands
+// template<typename T,typename U> typename result<T,U>::type fmax(T x, U y) { return binary_specialized<T,U>::fmax(x, y); }
+ inline half fmax(half x, half y) { return binary_specialized<half,half>::fmax(x, y); }
+ inline expr fmax(half x, expr y) { return binary_specialized<half,expr>::fmax(x, y); }
+ inline expr fmax(expr x, half y) { return binary_specialized<expr,half>::fmax(x, y); }
+ inline expr fmax(expr x, expr y) { return binary_specialized<expr,expr>::fmax(x, y); }
+
+ /// Minimum of half expressions.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return minimum of operands
+// template<typename T,typename U> typename result<T,U>::type fmin(T x, U y) { return binary_specialized<T,U>::fmin(x, y); }
+ inline half fmin(half x, half y) { return binary_specialized<half,half>::fmin(x, y); }
+ inline expr fmin(half x, expr y) { return binary_specialized<half,expr>::fmin(x, y); }
+ inline expr fmin(expr x, half y) { return binary_specialized<expr,half>::fmin(x, y); }
+ inline expr fmin(expr x, expr y) { return binary_specialized<expr,expr>::fmin(x, y); }
+
+ /// Positive difference.
+ /// \param x first operand
+ /// \param y second operand
+ /// \return \a x - \a y or 0 if difference negative
+// template<typename T,typename U> typename enable<expr,T,U>::type fdim(T x, U y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(half x, expr y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, half y) { return functions::fdim(x, y); }
+ inline expr fdim(expr x, expr y) { return functions::fdim(x, y); }
+
+ /// Get NaN value.
+ /// \return quiet NaN
+ inline half nanh(const char*) { return functions::nanh(); }
+
+ /// \}
+ /// \name Exponential functions
+ /// \{
+
+ /// Exponential function.
+ /// \param arg function argument
+ /// \return e raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp(T arg) { return functions::exp(arg); }
+ inline expr exp(half arg) { return functions::exp(arg); }
+ inline expr exp(expr arg) { return functions::exp(arg); }
+
+ /// Exponential minus one.
+ /// \param arg function argument
+ /// \return e raised to \a arg subtracted by 1
+// template<typename T> typename enable<expr,T>::type expm1(T arg) { return functions::expm1(arg); }
+ inline expr expm1(half arg) { return functions::expm1(arg); }
+ inline expr expm1(expr arg) { return functions::expm1(arg); }
+
+ /// Binary exponential.
+ /// \param arg function argument
+ /// \return 2 raised to \a arg
+// template<typename T> typename enable<expr,T>::type exp2(T arg) { return functions::exp2(arg); }
+ inline expr exp2(half arg) { return functions::exp2(arg); }
+ inline expr exp2(expr arg) { return functions::exp2(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base e
+// template<typename T> typename enable<expr,T>::type log(T arg) { return functions::log(arg); }
+ inline expr log(half arg) { return functions::log(arg); }
+ inline expr log(expr arg) { return functions::log(arg); }
+
+ /// Common logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 10
+// template<typename T> typename enable<expr,T>::type log10(T arg) { return functions::log10(arg); }
+ inline expr log10(half arg) { return functions::log10(arg); }
+ inline expr log10(expr arg) { return functions::log10(arg); }
+
+ /// Natural logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg plus 1 to base e
+// template<typename T> typename enable<expr,T>::type log1p(T arg) { return functions::log1p(arg); }
+ inline expr log1p(half arg) { return functions::log1p(arg); }
+ inline expr log1p(expr arg) { return functions::log1p(arg); }
+
+ /// Binary logorithm.
+ /// \param arg function argument
+ /// \return logarithm of \a arg to base 2
+// template<typename T> typename enable<expr,T>::type log2(T arg) { return functions::log2(arg); }
+ inline expr log2(half arg) { return functions::log2(arg); }
+ inline expr log2(expr arg) { return functions::log2(arg); }
+
+ /// \}
+ /// \name Power functions
+ /// \{
+
+ /// Square root.
+ /// \param arg function argument
+ /// \return square root of \a arg
+// template<typename T> typename enable<expr,T>::type sqrt(T arg) { return functions::sqrt(arg); }
+ inline expr sqrt(half arg) { return functions::sqrt(arg); }
+ inline expr sqrt(expr arg) { return functions::sqrt(arg); }
+
+ /// Cubic root.
+ /// \param arg function argument
+ /// \return cubic root of \a arg
+// template<typename T> typename enable<expr,T>::type cbrt(T arg) { return functions::cbrt(arg); }
+ inline expr cbrt(half arg) { return functions::cbrt(arg); }
+ inline expr cbrt(expr arg) { return functions::cbrt(arg); }
+
+ /// Hypotenuse function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return square root of sum of squares without internal over- or underflows
+// template<typename T,typename U> typename enable<expr,T,U>::type hypot(T x, U y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(half x, expr y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, half y) { return functions::hypot(x, y); }
+ inline expr hypot(expr x, expr y) { return functions::hypot(x, y); }
+
+ /// Power function.
+ /// \param base first argument
+ /// \param exp second argument
+ /// \return \a base raised to \a exp
+// template<typename T,typename U> typename enable<expr,T,U>::type pow(T base, U exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(half base, expr exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, half exp) { return functions::pow(base, exp); }
+ inline expr pow(expr base, expr exp) { return functions::pow(base, exp); }
+
+ /// \}
+ /// \name Trigonometric functions
+ /// \{
+
+ /// Sine function.
+ /// \param arg function argument
+ /// \return sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sin(T arg) { return functions::sin(arg); }
+ inline expr sin(half arg) { return functions::sin(arg); }
+ inline expr sin(expr arg) { return functions::sin(arg); }
+
+ /// Cosine function.
+ /// \param arg function argument
+ /// \return cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cos(T arg) { return functions::cos(arg); }
+ inline expr cos(half arg) { return functions::cos(arg); }
+ inline expr cos(expr arg) { return functions::cos(arg); }
+
+ /// Tangent function.
+ /// \param arg function argument
+ /// \return tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tan(T arg) { return functions::tan(arg); }
+ inline expr tan(half arg) { return functions::tan(arg); }
+ inline expr tan(expr arg) { return functions::tan(arg); }
+
+ /// Arc sine.
+ /// \param arg function argument
+ /// \return arc sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asin(T arg) { return functions::asin(arg); }
+ inline expr asin(half arg) { return functions::asin(arg); }
+ inline expr asin(expr arg) { return functions::asin(arg); }
+
+ /// Arc cosine function.
+ /// \param arg function argument
+ /// \return arc cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acos(T arg) { return functions::acos(arg); }
+ inline expr acos(half arg) { return functions::acos(arg); }
+ inline expr acos(expr arg) { return functions::acos(arg); }
+
+ /// Arc tangent function.
+ /// \param arg function argument
+ /// \return arc tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atan(T arg) { return functions::atan(arg); }
+ inline expr atan(half arg) { return functions::atan(arg); }
+ inline expr atan(expr arg) { return functions::atan(arg); }
+
+ /// Arc tangent function.
+ /// \param x first argument
+ /// \param y second argument
+ /// \return arc tangent value
+// template<typename T,typename U> typename enable<expr,T,U>::type atan2(T x, U y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(half x, expr y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, half y) { return functions::atan2(x, y); }
+ inline expr atan2(expr x, expr y) { return functions::atan2(x, y); }
+
+ /// \}
+ /// \name Hyperbolic functions
+ /// \{
+
+ /// Hyperbolic sine.
+ /// \param arg function argument
+ /// \return hyperbolic sine value of \a arg
+// template<typename T> typename enable<expr,T>::type sinh(T arg) { return functions::sinh(arg); }
+ inline expr sinh(half arg) { return functions::sinh(arg); }
+ inline expr sinh(expr arg) { return functions::sinh(arg); }
+
+ /// Hyperbolic cosine.
+ /// \param arg function argument
+ /// \return hyperbolic cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type cosh(T arg) { return functions::cosh(arg); }
+ inline expr cosh(half arg) { return functions::cosh(arg); }
+ inline expr cosh(expr arg) { return functions::cosh(arg); }
+
+ /// Hyperbolic tangent.
+ /// \param arg function argument
+ /// \return hyperbolic tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type tanh(T arg) { return functions::tanh(arg); }
+ inline expr tanh(half arg) { return functions::tanh(arg); }
+ inline expr tanh(expr arg) { return functions::tanh(arg); }
+
+ /// Hyperbolic area sine.
+ /// \param arg function argument
+ /// \return area sine value of \a arg
+// template<typename T> typename enable<expr,T>::type asinh(T arg) { return functions::asinh(arg); }
+ inline expr asinh(half arg) { return functions::asinh(arg); }
+ inline expr asinh(expr arg) { return functions::asinh(arg); }
+
+ /// Hyperbolic area cosine.
+ /// \param arg function argument
+ /// \return area cosine value of \a arg
+// template<typename T> typename enable<expr,T>::type acosh(T arg) { return functions::acosh(arg); }
+ inline expr acosh(half arg) { return functions::acosh(arg); }
+ inline expr acosh(expr arg) { return functions::acosh(arg); }
+
+ /// Hyperbolic area tangent.
+ /// \param arg function argument
+ /// \return area tangent value of \a arg
+// template<typename T> typename enable<expr,T>::type atanh(T arg) { return functions::atanh(arg); }
+ inline expr atanh(half arg) { return functions::atanh(arg); }
+ inline expr atanh(expr arg) { return functions::atanh(arg); }
+
+ /// \}
+ /// \name Error and gamma functions
+ /// \{
+
+ /// Error function.
+ /// \param arg function argument
+ /// \return error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erf(T arg) { return functions::erf(arg); }
+ inline expr erf(half arg) { return functions::erf(arg); }
+ inline expr erf(expr arg) { return functions::erf(arg); }
+
+ /// Complementary error function.
+ /// \param arg function argument
+ /// \return 1 minus error function value of \a arg
+// template<typename T> typename enable<expr,T>::type erfc(T arg) { return functions::erfc(arg); }
+ inline expr erfc(half arg) { return functions::erfc(arg); }
+ inline expr erfc(expr arg) { return functions::erfc(arg); }
+
+ /// Natural logarithm of gamma function.
+ /// \param arg function argument
+ /// \return natural logarith of gamma function for \a arg
+// template<typename T> typename enable<expr,T>::type lgamma(T arg) { return functions::lgamma(arg); }
+ inline expr lgamma(half arg) { return functions::lgamma(arg); }
+ inline expr lgamma(expr arg) { return functions::lgamma(arg); }
+
+ /// Gamma function.
+ /// \param arg function argument
+ /// \return gamma function value of \a arg
+// template<typename T> typename enable<expr,T>::type tgamma(T arg) { return functions::tgamma(arg); }
+ inline expr tgamma(half arg) { return functions::tgamma(arg); }
+ inline expr tgamma(expr arg) { return functions::tgamma(arg); }
+
+ /// \}
+ /// \name Rounding
+ /// \{
+
+ /// Nearest integer not less than half value.
+ /// \param arg half to round
+ /// \return nearest integer not less than \a arg
+// template<typename T> typename enable<half,T>::type ceil(T arg) { return functions::ceil(arg); }
+ inline half ceil(half arg) { return functions::ceil(arg); }
+ inline half ceil(expr arg) { return functions::ceil(arg); }
+
+ /// Nearest integer not greater than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater than \a arg
+// template<typename T> typename enable<half,T>::type floor(T arg) { return functions::floor(arg); }
+ inline half floor(half arg) { return functions::floor(arg); }
+ inline half floor(expr arg) { return functions::floor(arg); }
+
+ /// Nearest integer not greater in magnitude than half value.
+ /// \param arg half to round
+ /// \return nearest integer not greater in magnitude than \a arg
+// template<typename T> typename enable<half,T>::type trunc(T arg) { return functions::trunc(arg); }
+ inline half trunc(half arg) { return functions::trunc(arg); }
+ inline half trunc(expr arg) { return functions::trunc(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<half,T>::type round(T arg) { return functions::round(arg); }
+ inline half round(half arg) { return functions::round(arg); }
+ inline half round(expr arg) { return functions::round(arg); }
+
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long,T>::type lround(T arg) { return functions::lround(arg); }
+ inline long lround(half arg) { return functions::lround(arg); }
+ inline long lround(expr arg) { return functions::lround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type nearbyint(T arg) { return functions::nearbyint(arg); }
+ inline half nearbyint(half arg) { return functions::rint(arg); }
+ inline half nearbyint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<half,T>::type rint(T arg) { return functions::rint(arg); }
+ inline half rint(half arg) { return functions::rint(arg); }
+ inline half rint(expr arg) { return functions::rint(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long,T>::type lrint(T arg) { return functions::lrint(arg); }
+ inline long lrint(half arg) { return functions::lrint(arg); }
+ inline long lrint(expr arg) { return functions::lrint(arg); }
+ #if HALF_ENABLE_CPP11_LONG_LONG
+ /// Nearest integer.
+ /// \param arg half to round
+ /// \return nearest integer, rounded away from zero in half-way cases
+// template<typename T> typename enable<long long,T>::type llround(T arg) { return functions::llround(arg); }
+ inline long long llround(half arg) { return functions::llround(arg); }
+ inline long long llround(expr arg) { return functions::llround(arg); }
+
+ /// Nearest integer using half's internal rounding mode.
+ /// \param arg half expression to round
+ /// \return nearest integer using default rounding mode
+// template<typename T> typename enable<long long,T>::type llrint(T arg) { return functions::llrint(arg); }
+ inline long long llrint(half arg) { return functions::llrint(arg); }
+ inline long long llrint(expr arg) { return functions::llrint(arg); }
+ #endif
+
+ /// \}
+ /// \name Floating point manipulation
+ /// \{
+
+ /// Decompress floating point number.
+ /// \param arg number to decompress
+ /// \param exp address to store exponent at
+ /// \return significant in range [0.5, 1)
+// template<typename T> typename enable<half,T>::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); }
+ inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract integer and fractional parts.
+ /// \param arg number to decompress
+ /// \param iptr address to store integer part at
+ /// \return fractional part
+// template<typename T> typename enable<half,T>::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); }
+ inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); }
+ inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); }
+
+ /// Multiply by power of two.
+ /// \param arg number to modify
+ /// \param exp power of two to multiply with
+ /// \return \a arg multplied by 2 raised to \a exp
+// template<typename T> typename enable<half,T>::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); }
+ inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+ /// \retval FP_ILOGB0 for zero
+ /// \retval FP_ILOGBNAN for NaN
+ /// \retval MAX_INT for infinity
+// template<typename T> typename enable<int,T>::type ilogb(T arg) { return functions::ilogb(arg); }
+ inline int ilogb(half arg) { return functions::ilogb(arg); }
+ inline int ilogb(expr arg) { return functions::ilogb(arg); }
+
+ /// Extract exponent.
+ /// \param arg number to query
+ /// \return floating point exponent
+// template<typename T> typename enable<half,T>::type logb(T arg) { return functions::logb(arg); }
+ inline half logb(half arg) { return functions::logb(arg); }
+ inline half logb(expr arg) { return functions::logb(arg); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T,typename U> typename enable<half,T,U>::type nextafter(T from, U to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(half from, expr to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, half to) { return functions::nextafter(from, to); }
+ inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); }
+
+ /// Next representable value.
+ /// \param from value to compute next representable value for
+ /// \param to direction towards which to compute next value
+ /// \return next representable value after \a from in direction towards \a to
+// template<typename T> typename enable<half,T>::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); }
+ inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); }
+
+ /// Take sign.
+ /// \param x value to change sign for
+ /// \param y value to take sign from
+ /// \return value equal to \a x in magnitude and to \a y in sign
+// template<typename T,typename U> typename enable<half,T,U>::type copysign(T x, U y) { return functions::copysign(x, y); }
+ inline half copysign(half x, half y) { return functions::copysign(x, y); }
+ inline half copysign(half x, expr y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, half y) { return functions::copysign(x, y); }
+ inline half copysign(expr x, expr y) { return functions::copysign(x, y); }
+
+ /// \}
+ /// \name Floating point classification
+ /// \{
+
+
+ /// Classify floating point value.
+ /// \param arg number to classify
+ /// \retval FP_ZERO for positive and negative zero
+ /// \retval FP_SUBNORMAL for subnormal numbers
+ /// \retval FP_INFINITY for positive and negative infinity
+ /// \retval FP_NAN for NaNs
+ /// \retval FP_NORMAL for all other (normal) values
+// template<typename T> typename enable<int,T>::type fpclassify(T arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(half arg) { return functions::fpclassify(arg); }
+ inline int fpclassify(expr arg) { return functions::fpclassify(arg); }
+
+ /// Check if finite number.
+ /// \param arg number to check
+ /// \retval true if neither infinity nor NaN
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isfinite(T arg) { return functions::isfinite(arg); }
+ inline bool isfinite(half arg) { return functions::isfinite(arg); }
+ inline bool isfinite(expr arg) { return functions::isfinite(arg); }
+
+ /// Check for infinity.
+ /// \param arg number to check
+ /// \retval true for positive or negative infinity
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isinf(T arg) { return functions::isinf(arg); }
+ inline bool isinf(half arg) { return functions::isinf(arg); }
+ inline bool isinf(expr arg) { return functions::isinf(arg); }
+
+ /// Check for NaN.
+ /// \param arg number to check
+ /// \retval true for NaNs
+ /// \retval false else
+// template<typename T> typename enable<bool,T>::type isnan(T arg) { return functions::isnan(arg); }
+ inline bool isnan(half arg) { return functions::isnan(arg); }
+ inline bool isnan(expr arg) { return functions::isnan(arg); }
+
+ /// Check if normal number.
+ /// \param arg number to check
+ /// \retval true if normal number
+ /// \retval false if either subnormal, zero, infinity or NaN
+// template<typename T> typename enable<bool,T>::type isnormal(T arg) { return functions::isnormal(arg); }
+ inline bool isnormal(half arg) { return functions::isnormal(arg); }
+ inline bool isnormal(expr arg) { return functions::isnormal(arg); }
+
+ /// Check sign.
+ /// \param arg number to check
+ /// \retval true for negative number
+ /// \retval false for positive number
+// template<typename T> typename enable<bool,T>::type signbit(T arg) { return functions::signbit(arg); }
+ inline bool signbit(half arg) { return functions::signbit(arg); }
+ inline bool signbit(expr arg) { return functions::signbit(arg); }
+
+ /// \}
+ /// \name Comparison
+ /// \{
+
+ /// Comparison for greater than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreater(T x, U y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); }
+ inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); }
+
+ /// Comparison for greater equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x greater equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); }
+ inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); }
+
+ /// Comparison for less than.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less than \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isless(T x, U y) { return functions::isless(x, y); }
+ inline bool isless(half x, half y) { return functions::isless(x, y); }
+ inline bool isless(half x, expr y) { return functions::isless(x, y); }
+ inline bool isless(expr x, half y) { return functions::isless(x, y); }
+ inline bool isless(expr x, expr y) { return functions::isless(x, y); }
+
+ /// Comparison for less equal.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if \a x less equal \a y
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessequal(T x, U y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); }
+ inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); }
+
+ /// Comarison for less or greater.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if either less or greater
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type islessgreater(T x, U y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); }
+ inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); }
+
+ /// Check if unordered.
+ /// \param x first operand
+ /// \param y second operand
+ /// \retval true if unordered (one or two NaN operands)
+ /// \retval false else
+// template<typename T,typename U> typename enable<bool,T,U>::type isunordered(T x, U y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); }
+ inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); }
+
+ /// \name Casting
+ /// \{
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ /// It uses the default rounding mode.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,typename U> T half_cast(U arg) { return half_caster<T,U>::cast(arg); }
+
+ /// Cast to or from half-precision floating point number.
+ /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted
+ /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do.
+ ///
+ /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types
+ /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler
+ /// error and casting between [half](\ref half_float::half)s is just a no-op.
+ /// \tparam T destination type (half or built-in arithmetic type)
+ /// \tparam R rounding mode to use.
+ /// \tparam U source type (half or built-in arithmetic type)
+ /// \param arg value to cast
+ /// \return \a arg converted to destination type
+ template<typename T,std::float_round_style R,typename U> T half_cast(U arg) { return half_caster<T,U,R>::cast(arg); }
+ /// \}
+ }
+
+ using detail::operator==;
+ using detail::operator!=;
+ using detail::operator<;
+ using detail::operator>;
+ using detail::operator<=;
+ using detail::operator>=;
+ using detail::operator+;
+ using detail::operator-;
+ using detail::operator*;
+ using detail::operator/;
+ using detail::operator<<;
+ using detail::operator>>;
+
+ using detail::abs;
+ using detail::fabs;
+ using detail::fmod;
+ using detail::remainder;
+ using detail::remquo;
+ using detail::fma;
+ using detail::fmax;
+ using detail::fmin;
+ using detail::fdim;
+ using detail::nanh;
+ using detail::exp;
+ using detail::expm1;
+ using detail::exp2;
+ using detail::log;
+ using detail::log10;
+ using detail::log1p;
+ using detail::log2;
+ using detail::sqrt;
+ using detail::cbrt;
+ using detail::hypot;
+ using detail::pow;
+ using detail::sin;
+ using detail::cos;
+ using detail::tan;
+ using detail::asin;
+ using detail::acos;
+ using detail::atan;
+ using detail::atan2;
+ using detail::sinh;
+ using detail::cosh;
+ using detail::tanh;
+ using detail::asinh;
+ using detail::acosh;
+ using detail::atanh;
+ using detail::erf;
+ using detail::erfc;
+ using detail::lgamma;
+ using detail::tgamma;
+ using detail::ceil;
+ using detail::floor;
+ using detail::trunc;
+ using detail::round;
+ using detail::lround;
+ using detail::nearbyint;
+ using detail::rint;
+ using detail::lrint;
+#if HALF_ENABLE_CPP11_LONG_LONG
+ using detail::llround;
+ using detail::llrint;
+#endif
+ using detail::frexp;
+ using detail::ldexp;
+ using detail::modf;
+ using detail::scalbn;
+ using detail::scalbln;
+ using detail::ilogb;
+ using detail::logb;
+ using detail::nextafter;
+ using detail::nexttoward;
+ using detail::copysign;
+ using detail::fpclassify;
+ using detail::isfinite;
+ using detail::isinf;
+ using detail::isnan;
+ using detail::isnormal;
+ using detail::signbit;
+ using detail::isgreater;
+ using detail::isgreaterequal;
+ using detail::isless;
+ using detail::islessequal;
+ using detail::islessgreater;
+ using detail::isunordered;
+
+ using detail::half_cast;
+}
+
+
+/// Extensions to the C++ standard library.
+namespace std
+{
+ /// Numeric limits for half-precision floats.
+ /// Because of the underlying single-precision implementation of many operations, it inherits some properties from
+ /// `std::numeric_limits<float>`.
+ template<> class numeric_limits<half_float::half> : public numeric_limits<float>
+ {
+ public:
+ /// Supports signed values.
+ static HALF_CONSTEXPR_CONST bool is_signed = true;
+
+ /// Is not exact.
+ static HALF_CONSTEXPR_CONST bool is_exact = false;
+
+ /// Doesn't provide modulo arithmetic.
+ static HALF_CONSTEXPR_CONST bool is_modulo = false;
+
+ /// IEEE conformant.
+ static HALF_CONSTEXPR_CONST bool is_iec559 = true;
+
+ /// Supports infinity.
+ static HALF_CONSTEXPR_CONST bool has_infinity = true;
+
+ /// Supports quiet NaNs.
+ static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true;
+
+ /// Supports subnormal values.
+ static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present;
+
+ /// Rounding mode.
+ /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying
+ /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding
+ /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the
+ /// single-precision rounding mode.
+ static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits<float>::round_style==
+ half_float::half::round_style) ? half_float::half::round_style : round_indeterminate;
+
+ /// Significant digits.
+ static HALF_CONSTEXPR_CONST int digits = 11;
+
+ /// Significant decimal digits.
+ static HALF_CONSTEXPR_CONST int digits10 = 3;
+
+ /// Required decimal digits to represent all possible values.
+ static HALF_CONSTEXPR_CONST int max_digits10 = 5;
+
+ /// Number base.
+ static HALF_CONSTEXPR_CONST int radix = 2;
+
+ /// One more than smallest exponent.
+ static HALF_CONSTEXPR_CONST int min_exponent = -13;
+
+ /// Smallest normalized representable power of 10.
+ static HALF_CONSTEXPR_CONST int min_exponent10 = -4;
+
+ /// One more than largest exponent
+ static HALF_CONSTEXPR_CONST int max_exponent = 16;
+
+ /// Largest finitely representable power of 10.
+ static HALF_CONSTEXPR_CONST int max_exponent10 = 4;
+
+ /// Smallest positive normal value.
+ static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); }
+
+ /// Smallest finite value.
+ static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); }
+
+ /// Largest finite value.
+ static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); }
+
+ /// Difference between one and next representable value.
+ static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); }
+
+ /// Maximum rounding error.
+ static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW
+ { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); }
+
+ /// Positive infinity.
+ static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); }
+
+ /// Quiet NaN.
+ static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); }
+
+ /// Signalling NaN.
+ static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); }
+
+ /// Smallest positive subnormal value.
+ static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); }
+ };
+
+#if HALF_ENABLE_CPP11_HASH
+ /// Hash function for half-precision floats.
+ /// This is only defined if C++11 `std::hash` is supported and enabled.
+ template<> struct hash<half_float::half> //: unary_function<half_float::half,size_t>
+ {
+ /// Type of function argument.
+ typedef half_float::half argument_type;
+
+ /// Function return type.
+ typedef size_t result_type;
+
+ /// Compute hash function.
+ /// \param arg half to hash
+ /// \return hash value
+ result_type operator()(argument_type arg) const
+ { return hash<half_float::detail::uint16>()(static_cast<unsigned>(arg.data_)&-(arg.data_!=0x8000)); }
+ };
+#endif
+}
+
+
+#undef HALF_CONSTEXPR
+#undef HALF_CONSTEXPR_CONST
+#undef HALF_NOEXCEPT
+#undef HALF_NOTHROW
+#ifdef HALF_POP_WARNINGS
+ #pragma warning(pop)
+ #undef HALF_POP_WARNINGS
+#endif
+
+#endif
diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index e3b8970..31a33c6 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -25,7 +25,8 @@ // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +#include "half.h" +#include "half.hpp" #include "instructions.h" #include "ptx_ir.h" #include "opcodes.h" @@ -41,13 +42,23 @@ #include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" +#include <assert.h> +#include <string.h> +#include <sstream> +#include <stdio.h> +#include <string> +#include <map> +#include <stdlib.h> //Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include <stdarg.h> +using half_float::half; unsigned ptx_instruction::g_num_ptx_inst_uid=0; +bool debug_tensorcore = 0; + const char *g_opcode_string[NUM_OPCODES] = { #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, @@ -56,6 +67,104 @@ const char *g_opcode_string[NUM_OPCODES] = { #undef OP_DEF #undef OP_W_DEF }; +//Using profiled information::check the TensorCoreMatrixArrangement.xls for details +unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ + + unsigned offset; + unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; + unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; + unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; + unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; + unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; + unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; + unsigned thread_group = thread/4; + unsigned in_tg_index = thread%4; + + switch(wmma_type){ + case LOAD_A: + if(wmma_layout==ROW) + offset=load_a_row[thread_group]+16*in_tg_index; + else + offset=load_a_col[thread_group]+16*in_tg_index; + break; + + + case LOAD_B: + if(wmma_layout==ROW) + offset=load_b_row[thread_group]+16*in_tg_index; + else + offset=load_b_col[thread_group]+16*in_tg_index; + break; + + case LOAD_C: + case STORE_D: + if(type==F16_TYPE){ + if(wmma_layout==ROW) + offset=load_c_half_row[thread_group]+16*in_tg_index; + else + offset=load_c_half_col[thread_group]+in_tg_index; + } + else{ + if(wmma_layout==ROW) + offset=load_c_float_row[thread_group]; + else + offset=load_c_float_col[thread_group]; + + switch(in_tg_index){ + case 0: + break; + case 1: + if(wmma_layout==ROW) + offset+=16; + else + offset+=1; + break; + case 2: + if(wmma_layout==ROW) + offset+=2; + else + offset+=32; + break; + case 3: + if(wmma_layout==ROW) + offset+=18; + else + offset+=33; + break; + default: + abort(); + } + } + break; + + default: + abort(); + + } + offset = (offset/16)*stride+offset%16; + return offset; +} + +int acc_float_offset(int index,int wmma_layout,int stride){ + + int c_row_offset[]={0,1,32,33,4,5,36,37}; + int c_col_offset[]={0,16,2,18,64,80,66,82}; + int offset; + + + if(wmma_layout==ROW) + offset=c_row_offset[index]; + else if(wmma_layout==COL) + offset=c_col_offset[index]; + else{ + printf("wrong layout"); + abort(); + } + offset = (offset/16)*stride+offset%16; + return offset; +} void inst_not_implemented( const ptx_instruction * pI ) ; ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); @@ -74,6 +183,64 @@ void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) m_last_set_operand_value = value; } +void ptx_thread_info::print_reg_thread(char * fname) +{ + + FILE *fp= fopen(fname,"w"); + assert(fp!=NULL); + + int size = m_regs.size(); + + if(size>0) + { + reg_map_t reg = m_regs.back(); + + typename reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) + { + const std::string &name = it->first->name(); + const std::string &dec= it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp,"%s %llu %s %d\n",name.c_str(),it->second, dec.c_str(),size ); + + } + //m_regs.pop_back(); + } + fclose(fp); + + } + +void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) +{ + + + FILE * fp2 = fopen(fname, "r"); + assert(fp2!=NULL); + //m_regs.push_back( reg_map_t() ); + char line [ 200 ]; + while ( fgets ( line, sizeof line, fp2 ) != NULL ) + { + symbol *reg; + char * pch; + unsigned size; + pch = strtok (line," "); + char * name =pch; + reg= symtab->lookup(name); + ptx_reg_t data; + pch = strtok (NULL," "); + data = atoi(pch); + pch = strtok (NULL," "); + char * decl= pch; + pch = strtok (NULL," "); + size = atoi(pch); + + + m_regs.back()[reg] = data; + } + fclose ( fp2 ); +} + + ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) { static bool unfound_register_warned = false; @@ -136,10 +303,12 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in result.u64 = sym->get_address() + op.get_addr_offset(); } else if ( op.is_shared() ) { result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); } else { - const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); - abort(); + const char *name = op.name().c_str(); + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); + abort(); } } else if ( op.is_literal() ) { @@ -148,6 +317,8 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in result.u64 = op.get_symbol()->get_address(); } else if ( op.is_shared() ) { result.u64 = op.get_symbol()->get_address(); + } else if ( op.is_sstarr() ) { + result.u64 = op.get_symbol()->get_address(); } else if ( op.is_const() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_global() ) { @@ -155,11 +326,21 @@ ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_in } else if ( op.is_local() ) { result.u64 = op.get_symbol()->get_address(); } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if ( op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); + }else { const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); - assert(0); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if ( info2.is_param_kernel() ) { + result.u64 = sym2->get_address()+ op.get_addr_offset(); + } + else{ + printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); + assert(0); + } } if(op.get_operand_lohi() == 1) @@ -335,7 +516,7 @@ unsigned get_operand_nbits( const operand_info &op ) void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) { assert( op.is_vector() ); - assert( num_elements <= 4 ); // max 4 elements in a vector + assert( num_elements <= 8 ); for (int idx = num_elements - 1; idx >= 0; --idx) { const symbol *sym = NULL; @@ -640,6 +821,33 @@ void ptx_thread_info::set_vector_operand_values( const operand_info &dst, m_last_set_operand_value = data1; } +void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ) +{ + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } + else{ + printf("error:set_wmma_vector_operands"); + } + + m_last_set_operand_value = data8; +} #define my_abs(a) (((a)<0)?(-a):(a)) @@ -745,7 +953,7 @@ void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; default: assert(0); break; @@ -812,7 +1020,7 @@ void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; default: assert(0); break; @@ -1352,45 +1560,45 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &src1 = pI->src1(); const operand_info &src2 = pI->src2(); const operand_info &src3 = pI->src3(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; unsigned pos = b.u32 & 0xFF; unsigned len = c.u32 & 0xFF; - unsigned d = 0; switch (i_type) { case U32_TYPE: { unsigned mask; - d = a.u32 >> pos; + data.u32 = src.u32 >> pos; mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.u32 &= mask; break; } case U64_TYPE: { unsigned long mask; - d = a.u64 >> pos; + data.u64 = src.u64 >> pos; mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.u64 &= mask; break; } case S32_TYPE: { unsigned mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s32 >> min) & 0x1; - d = a.s32 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; if (sbit > 0) { mask = 0xFFFFFFFF << len; - d |= mask; + data.s32 |= mask; } else { mask = 0xFFFFFFFF >> (32 - len); - d &= mask; + data.s32 &= mask; } break; } @@ -1398,17 +1606,17 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { unsigned long mask; unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (a.s64 >> min) & 0x1; - d = a.s64 >> pos; + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; if (sbit > 0) { mask = 0xFFFFFFFFFFFFFFFF << len; - d |= mask; + data.s64 |= mask; } else { mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - d &= mask; + data.s64 &= mask; } break; } @@ -1417,10 +1625,47 @@ void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) abort(); return; } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + int i,max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; + + const operand_info &dst = pI->dst(); //get operand info of sources and destination + const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + + switch ( i_type ) { + case B32_TYPE: + max = 32; + break; + case B64_TYPE: + max = 64; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + data=src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for(i=0;i<len && pos+i<max;i++){ + data.u32=(~((0x00000001)<<(pos+i)))&data.u32; + data.u32=data.u32|((src1_data.u32&((0x00000001)<<(i)))<<(pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) @@ -1457,9 +1702,349 @@ void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + unsigned msb; + switch(i_type){ + case B32_TYPE: + msb = 31; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u32 & (1 << i))) + data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i=0; i<=msb; i++) { + if((src1_data.u64 & (1 << i))) + data.u64 |= 1 << (msb - i); + } + break; + default: assert(0); + } + thread->set_operand_value(dst,data, i_type, thread, pI); +} void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +unsigned trunc(unsigned num, unsigned precision) { + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 + //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); + //unsigned shifted_output = num >> (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + //if (shifted_output == (pow(2,precision)-1)) round_up = 0; + //num = shifted_output + round_up; + num >>= (latest_one-precision+1); + } + return num; +} +void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ + int offset; + int c_row_offset[]={0,8,0,8,4,12,4,12}; + int c_col_offset[]={0,0,8,8,0,0,8,8}; + int c_tg_inside_row_offset[]={0,1,0,1}; + int c_tg_inside_col_offset[]={0,0,2,2}; + int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; + int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; + + offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); + + if(wmma_type==LOAD_A){ + if(wmma_layout==ROW){ + offset+=index+8*((thread%16)/8); + } + else{ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if(wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + offset+=64*(index/4)+index%4+128*((thread%16)/8); + } + else{ + offset+=index+8*((thread%16)/8); + } + offset=(offset/16)*stride+offset%16; + assg_offset=index+8*((thread%16)/8); + } + else if( wmma_type==LOAD_C){ + if(type==F16_TYPE){ + row=c_row_offset[thread/4]+thread%4; + col=c_col_offset[thread/4]+index; + } + else{ + row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; + col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; + } + assg_offset=index; + } + + if(wmma_type==LOAD_A||wmma_type==LOAD_B){ + if(wmma_layout==ROW){ + row=offset/16; + col=offset%16; + } + else{ + col=offset/16; + row=offset%16; + } + } +} + +void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) +{ + int i,j,k,thrd; + int row,col,offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; + int stride; + + unsigned wmma_type = pI->get_wmma_type(); + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid ; + const operand_info &dst = pI->operand_lookup(0); + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + unsigned thread_group_index; + float temp; + half temp2; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + if(debug_tensorcore) + printf("THREAD=%d\n:",thrd); + for(int operand_num=1;operand_num<=3;operand_num++){ + const operand_info &src_a= pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + if(debug_tensorcore){ + printf("Thread%d_Iteration=%d\n:",thrd,operand_num); + for(k=0;k<nelem;k++){ + printf("%x ",v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + if(k%2==1) + hex_val=(v[k/2].s64&0xffff); + else + hex_val=((v[k/2].s64&0xffff0000)>>16); + nw_v[k].f16 =*((half *)&hex_val); + } + } + if(!((operand_num==3)&&(type2==F32_TYPE))){ + for(k=0;k<2*nelem;k++){ + temp=nw_v[k].f16; + if(debug_tensorcore) + printf("%.2f ",temp); + } + if(debug_tensorcore) + printf("\n"); + } + else{ + if(debug_tensorcore){ + for(k=0;k<8;k++){ + printf("%.2f ",v[k].f32); + } + printf("\n"); + } + } + switch(operand_num) { + case 1 ://operand 1 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); + if(debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_a[row][col]=nw_v[offset]; + } + break; + case 2 ://operand 2 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); + if(debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + matrix_b[row][col]=nw_v[offset]; + } + break; + case 3 ://operand 3 + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); + if(debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); + if(type2!=F16_TYPE){ + matrix_c[row][col]=v[offset]; + } + else { + matrix_c[row][col]=nw_v[offset]; + } + } + break; + default : + printf("Invalid Operand Index\n" ); + } + } + if(debug_tensorcore) + printf("\n"); + } + if(debug_tensorcore){ + printf("MATRIX_A\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_a[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + temp=matrix_b[i][j].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type2==F16_TYPE){ + temp=matrix_c[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + matrix_d[i][j].f16=0; + } + } + + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + for(k=0;k<16;k++){ + matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; + } + if((type==F16_TYPE)&&(type2==F16_TYPE)) + matrix_d[i][j].f16+=matrix_c[i][j].f16; + else if((type==F32_TYPE)&&(type2==F16_TYPE)){ + temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; + temp=temp2; + matrix_d[i][j].f32=temp; + } + else if((type==F16_TYPE)&&(type2==F32_TYPE)){ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f16=half(temp); + } + else{ + temp=matrix_d[i][j].f16; + temp+=matrix_c[i][j].f32; + matrix_d[i][j].f32=temp; + } + } + } + if(debug_tensorcore){ + printf("MATRIX_D\n"); + for (i=0;i<16;i++){ + for(j=0;j<16;j++){ + if(type==F16_TYPE){ + temp=matrix_d[i][j].f16; + printf("%.2f ",temp); + } + else + printf("%.2f ",matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + int row_t[8]; + int col_t[8]; + for(k=0;k<8;k++){ + mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); + if(debug_tensorcore) + printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); + } + thread = core->get_thread_info()[tid+thrd]; + + + if(type==F32_TYPE){ + thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); + + if(debug_tensorcore) + { + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(debug_tensorcore){ + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + temp=matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ",temp); + } + printf("\n"); + + printf("thread%d:",thrd); + for(k=0;k<8;k++){ + printf("%x ",matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); + nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); + nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); + nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); + thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); + if(debug_tensorcore) + printf("thread%d=%x,%x,%x,%x",thrd,nw_data1.s64,nw_data2.s64,nw_data3.s64,nw_data4.s64); + + } + else{ + printf("wmma:mma:wrong type\n"); + abort(); + } + } +} + void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { static unsigned call_uid_next = 1; @@ -1467,7 +2052,19 @@ void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &target = pI->func_addr(); assert( target.is_function_address() ); const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. + * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() >0) + target_func->do_pdom(); + target_func->set_pdom(); + } // check that number of args and return match function requirements if( pI->has_return() ^ target_func->has_return() ) { @@ -1698,7 +2295,10 @@ unsigned int saturatei(unsigned int a, unsigned int max) ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) { - assert( from_width == 32); + half mytemp; + float myfloat; + half_float::half tmp_h; + //assert( from_width == 32); enum cuda_math::cudaRoundMode mode = cuda_math::cudaRoundZero; switch (rounding_mode) { @@ -1740,8 +2340,12 @@ ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, } } else { switch ( to_width ) { - case 16: assert(0); break; - case 32: assert(0); break; // handled by f2f + case 16: + y.f16 =half_float::half_cast<half,std::numeric_limits<float>::round_style>(x.f32);//mytemp; + break; + case 32: + y.f32=float(x.f16); + break; // handled by f2f case 64: y.f64 = x.f32; break; @@ -1891,48 +2495,53 @@ ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) { ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); -#else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); -#endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } -#if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) -#else - if (cuda_math::__cuda___isnanf(y.f32)) -#endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); + if (from_width == 16){ + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float<float>(val); + }else{ + switch ( rounding_mode ) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: + #if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); + #else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); + #endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } + #if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) + #else + if (cuda_math::__cuda___isnanf(y.f32)) + #endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } } return y; @@ -1980,7 +2589,7 @@ ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_wid { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2x, f2x}, + { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} }; @@ -2002,7 +2611,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; case F32_TYPE: data.f32 = truncf(data.f32); break; @@ -2025,7 +2634,13 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE:// assert(0); break; +#if CUDART_VERSION >= 3000 + data.f16 = nearbyintf(data.f16); +#else + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); +#endif + break; case F32_TYPE: #if CUDART_VERSION >= 3000 data.f32 = nearbyintf(data.f32); @@ -2048,7 +2663,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; case F32_TYPE: data.f32 = floorf(data.f32); break; @@ -2067,7 +2682,7 @@ void ptx_round(ptx_reg_t& data, int rounding_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; case F32_TYPE: data.f32 = ceilf(data.f32); break; case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; default: assert(0); break; @@ -2108,7 +2723,10 @@ void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) case U32_TYPE: case U64_TYPE: printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: //assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; //negative + if (data.f16 < 0.0f) data.f16 = 0.0f; //positive + break; case F32_TYPE: if (data.f32 > 1.0f) data.f32 = 1.0f; //negative if (data.f32 < 0.0f) data.f32 = 0.0f; //positive @@ -2132,8 +2750,8 @@ void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) unsigned rounding_mode = pI->rounding_mode(); unsigned saturation_mode = pI->saturation_mode(); - if ( to_type == F16_TYPE || from_type == F16_TYPE ) - abort(); +// if ( to_type == F16_TYPE || from_type == F16_TYPE ) +// abort(); int to_sign, from_sign; size_t from_width, to_width; @@ -2268,7 +2886,7 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) data.u32 = src1_data.u32 / src2_data.u32; break; case B64_TYPE: data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; default: assert(0); break; @@ -2276,6 +2894,13 @@ void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) thread->set_operand_value(dst,data, i_type, thread,pI); } +void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + printf("DP4A instruction not implemented yet"); + assert(0); + +} + void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ptx_reg_t src1_data, src2_data, data; @@ -2355,7 +2980,12 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand space = param_space_kernel; else if( ti.is_param_local() ) { space = param_space_local; - } else { + } + //mov r1, param-label + else if (ti.is_reg() ){ + space = param_space_kernel; + } + else { printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); abort(); } @@ -2371,6 +3001,7 @@ void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand case surf_space: mem = thread->get_surf_memory(); break; case param_space_kernel: mem = thread->get_param_memory(); break; case shared_space: mem = thread->m_shared_mem; break; + case sstarr_space: mem = thread->m_sstarr_mem; break; case const_space: mem = thread->get_global_memory(); break; case generic_space: if( thread->get_ptx_version().ver() >= 2.0 ) { @@ -2446,6 +3077,332 @@ void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) ld_exec(pI,thread); } +void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + unsigned smid; + int t; + int thrd,odd,inx,k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid ; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + for (thrd=0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid+thrd]; + odd=thrd%2; + inx=thrd/2; + ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a= pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t* v= new ptx_reg_t[8]; + thread->get_vector_operand_values( src_a, v, nelem ); + stride=src2_data.u32; + + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; + + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } + decode_space(space,thread,src1,mem,addr); + + type_info_key::type_decode(type,size,t); + if(debug_tensorcore) + printf("mma_st: thrd=%d,addr=%x, fp(size=%d), stride=%d\n",thrd,addr_reg.u32,size,src2_data.u32); + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for(k=0;k<8;k++){ + if(k%2==0) + nw_v[k].s64=(v[k/2].s64&0xffff); + else + nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); + } + + for(k=0;k<8;k++){ + if(type==F32_TYPE){ + //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); + mem->write(push_addr,size/8,&v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + + if(debug_tensorcore){ + printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); + float temp; + int l; + printf("thread=%d:",thrd); + for(l=0;l<8;l++){ + temp=v[l].f32; + printf("%.2f",temp); + } + printf("\n"); + } + } + else if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + if(k%2==0) + mem_txn_addr[num_mem_txn++]=push_addr; + } + else if(wmma_layout==COL){ + //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr=new_addr+k*2*stride; + mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); + mem_txn_addr[num_mem_txn++]=push_addr; + } + + if(debug_tensorcore) + printf("wmma:store:thread%d=%x,%x,%x,%x,%x,%x,%x,%x\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); + } + } + + delete [] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + + if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + + assert( inst.memory_op == insn_memory_op ); + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } +} + +void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) +{ + size_t size; + int t,i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd,stride; + ptx_thread_info *thread; + + + if(core->get_gpu()->is_functional_sim()) + tid= inst.warp_id_func()*core->get_warp_size(); + else + tid= inst.warp_id()*core->get_warp_size(); + + _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; + + for (thrd=0; thrd < core->get_warp_size(); thrd++){ + thread = core->get_thread_info()[tid+thrd]; + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride=src2_data.u32; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if( whichspace(addr) == shared_space ) { + addr= generic_to_shared(smid,addr); + space = shared_space; + } + + decode_space(space,thread,src1,mem,addr); + type_info_key::type_decode(type,size,t); + + ptx_reg_t data[16]; + if(debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%d, stride=%d\n",thrd,src1_data.u32,size,src2_data.u32); + + addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn=0; + + if(wmma_type==LOAD_A){ + for(i=0;i<16;i++){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_B){ + for(i=0;i<16;i++){ + if(wmma_layout==COL){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + } + else if(wmma_layout==ROW){ + //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); + mem->read(fetch_addr,size/8,&data[i].s64); + } + else{ + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + } + else if(wmma_type==LOAD_C){ + for(i=0;i<8;i++){ + if(type==F16_TYPE){ + if(wmma_layout==ROW){ + //mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*i; + mem->read(fetch_addr,size/8,&data[i].s64); + if(i%2==0) + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else if(wmma_layout==COL){ + //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr=new_addr+2*stride*i; + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("mma_ld:wrong_type\n"); + abort(); + } + } + else if(type==F32_TYPE){ + //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); + mem->read(fetch_addr,size/8,&data[i].s64); + mem_txn_addr[num_mem_txn++]=fetch_addr; + } + else{ + printf("wrong type"); + abort(); + } + } + } + else{ + printf("wrong wmma type\n");; + abort(); + } + //generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); + + if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert( inst.memory_op == insn_memory_op ); + + if(debug_tensorcore){ + if(type==F16_TYPE){ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<16;i++){ + printf("%x ",data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ",thrd); + float temp; + for(i=0;i<16;i++){ + temp=data[i].f16; + printf("%.2f ",temp); + } + printf("\n"); + } + else{ + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%.2f ",data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ",thrd); + for(i=0;i<8;i++){ + printf("%x ",data[i].u64); + } + printf("\n"); + } + } + + if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ + thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); + } + else{ + ptx_reg_t nw_data[8]; + int num_reg; + + if(wmma_type==LOAD_C) + num_reg=4; + else + num_reg=8; + + for(i=0;i<num_reg;i++){ + nw_data[i].s64= ((data[2*i].s64 & 0xffff)<<16)| ((data[2*i+1].s64 & 0xffff)); + } + + if(wmma_type==LOAD_C) + thread->set_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); + else + thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); + if(debug_tensorcore){ + printf("mma_ld:data[0].s64=%x,data[1].s64=%x,new_data[0].s64=%x\n",data[0].u64,data[1].u64,nw_data[0].u64); + printf("mma_ld:data[2].s64=%x,data[3].s64=%x,new_data[1].s64=%x\n",data[2].u64,data[3].u64,nw_data[1].u64); + printf("mma_ld:data[4].s64=%x,data[5].s64=%x,new_data[2].s64=%x\n",data[4].u64,data[5].u64,nw_data[2].u64); + printf("mma_ld:data[6].s64=%x,data[7].s64=%x,new_data[3].s64=%x\n",data[6].u64,data[7].u64,nw_data[3].u64); + if(wmma_type!=LOAD_C){ + printf("mma_ld:data[8].s64=%x,data[9].s64=%x,new_data[4].s64=%x\n",data[8].u64,data[9].u64,nw_data[4].s64); + printf("mma_ld:data[10].s64=%x,data[11].s64=%x,new_data[5].s64=%x\n",data[10].u64,data[11].u64,nw_data[5].u64); + printf("mma_ld:data[12].s64=%x,data[13].s64=%x,new_data[6].s64=%x\n",data[12].u64,data[13].u64,nw_data[6].u64); + printf("mma_ld:data[14].s64=%x,data[15].s64=%x,new_data[7].s64=%x\n",data[14].u64,data[15].u64,nw_data[3].u64); + } + } + } + + //thread->m_last_effective_address = addr; + //thread->m_last_memory_space = space; + } +} + void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { ptx_reg_t a, d; @@ -2605,9 +3562,24 @@ void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; else assert(0); break; - case F16_TYPE: - assert(0); - break; + case F16_TYPE:{ + // assert(0); + // break; + assert( use_carry == false); + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } case F32_TYPE: { assert( use_carry == false); int orig_rm = fegetround(); @@ -2729,6 +3701,7 @@ void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) const operand_info &dst = pI->dst(); const operand_info &src1 = pI->src1(); unsigned i_type = pI->get_type(); + assert( src1.is_param_local() == 0 ); if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { // pack or unpack operation @@ -2907,9 +3880,25 @@ void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) if ( pI->is_lo() ) d.u64 = t.u64; else assert(0); break; - case F16_TYPE: - assert(0); - break; + case F16_TYPE:{ + //assert(0); + //break; + int orig_rm = fegetround(); + switch ( rounding_mode ) { + case RN_OPTION: break; + case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; + default: assert(0); break; + } + + d.f16 = a.f16 * b.f16; + + if ( pI->saturation_mode() ) { + if ( d.f16 < 0 ) d.f16 = 0; + else if ( d.f16 > 1.0f ) d.f16 = 1.0f; + } + fesetround( orig_rm ); + break; + } case F32_TYPE: { int orig_rm = fegetround(); switch ( rounding_mode ) { @@ -2972,7 +3961,7 @@ void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case U32_TYPE: case U64_TYPE: assert(0); break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; default: assert(0); break; @@ -3122,7 +4111,94 @@ void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) } void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } + +int prmt_mode_present(int mode) +{ + int returnval=0; + switch(mode){ + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval=1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode,int control,int d_sel_index,signed long long value){ + + int returnval; + int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; + int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; + int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; + int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; + int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; + int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; + + if(!prmt_mode_present(mode)){ + if(control&0x8){ + returnval=0xff; + } + else{ + returnval= (value>>(8*control)) & 0xff; + } + } + else{ + switch(mode){ + case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; + case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; + case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; + case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; + case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; + case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; + default: printf("ERROR\n");break; + } + } + return (returnval<<8*d_sel_index); +} + +void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { + + ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64=src1_data.s32|(src2_data.s64<<32); + int ctl[4]; + + if(!prmt_mode_present(mode)){ + ctl[0]=(src3_data.s32>>0)&0xf; + ctl[1]=(src3_data.s32>>4)&0xf; + ctl[2]=(src3_data.s32>>8)&0xf; + ctl[3]=(src3_data.s32>>12)&0xf; + } + else{ + ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; + } + + data.s32=0; + data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 + data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 + data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 + data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 + + thread->set_operand_value(dst,data, i_type, thread, pI); + + +} void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { @@ -3165,7 +4241,21 @@ void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - data.u64 = src1_data.u64 % src2_data.u64; + switch ( i_type ) { + case S32_TYPE: + data.s32 = src1_data.s32 % src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 % src2_data.s64; + break; + case U32_TYPE: + data.u32 = src1_data.u32 % src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 % src2_data.u64; + break; + default: assert(0); break; + } thread->set_operand_value(dst,data, i_type, thread, pI); } @@ -3543,7 +4633,13 @@ void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) { unsigned i_type = pI->get_type(); - int tid = inst.warp_id() * core->get_warp_size(); + int tid; + + if(core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + ptx_thread_info *thread = core->get_thread_info()[tid]; ptx_warp_info *warp_info = thread->m_warp_info; int lane = warp_info->get_done_threads(); @@ -3833,6 +4929,88 @@ void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) thread->set_operand_value(dst,d, i_type, thread, pI); } +void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) +{ + ptx_instruction * cpI = const_cast<ptx_instruction *>(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; + + decode_space(space,thread,src1,mem,addr); + + size_t size; + int t; + type_info_key::type_decode(type,size,t); + + // store data in sstarr memory + mem->write(addr,size/8,&src3_data.s64,thread,pI); + + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; + + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64=0; + mem->read(addr+(tid*4),size/8,&data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } + + // squeeze the zeros out of the array and store data back into original array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space,thread,src1,mem,addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); + mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); + offset++; + } + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4*(offset-1); + thread->set_operand_value(dst, data, type, thread, pI); + + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); + offset++; + } + + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; + thread->m_last_memory_space = space; + } +} + void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); @@ -3861,7 +5039,7 @@ void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) if (!vector_spec) { data = thread->get_operand_value(src1, dst, type, thread, 1); mem->write(addr,size/8,&data.s64,thread,pI); - } else { + } else { if (vector_spec == V2_TYPE) { ptx_reg_t* ptx_regs = new ptx_reg_t[2]; thread->get_vector_operand_values(src1, ptx_regs, 2); @@ -3943,7 +5121,7 @@ void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) case B64_TYPE: case U64_TYPE: data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: assert(0); break; + case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; default: assert(0); break; @@ -4083,9 +5261,9 @@ void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) gpgpu_t *gpu = thread->get_gpu(); const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = gpu->get_texarray(texref); - const struct textureInfo* texInfo = gpu->get_texinfo(texref); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texref); + const struct cudaArray* cuArray = gpu->get_texarray(texname); + const struct textureInfo* texInfo = gpu->get_texinfo(texname); + const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); //assume always 2D f32 input //access array with src2 coordinates diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 7bdf4d9..9554f55 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -44,9 +44,16 @@ template<unsigned BSIZE> memory_space_impl<BSIZE>::memory_space_impl( std::strin assert( m_log2_block_size != (unsigned)-1 ); } +template<unsigned BSIZE> void memory_space_impl<BSIZE>::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) +{ + m_data[index].write(offset,length,(const unsigned char*)data); +} + template<unsigned BSIZE> void memory_space_impl<BSIZE>::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; + if ( (addr+length) <= (index+1)*BSIZE ) { // fast route for intra-block access unsigned offset = addr & (BSIZE-1); @@ -142,8 +149,9 @@ template<unsigned BSIZE> void memory_space_impl<BSIZE>::read( mem_addr_t addr, s template<unsigned BSIZE> void memory_space_impl<BSIZE>::print( const char *format, FILE *fout ) const { typename map_t::const_iterator i_page; - for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s - %#x:", m_name.c_str(), i_page->first); + + for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); i_page->second.print(format, fout); } } diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index f785b8b..ab588bc 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -81,7 +81,7 @@ public: { unsigned int *i_data = (unsigned int*)m_data; for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 8 == 0) { + if (d % 1 == 0) { fprintf(fout, "\n"); } fprintf(fout, format, i_data[d]); @@ -104,6 +104,7 @@ class memory_space public: virtual ~memory_space() {} virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; virtual void print( const char *format, FILE *fout ) const = 0; virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; @@ -114,8 +115,10 @@ public: memory_space_impl( std::string name, unsigned hash_size ); virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); + virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); virtual void read( mem_addr_t addr, size_t length, void *data ) const; virtual void print( const char *format, FILE *fout ) const; + virtual void set_watch( addr_t addr, unsigned watchpoint ); private: diff --git a/src/cuda-sim/opcodes.def b/src/cuda-sim/opcodes.def index e1b1422..c4d6a83 100644 --- a/src/cuda-sim/opcodes.def +++ b/src/cuda-sim/opcodes.def @@ -52,6 +52,9 @@ OP_DEF(BRA_OP,bra_impl,"bra",0,3) OP_DEF(BRX_OP,brx_impl,"brx",0,3) OP_DEF(BREV_OP,brev_impl,"brev",1,1) OP_DEF(BRKPT_OP,brkpt_impl,"brkpt",1,9) +OP_W_DEF(MMA_OP,mma_impl,"mma",1,1) +OP_W_DEF(MMA_LD_OP,mma_ld_impl,"mma_load",1,5) +OP_W_DEF(MMA_ST_OP,mma_st_impl,"mma_store",0,5) OP_DEF(CALL_OP,call_impl,"call",1,3) OP_DEF(CALLP_OP,callp_impl,"callp",1,3) OP_DEF(CLZ_OP,clz_impl,"clz",1,1) @@ -60,6 +63,7 @@ OP_DEF(COS_OP,cos_impl,"cos",1,4) OP_DEF(CVT_OP,cvt_impl,"cvt",1,1) OP_DEF(CVTA_OP,cvta_impl,"cvta",1,1) OP_DEF(DIV_OP,div_impl,"div",1,1) +OP_DEF(DP4A_OP,dp4a_impl,"dp4a",1,1) OP_DEF(EX2_OP,ex2_impl,"ex2",1,4) OP_DEF(EXIT_OP,exit_impl,"exit",1,3) OP_DEF(FMA_OP,fma_impl,"fma",1,2) @@ -104,6 +108,7 @@ OP_DEF(SHR_OP,shr_impl,"shr",1,1) OP_DEF(SIN_OP,sin_impl,"sin",1,4) OP_DEF(SLCT_OP,slct_impl,"slct",1,1) OP_DEF(SQRT_OP,sqrt_impl,"sqrt",1,4) +OP_DEF(SST_OP,sst_impl,"sst",1,5) OP_DEF(SSY_OP,ssy_impl,"ssy",0,3) OP_DEF(ST_OP,st_impl,"st",0,5) OP_DEF(SUB_OP,sub_impl,"sub",1,1) diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index aa133da..b91d92f 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -60,5 +60,14 @@ enum special_regs { WARPID_REG, WARPSZ_REG }; - +enum wmma_type{ + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16 +}; #endif diff --git a/src/cuda-sim/ptx.l b/src/cuda-sim/ptx.l index af015c9..d4bf631 100644 --- a/src/cuda-sim/ptx.l +++ b/src/cuda-sim/ptx.l @@ -36,7 +36,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ptx.tab.h" #include <string.h> -char linebuf[4096]; +#define LINEBUF_SIZE (64*1024) +char linebuf[LINEBUF_SIZE]; unsigned col = 0; #define TC col+=strlen(ptx_text); #define CHECK_UNSIGNED \ @@ -60,7 +61,7 @@ addc TC; ptx_lval.int_value = ADDC_OP; return OPCODE; and TC; ptx_lval.int_value = AND_OP; return OPCODE; andn TC; ptx_lval.int_value = ANDN_OP; return OPCODE; atom TC; ptx_lval.int_value = ATOM_OP; return OPCODE; -bar TC; ptx_lval.int_value = BAR_OP; return OPCODE; +bar TC; ptx_lval.int_value = BAR_OP; return OPCODE; bfe TC; ptx_lval.int_value = BFE_OP; return OPCODE; bfi TC; ptx_lval.int_value = BFI_OP; return OPCODE; bfind TC; ptx_lval.int_value = BFIND_OP; return OPCODE; @@ -68,14 +69,20 @@ bra TC; ptx_lval.int_value = BRA_OP; return OPCODE; brx TC; ptx_lval.int_value = BRX_OP; return OPCODE; brev TC; ptx_lval.int_value = BREV_OP; return OPCODE; brkpt TC; ptx_lval.int_value = BRKPT_OP; return OPCODE; + +wmma TC; ptx_lval.int_value = MMA_OP; return OPCODE; +wmma\.load TC; ptx_lval.int_value = MMA_LD_OP; return OPCODE; +wmma\.store TC; ptx_lval.int_value = MMA_ST_OP; return OPCODE; + call TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALL_OP; return OPCODE; // blocking opcode token in case the callee has the same name as an opcode -callp TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALLP_OP; return OPCODE; +callp TC; BEGIN(NOT_OPCODE); ptx_lval.int_value = CALLP_OP; return OPCODE; clz TC; ptx_lval.int_value = CLZ_OP; return OPCODE; cnot TC; ptx_lval.int_value = CNOT_OP; return OPCODE; cos TC; ptx_lval.int_value = COS_OP; return OPCODE; cvt TC; ptx_lval.int_value = CVT_OP; return OPCODE; cvta TC; ptx_lval.int_value = CVTA_OP; return OPCODE; div TC; ptx_lval.int_value = DIV_OP; return OPCODE; +dp4a TC; ptx_lval.int_value = DP4A_OP; return OPCODE; ex2 TC; ptx_lval.int_value = EX2_OP; return OPCODE; exit TC; ptx_lval.int_value = EXIT_OP; return OPCODE; fma TC; ptx_lval.int_value = FMA_OP; return OPCODE; @@ -121,6 +128,7 @@ shr TC; ptx_lval.int_value = SHR_OP; return OPCODE; sin TC; ptx_lval.int_value = SIN_OP; return OPCODE; slct TC; ptx_lval.int_value = SLCT_OP; return OPCODE; sqrt TC; ptx_lval.int_value = SQRT_OP; return OPCODE; +sst TC; ptx_lval.int_value = SST_OP; return OPCODE; ssy TC; ptx_lval.int_value = SSY_OP; return OPCODE; st TC; ptx_lval.int_value = ST_OP; return OPCODE; st.volatile TC; ptx_lval.int_value = ST_OP; return OPCODE; @@ -148,7 +156,24 @@ nop TC; ptx_lval.int_value = NOP_OP; return OPCODE; break TC; ptx_lval.int_value = BREAK_OP; return OPCODE; breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; +"CPTX_END" printf("ENDING CUSTOM PTX.\n"); BEGIN(IN_COMMENT); + <INITIAL,NOT_OPCODE,IN_INST,IN_FUNC_DECL>{ +\.a\.sync TC; ptx_lval.int_value = LOAD_A; return WMMA_DIRECTIVE; +\.b\.sync TC; ptx_lval.int_value = LOAD_B; return WMMA_DIRECTIVE; +\.c\.sync TC; ptx_lval.int_value = LOAD_C; return WMMA_DIRECTIVE; +\.d\.sync TC; ptx_lval.int_value = STORE_D; return WMMA_DIRECTIVE; +\.mma\.sync TC;ptx_lval.int_value=MMA; return WMMA_DIRECTIVE; + +\.row TC; ptx_lval.int_value = ROW; return LAYOUT; +\.col TC; ptx_lval.int_value = COL; return LAYOUT; +\.m16n16k16 TC; ptx_lval.int_value = M16N16K16; return CONFIGURATION; +\.f4e TC; return PRMT_F4E_MODE; +\.b4e TC; return PRMT_B4E_MODE; +\.rc8 TC; return PRMT_RC8_MODE; +\.ecl TC; return PRMT_ECL_MODE; +\.ecr TC; return PRMT_ECR_MODE; +\.rc16 TC; return PRMT_RC16_MODE; \.align TC; return ALIGN_DIRECTIVE; \.branchtargets TC; return BRANCHTARGETS_DIRECTIVE; @@ -162,6 +187,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.file TC; BEGIN(INITIAL); return FILE_DIRECTIVE; \.func TC; BEGIN(IN_FUNC_DECL); return FUNC_DIRECTIVE; // blocking opcode parsing in case the function has the same name as an opcode (e.g. sin(), cos()) \.global TC; return GLOBAL_DIRECTIVE; +\.global.volatile TC; return GLOBAL_DIRECTIVE; //TODO: fix this! \.local TC; return LOCAL_DIRECTIVE; \.loc TC; return LOC_DIRECTIVE; \.maxnctapersm TC; return MAXNCTAPERSM_DIRECTIVE; @@ -175,6 +201,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.section TC; return SECTION_DIRECTIVE; \.shared TC; return SHARED_DIRECTIVE; \.sreg TC; return SREG_DIRECTIVE; +\.sstarr TC; return SSTARR_DIRECTIVE; \.struct TC; return STRUCT_DIRECTIVE; \.surf TC; return SURF_DIRECTIVE; /* not in PTX 2.1 */ \.target TC; return TARGET_DIRECTIVE; @@ -233,6 +260,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; \.u32 TC; return U32_TYPE; \.u64 TC; return U64_TYPE; \.f16 TC; return F16_TYPE; +\.f16x2 TC; return F16_TYPE; /* TODO: figure out what this should really be */ \.f32 TC; return F32_TYPE; \.f64 TC; return F64_TYPE; \.ff64 TC; return FF64_TYPE; @@ -382,7 +410,7 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; "//"[^\n]* TC; // eat single -\n.* col=0; strncpy(linebuf, yytext + 1, sizeof(linebuf)); yyless( 1 ); +\n.* col=0; strncpy(linebuf, yytext + 1, LINEBUF_SIZE); yyless( 1 ); " " TC; "\t" TC; @@ -399,9 +427,9 @@ breakaddr TC; ptx_lval.int_value = BREAKADDR_OP; return OPCODE; <IN_COMMENT>{ "*/" BEGIN(INITIAL); "CPTX_BEGIN" printf("BEGINNING CUSTOM PTX.\n"); BEGIN(INITIAL); -[^C*\n]+ // eat comment in chunks -"C" -"*" // eat the lone star +[^C*\n]+ // eat comment in chunks +"C" // eat the lone C +"*" // eat the lone star \n TC; } diff --git a/src/cuda-sim/ptx.y b/src/cuda-sim/ptx.y index 2783fc2..45392fb 100644 --- a/src/cuda-sim/ptx.y +++ b/src/cuda-sim/ptx.y @@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token <string_value> STRING %token <int_value> OPCODE +%token <int_value> WMMA_DIRECTIVE +%token <int_value> LAYOUT +%token <int_value> CONFIGURATION %token ALIGN_DIRECTIVE %token BRANCHTARGETS_DIRECTIVE %token BYTE_DIRECTIVE @@ -63,6 +66,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token SECTION_DIRECTIVE %token SHARED_DIRECTIVE %token SREG_DIRECTIVE +%token SSTARR_DIRECTIVE %token STRUCT_DIRECTIVE %token SURF_DIRECTIVE %token TARGET_DIRECTIVE @@ -198,6 +202,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token DOWN_OPTION; %token BFLY_OPTION; %token IDX_OPTION; +%token PRMT_F4E_MODE; +%token PRMT_B4E_MODE; +%token PRMT_RC8_MODE; +%token PRMT_RC16_MODE; +%token PRMT_ECL_MODE; +%token PRMT_ECR_MODE; %type <int_value> function_decl_header %type <ptr_value> function_decl @@ -349,6 +359,7 @@ addressable_spec: CONST_DIRECTIVE { add_space_spec(const_space,$1); } | LOCAL_DIRECTIVE { add_space_spec(local_space,0); } | PARAM_DIRECTIVE { add_space_spec(param_space_unclassified,0); } | SHARED_DIRECTIVE { add_space_spec(shared_space,0); } + | SSTARR_DIRECTIVE { add_space_spec(sstarr_space,0); } | SURF_DIRECTIVE { add_space_spec(surf_space,0); } | TEX_DIRECTIVE { add_space_spec(tex_space,0); } ; @@ -428,6 +439,8 @@ option: type_spec | compare_spec | addressable_spec | rounding_mode + | wmma_spec + | prmt_spec | SYNC_OPTION { add_option(SYNC_OPTION); } | ARRIVE_OPTION { add_option(ARRIVE_OPTION); } | RED_OPTION { add_option(RED_OPTION); } @@ -483,6 +496,7 @@ atomic_operation_spec: ATOMIC_AND { add_option(ATOMIC_AND); } rounding_mode: floating_point_rounding_mode | integer_rounding_mode; + floating_point_rounding_mode: RN_OPTION { add_option(RN_OPTION); } | RZ_OPTION { add_option(RZ_OPTION); } | RM_OPTION { add_option(RM_OPTION); } @@ -515,8 +529,25 @@ compare_spec:EQ_OPTION { add_option(EQ_OPTION); } | NAN_OPTION { add_option(NAN_OPTION); } ; -operand_list: /* empty*/ - | operand +prmt_spec: PRMT_F4E_MODE { add_option( PRMT_F4E_MODE); } + | PRMT_B4E_MODE { add_option( PRMT_B4E_MODE); } + | PRMT_RC8_MODE { add_option( PRMT_RC8_MODE); } + | PRMT_RC16_MODE{ add_option( PRMT_RC16_MODE);} + | PRMT_ECL_MODE { add_option( PRMT_ECL_MODE); } + | PRMT_ECR_MODE { add_option( PRMT_ECR_MODE); } + ; + +wmma_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space); add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} + | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);add_wmma_option($4);} + ; + +vp_spec: WMMA_DIRECTIVE LAYOUT CONFIGURATION{add_space_spec(global_space,0);add_ptr_spec(global_space);add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);} + | WMMA_DIRECTIVE LAYOUT LAYOUT CONFIGURATION{add_wmma_option($1);add_wmma_option($2);add_wmma_option($3);add_wmma_option($4);} + ; + + + +operand_list: operand | operand COMMA operand_list; operand: IDENTIFIER { add_scalar_operand( $1 ); } @@ -544,6 +575,7 @@ operand: IDENTIFIER { add_scalar_operand( $1 ); } vector_operand: LEFT_BRACE IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_2vector_operand($2,$4); } | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_3vector_operand($2,$4,$6); } | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_4vector_operand($2,$4,$6,$8); } + | LEFT_BRACE IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER COMMA IDENTIFIER RIGHT_BRACE { add_8vector_operand($2,$4,$6,$8,$10,$12,$14,$16); } | LEFT_BRACE IDENTIFIER RIGHT_BRACE { add_1vector_operand($2); } ; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index ba6d7ed..73db24d 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -258,6 +258,14 @@ bool symbol_table::add_function_decl( const char *name, int entry_point, functio return prior_decl; } +function_info *symbol_table::lookup_function( std::string name ) +{ + std::string key = std::string(name); + std::map<std::string,function_info*>::iterator it = m_function_info_lookup.find(key); + assert ( it != m_function_info_lookup.end() ); + return it->second; +} + type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) { if( space_spec == param_space_unclassified ) @@ -281,8 +289,10 @@ type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_di { type_info_key t = base_type->get_key(); t.set_array_dim(array_dim); - type_info *pt; - pt = m_types[t] = new type_info(this,t); + type_info *pt = new type_info(this,t); + //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. + //TODO: find where m_types can be used in future and solve the seg fault. + //pt = m_types[t] = new type_info(this,t); return pt; } @@ -576,6 +586,40 @@ bool function_info::connect_break_targets() //connecting break instructions with return modified; } +void function_info::do_pdom() +{ + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if ( g_debug_execution>=50 ) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if ( g_debug_execution>=2 ) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if ( g_debug_execution>=50 ) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); + for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); + fflush(stdout); + m_assembled = true; +} void intersect( std::set<int> &A, const std::set<int> &B ) { // return intersection of A and B in A @@ -996,7 +1040,7 @@ static std::list<operand_info> check_operands( int opcode, const std::list<operand_info> &operands ) { static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) ) { + if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { // just make sure these do not have have const operands... if( !g_warn_literal_operands_two_type_inst ) { std::list<operand_info>::const_iterator o; @@ -1044,6 +1088,7 @@ ptx_instruction::ptx_instruction( int opcode, const std::list<operand_info> &operands, const operand_info &return_var, const std::list<int> &options, + const std::list<int> &wmma_options, const std::list<int> &scalar_type, memory_space_t space_spec, const char *file, @@ -1062,6 +1107,7 @@ ptx_instruction::ptx_instruction( int opcode, m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); m_return_var = return_var; m_options = options; + m_wmma_options = wmma_options; m_wide = false; m_hi = false; m_lo = false; @@ -1079,9 +1125,33 @@ ptx_instruction::ptx_instruction( int opcode, m_atomic_spec = 0; m_membar_level = 0; m_inst_size = 8; // bytes - + int rr=0; std::list<int>::const_iterator i; unsigned n=1; + for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { + int last_ptx_inst_option = *i; + switch ( last_ptx_inst_option ) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type=last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++]=last_ptx_inst_option; + break; + case M16N16K16: + break; + default: + assert(0); + break; + } + } + rr=0; + n=1; for ( i=options.begin(); i!= options.end(); i++, n++ ) { int last_ptx_inst_option = *i; switch ( last_ptx_inst_option ) { @@ -1208,17 +1278,25 @@ ptx_instruction::ptx_instruction( int opcode, case HALF_OPTION: m_inst_size = 4; // bytes break; - case EXTP_OPTION: - break; - case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; - case UP_OPTION: - case DOWN_OPTION: - case BFLY_OPTION: - case IDX_OPTION: + case EXTP_OPTION: + break; + case NC_OPTION: + m_cache_option = last_ptx_inst_option; + break; + case UP_OPTION: + case DOWN_OPTION: + case BFLY_OPTION: + case IDX_OPTION: m_shfl_op = last_ptx_inst_option; break; + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + case PRMT_RC16_MODE: + m_prmt_op = last_ptx_inst_option; + break; default: assert(0); break; @@ -1307,6 +1385,7 @@ function_info::function_info(int entry_point ) m_kernel_info.smem = 0; m_local_mem_framesize = 0; m_args_aligned_size = -1; + pdom_done = false; //initialize it to false } unsigned function_info::print_insn( unsigned pc, FILE * fp ) const diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4d3641..1af85de 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -91,6 +91,7 @@ public: bool is_tex() const { return m_space_spec == tex_space;} bool is_func_addr() const { return m_is_function?true:false; } int scalar_type() const { return m_scalar_type_spec;} + int get_alignment_spec() const { return m_alignment_spec;} unsigned type_decode( size_t &size, int &t ) const; static unsigned type_decode( int type, size_t &size, int &t ); memory_space_t get_memory_space() const { return m_space_spec; } @@ -163,6 +164,7 @@ public: m_is_global = false; m_is_local = false; m_is_param_local = false; + m_is_param_kernel = false; m_is_tex = false; m_is_func_addr = false; m_reg_num_valid = false; @@ -176,6 +178,7 @@ public: if ( type ) m_is_global = type->get_key().is_global(); if ( type ) m_is_local = type->get_key().is_local(); if ( type ) m_is_param_local = type->get_key().is_param_local(); + if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); if ( type ) m_is_tex = type->get_key().is_tex(); if ( type ) m_is_func_addr = type->get_key().is_func_addr(); } @@ -222,10 +225,12 @@ public: bool is_label() const { return m_is_label;} bool is_shared() const { return m_is_shared;} + bool is_sstarr() const { return m_is_sstarr;} bool is_const() const { return m_is_const;} bool is_global() const { return m_is_global;} bool is_local() const { return m_is_local;} bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } bool is_tex() const { return m_is_tex;} bool is_func_addr() const { return m_is_func_addr; } bool is_reg() const @@ -279,10 +284,12 @@ private: bool m_address_valid; bool m_is_label; bool m_is_shared; + bool m_is_sstarr; bool m_is_const; bool m_is_global; bool m_is_local; bool m_is_param_local; + bool m_is_param_kernel; bool m_is_tex; bool m_is_func_addr; unsigned m_reg_num; @@ -308,16 +315,19 @@ public: symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); void add_function( function_info *func, const char *filename, unsigned linenumber ); bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); + function_info *lookup_function(std::string name); type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); type_info *add_type( function_info *func ); type_info *get_array_type( type_info *base_type, unsigned array_dim ); void set_label_address( const symbol *label, unsigned addr ); unsigned next_reg_num() { return ++m_reg_allocator;} addr_t get_shared_next() { return m_shared_next;} + addr_t get_sstarr_next() { return m_sstarr_next;} addr_t get_global_next() { return m_global_next;} addr_t get_local_next() { return m_local_next;} addr_t get_tex_next() { return m_tex_next;} void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} + void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} @@ -339,6 +349,7 @@ public: private: unsigned m_reg_allocator; unsigned m_shared_next; + unsigned m_sstarr_next; unsigned m_const_next; unsigned m_global_next; unsigned m_local_next; @@ -399,6 +410,8 @@ public: m_type = symbolic_t; } else if ( addr->is_param_local() ) { m_type = symbolic_t; + } else if ( addr->is_param_kernel() ) { + m_type = symbolic_t; } else if ( addr->is_tex() ) { m_type = symbolic_t; } else if ( addr->is_func_addr() ) { @@ -429,11 +442,15 @@ public: m_uid = get_uid(); m_valid = true; m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[4]; + m_value.m_vector_symbolic = new const symbol*[8]; m_value.m_vector_symbolic[0] = addr1; m_value.m_vector_symbolic[1] = addr2; m_value.m_vector_symbolic[2] = NULL; m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; m_addr_offset = 0; m_vector = false; m_neg_pred = false; @@ -567,16 +584,48 @@ public: m_valid = true; m_vector = true; m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[4]; + m_value.m_vector_symbolic = new const symbol*[8]; m_value.m_vector_symbolic[0] = s1; m_value.m_vector_symbolic[1] = s2; m_value.m_vector_symbolic[2] = s3; m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; m_addr_offset = 0; m_neg_pred = false; m_is_return_var = false; m_immediate_address=false; } + operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8) + { + init(); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol*[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address=false; + } + void init() { m_uid=(unsigned)-1; @@ -623,12 +672,16 @@ public: if( !m_value.m_vector_symbolic[1] ) return 1; if( !m_value.m_vector_symbolic[2] ) return 2; if( !m_value.m_vector_symbolic[3] ) return 3; - return 4; + if( !m_value.m_vector_symbolic[4] ) return 4; + if( !m_value.m_vector_symbolic[5] ) return 5; + if( !m_value.m_vector_symbolic[6] ) return 6; + if( !m_value.m_vector_symbolic[7] ) return 7; + return 8; } const symbol* vec_symbol(int idx) const { - assert(idx < 4); + assert(idx < 8); const symbol *result = m_value.m_vector_symbolic[idx]; assert( result != NULL ); return result; @@ -675,6 +728,13 @@ public: return m_value.m_symbolic->type()->get_key().is_param_local(); } + bool is_param_kernel() const + { + if ( m_type != symbolic_t ) + return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + bool is_vector() const { if ( m_vector) return true; @@ -685,6 +745,10 @@ public: int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } + int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } + int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } + int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } + int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } bool is_label() const { return m_type == label_t;} @@ -713,6 +777,7 @@ public: } return m_value.m_symbolic->is_shared(); } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} bool is_const() const { return m_value.m_symbolic->is_const();} bool is_global() const { return m_value.m_symbolic->is_global();} bool is_local() const { return m_value.m_symbolic->is_local();} @@ -731,7 +796,7 @@ public: { ptx_reg_t result; switch ( m_type ) { - case int_t: result.s32 = m_value.m_int; break; + case int_t: result.s64 = m_value.m_int; break; case float_op_t: result.f32 = m_value.m_float; break; case double_op_t: result.f64 = m_value.m_double; break; case unsigned_t: result.u32 = m_value.m_unsigned; break; @@ -860,6 +925,7 @@ public: const std::list<operand_info> &operands, const operand_info &return_var, const std::list<int> &options, + const std::list<int> &wmma_options, const std::list<int> &scalar_type, memory_space_t space_spec, const char *file, @@ -942,6 +1008,31 @@ public: assert( m_operands.size() > 3 ); return m_operands[3]; } + const operand_info &src4() const + { + assert( m_operands.size() > 4 ); + return m_operands[4]; + } + const operand_info &src5() const + { + assert( m_operands.size() > 5 ); + return m_operands[5]; + } + const operand_info &src6() const + { + assert( m_operands.size() > 6 ); + return m_operands[6]; + } + const operand_info &src7() const + { + assert( m_operands.size() > 7 ); + return m_operands[7]; + } + const operand_info &src8() const + { + assert( m_operands.size() > 8 ); + return m_operands[8]; + } const operand_info &operand_lookup( unsigned n ) const { @@ -957,6 +1048,12 @@ public: unsigned get_vector() const { return m_vector_spec;} unsigned get_atomic() const { return m_atomic_spec;} + int get_wmma_type() const { + return m_wmma_type; + } + int get_wmma_layout(int index) const { + return m_wmma_layout[index];//0->Matrix D,1->Matrix C + } int get_type() const { assert( !m_scalar_type.empty() ); @@ -1004,13 +1101,14 @@ public: unsigned dimension() const { return m_geom_spec;} unsigned barrier_op() const {return m_barrier_op;} unsigned shfl_op() const {return m_shfl_op;} + unsigned prmt_op() const {return m_prmt_op;} enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; enum vote_mode_t vote_mode() const { return m_vote_mode; } int membar_level() const { return m_membar_level; } bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP ) + if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) return true; // Check PTXPlus operand type below // Source operands are memory operands @@ -1022,7 +1120,7 @@ public: return false; } bool has_memory_write() const { - if( m_opcode == ST_OP ) return true; + if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; // Check PTXPlus operand type below // Destination operand is a memory operand ptx_instruction::const_iterator op=op_iter_begin(); @@ -1056,6 +1154,7 @@ private: operand_info m_return_var; std::list<int> m_options; + std::list<int> m_wmma_options; bool m_wide; bool m_hi; bool m_lo; @@ -1065,11 +1164,15 @@ private: bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) bool m_to_option; unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; unsigned m_rounding_mode; unsigned m_compare_op; unsigned m_saturation_mode; unsigned m_barrier_op; unsigned m_shfl_op; + unsigned m_prmt_op; std::list<int> m_scalar_type; memory_space_t m_space_spec; @@ -1178,7 +1281,7 @@ public: //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 void find_ipostdominators( ); void print_ipostdominators(); - + void do_pdom(); //function to call pdom analysis unsigned get_num_reconvergence_pairs(); @@ -1242,6 +1345,7 @@ public: void finalize( memory_space *param_mem ); void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); void list_param( FILE *fout ) const; + void ptx_jit_config(std::map<unsigned long long, size_t> mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; const struct gpgpu_ptx_sim_info* get_kernel_info () const { @@ -1277,6 +1381,20 @@ public: m_local_mem_framesize = sz; } bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } //return pdom flag + void set_pdom() { pdom_done = true; } //set pdom flag + + void add_config_param( size_t size, unsigned alignment ){ + unsigned offset = 0; + if (m_param_configs.size()>0){ + unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; + //ensure offset matches alignment requirements + offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; + } + m_param_configs.push_back(std::pair<size_t,unsigned>(size, offset)); + } + + std::pair<size_t, unsigned> get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} unsigned get_maxnt_id() { return maxnt_id;} @@ -1288,12 +1406,14 @@ private: bool m_entry_point; bool m_extern; bool m_assembled; + bool pdom_done; //flag to check whether pdom is completed or not std::string m_name; ptx_instruction **m_instr_mem; unsigned m_start_PC; unsigned m_instr_mem_size; std::map<std::string,param_t> m_kernel_params; std::map<unsigned,param_info> m_ptx_kernel_param_info; + std::vector< std::pair<size_t, unsigned> > m_param_configs; const symbol *m_return_var_sym; std::vector<const symbol*> m_args; std::list<ptx_instruction*> m_instructions; @@ -1312,6 +1432,8 @@ private: //parameter size for device kernels int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 11d9013..8c6f361 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -32,6 +32,7 @@ #include <unistd.h> #include <dirent.h> #include <fstream> +#include <sstream> /// globals @@ -189,6 +190,13 @@ symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source return symtab; } +symbol_table *gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) +{ + symbol_table *symtab=init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); + return symtab; +} + void fix_duplicate_errors(char fname2[1024]) { char tempfile[1024] = "_temp_ptx"; char commandline[1024]; @@ -292,38 +300,94 @@ void fix_duplicate_errors(char fname2[1024]) { } } -void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num ) + +//we need the application name here too. +char* get_app_binary_name(){ + char exe_path[1025]; + char *self_exe_path; +#ifdef __APPLE__ + //AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); +#else + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while(token !=NULL){ + self_exe_path = token; + token = strtok(NULL,"/"); + } +#endif + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; +} + +void gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) { - char fname[1024]; - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0]=0; + extern bool g_cdp_enabled; + if(!g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p_for_info); - fclose(ptxfile); + g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(g_ptxinfo_filename,"r"); + ptxinfo_parse(); + fclose(ptxinfo_in); +} - char fname2[1024]; - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - fprintf(stderr, " Ensure you have write access to simulation directory\n"); - fprintf(stderr, " and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } +void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version ) +{ + //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name=get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index=1; index <= no_of_ptx; index++){ + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); - char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char commandline[1024]; - char extra_flags[1024]; - extra_flags[0]=0; + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); + if (system(commandline) !=0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; #if CUDART_VERSION >= 3000 if ( g_occupancy_sm_number == 0 ) { @@ -355,26 +419,100 @@ void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + } + } + + //TODO: duplicate code! move it into a function so that it can be reused! + if(no_of_ptx==0) { + //For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname,1024,"_ptx_XXXXXX"); + int fd=mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); + FILE *ptxfile = fopen(fname,"w"); + fprintf(ptxfile,"%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2,1024,"_ptx2_XXXXXX"); + fd=mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf(" Ensure you have write access to simulation directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); } - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - fprintf(stderr, " Ensure ptxas is in your path.\n"); + //char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); + char extra_flags[1024]; + extra_flags[0]=0; + + #if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + extern bool g_cdp_enabled; + if(!g_cdp_enabled) + snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); + else + snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); + #endif + + snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if( result != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); exit(1); } } - ptxinfo_in = fopen(tempfile_ptxinfo,"r"); - g_ptxinfo_filename = tempfile_ptxinfo; + //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. + if(no_of_ptx>0){ + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); + snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3)!=0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n",commandline3); + exit(1); + } + } + + if(no_of_ptx>0) + g_ptxinfo_filename = final_tempfile_ptxinfo; + else + g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_in = fopen(g_ptxinfo_filename,"r"); + ptxinfo_parse(); + snprintf(commandline,1024,"rm -f *info"); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } if( ! g_save_embedded_ptx ) { - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); + if(no_of_ptx>0) + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); + else + snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - result = system(commandline); - if( result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while loading PTX (c) %d\n", result); + if( system(commandline) != 0 ) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); exit(1); } } -} - +}
\ No newline at end of file diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index cb02eda..e5df6a9 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -30,9 +30,12 @@ #include <string> extern bool g_override_embedded_ptx; +extern int no_of_ptx; //counter to track number of ptx files to be extracted in an application. class symbol_table *gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ); -void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num ); +class symbol_table *gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ); +void gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version=20 ); +void gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version ); char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); bool keep_intermediate_files(); diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index 6dc1f40..25758dd 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -32,6 +32,8 @@ extern int ptx_error( const char *s ); extern int ptx_lineno; +extern int ptx_parse(); +extern FILE *ptx_in; static const struct core_config *g_shader_core_config; void set_ptx_warp_size(const struct core_config * warp_size) @@ -59,6 +61,7 @@ memory_space_t g_ptr_spec = undefined_space; int g_scalar_type_spec = -1; int g_vector_spec = -1; int g_alignment_spec = -1; +int g_size = -1; int g_extern_spec = 0; // variable declaration stuff: @@ -72,6 +75,7 @@ symbol *g_label; int g_opcode = -1; std::list<operand_info> g_operands; std::list<int> g_options; +std::list<int> g_wmma_options; std::list<int> g_scalar_type; #define PTX_PARSE_DPRINTF(...) \ @@ -105,6 +109,36 @@ void read_parser_environment_variables() } } +void init_directive_state() +{ + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec=undefined_space; + g_ptr_spec=undefined_space; + g_scalar_type_spec=-1; + g_vector_spec=-1; + g_opcode=-1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void init_instruction_state() +{ + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(); + init_directive_state(); +} + symbol_table *init_parser( const char *ptx_filename ) { g_filename = strdup(ptx_filename); @@ -112,9 +146,9 @@ symbol_table *init_parser( const char *ptx_filename ) g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL); g_global_symbol_table = g_current_symbol_table = g_global_allfiles_symbol_table; } - else { + /*else { g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - } + }*/ ptx_lineno = 1; #define DEF(X,Y) g_ptx_token_decode[X] = Y; @@ -135,35 +169,13 @@ symbol_table *init_parser( const char *ptx_filename ) g_ptx_token_decode[generic_space] = "generic_space"; g_ptx_token_decode[instruction_space] = "instruction_space"; - return g_global_symbol_table; -} - -void init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_return_var = operand_info(); init_directive_state(); + init_instruction_state(); + + ptx_in = fopen(ptx_filename, "r"); + ptx_parse(); + fclose(ptx_in); + return g_global_symbol_table; } static int g_entry_point; @@ -300,6 +312,7 @@ void add_instruction() g_operands, g_return_var, g_options, + g_wmma_options, g_scalar_type, g_space_spec, g_filename, @@ -367,6 +380,9 @@ int pad_address (new_addr_type address, unsigned size, unsigned maxalign) { void add_identifier( const char *identifier, int array_dim, unsigned array_ident ) { + if(array_ident==ARRAY_IDENTIFIER){ + g_size *= array_dim; + } if( g_func_decl && (g_func_info == NULL) ) { // return variable decl... assert( g_add_identifier_cached__identifier == NULL ); @@ -440,6 +456,20 @@ void add_identifier( const char *identifier, int array_dim, unsigned array_ident g_last_symbol->set_address( addr+addr_pad ); g_current_symbol_table->alloc_shared( num_bits/8 + addr_pad ); break; + case sstarr_space: + printf("GPGPU-Sim PTX: allocating sstarr region for \"%s\" ", + identifier); + fflush(stdout); + assert( (num_bits%8) == 0 ); + addr = g_current_symbol_table->get_sstarr_next(); + addr_pad = pad_address(addr, num_bits/8, 128); + printf("from 0x%x to 0x%lx (sstarr memory space)\n", + addr+addr_pad, + addr+addr_pad + num_bits/8); + fflush(stdout); + g_last_symbol->set_address( addr+addr_pad ); + g_current_symbol_table->alloc_sstarr( num_bits/8 + addr_pad ); + break; case const_space: if( array_ident == ARRAY_IDENTIFIER_NO_DIM ) { printf("GPGPU-Sim PTX: deferring allocation of constant region for \"%s\" (need size information)\n", identifier ); @@ -556,10 +586,15 @@ void add_constptr(const char* identifier1, const char* identifier2, int offset) void add_function_arg() { + assert(g_size>0); if( g_func_info ) { PTX_PARSE_DPRINTF("add_function_arg \"%s\"", g_last_symbol->name().c_str() ); g_func_info->add_arg(g_last_symbol); + unsigned alignment = (g_alignment_spec==-1) ? g_size : g_alignment_spec; + assert(alignment==1||alignment==2||alignment==4||alignment==8||alignment==16);//known valid alignment values + g_func_info->add_config_param( g_size, alignment); } + } void add_extern_spec() @@ -611,12 +646,38 @@ void add_vector_spec(int spec ) void add_scalar_type_spec( int type_spec ) { + //save size of parameter + switch ( type_spec ) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + g_size = 1; break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + g_size = 2; break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + g_size = 4; break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + g_size = 8; break; + case BB128_TYPE: + g_size = 16; break; + } PTX_PARSE_DPRINTF("add_scalar_type_spec \"%s\"", g_ptx_token_decode[type_spec].c_str()); g_scalar_type.push_back( type_spec ); if ( g_scalar_type.size() > 1 ) { parse_assert( (g_opcode == -1) || (g_opcode == CVT_OP) || (g_opcode == SET_OP) || (g_opcode == SLCT_OP) - || (g_opcode == TEX_OP), - "only cvt, set, slct, and tex can have more than one type specifier."); + || (g_opcode == TEX_OP)|| (g_opcode==MMA_OP)|| (g_opcode == DP4A_OP), + "only cvt, set, slct, tex and dp4a can have more than one type specifier."); } g_scalar_type_spec = type_spec; } @@ -655,7 +716,11 @@ void add_option( int option ) PTX_PARSE_DPRINTF("add_option"); g_options.push_back( option ); } - +void add_wmma_option( int option ) +{ + PTX_PARSE_DPRINTF("add_option"); + g_wmma_options.push_back( option ); +} void add_double_operand( const char *d1, const char *d2 ) { //operands that access two variables. @@ -711,6 +776,28 @@ void add_4vector_operand( const char *d1, const char *d2, const char *d3, const if ( s4 == null_op ) s4 = NULL; g_operands.push_back( operand_info(s1,s2,s3,s4) ); } +void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4,const char *d5,const char *d6,const char *d7,const char *d8 ) +{ + PTX_PARSE_DPRINTF("add_8vector_operand"); + const symbol *s1 = g_current_symbol_table->lookup(d1); + const symbol *s2 = g_current_symbol_table->lookup(d2); + const symbol *s3 = g_current_symbol_table->lookup(d3); + const symbol *s4 = g_current_symbol_table->lookup(d4); + const symbol *s5 = g_current_symbol_table->lookup(d5); + const symbol *s6 = g_current_symbol_table->lookup(d6); + const symbol *s7 = g_current_symbol_table->lookup(d7); + const symbol *s8 = g_current_symbol_table->lookup(d8); + parse_assert( s1 != NULL && s2 != NULL && s3 != NULL && s4 != NULL && s5 !=NULL && s6 !=NULL && s7 !=NULL && s8 !=NULL, "v4 component(s) missing declarations."); + const symbol *null_op = g_current_symbol_table->lookup("_"); + if ( s2 == null_op ) s2 = NULL; + if ( s3 == null_op ) s3 = NULL; + if ( s4 == null_op ) s4 = NULL; + if ( s5 == null_op ) s5 = NULL; + if ( s6 == null_op ) s6 = NULL; + if ( s7 == null_op ) s7 = NULL; + if ( s8 == null_op ) s8 = NULL; + g_operands.push_back( operand_info(s1,s2,s3,s4,s5,s6,s7,s8) ); +} void add_builtin_operand( int builtin, int dim_modifier ) { diff --git a/src/cuda-sim/ptx_parser.h b/src/cuda-sim/ptx_parser.h index 13042e1..7b6e3a2 100644 --- a/src/cuda-sim/ptx_parser.h +++ b/src/cuda-sim/ptx_parser.h @@ -57,7 +57,9 @@ void add_1vector_operand( const char *d1 ); void add_2vector_operand( const char *d1, const char *d2 ); void add_3vector_operand( const char *d1, const char *d2, const char *d3 ); void add_4vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ); +void add_8vector_operand( const char *d1, const char *d2, const char *d3, const char *d4 ,const char *d5,const char *d6,const char *d7,const char *d8); void add_option(int option ); +void add_wmma_option(int option ); void add_builtin_operand( int builtin, int dim_modifier ); void add_memory_operand( ); void add_literal_int( int value ); diff --git a/src/cuda-sim/ptx_sim.cc b/src/cuda-sim/ptx_sim.cc index a3e43aa..820287d 100644 --- a/src/cuda-sim/ptx_sim.cc +++ b/src/cuda-sim/ptx_sim.cc @@ -44,6 +44,7 @@ ptx_cta_info::ptx_cta_info( unsigned sm_idx ) m_sm_idx = sm_idx; m_uid = g_ptx_cta_info_uid++; + m_bar_threads = 0; } void ptx_cta_info::add_thread( ptx_thread_info *thd ) @@ -128,6 +129,21 @@ unsigned ptx_cta_info::get_sm_idx() const return m_sm_idx; } +unsigned ptx_cta_info::get_bar_threads() const +{ + return m_bar_threads; +} + +void ptx_cta_info::inc_bar_threads() +{ + m_bar_threads++; +} + +void ptx_cta_info::reset_bar_threads() +{ + m_bar_threads = 0; +} + ptx_warp_info::ptx_warp_info() { reset_done_threads(); @@ -173,6 +189,7 @@ ptx_thread_info::ptx_thread_info( kernel_info_t &kernel ) m_last_memory_space = undefined_space; m_branch_taken = 0; m_shared_mem = NULL; + m_sstarr_mem = NULL; m_warp_info = NULL; m_cta_info = NULL; m_local_mem = NULL; diff --git a/src/cuda-sim/ptx_sim.h b/src/cuda-sim/ptx_sim.h index e6eb02e..d226fbe 100644 --- a/src/cuda-sim/ptx_sim.h +++ b/src/cuda-sim/ptx_sim.h @@ -28,7 +28,7 @@ #define ptx_sim_h_INCLUDED #include <stdlib.h> - +#include "half.h" #include "../abstract_hardware_model.h" #include "../tr1_hash_map.h" @@ -42,6 +42,12 @@ #include "memory.h" +#define GCC_VERSION (__GNUC__ * 10000 \ + + __GNUC_MINOR__ * 100 \ + + __GNUC_PATCHLEVEL__) + + + struct param_t { const void *pdata; int type; @@ -53,6 +59,8 @@ struct param_t { #include "memory.h" +using half_float::half; + union ptx_reg_t { ptx_reg_t() { bits.ms = 0; @@ -126,7 +134,12 @@ union ptx_reg_t { unsigned short u16; unsigned int u32; unsigned long long u64; - float f16; + //gcc 4.7.0 + #if GCC_VERSION >= 40700 + half f16; + #else + float f16; + #endif float f32; double f64; struct { @@ -158,8 +171,12 @@ public: void register_thread_exit( ptx_thread_info *thd ); void register_deleted_thread( ptx_thread_info *thd ); unsigned get_sm_idx() const; + unsigned get_bar_threads() const; + void inc_bar_threads(); + void reset_bar_threads(); private: + unsigned m_bar_threads; unsigned long long m_uid; unsigned m_sm_idx; std::set<ptx_thread_info*> m_threads_in_cta; @@ -169,7 +186,7 @@ private: class ptx_warp_info { public: - ptx_warp_info(); + ptx_warp_info(); // add get_core or something, or threads? unsigned get_done_threads() const; void inc_done_threads(); void reset_done_threads(); @@ -289,6 +306,8 @@ public: const ptx_version &get_ptx_version() const; void set_reg( const symbol *reg, const ptx_reg_t &value ); + void print_reg_thread (char * fname); + void resume_reg_thread(char * fname, symbol_table * symtab); ptx_reg_t get_reg( const symbol *reg ); ptx_reg_t get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ); void set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ); @@ -299,6 +318,15 @@ public: const ptx_reg_t &data2, const ptx_reg_t &data3, const ptx_reg_t &data4 ); + void set_wmma_vector_operand_values( const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4, + const ptx_reg_t &data5, + const ptx_reg_t &data6, + const ptx_reg_t &data7, + const ptx_reg_t &data8 ); function_info *func_info() { @@ -438,6 +466,7 @@ public: memory_space_t m_last_memory_space; dram_callback_t m_last_dram_callback; memory_space *m_shared_mem; + memory_space *m_sstarr_mem; memory_space *m_local_mem; ptx_warp_info *m_warp_info; ptx_cta_info *m_cta_info; diff --git a/src/gpgpu-sim/gpu-sim.cc b/src/gpgpu-sim/gpu-sim.cc index 8bbadf5..ec570bf 100644 --- a/src/gpgpu-sim/gpu-sim.cc +++ b/src/gpgpu-sim/gpu-sim.cc @@ -58,6 +58,7 @@ #include "../debug.h" #include "../gpgpusim_entrypoint.h" #include "../cuda-sim/cuda-sim.h" +#include "../cuda-sim/ptx_ir.h" #include "../trace.h" #include "mem_latency_stat.h" #include "power_stat.h" @@ -284,6 +285,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shader_registers", OPT_UINT32, &gpgpu_shader_registers, "Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)", "8192"); + option_parser_register(opp, "-gpgpu_registers_per_block", OPT_UINT32, &gpgpu_registers_per_block, + "Maximum number of registers per CTA. (default 8192)", + "8192"); option_parser_register(opp, "-gpgpu_ignore_resources_limitation", OPT_BOOL, &gpgpu_ignore_resources_limitation, "gpgpu_ignore_resources_limitation (default 0)", "0"); @@ -305,6 +309,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_n_ldst_response_buffer_size", OPT_UINT32, &ldst_unit_response_queue_size, "number of response packets in ld/st unit ejection buffer", "2"); + option_parser_register(opp, "-gpgpu_shmem_per_block", OPT_UINT32, &gpgpu_shmem_per_block, + "Size of shared memory per thread block or CTA (default 48kB)", + "49152"); option_parser_register(opp, "-gpgpu_shmem_size", OPT_UINT32, &gpgpu_shmem_size, "Size of shared memory per shader core (default 16kB)", "16384"); @@ -329,6 +336,9 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, "Number of portions a warp is divided into for shared memory bank conflict check ", "2"); + option_parser_register(opp, "-mem_unit_ports", OPT_INT32, &mem_unit_ports, + "The number of memory transactions allowed per core cycle", + "1"); option_parser_register(opp, "-gpgpu_shmem_warp_parts", OPT_INT32, &mem_warp_parts, "Number of portions a warp is divided into for shared memory bank conflict check ", "2"); @@ -347,8 +357,14 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_reg_bank_use_warp_id", OPT_BOOL, &gpgpu_reg_bank_use_warp_id, "Use warp ID in mapping registers to banks (default = off)", "0"); + option_parser_register(opp, "-sub_core_model", OPT_BOOL, &sub_core_model, + "Sub Core Volta/Pascal model (default = off)", + "0"); + option_parser_register(opp, "-enable_specialized_operand_collector", OPT_BOOL, &enable_specialized_operand_collector, + "enable_specialized_operand_collector", + "1"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_sp", OPT_INT32, &gpgpu_operand_collector_num_units_sp, - "number of collector units (default = 4)", + "number of collector units (default = 4)", "4"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_dp", OPT_INT32, &gpgpu_operand_collector_num_units_dp, "number of collector units (default = 0)", @@ -356,6 +372,12 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_units_sfu", OPT_INT32, &gpgpu_operand_collector_num_units_sfu, "number of collector units (default = 4)", "4"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_int", OPT_INT32, &gpgpu_operand_collector_num_units_int, + "number of collector units (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_units_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_units_tensor_core, + "number of collector units (default = 4)", + "4"); option_parser_register(opp, "-gpgpu_operand_collector_num_units_mem", OPT_INT32, &gpgpu_operand_collector_num_units_mem, "number of collector units (default = 2)", "2"); @@ -371,6 +393,12 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_in_ports_sfu, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_int", OPT_INT32, &gpgpu_operand_collector_num_in_ports_int, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_in_ports_tensor_core, + "number of collector unit in ports (default = 1)", + "1"); option_parser_register(opp, "-gpgpu_operand_collector_num_in_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_in_ports_mem, "number of collector unit in ports (default = 1)", "1"); @@ -386,6 +414,12 @@ void shader_core_config::reg_options(class OptionParser * opp) option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_sfu", OPT_INT32, &gpgpu_operand_collector_num_out_ports_sfu, "number of collector unit in ports (default = 1)", "1"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_int", OPT_INT32, &gpgpu_operand_collector_num_out_ports_int, + "number of collector unit in ports (default = 0)", + "0"); + option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_tensor_core", OPT_INT32, &gpgpu_operand_collector_num_out_ports_tensor_core, + "number of collector unit in ports (default = 1)", + "1"); option_parser_register(opp, "-gpgpu_operand_collector_num_out_ports_mem", OPT_INT32, &gpgpu_operand_collector_num_out_ports_mem, "number of collector unit in ports (default = 1)", "1"); @@ -409,17 +443,26 @@ void shader_core_config::reg_options(class OptionParser * opp) "1"); option_parser_register(opp, "-gpgpu_pipeline_widths", OPT_CSTR, &pipeline_widths_string, "Pipeline widths " - "ID_OC_SP,ID_OC_DP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_SFU,OC_EX_MEM,EX_WB", - "1,1,1,1,1,1,1,1,1" ); + "ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE", + "1,1,1,1,1,1,1,1,1,1,1,1,1" ); + option_parser_register(opp, "-gpgpu_tensor_core_avail", OPT_INT32, &gpgpu_tensor_core_avail, + "Tensor Core Available (default=0)", + "0"); option_parser_register(opp, "-gpgpu_num_sp_units", OPT_INT32, &gpgpu_num_sp_units, "Number of SP units (default=1)", "1"); option_parser_register(opp, "-gpgpu_num_dp_units", OPT_INT32, &gpgpu_num_dp_units, "Number of DP units (default=0)", "0"); + option_parser_register(opp, "-gpgpu_num_int_units", OPT_INT32, &gpgpu_num_int_units, + "Number of INT units (default=0)", + "0"); option_parser_register(opp, "-gpgpu_num_sfu_units", OPT_INT32, &gpgpu_num_sfu_units, "Number of SF units (default=1)", "1"); + option_parser_register(opp, "-gpgpu_num_tensor_core_units", OPT_INT32, &gpgpu_num_tensor_core_units, + "Number of tensor_core units (default=1)", + "1"); option_parser_register(opp, "-gpgpu_num_mem_units", OPT_INT32, &gpgpu_num_mem_units, "Number if ldst units (default=1) WARNING: not hooked up to anything", "1"); @@ -721,11 +764,21 @@ int gpgpu_sim::shared_mem_size() const return m_shader_config->gpgpu_shmem_size; } +int gpgpu_sim::shared_mem_per_block() const +{ + return m_shader_config->gpgpu_shmem_per_block; +} + int gpgpu_sim::num_registers_per_core() const { return m_shader_config->gpgpu_shader_registers; } +int gpgpu_sim::num_registers_per_block() const +{ + return m_shader_config->gpgpu_registers_per_block; +} + int gpgpu_sim::wrp_size() const { return m_shader_config->warp_size; @@ -1178,6 +1231,9 @@ void shader_core_ctx::mem_instruction_stats(const warp_inst_t &inst) case shared_space: m_stats->gpgpu_n_shmem_insn += active_count; break; + case sstarr_space: + m_stats->gpgpu_n_sstarr_insn += active_count; + break; case const_space: m_stats->gpgpu_n_const_insn += active_count; break; @@ -1379,22 +1435,44 @@ void shader_core_ctx::issue_block2core( kernel_info_t &kernel ) // bind functional simulation state of threads to hardware resources (simulation) warp_set_t warps; unsigned nthreads_in_block= 0; + function_info *kernel_func_info = kernel.entry(); + symbol_table * symtab= kernel_func_info->get_symtab(); + unsigned ctaid= kernel.get_next_cta_id_single(); + checkpoint *g_checkpoint= new checkpoint(); for (unsigned i = start_thread; i<end_thread; i++) { m_threadState[i].m_cta_id = free_cta_hw_id; unsigned warp_id = i/m_config->warp_size; nthreads_in_block += ptx_sim_init_thread(kernel,&m_thread[i],m_sid,i,cta_size-(i-start_thread),m_config->n_thread_per_shader,this,free_cta_hw_id,warp_id,m_cluster->get_gpu()); m_threadState[i].m_active = true; + // load thread local memory and register file + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t ) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i%cta_size,ctaid ); + m_thread[i]->resume_reg_thread(fname,symtab); + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i%cta_size,ctaid); + g_checkpoint->load_global_mem(m_thread[i]->m_local_mem, f1name); + } + // warps.set( warp_id ); } assert( nthreads_in_block > 0 && nthreads_in_block <= m_config->n_thread_per_shader); // should be at least one, but less than max m_cta_status[free_cta_hw_id]=nthreads_in_block; + if(m_gpu->resume_option==1 && kernel.get_uid()==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t ) + { + char f1name[2048]; + snprintf(f1name,2048,"checkpoint_files/shared_mem_%d.txt", ctaid); + + g_checkpoint->load_global_mem(m_thread[start_thread]->m_shared_mem, f1name); + } // now that we know which warps are used in this CTA, we can allocate // resources for use in CTA-wide barrier operations m_barriers.allocate_barrier(free_cta_hw_id,warps); // initialize the SIMT stacks and fetch hardware - init_warps( free_cta_hw_id, start_thread, end_thread); + init_warps( free_cta_hw_id, start_thread, end_thread, ctaid, cta_size, kernel.get_uid()); m_n_active_cta++; shader_CTA_count_log(m_sid, 1); @@ -1553,7 +1631,8 @@ void gpgpu_sim::cycle() if( g_single_step && ((gpu_sim_cycle+gpu_tot_sim_cycle) >= g_single_step) ) { raise(SIGTRAP); // Debug breakpoint } - gpu_sim_cycle++; + gpu_sim_cycle++; + if( g_interactive_debugger_enabled ) gpgpu_debug(); @@ -1643,7 +1722,7 @@ void gpgpu_sim::cycle() } } - if (!(gpu_sim_cycle % 20000)) { + if (!(gpu_sim_cycle % 50000)) { // deadlock detection if (m_config.gpu_deadlock_detect && gpu_sim_insn == last_gpu_sim_insn) { gpu_deadlock = true; diff --git a/src/gpgpu-sim/gpu-sim.h b/src/gpgpu-sim/gpu-sim.h index 1bae1fa..6ce5524 100644 --- a/src/gpgpu-sim/gpu-sim.h +++ b/src/gpgpu-sim/gpu-sim.h @@ -335,6 +335,7 @@ public: unsigned num_shader() const { return m_shader_config.num_shader(); } unsigned num_cluster() const { return m_shader_config.n_simt_clusters; } unsigned get_max_concurrent_kernel() const { return max_concurrent_kernel; } + unsigned checkpoint_option; private: void init_clock_domains(void ); @@ -436,7 +437,9 @@ public: void get_pdom_stack_top_info( unsigned sid, unsigned tid, unsigned *pc, unsigned *rpc ); int shared_mem_size() const; + int shared_mem_per_block() const; int num_registers_per_core() const; + int num_registers_per_block() const; int wrp_size() const; int shader_clock() const; const struct cudaDeviceProp *get_prop() const; diff --git a/src/gpgpu-sim/scoreboard.cc b/src/gpgpu-sim/scoreboard.cc index f412054..ebec891 100644 --- a/src/gpgpu-sim/scoreboard.cc +++ b/src/gpgpu-sim/scoreboard.cc @@ -82,7 +82,7 @@ const bool Scoreboard::islongop (unsigned warp_id,unsigned regnum) { void Scoreboard::reserveRegisters(const class warp_inst_t* inst) { - for( unsigned r=0; r < 4; r++) { + for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) { if(inst->out[r] > 0) { reserveRegister(inst->warp_id(), inst->out[r]); SHADER_DPRINTF( SCOREBOARD, @@ -100,7 +100,7 @@ void Scoreboard::reserveRegisters(const class warp_inst_t* inst) inst->space.get_type() == param_space_local || inst->space.get_type() == param_space_unclassified || inst->space.get_type() == tex_space)){ - for ( unsigned r=0; r<4; r++) { + for ( unsigned r=0; r<MAX_OUTPUT_VALUES; r++) { if(inst->out[r] > 0) { SHADER_DPRINTF( SCOREBOARD, "New longopreg marked - warp:%d, reg: %d\n", @@ -115,7 +115,7 @@ void Scoreboard::reserveRegisters(const class warp_inst_t* inst) // Release registers for an instruction void Scoreboard::releaseRegisters(const class warp_inst_t *inst) { - for( unsigned r=0; r < 4; r++) { + for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) { if(inst->out[r] > 0) { SHADER_DPRINTF( SCOREBOARD, "Register Released - warp:%d, reg: %d\n", @@ -138,15 +138,13 @@ bool Scoreboard::checkCollision( unsigned wid, const class inst_t *inst ) const // Get list of all input and output registers std::set<int> inst_regs; - if(inst->out[0] > 0) inst_regs.insert(inst->out[0]); - if(inst->out[1] > 0) inst_regs.insert(inst->out[1]); - if(inst->out[2] > 0) inst_regs.insert(inst->out[2]); - if(inst->out[3] > 0) inst_regs.insert(inst->out[3]); - if(inst->in[0] > 0) inst_regs.insert(inst->in[0]); - if(inst->in[1] > 0) inst_regs.insert(inst->in[1]); - if(inst->in[2] > 0) inst_regs.insert(inst->in[2]); - if(inst->in[3] > 0) inst_regs.insert(inst->in[3]); - if(inst->pred > 0) inst_regs.insert(inst->pred); + for(int iii=0;iii<inst->outcount;iii++) + inst_regs.insert(inst->out[iii]); + + for(int jjj=0;jjj<inst->incount;jjj++) + inst_regs.insert(inst->in[jjj]); + + if(inst->pred > 0) inst_regs.insert(inst->pred); if(inst->ar1 > 0) inst_regs.insert(inst->ar1); if(inst->ar2 > 0) inst_regs.insert(inst->ar2); diff --git a/src/gpgpu-sim/shader.cc b/src/gpgpu-sim/shader.cc index 8a84970..3db988b 100644 --- a/src/gpgpu-sim/shader.cc +++ b/src/gpgpu-sim/shader.cc @@ -90,6 +90,18 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, for (int j = 0; j<N_PIPELINE_STAGES; j++) { m_pipeline_reg.push_back(register_set(m_config->pipe_widths[j],pipeline_stage_name_decode[j])); } + if(m_config->sub_core_model) { + //in subcore model, each scheduler should has its own issue register, so num scheduler = reg width + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SP].get_size() ); + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_SFU].get_size() ); + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_MEM].get_size() ); + if(m_config->gpgpu_tensor_core_avail) + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_TENSOR_CORE].get_size() ); + if(m_config->gpgpu_num_dp_units > 0) + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_DP].get_size() ); + if(m_config->gpgpu_num_int_units > 0) + assert(m_config->gpgpu_num_sched_per_core == m_pipeline_reg[ID_OC_INT].get_size() ); + } m_threadState = (thread_ctx_t*) calloc(sizeof(thread_ctx_t), config->n_thread_per_shader); @@ -152,6 +164,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i ) @@ -167,6 +181,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i, config->gpgpu_scheduler_string @@ -183,6 +199,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i ) @@ -198,6 +216,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i ) @@ -213,6 +233,8 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, &m_pipeline_reg[ID_OC_SP], &m_pipeline_reg[ID_OC_DP], &m_pipeline_reg[ID_OC_SFU], + &m_pipeline_reg[ID_OC_INT], + &m_pipeline_reg[ID_OC_TENSOR_CORE], &m_pipeline_reg[ID_OC_MEM], i, config->gpgpu_scheduler_string @@ -233,53 +255,16 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, } //op collector configuration - enum { SP_CUS, DP_CUS, SFU_CUS, MEM_CUS, GEN_CUS }; - m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); - m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp); - m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); - m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); - m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen); - + + enum { SP_CUS, DP_CUS, SFU_CUS, TENSOR_CORE_CUS, INT_CUS, MEM_CUS, GEN_CUS }; + opndcoll_rfu_t::port_vector_t in_ports; opndcoll_rfu_t::port_vector_t out_ports; opndcoll_rfu_t::uint_vector_t cu_sets; - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); - cu_sets.push_back((unsigned)SP_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } - - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); - out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); - cu_sets.push_back((unsigned)DP_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); - out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); - cu_sets.push_back((unsigned)SFU_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } - - for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) { - in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); - out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); - cu_sets.push_back((unsigned)MEM_CUS); - cu_sets.push_back((unsigned)GEN_CUS); - m_operand_collector.add_port(in_ports,out_ports,cu_sets); - in_ports.clear(),out_ports.clear(),cu_sets.clear(); - } - - + //configure generic collectors + m_operand_collector.add_cu_set(GEN_CUS, m_config->gpgpu_operand_collector_num_units_gen, m_config->gpgpu_operand_collector_num_out_ports_gen); + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_gen; i++) { in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); @@ -287,15 +272,89 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); - cu_sets.push_back((unsigned)GEN_CUS); + if(m_config->gpgpu_tensor_core_avail) { + in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); + } + if(m_config->gpgpu_num_dp_units > 0) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); + } + if(m_config->gpgpu_num_int_units > 0) { + in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); + out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); + } + cu_sets.push_back((unsigned)GEN_CUS); m_operand_collector.add_port(in_ports,out_ports,cu_sets); in_ports.clear(),out_ports.clear(),cu_sets.clear(); } + + if(m_config->enable_specialized_operand_collector) { + m_operand_collector.add_cu_set(SP_CUS, m_config->gpgpu_operand_collector_num_units_sp, m_config->gpgpu_operand_collector_num_out_ports_sp); + m_operand_collector.add_cu_set(DP_CUS, m_config->gpgpu_operand_collector_num_units_dp, m_config->gpgpu_operand_collector_num_out_ports_dp); + m_operand_collector.add_cu_set(TENSOR_CORE_CUS, config->gpgpu_operand_collector_num_units_tensor_core, config->gpgpu_operand_collector_num_out_ports_tensor_core); + m_operand_collector.add_cu_set(SFU_CUS, m_config->gpgpu_operand_collector_num_units_sfu, m_config->gpgpu_operand_collector_num_out_ports_sfu); + m_operand_collector.add_cu_set(MEM_CUS, m_config->gpgpu_operand_collector_num_units_mem, m_config->gpgpu_operand_collector_num_out_ports_mem); + m_operand_collector.add_cu_set(INT_CUS, m_config->gpgpu_operand_collector_num_units_int, m_config->gpgpu_operand_collector_num_out_ports_int); + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sp; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_SP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SP]); + cu_sets.push_back((unsigned)SP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_dp; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_DP]); + out_ports.push_back(&m_pipeline_reg[OC_EX_DP]); + cu_sets.push_back((unsigned)DP_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_sfu; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_SFU]); + out_ports.push_back(&m_pipeline_reg[OC_EX_SFU]); + cu_sets.push_back((unsigned)SFU_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + + for (unsigned i = 0; i < config->gpgpu_operand_collector_num_in_ports_tensor_core; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_TENSOR_CORE]); + out_ports.push_back(&m_pipeline_reg[OC_EX_TENSOR_CORE]); + cu_sets.push_back((unsigned)TENSOR_CORE_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_mem; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_MEM]); + out_ports.push_back(&m_pipeline_reg[OC_EX_MEM]); + cu_sets.push_back((unsigned)MEM_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + + for (unsigned i = 0; i < m_config->gpgpu_operand_collector_num_in_ports_int; i++) { + in_ports.push_back(&m_pipeline_reg[ID_OC_INT]); + out_ports.push_back(&m_pipeline_reg[OC_EX_INT]); + cu_sets.push_back((unsigned)INT_CUS); + cu_sets.push_back((unsigned)GEN_CUS); + m_operand_collector.add_port(in_ports,out_ports,cu_sets); + in_ports.clear(),out_ports.clear(),cu_sets.clear(); + } + } m_operand_collector.init( m_config->gpgpu_num_reg_banks, this ); - // execute - m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + 1; // sp_unit, sfu, ldst_unit + m_num_function_units = m_config->gpgpu_num_sp_units + m_config->gpgpu_num_dp_units + m_config->gpgpu_num_sfu_units + m_config->gpgpu_num_tensor_core_units + m_config->gpgpu_num_int_units + 1; // sp_unit, sfu, dp, tensor, int, ldst_unit //m_dispatch_port = new enum pipeline_stage_name_t[ m_num_function_units ]; //m_issue_port = new enum pipeline_stage_name_t[ m_num_function_units ]; @@ -312,18 +371,29 @@ shader_core_ctx::shader_core_ctx( class gpgpu_sim *gpu, m_dispatch_port.push_back(ID_OC_DP); m_issue_port.push_back(OC_EX_DP); } + for (int k = 0; k < m_config->gpgpu_num_int_units; k++) { + m_fu.push_back(new int_unit( &m_pipeline_reg[EX_WB], m_config, this )); + m_dispatch_port.push_back(ID_OC_INT); + m_issue_port.push_back(OC_EX_INT); + } for (int k = 0; k < m_config->gpgpu_num_sfu_units; k++) { m_fu.push_back(new sfu( &m_pipeline_reg[EX_WB], m_config, this )); m_dispatch_port.push_back(ID_OC_SFU); m_issue_port.push_back(OC_EX_SFU); } + + for (int k = 0; k < config->gpgpu_num_tensor_core_units; k++) { + m_fu.push_back(new tensor_core( &m_pipeline_reg[EX_WB], m_config, this )); + m_dispatch_port.push_back(ID_OC_TENSOR_CORE); + m_issue_port.push_back(OC_EX_TENSOR_CORE); + } m_ldst_unit = new ldst_unit( m_icnt, m_mem_fetch_allocator, this, &m_operand_collector, m_scoreboard, config, mem_config, stats, shader_id, tpc_id ); m_fu.push_back(m_ldst_unit); m_dispatch_port.push_back(ID_OC_MEM); m_issue_port.push_back(OC_EX_MEM); - + assert(m_num_function_units == m_fu.size() and m_fu.size() == m_dispatch_port.size() and m_fu.size() == m_issue_port.size()); //there are as many result buses as the width of the EX_WB stage @@ -370,11 +440,12 @@ void shader_core_ctx::reinit(unsigned start_thread, unsigned end_thread, bool re } } -void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread ) +void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsigned end_thread, unsigned ctaid, int cta_size, unsigned kernel_id ) { address_type start_pc = next_pc(start_thread); if (m_config->model == POST_DOMINATOR) { unsigned start_warp = start_thread / m_config->warp_size; + unsigned warp_per_cta = cta_size / m_config->warp_size; unsigned end_warp = end_thread / m_config->warp_size + ((end_thread % m_config->warp_size)? 1 : 0); for (unsigned i = start_warp; i < end_warp; ++i) { unsigned n_active=0; @@ -389,6 +460,21 @@ void shader_core_ctx::init_warps( unsigned cta_id, unsigned start_thread, unsign } } m_simt_stack[i]->launch(start_pc,active_threads); + + if(m_gpu->resume_option==1 && kernel_id==m_gpu->resume_kernel && ctaid>=m_gpu->resume_CTA && ctaid<m_gpu->checkpoint_CTA_t ) + { + char fname[2048]; + snprintf(fname,2048,"checkpoint_files/warp_%d_%d_simt.txt",i%warp_per_cta,ctaid ); + unsigned pc,rpc; + m_simt_stack[i]->resume(fname); + m_simt_stack[i]->get_pdom_stack_top_info(&pc,&rpc); + for (unsigned t = 0; t < m_config->warp_size; t++) { + m_thread[i * m_config->warp_size + t]->set_npc(pc); + m_thread[i * m_config->warp_size + t]->update_pc(); + } + start_pc=pc; + } + m_warp[i].init(start_pc,cta_id,i,active_threads, m_dynamic_warp_id); ++m_dynamic_warp_id; m_not_completed += n_active; @@ -455,6 +541,7 @@ void shader_core_stats::print( FILE* fout ) const fprintf(fout, "gpgpu_n_load_insn = %d\n", gpgpu_n_load_insn); fprintf(fout, "gpgpu_n_store_insn = %d\n", gpgpu_n_store_insn); fprintf(fout, "gpgpu_n_shmem_insn = %d\n", gpgpu_n_shmem_insn); + fprintf(fout, "gpgpu_n_sstarr_insn = %d\n", gpgpu_n_sstarr_insn); fprintf(fout, "gpgpu_n_tex_insn = %d\n", gpgpu_n_tex_insn); fprintf(fout, "gpgpu_n_const_mem_insn = %d\n", gpgpu_n_const_insn); fprintf(fout, "gpgpu_n_param_mem_insn = %d\n", gpgpu_n_param_insn); @@ -758,18 +845,21 @@ void shader_core_ctx::func_exec_inst( warp_inst_t &inst ) { execute_warp_inst_t(inst); if( inst.is_load() || inst.is_store() ) - inst.generate_mem_accesses(); + { + inst.generate_mem_accesses(); + //inst.print_m_accessq(); + } } -void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* next_inst, const active_mask_t &active_mask, unsigned warp_id ) +void shader_core_ctx::issue_warp( register_set& pipe_reg_set, const warp_inst_t* next_inst, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id ) { - warp_inst_t** pipe_reg = pipe_reg_set.get_free(); + warp_inst_t** pipe_reg = pipe_reg = pipe_reg_set.get_free(m_config->sub_core_model, sch_id); assert(pipe_reg); m_warp[warp_id].ibuffer_free(); assert(next_inst->valid()); **pipe_reg = *next_inst; // static instruction information - (*pipe_reg)->issue( active_mask, warp_id, gpu_tot_sim_cycle + gpu_sim_cycle, m_warp[warp_id].get_dynamic_warp_id() ); // dynamic instruction information + (*pipe_reg)->issue( active_mask, warp_id, gpu_tot_sim_cycle + gpu_sim_cycle, m_warp[warp_id].get_dynamic_warp_id(), sch_id ); // dynamic instruction information m_stats->shader_cycle_distro[2+(*pipe_reg)->active_count()]++; func_exec_inst( **pipe_reg ); if( next_inst->op == BARRIER_OP ){ @@ -914,8 +1004,8 @@ void scheduler_unit::cycle() unsigned checked=0; unsigned issued=0; exec_unit_type_t previous_issued_inst_exec_type = exec_unit_type_t::NONE; - unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; - bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; + unsigned max_issue = m_shader->m_config->gpgpu_max_insn_issue_per_warp; + bool diff_exec_units = m_shader->m_config->gpgpu_dual_issue_diff_exec_units; //In tis mode, we only allow dual issue to diff execution units (as in Maxwell and Pascal) while( !warp(warp_id).waiting() && !warp(warp_id).ibuffer_empty() && (checked < max_issue) && (checked <= issued) && (issued < max_issue) ) { const warp_inst_t *pI = warp(warp_id).ibuffer_next_inst(); @@ -949,9 +1039,10 @@ void scheduler_unit::cycle() ready_inst = true; const active_mask_t &active_mask = m_simt_stack[warp_id]->get_active_mask(); assert( warp(warp_id).inst_in_pipeline() ); - if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP) ) { - if( m_mem_out->has_free() && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { - m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id); + + if ( (pI->op == LOAD_OP) || (pI->op == STORE_OP) || (pI->op == MEMORY_BARRIER_OP)||(pI->op==TENSOR_CORE_LOAD_OP)||(pI->op==TENSOR_CORE_STORE_OP) ) { + if( m_mem_out->has_free(m_shader->m_config->sub_core_model, m_id) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::MEM)) { + m_shader->issue_warp(*m_mem_out,pI,active_mask,warp_id,m_id); issued++; issued_inst=true; warp_inst_issued = true; @@ -959,58 +1050,98 @@ void scheduler_unit::cycle() } } else { - bool sp_pipe_avail = m_sp_out->has_free(); - bool sfu_pipe_avail = m_sfu_out->has_free(); - bool dp_pipe_avail = m_dp_out->has_free(); - if( sp_pipe_avail && (pI->op != SFU_OP && pI->op != DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SP)) { + bool sp_pipe_avail = m_sp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool sfu_pipe_avail = m_sfu_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool tensor_core_pipe_avail = m_tensor_core_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool dp_pipe_avail = m_dp_out->has_free(m_shader->m_config->sub_core_model, m_id); + bool int_pipe_avail = m_int_out->has_free(m_shader->m_config->sub_core_model, m_id); + + //This code need to be refactored + if(pI->op != TENSOR_CORE_OP && pI->op != SFU_OP && pI->op != DP_OP) { - //Jin: special for CDP api - if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { - assert(warp(warp_id).m_cdp_latency == 0); - - extern unsigned cdp_latency[5]; - if(pI->m_is_cdp == 1) - warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1]; - else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2 - warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1] - + cdp_latency[pI->m_is_cdp] * active_mask.count(); - warp(warp_id).m_cdp_dummy = true; - break; - } - else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) { - assert(warp(warp_id).m_cdp_latency == 0); - warp(warp_id).m_cdp_dummy = false; - } + bool execute_on_SP = false; + bool execute_on_INT = false; - // always prefer SP pipe for operations that can use both SP and SFU pipelines - m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id); - issued++; - issued_inst=true; - warp_inst_issued = true; - previous_issued_inst_exec_type = exec_unit_type_t::SP; - } else if ( (m_shader->m_config->gpgpu_num_dp_units != 0) && (pI->op == DP_OP) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::DP)) { + //if INT unit pipline exist, then execute ALU and INT operations on INT unit and SP-FPU on SP unit (like in Volta) + //if INT unit pipline does not exist, then execute all ALU, INT and SP operations on SP unit (as in Fermi, Pascal GPUs) + if(m_shader->m_config->gpgpu_num_int_units > 0 && + int_pipe_avail && + pI->op != SP_OP && + !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::INT)) + execute_on_INT = true; + else if (sp_pipe_avail && + (m_shader->m_config->gpgpu_num_int_units == 0 || + (m_shader->m_config->gpgpu_num_int_units > 0 && pI->op == SP_OP)) && + !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) + execute_on_SP = true; + + + if(execute_on_INT || execute_on_SP) { + //Jin: special for CDP api + if(pI->m_is_cdp && !warp(warp_id).m_cdp_dummy) { + assert(warp(warp_id).m_cdp_latency == 0); + + extern unsigned cdp_latency[5]; + if(pI->m_is_cdp == 1) + warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1]; + else //cudaLaunchDeviceV2 and cudaGetParameterBufferV2 + warp(warp_id).m_cdp_latency = cdp_latency[pI->m_is_cdp - 1] + + cdp_latency[pI->m_is_cdp] * active_mask.count(); + warp(warp_id).m_cdp_dummy = true; + break; + } + else if(pI->m_is_cdp && warp(warp_id).m_cdp_dummy) { + assert(warp(warp_id).m_cdp_latency == 0); + warp(warp_id).m_cdp_dummy = false; + } + } + + if(execute_on_SP) { + m_shader->issue_warp(*m_sp_out,pI,active_mask,warp_id,m_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::SP; + } else if (execute_on_INT) { + m_shader->issue_warp(*m_int_out,pI,active_mask,warp_id,m_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::INT; + } + } else if ( (m_shader->m_config->gpgpu_num_dp_units > 0) && (pI->op == DP_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::DP)) { if( dp_pipe_avail ) { - m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id); + m_shader->issue_warp(*m_dp_out,pI,active_mask,warp_id,m_id); issued++; issued_inst=true; warp_inst_issued = true; previous_issued_inst_exec_type = exec_unit_type_t::DP; } - } //If the DP units = 0 (like in Fermi archi), then change DP inst to SFU inst - else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && (!diff_exec_units || previous_issued_inst_exec_type != exec_unit_type_t::SFU)) { + } //If the DP units = 0 (like in Fermi archi), then execute DP inst on SFU unit + else if ( ((m_shader->m_config->gpgpu_num_dp_units == 0 && pI->op == DP_OP) || (pI->op == SFU_OP) || (pI->op == ALU_SFU_OP)) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SFU)) { if( sfu_pipe_avail ) { - m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id); + m_shader->issue_warp(*m_sfu_out,pI,active_mask,warp_id,m_id); issued++; issued_inst=true; warp_inst_issued = true; previous_issued_inst_exec_type = exec_unit_type_t::SFU; } - } - } - } else { + } + else if ( (pI->op == TENSOR_CORE_OP) && !(diff_exec_units && previous_issued_inst_exec_type == exec_unit_type_t::SP) ) { + if( tensor_core_pipe_avail ) { + m_shader->issue_warp(*m_tensor_core_out,pI,active_mask,warp_id,m_id); + issued++; + issued_inst=true; + warp_inst_issued = true; + previous_issued_inst_exec_type = exec_unit_type_t::TENSOR; + } + } + }//end of else + } else { + SCHED_DPRINTF( "Warp (warp_id %u, dynamic_warp_id %u) fails scoreboard\n", (*iter)->get_warp_id(), (*iter)->get_dynamic_warp_id() ); - } + } } } else if( valid ) { // this case can happen after a return instruction in diverged warp @@ -1144,7 +1275,7 @@ void two_level_active_scheduler::order_warps() for ( std::vector< shd_warp_t* >::iterator iter = m_next_cycle_prioritized_warps.begin(); iter != m_next_cycle_prioritized_warps.end(); ) { bool waiting = (*iter)->waiting(); - for (int i=0; i<4; i++){ + for (int i=0; i<MAX_INPUT_VALUES; i++){ const warp_inst_t* inst = (*iter)->ibuffer_next_inst(); //Is the instruction waiting on a long operation? if ( inst && inst->in[i] > 0 && this->m_scoreboard->islongop((*iter)->get_warp_id(), inst->in[i])){ @@ -1190,10 +1321,12 @@ swl_scheduler::swl_scheduler ( shader_core_stats* stats, shader_core_ctx* shader register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id, char* config_string ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ) + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ) { unsigned m_prioritization_readin; int ret = sscanf( config_string, @@ -1361,10 +1494,12 @@ void ldst_unit::get_L1T_sub_stats(struct cache_sub_stats &css) const{ void shader_core_ctx::warp_inst_complete(const warp_inst_t &inst) { - #if 0 - printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu issued@%llu\n", - inst.get_uid(), m_sid, inst.warp_id(), inst.pc, gpu_tot_sim_cycle + gpu_sim_cycle, inst.get_issue_cycle()); - #endif + + #if 0 + printf("[warp_inst_complete] uid=%u core=%u warp=%u pc=%#x @ time=%llu issued@%llu\n", + inst.get_uid(), m_sid, inst.warp_id(), inst.pc, gpu_tot_sim_cycle + gpu_sim_cycle, inst.get_issue_cycle()); + #endif + if(inst.op_pipe==SP__OP) m_stats->m_num_sp_committed[m_sid]++; else if(inst.op_pipe==SFU__OP) @@ -1467,7 +1602,7 @@ ldst_unit::process_cache_access( cache_t* cache, assert( !read_sent ); inst.accessq_pop_back(); if ( inst.is_load() ) { - for ( unsigned r=0; r < 4; r++) + for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) if (inst.out[r] > 0) m_pending_writes[inst.warp_id()][inst.out[r]]--; } @@ -1561,7 +1696,7 @@ void ldst_unit::L1_latency_queue_cycle() assert( !read_sent ); l1_latency_queue[0] = NULL; if ( mf_next->get_inst().is_load() ) { - for ( unsigned r=0; r < 4; r++) + for ( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) if (mf_next->get_inst().out[r] > 0) { assert(m_pending_writes[mf_next->get_inst().warp_id()][mf_next->get_inst().out[r]]>0); @@ -1661,11 +1796,11 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea if (m_core->get_config()->gmem_skip_L1D && (CACHE_L1 != inst.cache_op)) bypassL1D = true; } - if( bypassL1D ) { // bypass L1 cache unsigned control_size = inst.is_store() ? WRITE_PACKET_SIZE : READ_PACKET_SIZE; unsigned size = access.get_size() + control_size; + //printf("Interconnect:Addr: %x, size=%d\n",access.get_addr(),size); if( m_icnt->full(size, inst.is_store() || inst.isatomic()) ) { stall_cond = ICNT_RC_FAIL; } else { @@ -1674,7 +1809,7 @@ bool ldst_unit::memory_cycle( warp_inst_t &inst, mem_stage_stall_type &stall_rea inst.accessq_pop_back(); //inst.clear_active( access.get_warp_mask() ); if( inst.is_load() ) { - for( unsigned r=0; r < 4; r++) + for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++) if(inst.out[r] > 0) assert( m_pending_writes[inst.warp_id()][inst.out[r]] > 0 ); } else if( inst.is_store() ) @@ -1732,6 +1867,12 @@ sfu:: sfu( register_set* result_port, const shader_core_config *config,shader_c m_name = "SFU"; } +tensor_core:: tensor_core( register_set* result_port, const shader_core_config *config,shader_core_ctx *core ) + : pipelined_simd_unit(result_port,config,config->max_tensor_core_latency,core) +{ + m_name = "TENSOR_CORE"; +} + void sfu::issue( register_set& source_reg ) { warp_inst_t** ready_reg = source_reg.get_ready(); @@ -1742,11 +1883,23 @@ void sfu::issue( register_set& source_reg ) pipelined_simd_unit::issue(source_reg); } +void tensor_core::issue( register_set& source_reg ) +{ + warp_inst_t** ready_reg = source_reg.get_ready(); + //m_core->incexecstat((*ready_reg)); + + (*ready_reg)->op_pipe= TENSOR_CORE__OP; + m_core->incsfu_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); +} + + void ldst_unit::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); assert(active_count<=m_core->get_config()->warp_size); m_core->incfumemactivelanes_stat(active_count); } + void sp_unit::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); assert(active_count<=m_core->get_config()->warp_size); @@ -1762,6 +1915,13 @@ void dp_unit::active_lanes_in_pipeline(){ m_core->incfumemactivelanes_stat(active_count); } +void int_unit::active_lanes_in_pipeline(){ + unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count<=m_core->get_config()->warp_size); + m_core->incspactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); +} void sfu::active_lanes_in_pipeline(){ unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); assert(active_count<=m_core->get_config()->warp_size); @@ -1770,6 +1930,15 @@ void sfu::active_lanes_in_pipeline(){ m_core->incfumemactivelanes_stat(active_count); } +void tensor_core::active_lanes_in_pipeline(){ + unsigned active_count=pipelined_simd_unit::get_active_lanes_in_pipeline(); + assert(active_count<=m_core->get_config()->warp_size); + m_core->incsfuactivelanes_stat(active_count); + m_core->incfuactivelanes_stat(active_count); + m_core->incfumemactivelanes_stat(active_count); +} + + sp_unit::sp_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) : pipelined_simd_unit(result_port,config,config->max_sp_latency,core) { @@ -1782,6 +1951,12 @@ dp_unit::dp_unit( register_set* result_port, const shader_core_config *config,sh m_name = "DP "; } +int_unit::int_unit( register_set* result_port, const shader_core_config *config,shader_core_ctx *core) + : pipelined_simd_unit(result_port,config,config->max_sp_latency,core) +{ + m_name = "INT "; +} + void sp_unit :: issue(register_set& source_reg) { warp_inst_t** ready_reg = source_reg.get_ready(); @@ -1800,6 +1975,15 @@ void dp_unit :: issue(register_set& source_reg) pipelined_simd_unit::issue(source_reg); } +void int_unit :: issue(register_set& source_reg) +{ + warp_inst_t** ready_reg = source_reg.get_ready(); + //m_core->incexecstat((*ready_reg)); + (*ready_reg)->op_pipe=INTP__OP; + m_core->incsp_stat(m_core->get_config()->warp_size,(*ready_reg)->latency); + pipelined_simd_unit::issue(source_reg); +} + pipelined_simd_unit::pipelined_simd_unit( register_set* result_port, const shader_core_config *config, unsigned max_latency,shader_core_ctx *core ) : simd_function_unit(config) { @@ -1922,6 +2106,7 @@ ldst_unit::ldst_unit( mem_fetch_interface *icnt, l1_latency_queue.push_back((mem_fetch*)NULL); } } + m_name = "MEM "; } ldst_unit::ldst_unit( mem_fetch_interface *icnt, @@ -1958,7 +2143,7 @@ void ldst_unit:: issue( register_set ®_set ) if (inst->is_load() and inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id(); unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < 4; r++) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = inst->out[r]; if (reg_id > 0) { m_pending_writes[warp_id][reg_id] += n_accesses; @@ -1980,7 +2165,7 @@ void ldst_unit::writeback() if( !m_next_wb.empty() ) { if( m_operand_collector->writeback(m_next_wb) ) { bool insn_completed = false; - for( unsigned r=0; r < 4; r++ ) { + for( unsigned r=0; r < MAX_OUTPUT_VALUES; r++ ) { if( m_next_wb.out[r] > 0 ) { if( m_next_wb.space.get_type() != shared_space ) { assert( m_pending_writes[m_next_wb.warp_id()][m_next_wb.out[r]] > 0 ); @@ -2068,7 +2253,11 @@ void ldst_unit::writeback() unsigned ldst_unit::clock_multiplier() const { - return m_config->mem_warp_parts; + //to model multiple read port, we give multiple cycles for the memory units + if(m_config->mem_unit_ports) + return m_config->mem_unit_ports; + else + return m_config->mem_warp_parts; } /* void ldst_unit::issue( register_set ®_set ) @@ -2082,7 +2271,7 @@ void ldst_unit::issue( register_set ®_set ) if (inst->is_load() and inst->space.get_type() != shared_space) { unsigned warp_id = inst->warp_id(); unsigned n_accesses = inst->accessq_count(); - for (unsigned r = 0; r < 4; r++) { + for (unsigned r = 0; r < MAX_OUTPUT_VALUES; r++) { unsigned reg_id = inst->out[r]; if (reg_id > 0) { m_pending_writes[warp_id][reg_id] += n_accesses; @@ -2186,7 +2375,7 @@ void ldst_unit::cycle() //} bool pending_requests=false; - for( unsigned r=0; r<4; r++ ) { + for( unsigned r=0; r<MAX_OUTPUT_VALUES; r++ ) { unsigned reg_id = pipe_reg.out[r]; if( reg_id > 0 ) { if( m_pending_writes[warp_id].find(reg_id) != m_pending_writes[warp_id].end() ) { @@ -2494,7 +2683,7 @@ void shader_core_ctx::incexecstat(warp_inst_t *&inst) switch(inst->sp_op){ case INT__OP: - incialu_stat(inst->active_count(),25); + incialu_stat(inst->active_count(),32); break; case INT_MUL_OP: incimul_stat(inst->active_count(),7.2); @@ -3305,33 +3494,50 @@ void opndcoll_rfu_t::init( unsigned num_banks, shader_core_ctx *shader ) m_bank_warp_shift = (unsigned)(int) (log(m_warp_size+0.5) / log(2.0)); assert( (m_bank_warp_shift == 5) || (m_warp_size != 32) ); + sub_core_model = shader->get_config()->sub_core_model; + m_num_warp_sceds = shader->get_config()->gpgpu_num_sched_per_core; + if(sub_core_model) + assert(num_banks % shader->get_config()->gpgpu_num_sched_per_core == 0); + m_num_banks_per_sched = num_banks / shader->get_config()->gpgpu_num_sched_per_core; + for( unsigned j=0; j<m_cu.size(); j++) { - m_cu[j]->init(j,num_banks,m_bank_warp_shift,shader->get_config(),this); + m_cu[j]->init(j,num_banks,m_bank_warp_shift,shader->get_config(),this, sub_core_model, m_num_banks_per_sched ); } m_initialized=true; + + + + } -int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift) +int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id) { int bank = regnum; if (bank_warp_shift) bank += wid; - return bank % num_banks; + if(sub_core_model) { + unsigned bank_num = (bank % banks_per_sched) + (sched_id * banks_per_sched); + assert(bank_num < num_banks); + return bank_num; + } + else + return bank % num_banks; } -bool opndcoll_rfu_t::writeback( const warp_inst_t &inst ) +bool opndcoll_rfu_t::writeback( warp_inst_t &inst ) { assert( !inst.empty() ); std::list<unsigned> regs = m_shader->get_regs_written(inst); - std::list<unsigned>::iterator r; - unsigned n=0; - for( r=regs.begin(); r!=regs.end();r++,n++ ) { - unsigned reg = *r; - unsigned bank = register_bank(reg,inst.warp_id(),m_num_banks,m_bank_warp_shift); - if( m_arbiter.bank_idle(bank) ) { - m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg,m_num_banks,m_bank_warp_shift)); - } else { - return false; + for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) { + int reg_num = inst.arch_reg.dst[op]; // this math needs to match that used in function_info::ptx_decode_inst + if( reg_num >= 0 ){ // valid register + unsigned bank = register_bank(reg_num,inst.warp_id(),m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id()); + if( m_arbiter.bank_idle(bank) ) { + m_arbiter.allocate_bank_for_write(bank,op_t(&inst,reg_num,m_num_banks,m_bank_warp_shift, sub_core_model, m_num_banks_per_sched, inst.get_schd_id())); + inst.arch_reg.dst[op] = -1; + } else { + return false; + } } } for(unsigned i=0;i<(unsigned)regs.size();i++){ @@ -3413,7 +3619,7 @@ void opndcoll_rfu_t::allocate_reads() const op_t &rr = *r; unsigned reg = rr.get_reg(); unsigned wid = rr.get_wid(); - unsigned bank = register_bank(reg,wid,m_num_banks,m_bank_warp_shift); + unsigned bank = register_bank(reg,wid,m_num_banks,m_bank_warp_shift,sub_core_model, m_num_banks_per_sched, rr.get_sid()); m_arbiter.allocate_for_read(bank,rr); read_ops[bank] = rr; } @@ -3464,7 +3670,9 @@ void opndcoll_rfu_t::collector_unit_t::init( unsigned n, unsigned num_banks, unsigned log2_warp_size, const core_config *config, - opndcoll_rfu_t *rfu ) + opndcoll_rfu_t *rfu, + bool sub_core_model, + unsigned banks_per_sched) { m_rfu=rfu; m_cuid=n; @@ -3472,6 +3680,8 @@ void opndcoll_rfu_t::collector_unit_t::init( unsigned n, assert(m_warp==NULL); m_warp = new warp_inst_t(config); m_bank_warp_shift=log2_warp_size; + m_sub_core_model = sub_core_model; + m_num_banks_per_sched = banks_per_sched; } bool opndcoll_rfu_t::collector_unit_t::allocate( register_set* pipeline_reg_set, register_set* output_reg_set ) @@ -3486,7 +3696,7 @@ bool opndcoll_rfu_t::collector_unit_t::allocate( register_set* pipeline_reg_set, for( unsigned op=0; op < MAX_REG_OPERANDS; op++ ) { int reg_num = (*pipeline_reg)->arch_reg.src[op]; // this math needs to match that used in function_info::ptx_decode_inst if( reg_num >= 0 ) { // valid register - m_src_op[op] = op_t( this, op, reg_num, m_num_banks, m_bank_warp_shift ); + m_src_op[op] = op_t( this, op, reg_num, m_num_banks, m_bank_warp_shift, m_sub_core_model, m_num_banks_per_sched, (*pipeline_reg)->get_schd_id() ); m_not_ready.set(op); } else m_src_op[op] = op_t(); @@ -3664,6 +3874,7 @@ void simt_core_cluster::icnt_inject_request_packet(class mem_fetch *mf) case CONST_ACC_R: m_stats->gpgpu_n_mem_const++; break; case TEXTURE_ACC_R: m_stats->gpgpu_n_mem_texture++; break; case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; break; + //case GLOBAL_ACC_R: m_stats->gpgpu_n_mem_read_global++; printf("read_global%d\n",m_stats->gpgpu_n_mem_read_global); break; case GLOBAL_ACC_W: m_stats->gpgpu_n_mem_write_global++; break; case LOCAL_ACC_R: m_stats->gpgpu_n_mem_read_local++; break; case LOCAL_ACC_W: m_stats->gpgpu_n_mem_write_local++; break; diff --git a/src/gpgpu-sim/shader.h b/src/gpgpu-sim/shader.h index 4a87126..9abd223 100644 --- a/src/gpgpu-sim/shader.h +++ b/src/gpgpu-sim/shader.h @@ -76,7 +76,9 @@ enum exec_unit_type_t SP = 1, SFU = 2, MEM = 3, - DP = 4 + DP = 4, + INT = 5, + TENSOR = 6 }; class thread_ctx_t { @@ -291,7 +293,7 @@ inline unsigned wid_from_hw_tid(unsigned tid, unsigned warp_size){return tid/war const unsigned WARP_PER_CTA_MAX = 64; typedef std::bitset<WARP_PER_CTA_MAX> warp_set_t; -int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift); +int register_bank(int regnum, int wid, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ); class shader_core_ctx; struct shader_core_config; @@ -328,11 +330,13 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id) : m_supervised_warps(), m_stats(stats), m_shader(shader), m_scoreboard(scoreboard), m_simt_stack(simt), /*m_pipeline_reg(pipe_regs),*/ m_warp(warp), - m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_mem_out(mem_out), m_id(id){} + m_sp_out(sp_out),m_dp_out(dp_out),m_sfu_out(sfu_out),m_int_out(int_out),m_tensor_core_out(tensor_core_out),m_mem_out(mem_out), m_id(id){} virtual ~scheduler_unit(){} virtual void add_supervised_warp_id(int i) { m_supervised_warps.push_back(&warp(i)); @@ -378,6 +382,8 @@ public: // m_supervised_warps with their scheduling policies virtual void order_warps() = 0; + int get_schd_id() const {return m_id;} + protected: virtual void do_on_warp_issued( unsigned warp_id, unsigned num_issued, @@ -406,6 +412,8 @@ protected: register_set* m_sp_out; register_set* m_dp_out; register_set* m_sfu_out; + register_set* m_int_out; + register_set* m_tensor_core_out; register_set* m_mem_out; int m_id; @@ -419,9 +427,11 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} virtual ~lrr_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -437,9 +447,11 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} virtual ~gto_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -456,9 +468,11 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ){} + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ){} virtual ~oldest_scheduler () {} virtual void order_warps (); virtual void done_adding_supervised_warps() { @@ -475,10 +489,12 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id, char* config_str ) - : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, mem_out, id ), + : scheduler_unit ( stats, shader, scoreboard, simt, warp, sp_out, dp_out, sfu_out, int_out, tensor_core_out, mem_out, id ), m_pending_warps() { unsigned inner_level_readin; @@ -526,6 +542,8 @@ public: register_set* sp_out, register_set* dp_out, register_set* sfu_out, + register_set* int_out, + register_set* tensor_core_out, register_set* mem_out, int id, char* config_string ); @@ -558,7 +576,7 @@ public: void init( unsigned num_banks, shader_core_ctx *shader ); // modifiers - bool writeback( const warp_inst_t &warp ); // might cause stall + bool writeback( warp_inst_t &warp ); void step() { @@ -601,23 +619,25 @@ private: public: op_t() { m_valid = false; } - op_t( collector_unit_t *cu, unsigned op, unsigned reg, unsigned num_banks, unsigned bank_warp_shift ) + op_t( collector_unit_t *cu, unsigned op, unsigned reg, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ) { m_valid = true; m_warp=NULL; m_cu = cu; m_operand = op; m_register = reg; - m_bank = register_bank(reg,cu->get_warp_id(),num_banks,bank_warp_shift); + m_shced_id = sched_id; + m_bank = register_bank(reg,cu->get_warp_id(),num_banks,bank_warp_shift, sub_core_model, banks_per_sched, sched_id); } - op_t( const warp_inst_t *warp, unsigned reg, unsigned num_banks, unsigned bank_warp_shift ) + op_t( const warp_inst_t *warp, unsigned reg, unsigned num_banks, unsigned bank_warp_shift, bool sub_core_model, unsigned banks_per_sched, unsigned sched_id ) { m_valid=true; m_warp=warp; m_register=reg; m_cu=NULL; m_operand = -1; - m_bank = register_bank(reg,warp->warp_id(),num_banks,bank_warp_shift); + m_shced_id = sched_id; + m_bank = register_bank(reg,warp->warp_id(),num_banks,bank_warp_shift, sub_core_model, banks_per_sched, sched_id); } // accessors @@ -633,6 +653,10 @@ private: else if( m_cu ) return m_cu->get_warp_id(); else abort(); } + unsigned get_sid() const + { + return m_shced_id; + } unsigned get_active_count() const { if( m_warp ) return m_warp->active_count(); @@ -677,6 +701,7 @@ private: unsigned m_operand; // operand offset in instruction. e.g., add r1,r2,r3; r2 is oprd 0, r3 is 1 (r1 is dst) unsigned m_register; unsigned m_bank; + unsigned m_shced_id; //scheduler id that has issued this inst }; enum alloc_t { @@ -697,7 +722,7 @@ private: else if( m_allocation == WRITE_ALLOC ) { fprintf(fp,"wr: "); m_op.dump(fp); } fprintf(fp,"\n"); } - void alloc_read( const op_t &op ) { assert(is_free()); m_allocation=READ_ALLOC; m_op=op; } + void alloc_read( const op_t &op ) { assert(is_free()); m_allocation=READ_ALLOC; m_op=op; } void alloc_write( const op_t &op ) { assert(is_free()); m_allocation=WRITE_ALLOC; m_op=op; } void reset() { m_allocation = NO_ALLOC; } private: @@ -851,7 +876,9 @@ private: unsigned num_banks, unsigned log2_warp_size, const core_config *config, - opndcoll_rfu_t *rfu ); + opndcoll_rfu_t *rfu, + bool m_sub_core_model, + unsigned num_banks_per_sched); bool allocate( register_set* pipeline_reg, register_set* output_reg ); void collect_operand( unsigned op ) @@ -879,6 +906,9 @@ private: unsigned m_bank_warp_shift; opndcoll_rfu_t *m_rfu; + unsigned m_num_banks_per_sched; + bool m_sub_core_model; + }; class dispatch_unit_t { @@ -921,6 +951,10 @@ private: std::vector<collector_unit_t *> m_cu; arbiter_t m_arbiter; + unsigned m_num_banks_per_sched; + unsigned m_num_warp_sceds; + bool sub_core_model; + //unsigned m_num_ports; //std::vector<warp_inst_t**> m_input; //std::vector<warp_inst_t**> m_output; @@ -1021,6 +1055,9 @@ public: fprintf(fp,"%s dispatch= ", m_name.c_str() ); m_dispatch_reg->print(fp); } + const char* get_name() { + return m_name.c_str(); + } protected: std::string m_name; const shader_core_config *m_config; @@ -1112,6 +1149,46 @@ public: virtual void issue( register_set& source_reg ); }; +class tensor_core : public pipelined_simd_unit +{ +public: + tensor_core( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); + virtual bool can_issue( const warp_inst_t &inst ) const + { + switch(inst.op) { + case TENSOR_CORE_OP: break; + default: return false; + } + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue( register_set& source_reg ); +}; + + +class int_unit : public pipelined_simd_unit +{ +public: + int_unit( register_set* result_port, const shader_core_config *config, shader_core_ctx *core ); + virtual bool can_issue( const warp_inst_t &inst ) const + { + switch(inst.op) { + case SFU_OP: return false; + case LOAD_OP: return false; + case TENSOR_CORE_LOAD_OP: return false; + case STORE_OP: return false; + case TENSOR_CORE_STORE_OP: return false; + case MEMORY_BARRIER_OP: return false; + case SP_OP: return false; + case DP_OP: return false; + default: break; + } + return pipelined_simd_unit::can_issue(inst); + } + virtual void active_lanes_in_pipeline(); + virtual void issue( register_set& source_reg ); +}; + class sp_unit : public pipelined_simd_unit { public: @@ -1121,7 +1198,9 @@ public: switch(inst.op) { case SFU_OP: return false; case LOAD_OP: return false; + case TENSOR_CORE_LOAD_OP: return false; case STORE_OP: return false; + case TENSOR_CORE_STORE_OP: return false; case MEMORY_BARRIER_OP: return false; case DP_OP: return false; default: break; @@ -1165,7 +1244,9 @@ public: { switch(inst.op) { case LOAD_OP: break; + case TENSOR_CORE_LOAD_OP: break; case STORE_OP: break; + case TENSOR_CORE_STORE_OP: break; case MEMORY_BARRIER_OP: break; default: return false; } @@ -1257,26 +1338,34 @@ protected: enum pipeline_stage_name_t { ID_OC_SP=0, ID_OC_DP, + ID_OC_INT, ID_OC_SFU, ID_OC_MEM, OC_EX_SP, OC_EX_DP, + OC_EX_INT, OC_EX_SFU, OC_EX_MEM, EX_WB, + ID_OC_TENSOR_CORE, + OC_EX_TENSOR_CORE, N_PIPELINE_STAGES -}; + }; const char* const pipeline_stage_name_decode[] = { "ID_OC_SP", "ID_OC_DP", + "ID_OC_INT", "ID_OC_SFU", "ID_OC_MEM", "OC_EX_SP", "OC_EX_DP", + "OC_EX_INT", "OC_EX_SFU", "OC_EX_MEM", "EX_WB", + "ID_OC_TENSOR_CORE", + "OC_EX_TENSOR_CORE", "N_PIPELINE_STAGES" }; @@ -1299,16 +1388,24 @@ struct shader_core_config : public core_config char* toks = new char[100]; char* tokd = toks; strcpy(toks,pipeline_widths_string); - + toks = strtok(toks,","); - for (unsigned i = 0; i < N_PIPELINE_STAGES; i++) { + + /* Removing the tensorcore pipeline while reading the config files if the tensor core is not available. + If we won't remove it, old regression will be broken. + So to support the legacy config files it's best to handle in this way. + */ + int num_config_to_read=N_PIPELINE_STAGES-2*(!gpgpu_tensor_core_avail); + + for (unsigned i = 0; i <num_config_to_read; i++) { assert(toks); ntok = sscanf(toks,"%d", &pipe_widths[i]); assert(ntok == 1); toks = strtok(NULL,","); } - delete[] tokd; + delete[] tokd; + if (n_thread_per_shader > MAX_THREAD_PER_SM) { printf("GPGPU-Sim uArch: Error ** increase MAX_THREAD_PER_SM in abstract_hardware_model.h from %u to %u\n", MAX_THREAD_PER_SM, n_thread_per_shader); @@ -1318,7 +1415,10 @@ struct shader_core_config : public core_config assert( !(n_thread_per_shader % warp_size) ); max_sfu_latency = 512; max_sp_latency = 32; - m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone); + + max_tensor_core_latency = 64; + + m_L1I_config.init(m_L1I_config.m_config_string,FuncCachePreferNone); m_L1T_config.init(m_L1T_config.m_config_string,FuncCachePreferNone); m_L1C_config.init(m_L1C_config.m_config_string,FuncCachePreferNone); m_L1D_config.init(m_L1D_config.m_config_string,FuncCachePreferNone); @@ -1345,7 +1445,8 @@ struct shader_core_config : public core_config unsigned max_cta_per_core; //Limit on number of concurrent CTAs in shader core unsigned max_barriers_per_cta; char * gpgpu_scheduler_string; - + unsigned gpgpu_shmem_per_block; + unsigned gpgpu_registers_per_block; char* pipeline_widths_string; int pipe_widths[N_PIPELINE_STAGES]; @@ -1361,28 +1462,38 @@ struct shader_core_config : public core_config bool gpgpu_dual_issue_diff_exec_units; //op collector + bool enable_specialized_operand_collector; int gpgpu_operand_collector_num_units_sp; int gpgpu_operand_collector_num_units_dp; int gpgpu_operand_collector_num_units_sfu; + int gpgpu_operand_collector_num_units_tensor_core; int gpgpu_operand_collector_num_units_mem; int gpgpu_operand_collector_num_units_gen; + int gpgpu_operand_collector_num_units_int; unsigned int gpgpu_operand_collector_num_in_ports_sp; unsigned int gpgpu_operand_collector_num_in_ports_dp; unsigned int gpgpu_operand_collector_num_in_ports_sfu; + unsigned int gpgpu_operand_collector_num_in_ports_tensor_core; unsigned int gpgpu_operand_collector_num_in_ports_mem; unsigned int gpgpu_operand_collector_num_in_ports_gen; + unsigned int gpgpu_operand_collector_num_in_ports_int; unsigned int gpgpu_operand_collector_num_out_ports_sp; unsigned int gpgpu_operand_collector_num_out_ports_dp; unsigned int gpgpu_operand_collector_num_out_ports_sfu; + unsigned int gpgpu_operand_collector_num_out_ports_tensor_core; unsigned int gpgpu_operand_collector_num_out_ports_mem; unsigned int gpgpu_operand_collector_num_out_ports_gen; + unsigned int gpgpu_operand_collector_num_out_ports_int; int gpgpu_num_sp_units; + int gpgpu_tensor_core_avail; int gpgpu_num_dp_units; int gpgpu_num_sfu_units; + int gpgpu_num_tensor_core_units; int gpgpu_num_mem_units; + int gpgpu_num_int_units; //Shader core resources unsigned gpgpu_shader_registers; @@ -1392,9 +1503,11 @@ struct shader_core_config : public core_config bool gpgpu_reg_bank_use_warp_id; bool gpgpu_local_mem_map; bool gpgpu_ignore_resources_limitation; + bool sub_core_model; unsigned max_sp_latency; unsigned max_sfu_latency; + unsigned max_tensor_core_latency; unsigned n_simt_cores_per_cluster; unsigned n_simt_clusters; @@ -1436,12 +1549,14 @@ struct shader_core_stats_pod { unsigned *m_num_fpdiv_acesses; unsigned *m_num_sp_acesses; unsigned *m_num_sfu_acesses; + unsigned *m_num_tensor_core_acesses; unsigned *m_num_trans_acesses; unsigned *m_num_mem_acesses; unsigned *m_num_sp_committed; unsigned *m_num_tlb_hits; unsigned *m_num_tlb_accesses; unsigned *m_num_sfu_committed; + unsigned *m_num_tensor_core_committed; unsigned *m_num_mem_committed; unsigned *m_read_regfile_acesses; unsigned *m_write_regfile_acesses; @@ -1450,12 +1565,14 @@ struct shader_core_stats_pod { unsigned *m_num_imul32_acesses; unsigned *m_active_sp_lanes; unsigned *m_active_sfu_lanes; + unsigned *m_active_tensor_core_lanes; unsigned *m_active_fu_lanes; unsigned *m_active_fu_mem_lanes; unsigned *m_n_diverge; // number of divergence occurring in this shader unsigned gpgpu_n_load_insn; unsigned gpgpu_n_store_insn; unsigned gpgpu_n_shmem_insn; + unsigned gpgpu_n_sstarr_insn; unsigned gpgpu_n_tex_insn; unsigned gpgpu_n_const_insn; unsigned gpgpu_n_param_insn; @@ -1522,6 +1639,7 @@ public: m_num_fpdiv_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sp_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sfu_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_num_tensor_core_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_trans_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_mem_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sp_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); @@ -1529,9 +1647,11 @@ public: m_num_tlb_accesses=(unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_sp_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_sfu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_active_tensor_core_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_fu_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_active_fu_mem_lanes= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_sfu_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); + m_num_tensor_core_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_num_mem_committed= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_read_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); m_write_regfile_acesses= (unsigned*) calloc(config->num_shader(),sizeof(unsigned)); @@ -1836,7 +1956,7 @@ public: } int test_res_bus(int latency); - void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread); + void init_warps(unsigned cta_id, unsigned start_thread, unsigned end_thread,unsigned ctaid, int cta_size, unsigned kernel_id); virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid); address_type next_pc( int tid ) const; void fetch(); @@ -1848,7 +1968,7 @@ public: friend class scheduler_unit; //this is needed to use private issue warp. friend class TwoLevelScheduler; friend class LooseRoundRobbinScheduler; - void issue_warp( register_set& warp, const warp_inst_t *pI, const active_mask_t &active_mask, unsigned warp_id ); + void issue_warp( register_set& warp, const warp_inst_t *pI, const active_mask_t &active_mask, unsigned warp_id, unsigned sch_id ); void func_exec_inst( warp_inst_t &inst ); // Returns numbers of addresses in translated_addrs diff --git a/src/gpgpusim_entrypoint.cc b/src/gpgpusim_entrypoint.cc index 52e2f5e..270bace 100644 --- a/src/gpgpusim_entrypoint.cc +++ b/src/gpgpusim_entrypoint.cc @@ -108,7 +108,7 @@ void *gpgpu_sim_thread_concurrent(void*) printf("GPGPU-Sim: *** simulation thread starting and spinning waiting for work ***\n"); fflush(stdout); } - while( g_stream_manager->empty() && !g_sim_done ) + while( g_stream_manager->empty_protected() && !g_sim_done ) ; if(g_debug_execution >= 3) { printf("GPGPU-Sim: ** START simulation thread (detected work) **\n"); diff --git a/src/gpuwattch/cacti/Ucache.cc b/src/gpuwattch/cacti/Ucache.cc index e855238..8f733f7 100644 --- a/src/gpuwattch/cacti/Ucache.cc +++ b/src/gpuwattch/cacti/Ucache.cc @@ -224,7 +224,6 @@ void * calc_time_mt_wrapper(void * void_obj) data_arr.pop_back(); tag_arr.pop_back(); - pthread_exit(NULL); } diff --git a/src/stream_manager.cc b/src/stream_manager.cc index 3b6cbd5..6cd62a2 100644 --- a/src/stream_manager.cc +++ b/src/stream_manager.cc @@ -188,13 +188,21 @@ bool stream_operation::do_operation( gpgpu_sim *gpu ) } break; case stream_event: { - if(g_debug_execution >= 3) - printf("event update\n"); + printf("event update\n"); time_t wallclock = time((time_t *)NULL); m_event->update( gpu_tot_sim_cycle, wallclock ); m_stream->record_next_done(); } break; + case stream_wait_event: { + //only allows next op to go if event is done + //otherwise stays in the stream queue + printf("stream wait event processing...\n"); + if(m_event->done()) + printf("stream wait event done\n"); + m_stream->record_next_done(); + } + break; default: abort(); } @@ -369,6 +377,8 @@ void stream_manager::destroy_stream( CUstream_st *stream ) bool stream_manager::concurrent_streams_empty() { bool result = true; + if (m_streams.empty()) + return true; // called by gpu simulation thread std::list<struct CUstream_st *>::iterator s; for( s=m_streams.begin(); s!=m_streams.end();++s ) { @@ -376,6 +386,7 @@ bool stream_manager::concurrent_streams_empty() if( !stream->empty() ) { //stream->print(stdout); result = false; + break; } } return result; @@ -467,3 +478,10 @@ void stream_manager::push( stream_operation op ) } } +void stream_manager::pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ){ + std::list<CUstream_st *>::iterator s; + for( s=m_streams.begin(); s != m_streams.end(); s++ ) { + stream_operation op(*s,e,flags); + push(op); + } +} diff --git a/src/stream_manager.h b/src/stream_manager.h index d3a804f..91d1b36 100644 --- a/src/stream_manager.h +++ b/src/stream_manager.h @@ -51,7 +51,8 @@ enum stream_operation_type { stream_memcpy_to_symbol, stream_memcpy_from_symbol, stream_kernel_launch, - stream_event + stream_event, + stream_wait_event }; class stream_operation { @@ -101,6 +102,14 @@ public: m_stream=stream; m_done=false; } + stream_operation( struct CUstream_st *stream, class CUevent_st *e, unsigned int flags ) + { + m_kernel=NULL; + m_type=stream_wait_event; + m_event=e; + m_stream=stream; + m_done=false; + } stream_operation( const void *host_address_src, size_t device_address_dst, size_t cnt, struct CUstream_st *stream ) { m_kernel=NULL; @@ -246,6 +255,7 @@ public: bool empty(); void print( FILE *fp); void push( stream_operation op ); + void pushCudaStreamWaitEventToAllStreams( CUevent_st *e, unsigned int flags ); bool operation(bool * sim); void stop_all_running_kernels(); private: |
